[Bio] / FortyEightMeta / mg_check_sims.pl Repository:
ViewVC logotype

View of /FortyEightMeta/mg_check_sims.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.2 - (download) (as text) (annotate)
Thu Jun 12 21:35:53 2008 UTC (11 years, 7 months ago) by olson
Branch: MAIN
CVS Tags: mgrast_rel_2008_0806, mgrast_rel_2008_0923, mgrast_rel_2008_0924, mgrast_rel_2008_1110_v2, mgrast_rel_2008_0625, mgrast_rel_2008_0919, mgrast_rel_2008_1110, mgrast_rel_2008_0917
Changes since 1.1: +3 -6 lines
Replace the MLDBM/DB_File sim status database with a postgres version.

#
# Check on the status of the sims runs.
#
# If there are sims that have completed and aren't postprocessed yet,
# schedule a database loading run.
#
# When all sims are done running, submit a postproc_taxa_sims run
# that is held on the set of postprocessing runs that are still going.
#

use strict;
use FIG;
use FIG_Config;
use File::Basename;
use Data::Dumper;
use GenomeMeta;
use Job48;
use JobStage;
use SGE;
use FortyEightMeta::SimDB;
use FortyEightMeta::SimStatusDB;

my $STAGE = "check_sims";

@ARGV == 1 or die "Usage: $0 job-dir\n";

my $jobdir = shift;

-d $jobdir or die "$0: job dir $jobdir does not exist\n";

my $stage = new JobStage('Job48', $STAGE, $jobdir);
$stage or die "Cannot create job for $jobdir\n";

my $job_id = basename($jobdir);
my $job = $stage->job();

my $meta = $job->meta;

print "Running job! $jobdir\n";

my $load_exe = "$FIG_Config::bin/mg_load_sims";
-x $load_exe or $stage->fatal("Executable missing: $load_exe");

my $load_done_exe = "$FIG_Config::bin/mg_load_done";
-x $load_done_exe or $stage->fatal("Executable missing: $load_done_exe");

$stage->set_status("in_progress");

my $sge = new SGE;

my $proc = "$jobdir/proc";
chdir($proc) or $stage->fatal("cannot chdir $proc: $!");

my $status_db = FortyEightMeta::SimStatusDB->new($job_id);

my @completed = $status_db->get_tasks_with_status('complete');

for my $jent (@completed)
{
    my($sjob, $task, $rec) = @$jent;

    print "Checking $sjob $task " . Dumper($rec);

    if ($rec->{success})
    {
	#
	# Schedule the database load.
	#
	
 	my @sge_args;
	my $jobname = "mgl_$rec->{abbr}_${task}_$job_id";
	
	push(@sge_args, "-N $jobname");
	push(@sge_args, "-v PATH");
	push(@sge_args, "-e $jobdir/sge_output");
	push(@sge_args, "-o $jobdir/sge_output");
	push(@sge_args, "-b yes");

	push(@sge_args, "-l mg_postgres_load=1");

	my $sge_args = join(" ", @sge_args);
	
	my $sge_id;
	
	eval {
	    $sge_id = $sge->submit_job($meta, $sge_args, "$load_exe $jobdir $rec->{work_dir} $task");
	};

	if ($@)
	{
	    $stage->fatal("Error submitting sge job: $@");
	}
	$rec->{load_sge_id} = $sge_id;
	$jent->[3] = 'load_queued';
    }
    else
    {
	$rec->{error_message} = "Flagged as error in $STAGE: success=0";
	$jent->[3] = 'error';
    }
}

for my $jent (@completed)
{
    my($sjob, $task, $rec, $new_status) = @$jent;

    print "Updating $sjob $task " . Dumper($rec);

    $rec->{status} = $new_status;
    $status_db->set_task($sjob, $task, $rec);
}

my @not_started = $status_db->get_tasks_with_status('not_started');
my @running = $status_db->get_tasks_with_status('in_progress');
my @completed = $status_db->get_tasks_with_status('complete');

my $n_not_started = @not_started;
my $n_running = @running;
my $n_completed = @completed;

print "not_started=$n_not_started running=$n_running completed=$n_completed\n";

#
# If there are no sims tasks that are either not started yet, currently
# running, or completed (remember completed means the computation has
# finished but no postprocessing has run yet), we are done here.
#
# Schedule a job that is held on the SGE ids of all of the database
# load jobs; this job will verify that the loads have been
# completed successfully and allow the pipeline to continue.
#
if ($n_not_started == 0 and $n_running == 0 and $n_completed == 0)
{
    my @jobs = map { $status_db->get_tasks_with_status($_, 1) } qw(load_queued load_running);
    undef $status_db;

    my @ids;
    print "Setting up load-finished task\n";
    for my $jent (@jobs)
    {
	my($sjob, $task, $rec) = @$jent;
	my $sge_id = $rec->{load_sge_id};
	print "$sjob $task: sge_id = $sge_id\n";
	push(@ids, $sge_id);
    }

    my @sge_args;
    
    push(@sge_args, "-N mglc_$job_id");
    push(@sge_args, "-v PATH");
    push(@sge_args, "-e $jobdir/sge_output");
    push(@sge_args, "-o $jobdir/sge_output");
    push(@sge_args, "-b yes");
    push(@sge_args, "-l high");
    push(@sge_args, "-hold_jid " . join(",", @ids));
    
    my $sge_args = join(" ", @sge_args);
    
    my $sge_id;
    
    eval {
	
	$sge_id = $sge->submit_job($meta, $sge_args, "$load_done_exe $jobdir");
	#print "would submit '$sge_args' '$jobdir'\n";
	
    };
    
    if ($@)
    {
	$stage->fatal($meta, "error starting SGE job $load_done_exe $jobdir: $@\n");
    }

    $stage->set_status("complete");
}
else
{
    undef $status_db;
    $stage->set_status("not_started");
}

$stage->set_running("no");


MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3