[Bio] / FortyEightMeta / merge_repaired_job_results.pl Repository:
ViewVC logotype

View of /FortyEightMeta/merge_repaired_job_results.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.1 - (download) (as text) (annotate)
Fri May 30 23:23:51 2008 UTC (11 years, 10 months ago) by olson
Branch: MAIN
CVS Tags: mgrast_dev_08112011, mgrast_dev_08022011, mgrast_dev_05262011, mgrast_dev_04082011, mgrast_version_3_2, mgrast_dev_12152011, mgrast_dev_06072011, mgrast_rel_2008_0806, mgrast_dev_10262011, mgrast_dev_02212011, mgrast_rel_2008_0923, mgrast_release_3_0, mgrast_dev_03252011, mgrast_rel_2008_0924, mgrast_rel_2008_1110_v2, mgrast_rel_2008_0625, mgrast_release_3_0_4, mgrast_release_3_0_2, mgrast_release_3_0_3, mgrast_release_3_0_1, mgrast_dev_03312011, mgrast_release_3_1_2, mgrast_release_3_1_1, mgrast_release_3_1_0, mgrast_dev_04132011, mgrast_dev_04012011, mgrast_rel_2008_0919, mgrast_rel_2008_1110, myrast_33, mgrast_rel_2008_0917, mgrast_dev_04052011, mgrast_dev_02222011, HEAD
Development checkin of new metagenomics RAST pipeline.

#
# On a repaired job, see which of the chunks were repaired successfully. Move their
# err files into the main err files for the job.
#

use strict;
use FIG;
use FIG_Config;
use File::Basename;
use GenomeMeta;
use Job48;
use SGE;
use Carp;

my $STAGE = "sims";

@ARGV == 1 or die "Usage: $0 job-dir\n";

my $jobdir = shift;

if ($jobdir =~ /^\d+$/)
{
    $jobdir = "/vol/metagenome-48-hour/Jobs.prod/$jobdir";
}

-d $jobdir or die "$0: job dir $jobdir does not exist\n";

my $job_id = basename($jobdir);
my $job = new Job48($jobdir);

my $meta = $job->meta;

#
# Set of sims jobs to run.
#

my $opts = "-m 8 -e 0.1";
my @jobs = (["sims.gg",   10000, "/vol/metagenome-48-hour/Data/greengenes.fasta", "blastn", $opts],
	    ["sims.lsu", 450000, "/vol/metagenome-48-hour/Data/lsu.fa", "blastn", $opts],
	    ["sims.ssu",  45000, "/vol/metagenome-48-hour/Data/ssu.fa", "blastn", $opts],
	    ["sims.16s",  10000, "/vol/metagenome-48-hour/Data/16s.fa", "blastn", $opts],
#	    ["sims.seed",   500, "/home/redwards/databases/SEED_2006_07_01", "blastx", $opts],
	    ["sims.seed",   500, "/scratch/metagenome-48-hour/Data/SEED_2006_07_01", "blastx", $opts],
	   );

my $proc = "$jobdir/proc";
chdir($proc) or &fatal("cannot chdir $proc: $!");

for my $simj (@jobs)
{
    my($dir, $nseqs, $nr, $blastp, $opts) = @$simj;
    my $path = "$proc/$dir";
    -d $path or &fatal("Sims directory $path does not exist\n");

    my $repair = "$path/sims.repair";
    -d $repair or next;

    my $repair_err = "$path/sims.repair/err";
    -d $repair_err or die "missing $repair_err: $!";

    open(T, "<$path/task.list") or die "Cannot open $path/task.list: $!";
    my %in_to_err;
    while (my $tline = <T>)
    {
	chomp $tline;
	my($task, $in, $nr, $flags, $out, $err) = split(/\t/, $tline);
	$in_to_err{$in} = $err;
    }
    close(T);

    open(T, "<$repair/task.list") or die "Cannot open $repair/task.list: $!";

    my $bad = 0;

    while (my $tline = <T>)
    {
	chomp $tline;
	my($task, $in, $nr, $flags, $out, $err) = split(/\t/, $tline);

	if (!open(E, "<$err"))
	{
	    warn "cannot open err file $err: $!";
	    $bad++;
	    next;
	}

	my $fatal;
	my $success;
	while (<E>)
	{
	    if (/FATAL/)
	    {
		$fatal++;
		last;
	    }
	    elsif (/SUCCESS/)
	    {
		$success++;
	    }
	}
	close(E);
	if ($fatal or !$success)
	{
	    print  "repair task $task bad fatal=$fatal success=$success\n";
	    system("cat", $err);
	    $bad++;
	}
	else
	{
	    #
	    # Move the good err file into the original err file location.
	    #
	    my $orig = $in_to_err{$in};
	    my @s = stat($orig);
	    my $obak = "$orig.$s[9]";
	    rename($orig, $obak) or die "error moving $orig to $obak: $!";
	    rename($err, $orig) or die "Error moving $err to $orig: $!";
	}
    }

    close(T);

    if (!$bad)
    {
	#
	# Everything good, move the repair dir out of the way.
	#
	my @s = stat($repair);
	my $new = "$repair.$s[9]";
	rename($repair, $new)  or die "Error mvoing $repair to $new: $!";
	print "repaired $job_id\n";
    }
}

sub run
{
    my(@cmd) = @_;
    
    print "Run @cmd\n";
    my $rc = system(@cmd);
    if ($rc != 0)
    {
	&fatal("Failed with rc=$rc: @cmd");
    }
}

sub fatal
{
    my($msg) = @_;

    $meta->add_log_entry($0, ['fatal error', $msg]);
    $meta->set_metadata("status.$STAGE", "error");

    croak "$0: $msg";
}
    

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3