Parent Directory
| 
Revision Log
Revision 1.9 - (view) (download) (as text)
| 1 : | olson | 1.1 | |
| 2 : | use strict; | ||
| 3 : | use Carp; | ||
| 4 : | use Job48; | ||
| 5 : | use FIG_Config; | ||
| 6 : | use FIG; | ||
| 7 : | olson | 1.3 | use Getopt::Long; |
| 8 : | olson | 1.1 | |
| 9 : | # | ||
| 10 : | # Run a jobdirectory in one shot. For batch offload to a remote cluster that | ||
| 11 : | # doesn't have our scheduler, etc available. | ||
| 12 : | # | ||
| 13 : | |||
| 14 : | # | ||
| 15 : | # Stages are as follows; for now this is a copy and paste exercise from | ||
| 16 : | # FortyEight/check_jobs.pl. Use caution, don't run with scissors. | ||
| 17 : | # | ||
| 18 : | |||
| 19 : | # | ||
| 20 : | # upload | ||
| 21 : | # rp | ||
| 22 : | # Check status of keep_genecalls, then qc | ||
| 23 : | # Check status of correction, then correction | ||
| 24 : | # preprocess_sims | ||
| 25 : | # sims | ||
| 26 : | # bbhs | ||
| 27 : | # auto_assign | ||
| 28 : | # glue_contigs | ||
| 29 : | # pchs | ||
| 30 : | # scenario | ||
| 31 : | # export | ||
| 32 : | # | ||
| 33 : | |||
| 34 : | olson | 1.3 | my $parallel = 1; |
| 35 : | olson | 1.4 | my @phase; |
| 36 : | olson | 1.2 | |
| 37 : | olson | 1.4 | my $usage = "Usage: $0 [--parallel N] -phase N [--phase N ..] jobdir\n"; |
| 38 : | olson | 1.3 | |
| 39 : | if (!GetOptions("parallel=i" => \$parallel, | ||
| 40 : | olson | 1.4 | "phase=i" => \@phase)) |
| 41 : | olson | 1.3 | { |
| 42 : | die $usage; | ||
| 43 : | } | ||
| 44 : | |||
| 45 : | @ARGV == 1 or die $usage; | ||
| 46 : | olson | 1.1 | |
| 47 : | olson | 1.4 | @phase > 0 or die $usage; |
| 48 : | my %phase = map { $_ => 1 } @phase; | ||
| 49 : | |||
| 50 : | olson | 1.1 | my $job_dir = shift; |
| 51 : | |||
| 52 : | my $job = new Job48($job_dir); | ||
| 53 : | |||
| 54 : | olson | 1.2 | my $sims_data_dir = $FIG_Config::rast_sims_data; |
| 55 : | |||
| 56 : | if (!defined($sims_data_dir)) | ||
| 57 : | { | ||
| 58 : | $sims_data_dir = $FIG_Config::fortyeight_data; | ||
| 59 : | } | ||
| 60 : | |||
| 61 : | my $sims_nr = "$sims_data_dir/nr"; | ||
| 62 : | my $sims_peg_synonyms = "$sims_data_dir/peg.synonyms"; | ||
| 63 : | my $sims_keep_count = 300; | ||
| 64 : | |||
| 65 : | olson | 1.7 | my $job48 = new Job48($job_dir); |
| 66 : | my $meta = $job48->meta; | ||
| 67 : | |||
| 68 : | my $host = `hostname`; | ||
| 69 : | chomp $host; | ||
| 70 : | $meta->add_log_entry($0, "Running phases @phase on $host"); | ||
| 71 : | |||
| 72 : | olson | 1.4 | # |
| 73 : | # Emulate execution of SGE parallel environment via the | ||
| 74 : | # --parallel N argument. | ||
| 75 : | # | ||
| 76 : | olson | 1.3 | if ($parallel > 1) |
| 77 : | { | ||
| 78 : | $ENV{PE} = 'cluster'; | ||
| 79 : | $ENV{NSLOTS} = $parallel; | ||
| 80 : | } | ||
| 81 : | olson | 1.2 | |
| 82 : | olson | 1.5 | if ($phase{1}) |
| 83 : | olson | 1.3 | { |
| 84 : | &do_upload($job); | ||
| 85 : | &do_rp($job); | ||
| 86 : | } | ||
| 87 : | |||
| 88 : | olson | 1.5 | if ($phase{2}) |
| 89 : | olson | 1.3 | { |
| 90 : | &do_qc($job); | ||
| 91 : | &do_correction($job); | ||
| 92 : | &do_sims_preprocess($job); | ||
| 93 : | } | ||
| 94 : | |||
| 95 : | olson | 1.5 | if ($phase{3}) |
| 96 : | olson | 1.3 | { |
| 97 : | olson | 1.4 | # |
| 98 : | # If running inside a SGE task array job, execute | ||
| 99 : | # our task. Otherwise run all of them. | ||
| 100 : | # | ||
| 101 : | if ($ENV{SGE_TASK_ID}) | ||
| 102 : | { | ||
| 103 : | &run("$FIG_Config::bin/rp_compute_sims", $job->dir); | ||
| 104 : | } | ||
| 105 : | else | ||
| 106 : | { | ||
| 107 : | &do_sims($job); | ||
| 108 : | } | ||
| 109 : | olson | 1.3 | } |
| 110 : | |||
| 111 : | olson | 1.5 | if ($phase{4}) |
| 112 : | olson | 1.3 | { |
| 113 : | olson | 1.6 | &do_sims_postprocess($job); |
| 114 : | olson | 1.3 | &do_bbhs($job); |
| 115 : | &do_auto_assign($job); | ||
| 116 : | &do_glue_contigs($job); | ||
| 117 : | &do_pchs($job); | ||
| 118 : | &do_scenario($job); | ||
| 119 : | &do_export($job); | ||
| 120 : | olson | 1.8 | &mark_job_user_complete($job); |
| 121 : | olson | 1.3 | } |
| 122 : | olson | 1.1 | |
| 123 : | sub do_upload | ||
| 124 : | { | ||
| 125 : | my($job) = @_; | ||
| 126 : | return; | ||
| 127 : | } | ||
| 128 : | |||
| 129 : | sub do_rp | ||
| 130 : | { | ||
| 131 : | my($job) = @_; | ||
| 132 : | &run("$FIG_Config::bin/rp_rapid_propagation", $job->dir); | ||
| 133 : | } | ||
| 134 : | |||
| 135 : | sub do_qc | ||
| 136 : | { | ||
| 137 : | my($job) = @_; | ||
| 138 : | |||
| 139 : | if ($job->meta->get_metadata("keep_genecalls")) | ||
| 140 : | { | ||
| 141 : | $job->meta->add_log_entry($0, "keep_genecalls is enabled: marking qc as complete"); | ||
| 142 : | $job->meta->set_metadata("status.qc", "complete"); | ||
| 143 : | return; | ||
| 144 : | } | ||
| 145 : | |||
| 146 : | &run("$FIG_Config::bin/rp_quality_check", $job->dir); | ||
| 147 : | } | ||
| 148 : | |||
| 149 : | sub do_correction | ||
| 150 : | { | ||
| 151 : | my($job) = @_; | ||
| 152 : | |||
| 153 : | if ($job->meta->get_metadata("keep_genecalls")) | ||
| 154 : | { | ||
| 155 : | $job->meta->add_log_entry($0, "keep_genecalls is enabled: marking correction as complete"); | ||
| 156 : | $job->meta->set_metadata("status.correction", "complete"); | ||
| 157 : | return; | ||
| 158 : | } | ||
| 159 : | |||
| 160 : | my $correction_list = $job->meta->get_metadata("correction.request"); | ||
| 161 : | olson | 1.3 | |
| 162 : | if (ref($correction_list)) | ||
| 163 : | { | ||
| 164 : | my $correction_str = join(",", @$correction_list); | ||
| 165 : | &run("$FIG_Config::bin/rp_correction", $job->dir, $correction_str); | ||
| 166 : | } | ||
| 167 : | olson | 1.1 | } |
| 168 : | |||
| 169 : | sub do_sims_preprocess | ||
| 170 : | { | ||
| 171 : | my($job) = @_; | ||
| 172 : | olson | 1.2 | |
| 173 : | &run("$FIG_Config::bin/rp_preprocess_sims", $job->dir, $sims_nr, $sims_peg_synonyms); | ||
| 174 : | |||
| 175 : | olson | 1.1 | } |
| 176 : | |||
| 177 : | sub do_sims | ||
| 178 : | { | ||
| 179 : | my($job) = @_; | ||
| 180 : | olson | 1.2 | |
| 181 : | if (!open(CHUNK, "<", $job->dir. "/sims.job/chunk.out")) | ||
| 182 : | { | ||
| 183 : | die "Error opening $job_dir/sims.job/chunk.out: $!"; | ||
| 184 : | } | ||
| 185 : | |||
| 186 : | # | ||
| 187 : | # Extract created task ids | ||
| 188 : | # | ||
| 189 : | |||
| 190 : | my($task_start, $task_end); | ||
| 191 : | while (<CHUNK>) | ||
| 192 : | { | ||
| 193 : | print; | ||
| 194 : | chomp; | ||
| 195 : | if (/^tasks\s+(\d+)\s+(\d+)/) | ||
| 196 : | { | ||
| 197 : | $task_start = $1; | ||
| 198 : | $task_end = $2; | ||
| 199 : | } | ||
| 200 : | } | ||
| 201 : | close(CHUNK); | ||
| 202 : | |||
| 203 : | if (!defined($task_start)) | ||
| 204 : | { | ||
| 205 : | die "Tasks not found"; | ||
| 206 : | } | ||
| 207 : | |||
| 208 : | olson | 1.6 | for my $task ($task_start .. $task_end) |
| 209 : | { | ||
| 210 : | $ENV{SGE_TASK_ID} = $task; | ||
| 211 : | &run("$FIG_Config::bin/rp_compute_sims", $job->dir); | ||
| 212 : | } | ||
| 213 : | } | ||
| 214 : | |||
| 215 : | sub do_sims_postprocess | ||
| 216 : | { | ||
| 217 : | my($job) = @_; | ||
| 218 : | |||
| 219 : | olson | 1.2 | my $sims_nr_len = $sims_nr; |
| 220 : | if (-f "$sims_nr-len.btree") | ||
| 221 : | { | ||
| 222 : | $sims_nr_len = "$sims_nr-len.btree"; | ||
| 223 : | } | ||
| 224 : | |||
| 225 : | olson | 1.6 | &run("$FIG_Config::bin/rp_postproc_sims", $job->dir, $sims_nr_len, $sims_peg_synonyms, $sims_keep_count); |
| 226 : | olson | 1.1 | } |
| 227 : | |||
| 228 : | sub do_bbhs | ||
| 229 : | { | ||
| 230 : | my($job) = @_; | ||
| 231 : | olson | 1.2 | &run("$FIG_Config::bin/rp_compute_bbhs", $job->dir); |
| 232 : | olson | 1.1 | } |
| 233 : | |||
| 234 : | sub do_auto_assign | ||
| 235 : | { | ||
| 236 : | my($job) = @_; | ||
| 237 : | olson | 1.2 | &run("$FIG_Config::bin/rp_auto_assign", $job->dir); |
| 238 : | olson | 1.1 | } |
| 239 : | |||
| 240 : | sub do_glue_contigs | ||
| 241 : | { | ||
| 242 : | my($job) = @_; | ||
| 243 : | olson | 1.2 | &run("$FIG_Config::bin/rp_glue_contigs", $job->dir); |
| 244 : | olson | 1.1 | } |
| 245 : | |||
| 246 : | sub do_pchs | ||
| 247 : | { | ||
| 248 : | my($job) = @_; | ||
| 249 : | olson | 1.2 | &run("$FIG_Config::bin/rp_compute_pchs", $job->dir); |
| 250 : | olson | 1.1 | } |
| 251 : | |||
| 252 : | sub do_scenario | ||
| 253 : | { | ||
| 254 : | my($job) = @_; | ||
| 255 : | olson | 1.2 | &run("$FIG_Config::bin/rp_scenarios", $job->dir); |
| 256 : | olson | 1.1 | } |
| 257 : | |||
| 258 : | sub do_export | ||
| 259 : | { | ||
| 260 : | my($job) = @_; | ||
| 261 : | olson | 1.2 | &run("$FIG_Config::bin/rp_write_exports", $job->dir); |
| 262 : | olson | 1.1 | } |
| 263 : | |||
| 264 : | olson | 1.8 | sub mark_job_user_complete |
| 265 : | { | ||
| 266 : | my($job) = @_; | ||
| 267 : | |||
| 268 : | my $job_dir = $job->dir; | ||
| 269 : | my $meta = $job->meta; | ||
| 270 : | my $job_id = $job->id; | ||
| 271 : | |||
| 272 : | system("$FIG_Config::bin/send_job_completion_email", $job_dir); | ||
| 273 : | |||
| 274 : | $meta->set_metadata("status.final", "complete"); | ||
| 275 : | |||
| 276 : | # | ||
| 277 : | # If the job is a SEED candidate, send VV email. | ||
| 278 : | # | ||
| 279 : | |||
| 280 : | if ($meta->get_metadata("import.suggested") or | ||
| 281 : | $meta->get_metadata("import.candidate")) | ||
| 282 : | { | ||
| 283 : | my $gname = $job->genome_name; | ||
| 284 : | my $mail = Mail::Mailer->new(); | ||
| 285 : | $mail->open({ | ||
| 286 : | To => 'Veronika Vonstein <veronika@thefig.info>, Robert Olson<olson@mcs.anl.gov>, Andreas Wilke<wilke@mcs.anl.gov>', | ||
| 287 : | From => 'Annotation Server <rast@mcs.anl.gov>', | ||
| 288 : | Subject => "RAST job $job_id marked for SEED inclusion", | ||
| 289 : | }); | ||
| 290 : | |||
| 291 : | print $mail <<END; | ||
| 292 : | RAST job #$job_id ($gname) was submitted for inclusion into the SEED, and has finished its processing. | ||
| 293 : | END | ||
| 294 : | $mail->close(); | ||
| 295 : | |||
| 296 : | # | ||
| 297 : | # We also mark the job as ACTIVE again so that the | ||
| 298 : | # normal post-seed-acceptance pipeline stages may execute. | ||
| 299 : | # | ||
| 300 : | open(F, ">$job_dir/ACTIVE"); | ||
| 301 : | close(F); | ||
| 302 : | } | ||
| 303 : | else | ||
| 304 : | { | ||
| 305 : | # | ||
| 306 : | # Otherwise it is completely done. | ||
| 307 : | # | ||
| 308 : | &mark_job_done($job); | ||
| 309 : | } | ||
| 310 : | } | ||
| 311 : | |||
| 312 : | sub mark_job_done | ||
| 313 : | { | ||
| 314 : | my($job) = @_; | ||
| 315 : | |||
| 316 : | # | ||
| 317 : | # If we spooled the job out onto the lustre disk, we need to | ||
| 318 : | # spool it back. | ||
| 319 : | # | ||
| 320 : | |||
| 321 : | my $meta = $job->meta; | ||
| 322 : | my $job_dir = $job->dir; | ||
| 323 : | |||
| 324 : | if ($meta->get_metadata("lustre_required")) | ||
| 325 : | { | ||
| 326 : | &run("$FIG_Config::bin/rp_lustre_finish", $job_dir); | ||
| 327 : | } | ||
| 328 : | if (open(D, ">$job_dir/DONE")) | ||
| 329 : | { | ||
| 330 : | print D time . "\n"; | ||
| 331 : | close(D); | ||
| 332 : | } | ||
| 333 : | else | ||
| 334 : | { | ||
| 335 : | warn "Error opening $job_dir/DONE: $!\n"; | ||
| 336 : | } | ||
| 337 : | |||
| 338 : | unlink("$job_dir/ACTIVE"); | ||
| 339 : | } | ||
| 340 : | olson | 1.1 | |
| 341 : | sub run | ||
| 342 : | { | ||
| 343 : | my(@cmd) = @_; | ||
| 344 : | |||
| 345 : | olson | 1.9 | my $cmd_str = join(" ", @cmd); |
| 346 : | print "Start: $cmd_str\n"; | ||
| 347 : | $meta->add_log_entry($0, ['Start', $cmd_str)]); | ||
| 348 : | olson | 1.1 | my $rc = system(@cmd); |
| 349 : | if ($rc != 0) | ||
| 350 : | { | ||
| 351 : | olson | 1.9 | $meta->add_log_entry($0, ['Failed', $rc, $cmd_str]); |
| 352 : | confess "Cmd failed with rc=$rc: $cmd_str\n"; | ||
| 353 : | olson | 1.1 | } |
| 354 : | olson | 1.9 | $meta->add_log_entry($0, ['Done', $cmd_str]); |
| 355 : | print "Done: $cmd_str\n"; | ||
| 356 : | olson | 1.1 | } |
| MCS Webmaster | ViewVC Help |
| Powered by ViewVC 1.0.3 |