Parent Directory
| 
Revision Log
Revision 1.9 - (view) (download) (as text)
| 1 : | overbeek | 1.1 | ######################################################################## |
| 2 : | |||
| 3 : | olson | 1.8 | use strict; |
| 4 : | overbeek | 1.1 | use FIG; |
| 5 : | olson | 1.8 | use IPC::Run qw(start finish); |
| 6 : | use Cache::Memcached::Fast; | ||
| 7 : | overbeek | 1.1 | |
| 8 : | olson | 1.8 | my $usage = "usage: FFB2_build_updated_FF OldReleaseDir NewReleaseDir memcache-host memcache-port [ssfam]"; |
| 9 : | overbeek | 1.1 | |
| 10 : | olson | 1.8 | my($oldD,$newD,$mchost, $mcport); |
| 11 : | overbeek | 1.1 | |
| 12 : | ( | ||
| 13 : | ($oldD = shift @ARGV) && (-d $oldD) && | ||
| 14 : | olson | 1.8 | ($newD = shift @ARGV) && |
| 15 : | ($mchost = shift @ARGV) && | ||
| 16 : | ($mcport = shift @ARGV) | ||
| 17 : | overbeek | 1.1 | ) |
| 18 : | || die $usage; | ||
| 19 : | |||
| 20 : | olson | 1.8 | my $in_ssfam = shift; |
| 21 : | |||
| 22 : | olson | 1.3 | my $sort_args = "-T $FIG_Config::temp -S 4G"; |
| 23 : | |||
| 24 : | overbeek | 1.1 | mkdir($newD,0777) || die "You have an existing $newD; remove it and try again"; |
| 25 : | olson | 1.7 | |
| 26 : | olson | 1.8 | my $mc = new Cache::Memcached::Fast({ servers => ["$mchost:$mcport"] } ); |
| 27 : | $mc or die "Could not connect to memcached at $mchost:$mcport\n"; | ||
| 28 : | |||
| 29 : | olson | 1.9 | $mc->flush_all(); |
| 30 : | olson | 1.8 | |
| 31 : | # | ||
| 32 : | # Do some work up front to ensure we can load the memcache with all the data we need. | ||
| 33 : | # | ||
| 34 : | |||
| 35 : | my $ssfam; | ||
| 36 : | if (-f $in_ssfam) | ||
| 37 : | { | ||
| 38 : | $ssfam = $in_ssfam; | ||
| 39 : | } | ||
| 40 : | else | ||
| 41 : | { | ||
| 42 : | $ssfam = "$FIG_Config::temp/subsys.based.families.$$"; | ||
| 43 : | &FIG::run("FFB2_make_subsys_based_families > $ssfam"); | ||
| 44 : | } | ||
| 45 : | |||
| 46 : | olson | 1.9 | print "Loading memcache\n"; |
| 47 : | &FIG::run("FFB2_load_memcache $ssfam $oldD/families.2c $mchost $mcport"); | ||
| 48 : | print "Done\n"; | ||
| 49 : | olson | 1.8 | |
| 50 : | my $parallel = 1; | ||
| 51 : | olson | 1.7 | my $child_pid; |
| 52 : | if ($parallel) | ||
| 53 : | { | ||
| 54 : | # | ||
| 55 : | # Run the FF update in a child process while we manage getting the phylo oligos here. | ||
| 56 : | # | ||
| 57 : | |||
| 58 : | $child_pid = fork; | ||
| 59 : | |||
| 60 : | if ($child_pid == 0) | ||
| 61 : | { | ||
| 62 : | figfam_processing(); | ||
| 63 : | exit; | ||
| 64 : | } | ||
| 65 : | } | ||
| 66 : | else | ||
| 67 : | { | ||
| 68 : | figfam_processing(); | ||
| 69 : | } | ||
| 70 : | overbeek | 1.1 | |
| 71 : | olson | 1.8 | # |
| 72 : | # Run the new parallel get-prots | ||
| 73 : | # | ||
| 74 : | my $n_get_prots = 3; | ||
| 75 : | |||
| 76 : | my @handles; | ||
| 77 : | for my $i (0..$n_get_prots - 1) | ||
| 78 : | { | ||
| 79 : | my $outfile = "$FIG_Config::temp/ffb2_xx.$$.$i"; | ||
| 80 : | |||
| 81 : | my $h = start(['FFB2_xx', 12, "$newD/setI", $n_get_prots, $i, $mchost, $mcport], | ||
| 82 : | "<", "/dev/null", ">", $outfile); | ||
| 83 : | push(@handles, [$i, $h, $outfile]); | ||
| 84 : | } | ||
| 85 : | for my $e (@handles) | ||
| 86 : | { | ||
| 87 : | my($i, $h, $file) = @$e; | ||
| 88 : | print "Wait for finish $i $file\n"; | ||
| 89 : | $h->finish(); | ||
| 90 : | } | ||
| 91 : | |||
| 92 : | my @files = map { $_->[2] } @handles; | ||
| 93 : | print STDERR "Merging phylo @files\n"; | ||
| 94 : | &FIG::run("sort -m @files | FFB2_usable_motifs $newD/PhyloSigs"); | ||
| 95 : | |||
| 96 : | #&FIG::run("FFB2_get_prots $newD/ProtsByGenomeSet $newD/setI"); | ||
| 97 : | #&FIG::run("FFB2_get_prot_gs_oligos $newD/ProtsByGenomeSet 12 | sort $sort_args | FFB2_usable_motifs $newD/PhyloSigs"); | ||
| 98 : | olson | 1.7 | |
| 99 : | if ($parallel) | ||
| 100 : | { | ||
| 101 : | my $rc = waitpid($child_pid, 0); | ||
| 102 : | if ($? != 0) | ||
| 103 : | { | ||
| 104 : | die "Error running parallel child: $?\n"; | ||
| 105 : | } | ||
| 106 : | } | ||
| 107 : | |||
| 108 : | |||
| 109 : | overbeek | 1.1 | mkdir("$newD/Merged",0777) || die "could not make $newD/Merged: $!"; |
| 110 : | olson | 1.8 | |
| 111 : | my @merge_handles; | ||
| 112 : | |||
| 113 : | overbeek | 1.1 | foreach my $i (7..12) |
| 114 : | { | ||
| 115 : | mkdir("$newD/Merged/$i",0777) || die "could not make $newD/Merged/$i: $!"; | ||
| 116 : | olson | 1.8 | my $h = start(["FFB2_merge_oligos $newD/Kmers/$i/good.oligos $newD/PhyloSigs/$i/good.oligos"], "|", |
| 117 : | ["FFB2_create_binary_kmers", "-l", $i, "-s", "4,2", "-", "$newD/Merged/$i/table.binary"]); | ||
| 118 : | |||
| 119 : | push(@merge_handles, [$h, $i]); | ||
| 120 : | |||
| 121 : | # &FIG::run("FFB2_merge_oligos $newD/Kmers/$i/good.oligos $newD/PhyloSigs/$i/good.oligos > $newD/Merged/$i/merged.oligos"); | ||
| 122 : | overbeek | 1.1 | } |
| 123 : | |||
| 124 : | olson | 1.8 | for my $ent (@merge_handles) |
| 125 : | olson | 1.2 | { |
| 126 : | olson | 1.8 | my($h, $k) = @$ent; |
| 127 : | print "Wait for finish for k=$k\n"; | ||
| 128 : | my $r = $h->finish(); | ||
| 129 : | print "done r=$r\n"; | ||
| 130 : | olson | 1.2 | } |
| 131 : | olson | 1.5 | |
| 132 : | olson | 1.8 | my $load1 = start(["FFB2_load_oligo_index", "$newD/function.index", "$newD/FRI.db"]); |
| 133 : | my $load2 = start(["FFB2_load_oligo_index", "$newD/setI", "$newD/setI.db"]); | ||
| 134 : | my $build = start(["build_prok_nonff_fasta $newD > $newD/extra_prok_seqs.fasta"]); | ||
| 135 : | |||
| 136 : | |||
| 137 : | #foreach my $i (7..12) | ||
| 138 : | #{ | ||
| 139 : | # &FIG::run("FFB2_create_binary_kmers -s 4,2 $newD/Merged/$i/merged.oligos $newD/Merged/$i/table.binary"); | ||
| 140 : | #} | ||
| 141 : | |||
| 142 : | olson | 1.7 | system("FFB2_run_tests $newD"); |
| 143 : | system("FFB2_compare_tests $oldD $newD"); | ||
| 144 : | olson | 1.8 | $build->finish(); |
| 145 : | overbeek | 1.4 | &FIG::run("formatdb -p T -i $newD/extra_prok_seqs.fasta"); |
| 146 : | overbeek | 1.6 | |
| 147 : | olson | 1.8 | $load1->finish(); |
| 148 : | $load2->finish(); | ||
| 149 : | |||
| 150 : | overbeek | 1.6 | &FIG::run("FFB2_make_FF_index $newD $newD/fam.func.index $newD/FamFuncBlastD"); |
| 151 : | olson | 1.7 | &FIG::run("FFB2_build_ff_indexes $newD"); |
| 152 : | |||
| 153 : | sub figfam_processing | ||
| 154 : | { | ||
| 155 : | olson | 1.8 | &FIG::run("FFB2_update_FIGfams -f $ssfam $newD/families.2c $newD/family.functions $oldD"); |
| 156 : | olson | 1.7 | &FIG::run("FFB2_get_oligos $newD/families.2c $newD/family.functions 2> $newD/function.index |" . |
| 157 : | "sort $sort_args | FFB2_usable_motifs $newD/Kmers"); | ||
| 158 : | } | ||
| 159 : | |||
| 160 : |
| MCS Webmaster | ViewVC Help |
| Powered by ViewVC 1.0.3 |