[Bio] / Sprout / SproutLoad.pm Repository:
ViewVC logotype

Diff of /Sprout/SproutLoad.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.32, Sun Apr 2 17:34:44 2006 UTC revision 1.48, Fri Jul 7 00:24:16 2006 UTC
# Line 30  Line 30 
30      $stats->Accumulate($spl->LoadFeatureData());      $stats->Accumulate($spl->LoadFeatureData());
31      print $stats->Show();      print $stats->Show();
32    
 This module makes use of the internal Sprout property C<_erdb>.  
   
33  It is worth noting that the FIG object does not need to be a real one. Any object  It is worth noting that the FIG object does not need to be a real one. Any object
34  that implements the FIG methods for data retrieval could be used. So, for example,  that implements the FIG methods for data retrieval could be used. So, for example,
35  this object could be used to copy data from one Sprout database to another, or  this object could be used to copy data from one Sprout database to another, or
# Line 80  Line 78 
78  =item subsysFile  =item subsysFile
79    
80  Either the name of the file containing the list of trusted subsystems or a reference  Either the name of the file containing the list of trusted subsystems or a reference
81  to a list of subsystem names. If nothing is specified, all known subsystems will be  to a list of subsystem names. If nothing is specified, all NMPDR subsystems will be
82  considered trusted. Only subsystem data related to the trusted subsystems is loaded.  considered trusted. (A subsystem is considered NMPDR if it has a file named C<NMPDR>
83    in its data directory.) Only subsystem data related to the trusted subsystems is loaded.
84    
85  =item options  =item options
86    
# Line 94  Line 93 
93  sub new {  sub new {
94      # Get the parameters.      # Get the parameters.
95      my ($class, $sprout, $fig, $genomeFile, $subsysFile, $options) = @_;      my ($class, $sprout, $fig, $genomeFile, $subsysFile, $options) = @_;
96      # Load the list of genomes into a hash.      # Create the genome hash.
97      my %genomes;      my %genomes = ();
98        # We only need it if load-only is NOT specified.
99        if (! $options->{loadOnly}) {
100      if (! defined($genomeFile) || $genomeFile eq '') {      if (! defined($genomeFile) || $genomeFile eq '') {
101          # Here we want all the complete genomes and an access code of 1.          # Here we want all the complete genomes and an access code of 1.
102          my @genomeList = $fig->genomes(1);          my @genomeList = $fig->genomes(1);
# Line 129  Line 130 
130              Confess("Invalid genome parameter ($type) in SproutLoad constructor.");              Confess("Invalid genome parameter ($type) in SproutLoad constructor.");
131          }          }
132      }      }
133        }
134      # Load the list of trusted subsystems.      # Load the list of trusted subsystems.
135      my %subsystems = ();      my %subsystems = ();
136        # We only need it if load-only is NOT specified.
137        if (! $options->{loadOnly}) {
138      if (! defined $subsysFile || $subsysFile eq '') {      if (! defined $subsysFile || $subsysFile eq '') {
139          # Here we want all the subsystems.              # Here we want all the NMPDR subsystems. First we get the whole list.
140          %subsystems = map { $_ => 1 } $fig->all_subsystems();              my @subs = $fig->all_subsystems();
141                # Loop through, checking for the NMPDR file.
142                for my $sub (@subs) {
143                    if (-e "$FIG_Config::data/Subsystems/$sub/NMPDR") {
144                        $subsystems{$sub} = 1;
145                    }
146                }
147      } else {      } else {
148          my $type = ref $subsysFile;          my $type = ref $subsysFile;
149          if ($type eq 'ARRAY') {          if ($type eq 'ARRAY') {
# Line 153  Line 163 
163              Confess("Invalid subsystem parameter in SproutLoad constructor.");              Confess("Invalid subsystem parameter in SproutLoad constructor.");
164          }          }
165      }      }
166        }
167      # Get the data directory from the Sprout object.      # Get the data directory from the Sprout object.
168      my ($directory) = $sprout->LoadInfo();      my ($directory) = $sprout->LoadInfo();
169      # Create the Sprout load object.      # Create the Sprout load object.
# Line 162  Line 173 
173                    subsystems => \%subsystems,                    subsystems => \%subsystems,
174                    sprout => $sprout,                    sprout => $sprout,
175                    loadDirectory => $directory,                    loadDirectory => $directory,
176                    erdb => $sprout->{_erdb},                    erdb => $sprout,
177                    loaders => [],                    loaders => [],
178                    options => $options                    options => $options
179                   };                   };
# Line 364  Line 375 
375                  for my $coupleData (@couplings) {                  for my $coupleData (@couplings) {
376                      my ($peg2, $score) = @{$coupleData};                      my ($peg2, $score) = @{$coupleData};
377                      # Compute the coupling ID.                      # Compute the coupling ID.
378                      my $coupleID = Sprout::CouplingID($peg1, $peg2);                      my $coupleID = $self->{erdb}->CouplingID($peg1, $peg2);
379                      if (! exists $dupHash{$coupleID}) {                      if (! exists $dupHash{$coupleID}) {
380                          $loadCoupling->Add("couplingIn");                          $loadCoupling->Add("couplingIn");
381                          # Here we have a new coupling to store in the load files.                          # Here we have a new coupling to store in the load files.
# Line 623  Line 634 
634  The following relations are loaded by this method.  The following relations are loaded by this method.
635    
636      Subsystem      Subsystem
637        SubsystemClass
638      Role      Role
639      RoleEC      RoleEC
640      SSCell      SSCell
# Line 685  Line 697 
697      my $loadConsistsOfGenomes = $self->_TableLoader('ConsistsOfGenomes', $self->PrimaryOnly);      my $loadConsistsOfGenomes = $self->_TableLoader('ConsistsOfGenomes', $self->PrimaryOnly);
698      my $loadHasRoleSubset = $self->_TableLoader('HasRoleSubset', $self->PrimaryOnly);      my $loadHasRoleSubset = $self->_TableLoader('HasRoleSubset', $self->PrimaryOnly);
699      my $loadHasGenomeSubset = $self->_TableLoader('HasGenomeSubset', $self->PrimaryOnly);      my $loadHasGenomeSubset = $self->_TableLoader('HasGenomeSubset', $self->PrimaryOnly);
700        my $loadSubsystemClass = $self->_TableLoader('SubsystemClass', $self->PrimaryOnly);
701      if ($self->{options}->{loadOnly}) {      if ($self->{options}->{loadOnly}) {
702          Trace("Loading from existing files.") if T(2);          Trace("Loading from existing files.") if T(2);
703      } else {      } else {
# Line 710  Line 723 
723                  my $curator = $sub->get_curator();                  my $curator = $sub->get_curator();
724                  my $notes = $sub->get_notes();                  my $notes = $sub->get_notes();
725                  $loadSubsystem->Put($subsysID, $curator, $notes);                  $loadSubsystem->Put($subsysID, $curator, $notes);
726                    my $class = $fig->subsystem_classification($subsysID);
727                    if ($class) {
728                        $loadSubsystemClass->Put($subsysID, $class);
729                    }
730                  # Connect it to its roles. Each role is a column in the subsystem spreadsheet.                  # Connect it to its roles. Each role is a column in the subsystem spreadsheet.
731                  for (my $col = 0; defined($roleID = $sub->get_role($col)); $col++) {                  for (my $col = 0; defined($roleID = $sub->get_role($col)); $col++) {
732                      # Connect to this role.                      # Connect to this role.
# Line 753  Line 770 
770                          # part of the spreadsheet cell ID.                          # part of the spreadsheet cell ID.
771                          for (my $col = 0; defined($roleID = $sub->get_role($col)); $col++) {                          for (my $col = 0; defined($roleID = $sub->get_role($col)); $col++) {
772                              # Get the features in the spreadsheet cell for this genome and role.                              # Get the features in the spreadsheet cell for this genome and role.
773                              my @pegs = $sub->get_pegs_from_cell($row, $col);                              my @pegs = grep { !$fig->is_deleted_fid($_) } $sub->get_pegs_from_cell($row, $col);
774                              # Only proceed if features exist.                              # Only proceed if features exist.
775                              if (@pegs > 0) {                              if (@pegs > 0) {
776                                  # Create the spreadsheet cell.                                  # Create the spreadsheet cell.
# Line 774  Line 791 
791                          if ($pegCount > 0) {                          if ($pegCount > 0) {
792                              Trace("$pegCount PEGs in $cellCount cells for $genomeID.") if T(3);                              Trace("$pegCount PEGs in $cellCount cells for $genomeID.") if T(3);
793                              $loadParticipatesIn->Put($genomeID, $subsysID, $variantCode);                              $loadParticipatesIn->Put($genomeID, $subsysID, $variantCode);
                             # Partition the PEGs found into clusters.  
                             my @clusters = $fig->compute_clusters(\@pegsFound, $sub);  
794                              # Create a hash mapping PEG IDs to cluster numbers.                              # Create a hash mapping PEG IDs to cluster numbers.
795                              # We default to -1 for all of them.                              # We default to -1 for all of them.
796                              my %clusterOf = map { $_ => -1 } @pegsFound;                              my %clusterOf = map { $_ => -1 } @pegsFound;
797                                # Partition the PEGs found into clusters.
798                                my @clusters = $fig->compute_clusters([keys %clusterOf], $sub);
799                              for (my $i = 0; $i <= $#clusters; $i++) {                              for (my $i = 0; $i <= $#clusters; $i++) {
800                                  my $subList = $clusters[$i];                                  my $subList = $clusters[$i];
801                                  for my $peg (@{$subList}) {                                  for my $peg (@{$subList}) {
# Line 1021  Line 1038 
1038          # Loop through the genomes.          # Loop through the genomes.
1039          for my $genomeID (sort keys %{$genomeHash}) {          for my $genomeID (sort keys %{$genomeHash}) {
1040              Trace("Processing $genomeID.") if T(3);              Trace("Processing $genomeID.") if T(3);
             # Get the genome's PEGs.  
             my @pegs = $fig->pegs_of($genomeID);  
             for my $peg (@pegs) {  
                 Trace("Processing $peg.") if T(4);  
1041                  # Create a hash of timestamps. We use this to prevent duplicate time stamps                  # Create a hash of timestamps. We use this to prevent duplicate time stamps
1042                  # from showing up for a single PEG's annotations.                  # from showing up for a single PEG's annotations.
1043                  my %seenTimestamps = ();                  my %seenTimestamps = ();
1044                  # Loop through the annotations.              # Get the genome's annotations.
1045                  for my $tuple ($fig->feature_annotations($peg, "raw")) {              my @annotations = $fig->read_all_annotations($genomeID);
1046                      my ($fid, $timestamp, $user, $text) = @{$tuple};              Trace("Processing annotations.") if T(2);
1047                for my $tuple (@annotations) {
1048                    # Get the annotation tuple.
1049                    my ($peg, $timestamp, $user, $text) = @{$tuple};
1050                      # Here we fix up the annotation text. "\r" is removed,                      # Here we fix up the annotation text. "\r" is removed,
1051                      # and "\t" and "\n" are escaped. Note we use the "s"                  # and "\t" and "\n" are escaped. Note we use the "gs"
1052                      # modifier so that new-lines inside the text do not                      # modifier so that new-lines inside the text do not
1053                      # stop the substitution search.                      # stop the substitution search.
1054                      $text =~ s/\r//gs;                      $text =~ s/\r//gs;
# Line 1045  Line 1061 
1061                          # Here it's a number. We need to insure the one we use to form                          # Here it's a number. We need to insure the one we use to form
1062                          # the key is unique.                          # the key is unique.
1063                          my $keyStamp = $timestamp;                          my $keyStamp = $timestamp;
1064                          while ($seenTimestamps{$keyStamp}) {                      while ($seenTimestamps{"$peg:$keyStamp"}) {
1065                              $keyStamp++;                              $keyStamp++;
1066                          }                          }
                         $seenTimestamps{$keyStamp} = 1;  
1067                          my $annotationID = "$peg:$keyStamp";                          my $annotationID = "$peg:$keyStamp";
1068                        $seenTimestamps{$annotationID} = 1;
1069                          # Insure the user exists.                          # Insure the user exists.
1070                          if (! $users{$user}) {                          if (! $users{$user}) {
1071                              $loadSproutUser->Put($user, "SEED user");                              $loadSproutUser->Put($user, "SEED user");
# Line 1067  Line 1083 
1083                  }                  }
1084              }              }
1085          }          }
     }  
1086      # Finish the load.      # Finish the load.
1087      my $retVal = $self->_FinishAll();      my $retVal = $self->_FinishAll();
1088      return $retVal;      return $retVal;
# Line 1387  Line 1402 
1402      return $retVal;      return $retVal;
1403  }  }
1404    
1405    =head3 LoadSynonymData
1406    
1407    C<< my $stats = $spl->LoadSynonymData(); >>
1408    
1409    Load the synonym groups into Sprout.
1410    
1411    The following relations are loaded by this method.
1412    
1413        SynonymGroup
1414        IsSynonymGroupFor
1415    
1416    The source information for these relations is taken from the C<maps_to_id> method
1417    of the B<FIG> object. The process starts from the features, so it is possible
1418    that there will be duplicates in the SynonymGroup load file, since the relationship
1419    is one-to-many toward the features. The automatic sort on primary entity relations
1420    will fix this for us.
1421    
1422    =over 4
1423    
1424    =item RETURNS
1425    
1426    Returns a statistics object for the loads.
1427    
1428    =back
1429    
1430    =cut
1431    #: Return Type $%;
1432    sub LoadSynonymData {
1433        # Get this object instance.
1434        my ($self) = @_;
1435        # Get the FIG object.
1436        my $fig = $self->{fig};
1437        # Get the genome hash.
1438        my $genomeHash = $self->{genomes};
1439        # Create a load object for the table we're loading.
1440        my $loadSynonymGroup = $self->_TableLoader('SynonymGroup');
1441        my $loadIsSynonymGroupFor = $self->_TableLoader('IsSynonymGroupFor');
1442        if ($self->{options}->{loadOnly}) {
1443            Trace("Loading from existing files.") if T(2);
1444        } else {
1445            Trace("Generating synonym group data.") if T(2);
1446            # Loop through the genomes.
1447            for my $genomeID (sort keys %{$genomeHash}) {
1448                Trace("Processing $genomeID.") if T(3);
1449                # Get all of the features for this genome. The only method that does this is
1450                # all_features_detailed, which returns extra baggage that we discard.
1451                my $featureData = $fig->all_features_detailed($genomeID);
1452                my @fids = map { $_->[0] } @{$featureData};
1453                Trace(scalar(@fids) . " features found for genome $genomeID.") if T(3);
1454                # Loop through the feature IDs.
1455                for my $fid (@fids) {
1456                    # Get the group for this feature.
1457                    my $synonym = $fig->maps_to_id($fid);
1458                    # Only proceed if the synonym is a real group.
1459                    if ($synonym ne $fid) {
1460                        $loadSynonymGroup->Put($synonym);
1461                        $loadIsSynonymGroupFor->Put($synonym, $fid);
1462                    }
1463                }
1464            }
1465        }
1466        # Finish the load.
1467        my $retVal = $self->_FinishAll();
1468        return $retVal;
1469    }
1470    
1471    
1472  =head2 Internal Utility Methods  =head2 Internal Utility Methods
1473    
1474  =head3 TableLoader  =head3 TableLoader
# Line 1454  Line 1536 
1536      my $retVal = Stats->new();      my $retVal = Stats->new();
1537      # Get the loader list.      # Get the loader list.
1538      my $loadList = $self->{loaders};      my $loadList = $self->{loaders};
1539        # Create a hash to hold the statistics objects, keyed on relation name.
1540        my %loaderHash = ();
1541      # Loop through the list, finishing the loads. Note that if the finish fails, we die      # Loop through the list, finishing the loads. Note that if the finish fails, we die
1542      # ignominiously. At some future point, we want to make the loads restartable.      # ignominiously. At some future point, we want to make the loads more restartable.
1543      while (my $loader = pop @{$loadList}) {      while (my $loader = pop @{$loadList}) {
1544          # Get the relation name.          # Get the relation name.
1545          my $relName = $loader->RelName;          my $relName = $loader->RelName;
# Line 1466  Line 1550 
1550              # Here we really need to finish.              # Here we really need to finish.
1551              Trace("Finishing $relName.") if T(2);              Trace("Finishing $relName.") if T(2);
1552              my $stats = $loader->Finish();              my $stats = $loader->Finish();
1553                $loaderHash{$relName} = $stats;
1554            }
1555        }
1556        # Now we loop through again, actually loading the tables. We want to finish before
1557        # loading so that if something goes wrong at this point, all the load files are usable
1558        # and we don't have to redo all that work.
1559        for my $relName (sort keys %loaderHash) {
1560            # Get the statistics for this relation.
1561            my $stats = $loaderHash{$relName};
1562            # Check for a database load.
1563              if ($self->{options}->{dbLoad}) {              if ($self->{options}->{dbLoad}) {
1564                  # Here we want to use the load file just created to load the database.                  # Here we want to use the load file just created to load the database.
1565                  Trace("Loading relation $relName.") if T(2);                  Trace("Loading relation $relName.") if T(2);
Legend:
Removed from v.1.32  
changed lines
  Added in v.1.48
MCS Webmaster
 ViewVC Help
Powered by ViewVC 1.0.3