[Bio] / Sprout / SaplingGenomeLoader.pm Repository:
ViewVC logotype

Diff of /Sprout/SaplingGenomeLoader.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2, Tue Jan 11 15:04:03 2011 UTC revision 1.9, Thu Apr 21 19:58:18 2011 UTC
# Line 26  Line 26 
26      use SAPserver;      use SAPserver;
27      use Sapling;      use Sapling;
28      use AliasAnalysis;      use AliasAnalysis;
29        use base qw(SaplingDataLoader);
30    
31  =head1 Sapling Genome Loader  =head1 Sapling Genome Loader
32    
# Line 71  Line 72 
72      # Load the features.      # Load the features.
73      Trace("Loading features for $genome.") if T(2);      Trace("Loading features for $genome.") if T(2);
74      $loaderObject->LoadFeatures();      $loaderObject->LoadFeatures();
75        # Check for annotation history. If we have it, load the history records into the
76        # database.
77        if (-f "$directory/annotations") {
78            Trace("Processing annotations.") if T(3);
79            $loaderObject->LoadAnnotations("$directory/annotations");
80        }
81      # Load the subsystem bindings.      # Load the subsystem bindings.
82      Trace("Loading subsystems for $genome.") if T(2);      Trace("Loading subsystems for $genome.") if T(2);
83      $loaderObject->LoadSubsystems();      $loaderObject->LoadSubsystems();
# Line 111  Line 118 
118      my ($sap, $genome) = @_;      my ($sap, $genome) = @_;
119      # Create the statistics object.      # Create the statistics object.
120      my $stats = Stats->new();      my $stats = Stats->new();
121      # Delete the DNA.      # Delete the DNA sequences.
122      DeleteRelatedRecords($sap, $genome, $stats, 'HasSection', 'DNASequence');      my @seqs = $sap->GetFlat('DNASequence', 'DNASequence(id) LIKE ?', ["$genome:%"], 'id');
123        for my $seq (@seqs) {
124            my $delStats = $sap->Delete(DNASequence => $seq);
125            $stats->Accumulate($delStats);
126        }
127      # Delete the contigs.      # Delete the contigs.
128      DeleteRelatedRecords($sap, $genome, $stats, 'IsMadeUpOf', 'Contig');      SaplingDataLoader::DeleteRelatedRecords($sap, $genome, $stats, 'IsMadeUpOf', 'Contig');
129      # Delete the features.      # Delete the features.
130      DeleteRelatedRecords($sap, $genome, $stats, 'IsOwnerOf', 'Feature');      SaplingDataLoader::DeleteRelatedRecords($sap, $genome, $stats, 'IsOwnerOf', 'Feature');
131      # Delete the molecular machines.      # Delete the molecular machines.
132      DeleteRelatedRecords($sap, $genome, $stats, 'Uses', 'MolecularMachine');      SaplingDataLoader::DeleteRelatedRecords($sap, $genome, $stats, 'Uses', 'MolecularMachine');
133        # Delete the annotations.
134        SaplingDataLoader::DeleteRelatedRecords($sap, $genome, $stats, 'IsAnnotatedBy', 'Annotation');
135      # Delete the genome itself.      # Delete the genome itself.
136      my $subStats = $sap->Delete(Genome => $genome);      my $subStats = $sap->Delete(Genome => $genome);
137      # Accumulate the statistics from the delete.      # Accumulate the statistics from the delete.
# Line 127  Line 140 
140      return $stats;      return $stats;
141  }  }
142    
143    
144    =head3 Process
145    
146        my $stats = SaplingGenomeLoader::Process($sap, $genome, $directory);
147    
148    Load genome data from the specified directory. If the genome data already
149    exists in the database, it will be deleted first.
150    
151    =over 4
152    
153    =item sap
154    
155    L</Sapling> object for accessing the database.
156    
157    =item genome
158    
159    ID of the genome whose  data is being loaded.
160    
161    =item directory
162    
163    Name of the directory containing the genome data files.
164    
165    =item RETURN
166    
167    Returns a statistics object describing the activity during the reload.
168    
169    =back
170    
171    =cut
172    
173    sub Process {
174        # Get the parameters.
175        my ($sap, $genome, $directory) = @_;
176        # Clear the existing data for the specified genome.
177        my $stats = ClearGenome($sap, $genome);
178        # Load the new expression data from the specified directory.
179        my $newStats = Load($sap, $genome, $directory);
180        # Merge the statistics.
181        $stats->Accumulate($newStats);
182        # Return the result.
183        return $stats;
184    }
185    
186    
187  =head2 Loader Object Methods  =head2 Loader Object Methods
188    
189  =head3 new  =head3 new
# Line 176  Line 233 
233    
234  L<Stats> object for tracking statistical information about the load.  L<Stats> object for tracking statistical information about the load.
235    
236    =item timestamps
237    
238    A hash of hashes, keyed by feature ID. The sub-hashes are keyed by annotation timestamp,
239    and used to prevent duplicate timestamps.
240    
241  =back  =back
242    
243  =cut  =cut
# Line 184  Line 246 
246      # Get the parameters.      # Get the parameters.
247      my ($class, $sap, $genome, $directory) = @_;      my ($class, $sap, $genome, $directory) = @_;
248      # Create the object.      # Create the object.
249      my $retVal = {      my $retVal = SaplingDataLoader::new($class, $sap, qw(contigs dna pegs rnas));
250          sap => $sap,      # Add our specialized data.
251          genome => $genome,      $retVal->{genome} = $genome;
252          directory => $directory,      $retVal->{directory} = $directory;
253          stats => Stats->new(qw(contigs dna pegs rnas)),      $retVal->{timestamps} = {};
254          supportRecords => {}      # Return the result.
     };  
     # Bless and return it.  
     bless $retVal, $class;  
255      return $retVal;      return $retVal;
256  }  }
257    
# Line 360  Line 419 
419  sub LoadFeatures {  sub LoadFeatures {
420      # Get the parameters.      # Get the parameters.
421      my ($self) = @_;      my ($self) = @_;
422        # Read in the functional assignments.
423        Trace("Reading functional assignments.") if T(3);
424        my $assignHash = $self->ReadAssignments();
425      # Get the directory of feature types.      # Get the directory of feature types.
426      my $featureDir = "$self->{directory}/Features";      my $featureDir = "$self->{directory}/Features";
427      my @types = Tracer::OpenDir("$self->{directory}/Features", 1);      my @types = Tracer::OpenDir("$self->{directory}/Features", 1);
# Line 368  Line 430 
430          # Insure this is a genuine feature directory.          # Insure this is a genuine feature directory.
431          if (-f "$featureDir/$type/tbl") {          if (-f "$featureDir/$type/tbl") {
432              # Yes, load the feature data.              # Yes, load the feature data.
433              $self->LoadFeatureData($featureDir, $type);              $self->LoadFeatureData($featureDir, $type, $assignHash);
434          }          }
435      }      }
436      # Check for protein sequences. If we have some, load them into the database.      # Check for protein sequences. If we have some, load them into the database.
437      if (-f "$featureDir/peg/fasta") {      if (-f "$featureDir/peg/fasta") {
438            Trace("Processing protein sequences.") if T(3);
439          $self->LoadProteinData("$featureDir/peg/fasta");          $self->LoadProteinData("$featureDir/peg/fasta");
440      }      }
441        # Now loop through the features, connecting them to their roles. Note that deleted
442        # features will not be in the assignment hash.
443        Trace("Connecting features to roles.") if T(3);
444        for my $fid (keys %$assignHash) {
445            $self->ConnectFunctionRoles($fid, $assignHash->{$fid});
446        }
447  }  }
448    
449  =head3 LoadFeatureData  =head3 LoadFeatureData
450    
451      $self->LoadFeatureData($featureDir, $type);      $loaderObject->LoadFeatureData($featureDir, $type, $assignHash);
452    
453  Load the basic data for each feature into the database. The number of features of  Load the basic data for each feature into the database. The number of features of
454  the type found will be recorded in the statistics object.  the type found will be recorded in the statistics object.
# Line 394  Line 463 
463    
464  Type of feature to load.  Type of feature to load.
465    
466    =item assignHash
467    
468    Reference to a hash mapping each feature ID to its functional assignment.
469    
470  =back  =back
471    
472  =cut  =cut
473    
474  sub LoadFeatureData {  sub LoadFeatureData {
475      # Get the parameters.      # Get the parameters.
476      my ($self, $featureDir, $type) = @_;      my ($self, $featureDir, $type, $assignHash) = @_;
477      # Get the sapling database.      # Get the sapling database.
478      my $sap = $self->{sap};      my $sap = $self->{sap};
479      # Get the maximum location  segment length. We'll need this later.      # Get the maximum location  segment length. We'll need this later.
480      my $maxLength = $sap->TuningParameter('maxLocationLength');      my $maxLength = $sap->TuningParameter('maxLocationLength');
481      # Get the statistics object.      # Get the statistics object.
482      my $stats = $self->{stats};      my $stats = $self->{stats};
     # Read in the functional assignments.  
     my $assignHash = $self->ReadAssignments();  
483      # This hash will track the features we've created. If a feature is found a second      # This hash will track the features we've created. If a feature is found a second
484      # time, it overwrites the original.      # time, it overwrites the original.
485      my %fids;      my $fidHash = $self->{timestamps};
486        # Finally, we need the timestamp hash. The initial feature population
487      # Insure we have a tbl file for this feature type.      # Insure we have a tbl file for this feature type.
488      my $fileName = "$featureDir/$type/tbl";      my $fileName = "$featureDir/$type/tbl";
489      if (-f $fileName) {      if (-f $fileName) {
490          # We have one, so open it for input.          # We have one, so we can read through it. First, however, we need to get the list
491            # of deleted features.
492            my %deletedFids;
493            my $deleteFile = "$featureDir/$type/deleted.features";
494            if (-f $deleteFile) {
495                %deletedFids = map { $_ => 1 } Tracer::GetFile($deleteFile);
496            }
497            # Open the main file for input.
498            Trace("Reading features from $fileName.") if T(3);
499          my $ih = Open(undef, "<$fileName");          my $ih = Open(undef, "<$fileName");
500          while (! eof $ih) {          while (! eof $ih) {
501              # Read this feature's information.              # Read this feature's information.
502              my ($fid, $locations, @aliases) = Tracer::GetLine($ih);              my ($fid, $locations, @aliases) = Tracer::GetLine($ih);
503              # If the feature already exists, delete it.              # Only proceed if the feature is NOT deleted.
504              if (exists $fids{$fid}) {              if (! exists $deletedFids{$fid}) {
505                    # If the feature already exists, delete it. (This should be extremely rare.)
506                    if (exists $fidHash->{$fid}) {
507                  $sap->Delete(Feature => $fid);                  $sap->Delete(Feature => $fid);
508                  $stats->Add(duplicateFid => 1);                  $stats->Add(duplicateFid => 1);
509              }              }
# Line 455  Line 537 
537              $sap->InsertObject('Feature', id => $fid, feature_type => $type,              $sap->InsertObject('Feature', id => $fid, feature_type => $type,
538                                 function => $assignHash->{$fid}, locked => 0,                                 function => $assignHash->{$fid}, locked => 0,
539                                 sequence_length => $length);                                 sequence_length => $length);
540              $fids{$fid} = 1;                  $fidHash->{$fid} = {};
541              $stats->Add($type => 1);              $stats->Add($type => 1);
542              # The next step is to connect the feature record to its locations. This              # The next step is to connect the feature record to its locations. This
543              # involves dividing the location into segments. The following variable              # involves dividing the location into segments. The following variable
# Line 534  Line 616 
616          }          }
617      }      }
618  }  }
619    }
620    
621  =head3 LoadProteinData  =head3 LoadProteinData
622    
# Line 589  Line 672 
672      }      }
673  }  }
674    
675    
676    =head3 LoadAnnotations
677    
678        $loaderObject->LoadAnnotations($fileName);
679    
680    Read in the annotation history information and use it to create annotation records.
681    
682    =over 4
683    
684    =item fileName
685    
686    Name of the annotation history file. This file is formatted with four fields per
687    record. Each field is on a separate line, with a double slash (C<//>) used as the
688    line terminator. The fields, in order, are (0) the feature ID, (1) the timestamp
689    (formatted as an integer), (2) the user name, and (3) the annotation text.
690    
691    =back
692    
693    =cut
694    
695    sub LoadAnnotations {
696        # Get the parameters.
697        my ($self, $fileName) = @_;
698        # Get the timestamp hash.
699        my $timeHash = $self->{timestamps};
700        # Get the Sapling database.
701        my $sap = $self->{sap};
702        # Get the statistics object.
703        my $stats = $self->{stats};
704        # Open the input file.
705        my $ih = Tracer::Open(undef, "<$fileName");
706        # Loop through the input.
707        while (! eof $ih) {
708            # Read in the peg, timestamp, and user ID.
709            my ($fid, $timestamp, $user, $text) = ReadAnnotation($ih);
710            # Only proceed if the feature exists.
711            if (! exists $timeHash->{$fid}) {
712                $stats->Add(skippedAnnotation => 1);
713            } else {
714                # Change assignments by the master user to FIG assignments.
715                $text =~ s/Set master function/Set FIG function/s;
716                # Insure the time stamp is valid.
717                if ($timestamp =~ /^\d+$/) {
718                    # Here it's a number. We need to insure the one we use to form
719                    # the key is unique.
720                    my $keyStamp = $timestamp;
721                    while ($timeHash->{$fid}{$keyStamp}) {
722                        $keyStamp++;
723                        $stats->Add(skippedStamp => 1);
724                    }
725                    # Form the annotation ID.
726                    my $annotationID = SaplingDataLoader::ComputeAnnotationID($fid, $keyStamp);
727                    $timeHash->{$fid}{$keyStamp} = 1;
728                    # Generate the annotation.
729                    $sap->InsertObject('IsAnnotatedBy', from_link => $fid, to_link => $annotationID);
730                    $sap->InsertObject('Annotation', id => $annotationID, annotation_time => $timestamp,
731                                comment => $text, annotator => $user);
732                } else {
733                    # Here we have an invalid time stamp.
734                    Trace("Invalid time stamp \"$timestamp\" in annotations for $fid.") if T(1);
735                }
736            }
737        }
738    }
739    
740    
741  =head3 WriteProtein  =head3 WriteProtein
742    
743      $loaderObject->WriteProtein($fid, $sequence);      $loaderObject->WriteProtein($fid, $sequence);
# Line 614  Line 763 
763      # Get the parameters.      # Get the parameters.
764      my ($self, $fid, $sequence) = @_;      my ($self, $fid, $sequence) = @_;
765      # Compute the key of the protein sequence.      # Compute the key of the protein sequence.
766      my $protID = ERDB::DigestKey($sequence);      my $protID = $self->{sap}->ProteinID($sequence);
767      # Insure the protein exists.      # Insure the protein exists.
768      $self->InsureEntity(ProteinSequence => $protID, sequence => $sequence);      $self->InsureEntity(ProteinSequence => $protID, sequence => $sequence);
769      # Connect the feature to it.      # Connect the feature to it.
770      $self->{sap}->InsertObject('IsProteinFor', from_link => $protID, to_link => $fid);      $self->{sap}->InsertObject('IsProteinFor', from_link => $protID, to_link => $fid);
771  }  }
772    
 =head3 InsureEntity  
   
     my $createdFlag = $loaderObject->InsureEntity($entityType => $id, %fields);  
   
 Insure that the specified record exists in the database. If no record is found of the  
 specified type with the specified ID, one will be created with the indicated fields.  
   
 =over 4  
   
 =item $entityType  
   
 Type of entity to check.  
   
 =item id  
   
 ID of the entity instance in question.  
   
 =item fields  
   
 Hash mapping field names to values for all the fields in the desired entity record except  
 for the ID.  
   
 =item RETURN  
   
 Returns TRUE if a new object was created, FALSE if it already existed.  
   
 =back  
   
 =cut  
   
 sub InsureEntity {  
     # Get the parameters.  
     my ($self, $entityType, $id, %fields) = @_;  
     # Get the database.  
     my $sap = $self->{sap};  
     # Get the support record ID hash.  
     my $supportHash = $self->{supportRecords};  
     # Denote we haven't created a new record.  
     my $retVal = 0;  
     # Get the sub-hash for this entity type.  
     my $entityHash = $supportHash->{$entityType};  
     if (! defined $entityHash) {  
         $entityHash = {};  
         $supportHash->{$entityType} = $entityHash;  
     }  
     # Check for this instance.  
     if (! $entityHash->{$id}) {  
         # It's not found. Check the database.  
         if (! $sap->Exists($entityType => $id)) {  
             # It's not in the database either, so create it.  
             $sap->InsertObject($entityType, id => $id, %fields);  
             $self->{stats}->Add(insertSupport => 1);  
             $retVal = 1;  
         }  
         # Mark the record in the hash so we know we have it.  
         $entityHash->{$id} = 1;  
     }  
     # Return the insertion indicator.  
     return $retVal;  
 }  
   
773  =head3 LoadSubsystems  =head3 LoadSubsystems
774    
775      $loaderObject->LoadSubsystems();      $loaderObject->LoadSubsystems();
# Line 737  Line 825 
825              if (@$roleList > 0) {              if (@$roleList > 0) {
826                  # Get the subsystem information from the first role and create the subsystem.                  # Get the subsystem information from the first role and create the subsystem.
827                  my $roleH = $roleList->[0];                  my $roleH = $roleList->[0];
828                  my %subFields = ExtractFields(Subsystem => $roleH);                  my %subFields = SaplingDataLoader::ExtractFields(Subsystem => $roleH);
829                  $sap->InsertObject('Subsystem', %subFields);                  $sap->InsertObject('Subsystem', %subFields);
830                  # Now loop through the roles. The Includes records are always inserted, but the                  # Now loop through the roles. The Includes records are always inserted, but the
831                  # roles are only inserted if they don't already exist.                  # roles are only inserted if they don't already exist.
832                  for $roleH (@$roleList) {                  for $roleH (@$roleList) {
833                      # Create the Includes record.                      # Create the Includes record.
834                      my %incFields = ExtractFields(Includes => $roleH);                      my %incFields = SaplingDataLoader::ExtractFields(Includes => $roleH);
835                      $sap->InsertObject('Includes', %incFields);                      $sap->InsertObject('Includes', %incFields);
836                      # Insure we have the role in place.                      # Insure we have the role in place.
837                      my %roleFields = ExtractFields(Role => $roleH);                      my %roleFields = SaplingDataLoader::ExtractFields(Role => $roleH);
838                      my $roleID = $roleFields{id};                      my $roleID = $roleFields{id};
839                      delete $roleFields{id};                      delete $roleFields{id};
840                      $self->InsureEntity('Role', $roleID, %roleFields);                      $self->InsureEntity('Role', $roleID, %roleFields);
# Line 984  Line 1072 
1072      $self->{stats}->Add(segment => 1);      $self->{stats}->Add(segment => 1);
1073  }  }
1074    
 =head2 Internal Utility Methods  
   
 =head3 DeleteRelatedRecords  
   
     DeleteRelatedRecords($sap, $genome, $stats, $relName, $entityName);  
   
 Delete all the records in the named entity and relationship relating to the  
 specified genome and roll up the statistics in the specified statistics object.  
   
 =over 4  
   
 =item sap  
   
 L<Sapling> object for accessing the database.  
   
 =item genome  
   
 ID of the relevant genome.  
   
 =item stats  
   
 L<Stats> object for tracking the delete activity.  
   
 =item relName  
   
 Name of a relationship from the B<Genome> table.  
   
 =item entityName  
   
 Name of the entity on the other side of the relationship.  
   
 =back  
   
 =cut  
   
 sub DeleteRelatedRecords {  
     # Get the parameters.  
     my ($sap, $genome, $stats, $relName, $entityName) = @_;  
     # Get all the relationship records.  
     my (@targets) = $sap->GetFlat($relName, "$relName(from-link) = ?", [$genome],  
                                   "to-link");  
     # Loop through the relationship records, deleting them and the target entity  
     # records.  
     for my $target (@targets) {  
         # Delete the relationship instance.  
         $sap->DeleteRow($relName, $genome, $target);  
         $stats->Add($relName => 1);  
         # Delete the entity instance.  
         my $subStats = $sap->Delete($entityName, $target);  
         # Roll up the statistics.  
         $stats->Accumulate($subStats);  
     }  
 }  
   
 =head3 ExtractFields  
   
     my %fieldHash = SaplingGenomeLoader::ExtractFields($tableName, $dataHash);  
   
 Extract from the incoming hash the field names and values from the specified table.  
   
 =over 4  
   
 =item tableName  
   
 Name of the table whose field names and values are desired.  
   
 =item dataHash  
   
 Reference to a hash mapping fully-qualified ERDB field names to values.  
   
 =item RETURN  
   
 Returns a hash containing only the fields from the specified table and their values.  
   
 =back  
   
 =cut  
   
 sub ExtractFields {  
     # Get the parameters.  
     my ($tableName, $dataHash) = @_;  
     # Declare the return variable.  
     my %retVal;  
     # Extract the desired fields.  
     for my $field (keys %$dataHash) {  
         # Is this a field for the specified table?  
         if ($field =~ /^$tableName\(([^)]+)/) {  
             # Yes, put it in the output hash.  
             $retVal{$1} = $dataHash->{$field};  
         }  
     }  
     # Return the computed hash.  
     return %retVal;  
 }  
   
1075  =head3 CreateIdentifier  =head3 CreateIdentifier
1076    
1077      $loaderObject->CreateIdentifier($alias, $conf, $aliasType, $fid);      $loaderObject->CreateIdentifier($alias, $conf, $aliasType, $fid);
# Line 1121  Line 1114 
1114      # Insure the identifier exists in the database.      # Insure the identifier exists in the database.
1115      $self->InsureEntity(Identifier => $alias, source => $aliasType, natural_form => $natural);      $self->InsureEntity(Identifier => $alias, source => $aliasType, natural_form => $natural);
1116      # Connect the identifier to the feature.      # Connect the identifier to the feature.
1117      $sap->InsertObject('Identifies', from_link => $alias, to_link => $fid, conf => $conf);      $sap->InsertObject('IsIdentifiedBy', to_link => $alias, from_link => $fid, conf => $conf);
1118    }
1119    
1120    =head2 Internal Utility Methods
1121    
1122    =head3 ReadAnnotation
1123    
1124        my ($fid, $timestamp, $user, $text) = SaplingGenomeLoader::ReadAnnotation($ih);
1125    
1126    Read the next record from an annotation file. The next record must exist (that is, an
1127    end-of-file check should have been performed before calling this method).
1128    
1129    =over 4
1130    
1131    =item ih
1132    
1133    Open file handle for the annotation file.
1134    
1135    =item RETURN
1136    
1137    Returns a list containing the four fields of the record read-- (0) the feature ID, (1) the
1138    timestamp, (2) the user ID, and (3) the annotation text.
1139    
1140    =back
1141    
1142    =cut
1143    
1144    sub ReadAnnotation {
1145        # Get the parameter.
1146        my ($ih) = @_;
1147        # Read the three fixed fields.
1148        my $fid = <$ih>; chomp $fid;
1149        my $timestamp = <$ih>; chomp $timestamp;
1150        my $user = <$ih>; chomp $user;
1151        # Loop through the lines of the text field.
1152        my $text = "";
1153        my $line = <$ih>;
1154        while ($line ne "//\n") {
1155            $text .= $line;
1156            $line = <$ih>;
1157        }
1158        # Remove the trailing new-line from the text.
1159        chomp $text;
1160        # Return the fields.
1161        return ($fid, $timestamp, $user, $text);
1162  }  }
1163    
1164  1;  1;

Legend:
Removed from v.1.2  
changed lines
  Added in v.1.9

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3