Diff of /Sprout/SproutLoad.pm

-revision 1.48, Fri Jul  7 00:24:16 2006 UTC
+revision 1.90, Thu Dec  6 14:53:50 2007 UTC
 Line 7
      use PageBuilder;
      use ERDBLoad;
      use FIG;
+     use FIGRules;
      use Sprout;
      use Stats;
      use BasicLocation;
      use HTML;
+     use AliasAnalysis;
  =head1 Sprout Load Methods
-Line 50
+Line 52
  =head3 new
- C<< my $spl = SproutLoad->new($sprout, $fig, $genomeFile, $subsysFile, $options); >>
+     my $spl = SproutLoad->new($sprout, $fig, $genomeFile, $subsysFile, $options);
  Construct a new Sprout Loader object, specifying the two participating databases and
  the name of the files containing the list of genomes and subsystems to use.
-Line 80
+Line 82
  Either the name of the file containing the list of trusted subsystems or a reference
  to a list of subsystem names. If nothing is specified, all NMPDR subsystems will be
  considered trusted. (A subsystem is considered NMPDR if it has a file named C<NMPDR>
- in its data directory.) Only subsystem data related to the trusted subsystems is loaded.
+ in its data directory.) Only subsystem data related to the NMPDR subsystems is loaded.
  =item options
-Line 101
+Line 103
              # Here we want all the complete genomes and an access code of 1.
              my @genomeList = $fig->genomes(1);
              %genomes = map { $_ => 1 } @genomeList;
+             Trace(scalar(keys %genomes) . " genomes found.") if T(3);
          } else {
              my $type = ref $genomeFile;
              Trace("Genome file parameter type is \"$type\".") if T(3);
-Line 120
+Line 123
                      # an omitted access code can be defaulted to 1.
                      for my $genomeLine (@genomeList) {
                          my ($genomeID, $accessCode) = split("\t", $genomeLine);
-                         if (undef $accessCode) {
+                         if (! defined($accessCode)) {
                              $accessCode = 1;
                          }
                          $genomes{$genomeID} = $accessCode;
-Line 136
+Line 139
      # We only need it if load-only is NOT specified.
      if (! $options->{loadOnly}) {
          if (! defined $subsysFile || $subsysFile eq '') {
-             # Here we want all the NMPDR subsystems. First we get the whole list.
+             # Here we want all the usable subsystems. First we get the whole list.
              my @subs = $fig->all_subsystems();
              # Loop through, checking for the NMPDR file.
              for my $sub (@subs) {
-                 if (-e "$FIG_Config::data/Subsystems/$sub/NMPDR") {
+                 if ($fig->nmpdr_subsystem($sub)) {
                      $subsystems{$sub} = 1;
                  }
              }
-Line 163
+Line 166
                  Confess("Invalid subsystem parameter in SproutLoad constructor.");
              }
          }
+         # Go through the subsys hash again, creating the keyword list for each subsystem.
+         for my $subsystem (keys %subsystems) {
+             my $name = $subsystem;
+             $name =~ s/_/ /g;
+ #            my $classes = $fig->subsystem_classification($subsystem);
+ #            $name .= " " . join(" ", @{$classes});
+             $subsystems{$subsystem} = $name;
      }
+     }
+     # Get the list of NMPDR-oriented attribute keys.
+     my @propKeys = $fig->get_group_keys("NMPDR");
      # Get the data directory from the Sprout object.
      my ($directory) = $sprout->LoadInfo();
      # Create the Sprout load object.
-Line 175
+Line 188
                    loadDirectory => $directory,
                    erdb => $sprout,
                    loaders => [],
-                   options => $options
+                   options => $options,
+                   propKeys => \@propKeys,
                   };
      # Bless and return it.
      bless $retVal, $class;
-Line 184
+Line 198
  =head3 LoadOnly
- C<< my $flag = $spl->LoadOnly; >>
+     my $flag = $spl->LoadOnly;
  Return TRUE if we are in load-only mode, else FALSE.
-Line 195
+Line 209
      return $self->{options}->{loadOnly};
  }
- =head3 PrimaryOnly
- C<< my $flag = $spl->PrimaryOnly; >>
- Return TRUE if only the main entity is to be loaded, else FALSE.
- =cut
- sub PrimaryOnly {
-     my ($self) = @_;
-     return $self->{options}->{primaryOnly};
- }
  =head3 LoadGenomeData
- C<< my $stats = $spl->LoadGenomeData(); >>
+     my $stats = $spl->LoadGenomeData();
  Load the Genome, Contig, and Sequence data from FIG into Sprout.
-Line 247
+Line 249
      my $genomeCount = (keys %{$genomeHash});
      # Create load objects for each of the tables we're loading.
      my $loadGenome = $self->_TableLoader('Genome');
-     my $loadHasContig = $self->_TableLoader('HasContig', $self->PrimaryOnly);
+     my $loadHasContig = $self->_TableLoader('HasContig');
-     my $loadContig = $self->_TableLoader('Contig', $self->PrimaryOnly);
+     my $loadContig = $self->_TableLoader('Contig');
-     my $loadIsMadeUpOf = $self->_TableLoader('IsMadeUpOf', $self->PrimaryOnly);
+     my $loadIsMadeUpOf = $self->_TableLoader('IsMadeUpOf');
-     my $loadSequence = $self->_TableLoader('Sequence', $self->PrimaryOnly);
+     my $loadSequence = $self->_TableLoader('Sequence');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-Line 266
+Line 268
              my $extra = join " ", @extraData;
              # Get the full taxonomy.
              my $taxonomy = $fig->taxonomy_of($genomeID);
+             # Get the version. If no version is specified, we default to the genome ID by itself.
+             my $version = $fig->genome_version($genomeID);
+             if (! defined($version)) {
+                 $version = $genomeID;
+             }
+             # Get the DNA size.
+             my $dnaSize = $fig->genome_szdna($genomeID);
+             # Open the NMPDR group file for this genome.
+             my $group;
+             if (open(TMP, "<$FIG_Config::organisms/$genomeID/NMPDR") &&
+                 defined($group = <TMP>)) {
+                 # Clean the line ending.
+                 chomp $group;
+             } else {
+                 # No group, so use the default.
+                 $group = $FIG_Config::otherGroup;
+             }
+             close TMP;
              # Output the genome record.
-             $loadGenome->Put($genomeID, $accessCode, $fig->is_complete($genomeID), $genus,
+             $loadGenome->Put($genomeID, $accessCode, $fig->is_complete($genomeID),
-                              $species, $extra, $taxonomy);
+                              $dnaSize, $genus, $group, $species, $extra, $version, $taxonomy);
              # Now we loop through each of the genome's contigs.
              my @contigs = $fig->all_contigs($genomeID);
              for my $contigID (@contigs) {
-Line 306
+Line 326
      return $retVal;
  }
- =head3 LoadCouplingData
- C<< my $stats = $spl->LoadCouplingData(); >>
- Load the coupling and evidence data from FIG into Sprout.
- The coupling data specifies which genome features are functionally coupled. The
- evidence data explains why the coupling is functional.
- The following relations are loaded by this method.
-     Coupling
-     IsEvidencedBy
-     PCH
-     ParticipatesInCoupling
-     UsesAsEvidence
- =over 4
- =item RETURNS
- Returns a statistics object for the loads.
- =back
- =cut
- #: Return Type $%;
- sub LoadCouplingData {
-     # Get this object instance.
-     my ($self) = @_;
-     # Get the FIG object.
-     my $fig = $self->{fig};
-     # Get the genome hash.
-     my $genomeFilter = $self->{genomes};
-     my $genomeCount = (keys %{$genomeFilter});
-     my $featureCount = $genomeCount * 4000;
-     # Start the loads.
-     my $loadCoupling = $self->_TableLoader('Coupling');
-     my $loadIsEvidencedBy = $self->_TableLoader('IsEvidencedBy', $self->PrimaryOnly);
-     my $loadPCH = $self->_TableLoader('PCH', $self->PrimaryOnly);
-     my $loadParticipatesInCoupling = $self->_TableLoader('ParticipatesInCoupling', $self->PrimaryOnly);
-     my $loadUsesAsEvidence = $self->_TableLoader('UsesAsEvidence', $self->PrimaryOnly);
-     if ($self->{options}->{loadOnly}) {
-         Trace("Loading from existing files.") if T(2);
-     } else {
-         Trace("Generating coupling data.") if T(2);
-         # Loop through the genomes found.
-         for my $genome (sort keys %{$genomeFilter}) {
-             Trace("Generating coupling data for $genome.") if T(3);
-             $loadCoupling->Add("genomeIn");
-             # Create a hash table for holding coupled pairs. We use this to prevent
-             # duplicates. For example, if A is coupled to B, we don't want to also
-             # assert that B is coupled to A, because we already know it. Fortunately,
-             # all couplings occur within a genome, so we can keep the hash table
-             # size reasonably small.
-             my %dupHash = ();
-             # Get all of the genome's PEGs.
-             my @pegs = $fig->pegs_of($genome);
-             # Loop through the PEGs.
-             for my $peg1 (@pegs) {
-                 $loadCoupling->Add("pegIn");
-                 Trace("Processing PEG $peg1 for $genome.") if T(4);
-                 # Get a list of the coupled PEGs.
-                 my @couplings = $fig->coupled_to($peg1);
-                 # For each coupled PEG, we need to verify that a coupling already
-                 # exists. If not, we have to create one.
-                 for my $coupleData (@couplings) {
-                     my ($peg2, $score) = @{$coupleData};
-                     # Compute the coupling ID.
-                     my $coupleID = $self->{erdb}->CouplingID($peg1, $peg2);
-                     if (! exists $dupHash{$coupleID}) {
-                         $loadCoupling->Add("couplingIn");
-                         # Here we have a new coupling to store in the load files.
-                         Trace("Storing coupling ($coupleID) with score $score.") if T(4);
-                         # Ensure we don't do this again.
-                         $dupHash{$coupleID} = $score;
-                         # Write the coupling record.
-                         $loadCoupling->Put($coupleID, $score);
-                         # Connect it to the coupled PEGs.
-                         $loadParticipatesInCoupling->Put($peg1, $coupleID, 1);
-                         $loadParticipatesInCoupling->Put($peg2, $coupleID, 2);
-                         # Get the evidence for this coupling.
-                         my @evidence = $fig->coupling_evidence($peg1, $peg2);
-                         # Organize the evidence into a hash table.
-                         my %evidenceMap = ();
-                         # Process each evidence item.
-                         for my $evidenceData (@evidence) {
-                             $loadPCH->Add("evidenceIn");
-                             my ($peg3, $peg4, $usage) = @{$evidenceData};
-                             # Only proceed if the evidence is from a Sprout
-                             # genome.
-                             if ($genomeFilter->{$fig->genome_of($peg3)}) {
-                                 $loadUsesAsEvidence->Add("evidenceChosen");
-                                 my $evidenceKey = "$coupleID $peg3 $peg4";
-                                 # We store this evidence in the hash if the usage
-                                 # is nonzero or no prior evidence has been found. This
-                                 # insures that if there is duplicate evidence, we
-                                 # at least keep the meaningful ones. Only evidence in
-                                 # the hash makes it to the output.
-                                 if ($usage || ! exists $evidenceMap{$evidenceKey}) {
-                                     $evidenceMap{$evidenceKey} = $evidenceData;
-                                 }
-                             }
-                         }
-                         for my $evidenceID (keys %evidenceMap) {
-                             # Create the evidence record.
-                             my ($peg3, $peg4, $usage) = @{$evidenceMap{$evidenceID}};
-                             $loadPCH->Put($evidenceID, $usage);
-                             # Connect it to the coupling.
-                             $loadIsEvidencedBy->Put($coupleID, $evidenceID);
-                             # Connect it to the features.
-                             $loadUsesAsEvidence->Put($evidenceID, $peg3, 1);
-                             $loadUsesAsEvidence->Put($evidenceID, $peg4, 2);
-                         }
-                     }
-                 }
-             }
-         }
-     }
-     # All done. Finish the load.
-     my $retVal = $self->_FinishAll();
-     return $retVal;
- }
  =head3 LoadFeatureData
- C<< my $stats = $spl->LoadFeatureData(); >>
+     my $stats = $spl->LoadFeatureData();
  Load the feature data from FIG into Sprout.
-Line 442
+Line 338
      Feature
      FeatureAlias
+     IsAliasOf
      FeatureLink
      FeatureTranslation
      FeatureUpstream
      IsLocatedIn
      HasFeature
+     HasRoleInSubsystem
+     FeatureEssential
+     FeatureVirulent
+     FeatureIEDB
+     CDD
+     IsPresentOnProteinOf
  =over 4
-Line 461
+Line 364
  sub LoadFeatureData {
      # Get this object instance.
      my ($self) = @_;
-     # Get the FIG object.
+     # Get the FIG and Sprout objects.
      my $fig = $self->{fig};
+     my $sprout = $self->{sprout};
      # Get the table of genome IDs.
      my $genomeHash = $self->{genomes};
      # Create load objects for each of the tables we're loading.
      my $loadFeature = $self->_TableLoader('Feature');
-     my $loadIsLocatedIn = $self->_TableLoader('IsLocatedIn', $self->PrimaryOnly);
+     my $loadIsLocatedIn = $self->_TableLoader('IsLocatedIn');
      my $loadFeatureAlias = $self->_TableLoader('FeatureAlias');
+     my $loadIsAliasOf = $self->_TableLoader('IsAliasOf');
      my $loadFeatureLink = $self->_TableLoader('FeatureLink');
      my $loadFeatureTranslation = $self->_TableLoader('FeatureTranslation');
      my $loadFeatureUpstream = $self->_TableLoader('FeatureUpstream');
      my $loadHasFeature = $self->_TableLoader('HasFeature');
+     my $loadHasRoleInSubsystem = $self->_TableLoader('HasRoleInSubsystem');
+     my $loadFeatureEssential = $self->_TableLoader('FeatureEssential');
+     my $loadFeatureVirulent = $self->_TableLoader('FeatureVirulent');
+     my $loadFeatureIEDB = $self->_TableLoader('FeatureIEDB');
+     my $loadCDD = $self->_TableLoader('CDD');
+     my $loadIsPresentOnProteinOf = $self->_TableLoader('IsPresentOnProteinOf');
+     # Get the subsystem hash.
+     my $subHash = $self->{subsystems};
+     # Get the property keys.
+     my $propKeys = $self->{propKeys};
+     # Create a hashes to hold CDD and alias values.
+     my %CDD = ();
+     my %alias = ();
      # Get the maximum sequence size. We need this later for splitting up the
      # locations.
      my $chunkSize = $self->{sprout}->MaxSegment();
-Line 481
+Line 399
      } else {
          Trace("Generating feature data.") if T(2);
          # Now we loop through the genomes, generating the data for each one.
-         for my $genomeID (sort keys %{$genomeHash}) {
+         my @allGenomes = sort keys %{$genomeHash};
+         Trace(scalar(@allGenomes) . " genomes found in list.") if T(3);
+         for my $genomeID (@allGenomes) {
              Trace("Loading features for genome $genomeID.") if T(3);
              $loadFeature->Add("genomeIn");
              # Get the feature list for this genome.
-             my $features = $fig->all_features_detailed($genomeID);
+             my $features = $fig->all_features_detailed_fast($genomeID);
+             # Sort and count the list.
+             my @featureTuples = sort { $a->[0] cmp $b->[0] } @{$features};
+             my $count = scalar @featureTuples;
+             my @fids = map { $_->[0] } @featureTuples;
+             Trace("$count features found for genome $genomeID.") if T(3);
+             # Get the attributes for this genome and put them in a hash by feature ID.
+             my $attributes = GetGenomeAttributes($fig, $genomeID, \@fids, $propKeys);
+             Trace("Looping through features for $genomeID.") if T(3);
+             # Set up for our duplicate-feature check.
+             my $oldFeatureID = "";
              # Loop through the features.
-             for my $featureData (@{$features}) {
+             for my $featureTuple (@featureTuples) {
-                 $loadFeature->Add("featureIn");
                  # Split the tuple.
-                 my ($featureID, $locations, undef, $type) = @{$featureData};
+                 my ($featureID, $locations, undef, $type, $minloc, $maxloc, $assignment, $user, $quality) = @{$featureTuple};
-                 # Create the feature record.
+                 # Check for duplicates.
-                 $loadFeature->Put($featureID, 1, $type);
+                 if ($featureID eq $oldFeatureID) {
-                 # Link it to the parent genome.
+                     Trace("Duplicate feature $featureID found.") if T(1);
-                 $loadHasFeature->Put($genomeID, $featureID, $type);
+                 } else {
+                     $oldFeatureID = $featureID;
+                     # Count this feature.
+                     $loadFeature->Add("featureIn");
+                     # Fix the quality. It is almost always a space, but some odd stuff might sneak through, and the
+                     # Sprout database requires a single character.
+                     if (! defined($quality) || $quality eq "") {
+                         $quality = " ";
+                     }
+                     # Begin building the keywords. We start with the genome ID, the
+                     # feature ID, the taxonomy, and the organism name.
+                     my @keywords = ($genomeID, $featureID, $fig->genus_species($genomeID),
+                                     $fig->taxonomy_of($genomeID));
                  # Create the aliases.
                  for my $alias ($fig->feature_aliases($featureID)) {
-                     $loadFeatureAlias->Put($featureID, $alias);
+                         #Connect this alias to this feature.
-                 }
+                         $loadIsAliasOf->Put($alias, $featureID);
+                         push @keywords, $alias;
+                         # If this is a locus tag, also add its natural form as a keyword.
+                         my $naturalName = AliasAnalysis::Type(LocusTag => $alias);
+                         if ($naturalName) {
+                             push @keywords, $naturalName;
+                         }
+                         # If this is the first time for the specified alias, create its
+                         # alias record.
+                         if (! exists $alias{$alias}) {
+                             $loadFeatureAlias->Put($alias);
+                             $alias{$alias} = 1;
+                         }
+                     }
+                     Trace("Assignment for $featureID is: $assignment") if T(4);
+                     # Break the assignment into words and shove it onto the
+                     # keyword list.
+                     push @keywords, split(/\s+/, $assignment);
+                     # Link this feature to the parent genome.
+                     $loadHasFeature->Put($genomeID, $featureID, $type);
                  # Get the links.
                  my @links = $fig->fid_links($featureID);
                  for my $link (@links) {
-Line 517
+Line 477
                          $loadFeatureUpstream->Put($featureID, $upstream);
                      }
                  }
+                     # Now we need to find the subsystems this feature participates in.
+                     # We also add the subsystems to the keyword list. Before we do that,
+                     # we must convert underscores to spaces.
+                     my @subsystems = $fig->peg_to_subsystems($featureID);
+                     for my $subsystem (@subsystems) {
+                         # Only proceed if we like this subsystem.
+                         if (exists $subHash->{$subsystem}) {
+                             # Store the has-role link.
+                             $loadHasRoleInSubsystem->Put($featureID, $subsystem, $genomeID, $type);
+                             # Save the subsystem's keyword data.
+                             my $subKeywords = $subHash->{$subsystem};
+                             push @keywords, split /\s+/, $subKeywords;
+                             # Now we need to get this feature's role in the subsystem.
+                             my $subObject = $fig->get_subsystem($subsystem);
+                             my @roleColumns = $subObject->get_peg_roles($featureID);
+                             my @allRoles = $subObject->get_roles();
+                             for my $col (@roleColumns) {
+                                 my $role = $allRoles[$col];
+                                 push @keywords, split /\s+/, $role;
+                                 push @keywords, $subObject->get_role_abbr($col);
+                             }
+                         }
+                     }
+                     # There are three special attributes computed from property
+                     # data that we build next. If the special attribute is non-empty,
+                     # its name will be added to the keyword list. First, we get all
+                     # the attributes for this feature. They will come back as
+                     # 4-tuples: [peg, name, value, URL]. We use a 3-tuple instead:
+                     # [name, value, value with URL]. (We don't need the PEG, since
+                     # we already know it.)
+                     my @attributes = map { [$_->[1], $_->[2], Tracer::CombineURL($_->[2], $_->[3])] }
+                                          @{$attributes->{$featureID}};
+                     # Now we process each of the special attributes.
+                     if (SpecialAttribute($featureID, \@attributes,
+, [0,2], '^(essential|potential_essential)$',
+                                          $loadFeatureEssential)) {
+                         push @keywords, 'essential';
+                         $loadFeature->Add('essential');
+                     }
+                     if (SpecialAttribute($featureID, \@attributes,
+, [2], '^virulen',
+                                          $loadFeatureVirulent)) {
+                         push @keywords, 'virulent';
+                         $loadFeature->Add('virulent');
+                     }
+                     if (SpecialAttribute($featureID, \@attributes,
+, [0,2], '^iedb_',
+                                          $loadFeatureIEDB)) {
+                         push @keywords, 'iedb';
+                         $loadFeature->Add('iedb');
+                     }
+                     # Now we have some other attributes we need to process. Currently,
+                     # this is CDD and CELLO, but we expect the number to increase.
+                     my %attributeHash = ();
+                     for my $attrRow (@{$attributes->{$featureID}}) {
+                         my (undef, $key, @values) = @{$attrRow};
+                         $key =~ /^([^:]+)::(.+)/;
+                         if (exists $attributeHash{$1}) {
+                             $attributeHash{$1}->{$2} = \@values;
+                         } else {
+                             $attributeHash{$1} = {$2 => \@values};
+                         }
+                     }
+                     my $celloValue = "unknown";
+                     # Pull in the CELLO attribute. There will never be more than one.
+                     # If we have one, it's a feature attribute AND a keyword.
+                     my @celloData = keys %{$attributeHash{CELLO}};
+                     if (@celloData) {
+                         $celloValue = $celloData[0];
+                         push @keywords, $celloValue;
+                     }
+                     # Now we handle CDD. This is a bit more complicated, because
+                     # there are multiple CDDs per protein.
+                     if (exists $attributeHash{CDD}) {
+                         # Get the hash of CDD IDs to scores for this feature. We
+                         # already know it exists because of the above IF.
+                         my $cddHash = $attributeHash{CDD};
+                         my @cddData = sort keys %{$cddHash};
+                         for my $cdd (@cddData) {
+                             # Extract the score for this CDD and decode it.
+                             my ($codeScore) = split(/\s*,\s*/, $cddHash->{$cdd}->[1]);
+                             my $realScore = FIGRules::DecodeScore($codeScore);
+                             # We can't afford to crash because of a bad attribute
+                             # value, hence the IF below.
+                             if (! defined($realScore)) {
+                                 # Bad score, so count it.
+                                 $loadFeature->Add('badCDDscore');
+                             } else {
+                                 # Create the connection.
+                                 $loadIsPresentOnProteinOf->Put($cdd, $featureID, $realScore);
+                                 # If this CDD does not yet exist, create its record.
+                                 if (! exists $CDD{$cdd}) {
+                                     $CDD{$cdd} = 1;
+                                     $loadCDD->Put($cdd);
+                                 }
+                             }
+                         }
+                     }
+                     # Now we need to bust up hyphenated words in the keyword
+                     # list. We keep them separate and put them at the end so
+                     # the original word order is available.
+                     my $keywordString = "";
+                     my $bustedString = "";
+                     for my $keyword (@keywords) {
+                         if (length $keyword >= 3) {
+                             $keywordString .= " $keyword";
+                             if ($keyword =~ /-/) {
+                                 my @words = split /-/, $keyword;
+                                 $bustedString .= join(" ", "", @words);
+                             }
+                         }
+                     }
+                     $keywordString .= $bustedString;
+                     # Get rid of annoying punctuation.
+                     $keywordString =~ s/[();]//g;
+                     # Clean the keyword list.
+                     my $cleanWords = $sprout->CleanKeywords($keywordString);
+                     Trace("Keyword string for $featureID: $cleanWords") if T(4);
+                     # Now we need to process the feature's locations. First, we split them up.
+                     my @locationList = split /\s*,\s*/, $locations;
+                     # Next, we convert them to Sprout location objects.
+                     my @locObjectList = map { BasicLocation->new("$genomeID:$_") } @locationList;
+                     # Assemble them into a sprout location string for later.
+                     my $locationString = join(", ", map { $_->String } @locObjectList);
                  # This part is the roughest. We need to relate the features to contig
                  # locations, and the locations must be split so that none of them exceed
                  # the maximum segment size. This simplifies the genes_in_region processing
-                 # for Sprout.
+                     # for Sprout. To start, we create the location position indicator.
-                 my @locationList = split /\s*,\s*/, $locations;
-                 # Create the location position indicator.
                  my $i = 1;
                  # Loop through the locations.
-                 for my $location (@locationList) {
+                     for my $locObject (@locObjectList) {
-                     # Parse the location.
+                         # Split this location into a list of chunks.
-                     my $locObject = BasicLocation->new("$genomeID:$location");
-                     # Split it into a list of chunks.
                      my @locOList = ();
                      while (my $peeling = $locObject->Peel($chunkSize)) {
                          $loadIsLocatedIn->Add("peeling");
-Line 543
+Line 623
                          $i++;
                      }
                  }
+                     # Finally, reassemble the location objects into a list of Sprout location strings.
+                     # Create the feature record.
+                     $loadFeature->Put($featureID, 1, $user, $quality, $celloValue, $type, $assignment, $cleanWords, $locationString);
              }
          }
-     }
+             Trace("Genome $genomeID processed.") if T(3);
-     # Finish the loads.
-     my $retVal = $self->_FinishAll();
-     return $retVal;
- }
- =head3 LoadBBHData
- C<< my $stats = $spl->LoadBBHData(); >>
- Load the bidirectional best hit data from FIG into Sprout.
- Sprout does not store information on similarities. Instead, it has only the
- bi-directional best hits. Even so, the BBH table is one of the largest in
- the database.
- The following relations are loaded by this method.
-     IsBidirectionalBestHitOf
- =over 4
- =item RETURNS
- Returns a statistics object for the loads.
- =back
- =cut
- #: Return Type $%;
- sub LoadBBHData {
-     # Get this object instance.
-     my ($self) = @_;
-     # Get the FIG object.
-     my $fig = $self->{fig};
-     # Get the table of genome IDs.
-     my $genomeHash = $self->{genomes};
-     # Create load objects for each of the tables we're loading.
-     my $loadIsBidirectionalBestHitOf = $self->_TableLoader('IsBidirectionalBestHitOf');
-     if ($self->{options}->{loadOnly}) {
-         Trace("Loading from existing files.") if T(2);
-     } else {
-         Trace("Generating BBH data.") if T(2);
-         # Now we loop through the genomes, generating the data for each one.
-         for my $genomeID (sort keys %{$genomeHash}) {
-             $loadIsBidirectionalBestHitOf->Add("genomeIn");
-             Trace("Processing features for genome $genomeID.") if T(3);
-             # Get the feature list for this genome.
-             my $features = $fig->all_features_detailed($genomeID);
-             # Loop through the features.
-             for my $featureData (@{$features}) {
-                 # Split the tuple.
-                 my ($featureID, $locations, $aliases, $type) = @{$featureData};
-                 # Get the bi-directional best hits.
-                 my @bbhList = $fig->bbhs($featureID);
-                 for my $bbhEntry (@bbhList) {
-                     # Get the target feature ID and the score.
-                     my ($targetID, $score) = @{$bbhEntry};
-                     # Check the target feature's genome.
-                     my $targetGenomeID = $fig->genome_of($targetID);
-                     # Only proceed if it's one of our genomes.
-                     if ($genomeHash->{$targetGenomeID}) {
-                         $loadIsBidirectionalBestHitOf->Put($featureID, $targetID, $targetGenomeID,
-                                                            $score);
-                     }
-                 }
-             }
          }
      }
      # Finish the loads.
-Line 621
+Line 638
  =head3 LoadSubsystemData
- C<< my $stats = $spl->LoadSubsystemData(); >>
+     my $stats = $spl->LoadSubsystemData();
  Load the subsystem data from FIG into Sprout.
-Line 637
+Line 654
      SubsystemClass
      Role
      RoleEC
+     IsIdentifiedByEC
      SSCell
      ContainsFeature
      IsGenomeOf
-Line 678
+Line 696
      # Get the map list.
      my @maps = $fig->all_maps;
      # Create load objects for each of the tables we're loading.
-     my $loadDiagram = $self->_TableLoader('Diagram', $self->PrimaryOnly);
+     my $loadDiagram = $self->_TableLoader('Diagram');
-     my $loadRoleOccursIn = $self->_TableLoader('RoleOccursIn', $self->PrimaryOnly);
+     my $loadRoleOccursIn = $self->_TableLoader('RoleOccursIn');
      my $loadSubsystem = $self->_TableLoader('Subsystem');
-     my $loadRole = $self->_TableLoader('Role', $self->PrimaryOnly);
+     my $loadRole = $self->_TableLoader('Role');
-     my $loadRoleEC = $self->_TableLoader('RoleEC', $self->PrimaryOnly);
+     my $loadRoleEC = $self->_TableLoader('RoleEC');
-     my $loadCatalyzes = $self->_TableLoader('Catalyzes', $self->PrimaryOnly);
+     my $loadIsIdentifiedByEC = $self->_TableLoader('IsIdentifiedByEC');
-     my $loadSSCell = $self->_TableLoader('SSCell', $self->PrimaryOnly);
+     my $loadCatalyzes = $self->_TableLoader('Catalyzes');
-     my $loadContainsFeature = $self->_TableLoader('ContainsFeature', $self->PrimaryOnly);
+     my $loadSSCell = $self->_TableLoader('SSCell');
-     my $loadIsGenomeOf = $self->_TableLoader('IsGenomeOf', $self->PrimaryOnly);
+     my $loadContainsFeature = $self->_TableLoader('ContainsFeature');
-     my $loadIsRoleOf = $self->_TableLoader('IsRoleOf', $self->PrimaryOnly);
+     my $loadIsGenomeOf = $self->_TableLoader('IsGenomeOf');
-     my $loadOccursInSubsystem = $self->_TableLoader('OccursInSubsystem', $self->PrimaryOnly);
+     my $loadIsRoleOf = $self->_TableLoader('IsRoleOf');
-     my $loadParticipatesIn = $self->_TableLoader('ParticipatesIn', $self->PrimaryOnly);
+     my $loadOccursInSubsystem = $self->_TableLoader('OccursInSubsystem');
-     my $loadHasSSCell = $self->_TableLoader('HasSSCell', $self->PrimaryOnly);
+     my $loadParticipatesIn = $self->_TableLoader('ParticipatesIn');
-     my $loadRoleSubset = $self->_TableLoader('RoleSubset', $self->PrimaryOnly);
+     my $loadHasSSCell = $self->_TableLoader('HasSSCell');
-     my $loadGenomeSubset = $self->_TableLoader('GenomeSubset', $self->PrimaryOnly);
+     my $loadRoleSubset = $self->_TableLoader('RoleSubset');
-     my $loadConsistsOfRoles = $self->_TableLoader('ConsistsOfRoles', $self->PrimaryOnly);
+     my $loadGenomeSubset = $self->_TableLoader('GenomeSubset');
-     my $loadConsistsOfGenomes = $self->_TableLoader('ConsistsOfGenomes', $self->PrimaryOnly);
+     my $loadConsistsOfRoles = $self->_TableLoader('ConsistsOfRoles');
-     my $loadHasRoleSubset = $self->_TableLoader('HasRoleSubset', $self->PrimaryOnly);
+     my $loadConsistsOfGenomes = $self->_TableLoader('ConsistsOfGenomes');
-     my $loadHasGenomeSubset = $self->_TableLoader('HasGenomeSubset', $self->PrimaryOnly);
+     my $loadHasRoleSubset = $self->_TableLoader('HasRoleSubset');
-     my $loadSubsystemClass = $self->_TableLoader('SubsystemClass', $self->PrimaryOnly);
+     my $loadHasGenomeSubset = $self->_TableLoader('HasGenomeSubset');
+     my $loadSubsystemClass = $self->_TableLoader('SubsystemClass');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
          Trace("Generating subsystem data.") if T(2);
-         # This hash will contain the role for each EC. When we're done, this
+         # This hash will contain the roles for each EC. When we're done, this
          # information will be used to generate the Catalyzes table.
          my %ecToRoles = ();
          # Loop through the subsystems. Our first task will be to create the
-Line 716
+Line 735
              # Get the subsystem object.
              my $sub = $fig->get_subsystem($subsysID);
              # Only proceed if the subsystem has a spreadsheet.
-             if (! $sub->{empty_ss}) {
+             if (defined($sub) && ! $sub->{empty_ss}) {
                  Trace("Creating subsystem $subsysID.") if T(3);
                  $loadSubsystem->Add("subsystemIn");
                  # Create the subsystem record.
                  my $curator = $sub->get_curator();
                  my $notes = $sub->get_notes();
                  $loadSubsystem->Put($subsysID, $curator, $notes);
-                 my $class = $fig->subsystem_classification($subsysID);
+                 # Now for the classification string. This comes back as a list
-                 if ($class) {
+                 # reference and we convert it to a space-delimited string.
-                     $loadSubsystemClass->Put($subsysID, $class);
+                 my $classList = $fig->subsystem_classification($subsysID);
-                 }
+                 my $classString = join($FIG_Config::splitter, grep { $_ } @$classList);
+                 $loadSubsystemClass->Put($subsysID, $classString);
                  # Connect it to its roles. Each role is a column in the subsystem spreadsheet.
                  for (my $col = 0; defined($roleID = $sub->get_role($col)); $col++) {
+                     # Get the role's abbreviation.
+                     my $abbr = $sub->get_role_abbr($col);
                      # Connect to this role.
                      $loadOccursInSubsystem->Add("roleIn");
-                     $loadOccursInSubsystem->Put($roleID, $subsysID, $col);
+                     $loadOccursInSubsystem->Put($roleID, $subsysID, $abbr, $col);
                      # If it's a new role, add it to the role table.
                      if (! exists $roleData{$roleID}) {
                          # Get the role's abbreviation.
-                         my $abbr = $sub->get_role_abbr($col);
                          # Add the role.
-                         $loadRole->Put($roleID, $abbr);
+                         $loadRole->Put($roleID);
                          $roleData{$roleID} = 1;
                          # Check for an EC number.
-                         if ($roleID =~ /\(EC ([^.]+\.[^.]+\.[^.]+\.[^)]+)\)\s*$/) {
+                         if ($roleID =~ /\(EC (\d+\.\d+\.\d+\.\d+)\s*\)\s*$/) {
                              my $ec = $1;
-                             $loadRoleEC->Put($roleID, $ec);
+                             $loadIsIdentifiedByEC->Put($roleID, $ec);
-                             $ecToRoles{$ec} = $roleID;
+                             # Check to see if this is our first encounter with this EC.
+                             if (exists $ecToRoles{$ec}) {
+                                 # No, so just add this role to the EC list.
+                                 push @{$ecToRoles{$ec}}, $roleID;
+                             } else {
+                                 # Output this EC.
+                                 $loadRoleEC->Put($ec);
+                                 # Create its role list.
+                                 $ecToRoles{$ec} = [$roleID];
+                             }
                          }
                      }
                  }
-Line 843
+Line 873
                      }
                  }
              }
+         }
              # Now we loop through the diagrams. We need to create the diagram records
              # and link each diagram to its roles. Note that only roles which occur
              # in subsystems (and therefore appear in the %ecToRoles hash) are
-Line 855
+Line 886
                  # Now we need to link all the map's roles to it.
                  # A hash is used to prevent duplicates.
                  my %roleHash = ();
-                 for my $role ($fig->map_to_ecs($map)) {
+             for my $ec ($fig->map_to_ecs($map)) {
-                     if (exists $ecToRoles{$role} && ! $roleHash{$role}) {
+                 if (exists $ecToRoles{$ec}) {
-                         $loadRoleOccursIn->Put($ecToRoles{$role}, $map);
+                     for my $role (@{$ecToRoles{$ec}}) {
+                         if (! $roleHash{$role}) {
+                             $loadRoleOccursIn->Put($role, $map);
                          $roleHash{$role} = 1;
                      }
                  }
              }
+             }
+         }
              # Before we leave, we must create the Catalyzes table. We start with the reactions,
              # then use the "ecToRoles" table to convert EC numbers to role IDs.
              my @reactions = $fig->all_reactions();
              for my $reactionID (@reactions) {
                  # Get this reaction's list of roles. The results will be EC numbers.
-                 my @roles = $fig->catalyzed_by($reactionID);
+             my @ecs = $fig->catalyzed_by($reactionID);
                  # Loop through the roles, creating catalyzation records.
-                 for my $thisRole (@roles) {
+             for my $thisEC (@ecs) {
-                     if (exists $ecToRoles{$thisRole}) {
+                 if (exists $ecToRoles{$thisEC}) {
-                         $loadCatalyzes->Put($ecToRoles{$thisRole}, $reactionID);
+                     for my $thisRole (@{$ecToRoles{$thisEC}}) {
+                         $loadCatalyzes->Put($thisRole, $reactionID);
                      }
                  }
              }
-Line 884
+Line 920
  =head3 LoadPropertyData
- C<< my $stats = $spl->LoadPropertyData(); >>
+     my $stats = $spl->LoadPropertyData();
  Load the attribute data from FIG into Sprout.
-Line 920
+Line 956
      my $genomeHash = $self->{genomes};
      # Create load objects for each of the tables we're loading.
      my $loadProperty = $self->_TableLoader('Property');
-     my $loadHasProperty = $self->_TableLoader('HasProperty', $self->PrimaryOnly);
+     my $loadHasProperty = $self->_TableLoader('HasProperty');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-Line 928
+Line 964
          # Create a hash for storing property IDs.
          my %propertyKeys = ();
          my $nextID = 1;
+         # Get the attributes we intend to store in the property table.
+         my $propKeys = $self->{propKeys};
          # Loop through the genomes.
-         for my $genomeID (keys %{$genomeHash}) {
+         for my $genomeID (sort keys %{$genomeHash}) {
              $loadProperty->Add("genomeIn");
              Trace("Generating properties for $genomeID.") if T(3);
-             # Get the genome's features. The feature ID is the first field in the
+             # Initialize a counter.
-             # tuples returned by "all_features_detailed". We use "all_features_detailed"
-             # rather than "all_features" because we want all features regardless of type.
-             my @features = map { $_->[0] } @{$fig->all_features_detailed($genomeID)};
-             my $featureCount = 0;
              my $propertyCount = 0;
-             # Loop through the features, creating HasProperty records.
+             # Get the properties for this genome's features.
-             for my $fid (@features) {
+             my @attributes = $fig->get_attributes("fig|$genomeID%", $propKeys);
-                 # Get all attributes for this feature. We do this one feature at a time
+             Trace("Property list built for $genomeID.") if T(3);
-                 # to insure we do not get any genome attributes.
+             # Loop through the results, creating HasProperty records.
-                 my @attributeList = $fig->get_attributes($fid, '', '', '');
+             for my $attributeData (@attributes) {
-                 if (scalar @attributeList) {
+                 # Pull apart the attribute tuple.
-                     $featureCount++;
+                 my ($fid, $key, $value, $url) = @{$attributeData};
-                 }
-                 # Loop through the attributes.
-                 for my $tuple (@attributeList) {
-                     $propertyCount++;
-                     # Get this attribute value's data. Note that we throw away the FID,
-                     # since it will always be the same as the value if "$fid".
-                     my (undef, $key, $value, $url) = @{$tuple};
                      # Concatenate the key and value and check the "propertyKeys" hash to
                      # see if we already have an ID for it. We use a tab for the separator
                      # character.
-Line 969
+Line 996
                      # Create the HasProperty entry for this feature/property association.
                      $loadHasProperty->Put($fid, $propertyID, $url);
                  }
-             }
              # Update the statistics.
-             Trace("$propertyCount attributes processed for $featureCount features.") if T(3);
+             Trace("$propertyCount attributes processed.") if T(3);
-             $loadHasProperty->Add("featuresIn", $featureCount);
              $loadHasProperty->Add("propertiesIn", $propertyCount);
          }
      }
-Line 983
+Line 1008
  =head3 LoadAnnotationData
- C<< my $stats = $spl->LoadAnnotationData(); >>
+     my $stats = $spl->LoadAnnotationData();
  Load the annotation data from FIG into Sprout.
-Line 1017
+Line 1042
      my $genomeHash = $self->{genomes};
      # Create load objects for each of the tables we're loading.
      my $loadAnnotation = $self->_TableLoader('Annotation');
-     my $loadIsTargetOfAnnotation = $self->_TableLoader('IsTargetOfAnnotation', $self->PrimaryOnly);
+     my $loadIsTargetOfAnnotation = $self->_TableLoader('IsTargetOfAnnotation');
-     my $loadSproutUser = $self->_TableLoader('SproutUser', $self->PrimaryOnly);
+     my $loadSproutUser = $self->_TableLoader('SproutUser');
-     my $loadUserAccess = $self->_TableLoader('UserAccess', $self->PrimaryOnly);
+     my $loadUserAccess = $self->_TableLoader('UserAccess');
-     my $loadMadeAnnotation = $self->_TableLoader('MadeAnnotation', $self->PrimaryOnly);
+     my $loadMadeAnnotation = $self->_TableLoader('MadeAnnotation');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-Line 1090
+Line 1115
  =head3 LoadSourceData
- C<< my $stats = $spl->LoadSourceData(); >>
+     my $stats = $spl->LoadSourceData();
  Load the source data from FIG into Sprout.
-Line 1124
+Line 1149
      # Get the genome hash.
      my $genomeHash = $self->{genomes};
      # Create load objects for each of the tables we're loading.
-     my $loadComesFrom = $self->_TableLoader('ComesFrom', $self->PrimaryOnly);
+     my $loadComesFrom = $self->_TableLoader('ComesFrom');
      my $loadSource = $self->_TableLoader('Source');
      my $loadSourceURL = $self->_TableLoader('SourceURL');
      if ($self->{options}->{loadOnly}) {
-Line 1168
+Line 1193
  =head3 LoadExternalData
- C<< my $stats = $spl->LoadExternalData(); >>
+     my $stats = $spl->LoadExternalData();
  Load the external data from FIG into Sprout.
-Line 1211
+Line 1236
      } else {
          Trace("Generating external data.") if T(2);
          # We loop through the files one at a time. First, the organism file.
-         Open(\*ORGS, "<$FIG_Config::global/ext_org.table");
+         Open(\*ORGS, "sort +0 -1 -u -t\"\t\" $FIG_Config::global/ext_org.table |");
          my $orgLine;
          while (defined($orgLine = <ORGS>)) {
              # Clean the input line.
-Line 1223
+Line 1248
          close ORGS;
          # Now the function file.
          my $funcLine;
-         Open(\*FUNCS, "<$FIG_Config::global/ext_func.table");
+         Open(\*FUNCS, "sort +0 -1 -u -t\"\t\" $FIG_Config::global/ext_func.table |");
          while (defined($funcLine = <FUNCS>)) {
              # Clean the line ending.
              chomp $funcLine;
-Line 1248
+Line 1273
  =head3 LoadReactionData
- C<< my $stats = $spl->LoadReactionData(); >>
+     my $stats = $spl->LoadReactionData();
  Load the reaction data from FIG into Sprout.
-Line 1261
+Line 1286
      Compound
      CompoundName
      CompoundCAS
+     IsIdentifiedByCAS
+     HasCompoundName
      IsAComponentOf
  This method proceeds reaction by reaction rather than genome by genome.
-Line 1282
+Line 1309
      my $fig = $self->{fig};
      # Create load objects for each of the tables we're loading.
      my $loadReaction = $self->_TableLoader('Reaction');
-     my $loadReactionURL = $self->_TableLoader('ReactionURL', $self->PrimaryOnly);
+     my $loadReactionURL = $self->_TableLoader('ReactionURL');
-     my $loadCompound = $self->_TableLoader('Compound', $self->PrimaryOnly);
+     my $loadCompound = $self->_TableLoader('Compound');
-     my $loadCompoundName = $self->_TableLoader('CompoundName', $self->PrimaryOnly);
+     my $loadCompoundName = $self->_TableLoader('CompoundName');
-     my $loadCompoundCAS = $self->_TableLoader('CompoundCAS', $self->PrimaryOnly);
+     my $loadCompoundCAS = $self->_TableLoader('CompoundCAS');
-     my $loadIsAComponentOf = $self->_TableLoader('IsAComponentOf', $self->PrimaryOnly);
+     my $loadIsAComponentOf = $self->_TableLoader('IsAComponentOf');
+     my $loadIsIdentifiedByCAS = $self->_TableLoader('IsIdentifiedByCAS');
+     my $loadHasCompoundName = $self->_TableLoader('HasCompoundName');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-         Trace("Generating annotation data.") if T(2);
+         Trace("Generating reaction data.") if T(2);
+         # We need some hashes to prevent duplicates.
+         my %compoundNames = ();
+         my %compoundCASes = ();
          # First we create the compounds.
          my @compounds = $fig->all_compounds();
          for my $cid (@compounds) {
-Line 1299
+Line 1331
              # Each name will be given a priority number, starting with 1.
              my $prio = 1;
              for my $name (@names) {
-                 $loadCompoundName->Put($cid, $name, $prio++);
+                 if (! exists $compoundNames{$name}) {
+                     $loadCompoundName->Put($name);
+                     $compoundNames{$name} = 1;
+                 }
+                 $loadHasCompoundName->Put($cid, $name, $prio++);
              }
              # Create the main compound record. Note that the first name
              # becomes the label.
-Line 1308
+Line 1344
              # Check for a CAS ID.
              my $cas = $fig->cas($cid);
              if ($cas) {
-                 $loadCompoundCAS->Put($cid, $cas);
+                 $loadIsIdentifiedByCAS->Put($cid, $cas);
+                 if (! exists $compoundCASes{$cas}) {
+                     $loadCompoundCAS->Put($cas);
+                     $compoundCASes{$cas} = 1;
+                 }
              }
          }
          # All the compounds are set up, so we need to loop through the reactions next. First,
-Line 1345
+Line 1385
      return $retVal;
  }
- =head3 LoadGroupData
+ =head3 LoadSynonymData
- C<< my $stats = $spl->LoadGroupData(); >>
+     my $stats = $spl->LoadSynonymData();
- Load the genome Groups into Sprout.
+ Load the synonym groups into Sprout.
  The following relations are loaded by this method.
-     GenomeGroups
+     SynonymGroup
+     IsSynonymGroupFor
- There is no direct support for genome groups in FIG, so we access the SEED
+ The source information for these relations is taken from the C<maps_to_id> method
- files directly.
+ of the B<FIG> object. Unfortunately, to make this work, we need to use direct
+ SQL against the FIG database.
  =over 4
-Line 1368
+Line 1410
  =cut
  #: Return Type $%;
- sub LoadGroupData {
+ sub LoadSynonymData {
      # Get this object instance.
      my ($self) = @_;
      # Get the FIG object.
-Line 1376
+Line 1418
      # Get the genome hash.
      my $genomeHash = $self->{genomes};
      # Create a load object for the table we're loading.
-     my $loadGenomeGroups = $self->_TableLoader('GenomeGroups');
+     my $loadSynonymGroup = $self->_TableLoader('SynonymGroup');
+     my $loadIsSynonymGroupFor = $self->_TableLoader('IsSynonymGroupFor');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-         Trace("Generating group data.") if T(2);
+         Trace("Generating synonym group data.") if T(2);
-         # Loop through the genomes.
+         # Get the database handle.
-         my $line;
+         my $dbh = $fig->db_handle();
-         for my $genomeID (keys %{$genomeHash}) {
+         # Ask for the synonyms. Note that "maps_to" is a group name, and "syn_id" is a PEG ID or alias.
-             Trace("Processing $genomeID.") if T(3);
+         my $sth = $dbh->prepare_command("SELECT maps_to, syn_id FROM peg_synonyms ORDER BY maps_to");
-             # Open the NMPDR group file for this genome.
+         my $result = $sth->execute();
-             if (open(TMP, "<$FIG_Config::organisms/$genomeID/NMPDR") &&
+         if (! defined($result)) {
-                 defined($line = <TMP>)) {
+             Confess("Database error in Synonym load: " . $sth->errstr());
-                 # Clean the line ending.
+         } else {
-                 chomp $line;
+             Trace("Processing synonym results.") if T(2);
-                 # Add the group to the table. Note that there can only be one group
+             # Remember the current synonym.
-                 # per genome.
+             my $current_syn = "";
-                 $loadGenomeGroups->Put($genomeID, $line);
+             # Count the features.
+             my $featureCount = 0;
+             my $entryCount = 0;
+             # Loop through the synonym/peg pairs.
+             while (my @row = $sth->fetchrow()) {
+                 # Get the synonym group ID and feature ID.
+                 my ($syn_id, $peg) = @row;
+                 # Count this row.
+                 $entryCount++;
+                 if ($entryCount % 1000 == 0) {
+                     Trace("$entryCount rows processed.") if T(3);
              }
-             close TMP;
+                 # Insure it's for one of our genomes.
+                 my $genomeID = FIG::genome_of($peg);
+                 if (exists $genomeHash->{$genomeID}) {
+                     # Verify the synonym.
+                     if ($syn_id ne $current_syn) {
+                         # It's new, so put it in the group table.
+                         $loadSynonymGroup->Put($syn_id);
+                         $current_syn = $syn_id;
+                     }
+                     # Connect the synonym to the peg.
+                     $loadIsSynonymGroupFor->Put($syn_id, $peg);
+                     # Count this feature.
+                     $featureCount++;
+                     if ($featureCount % 1000 == 0) {
+                         Trace("$featureCount features processed.") if T(3);
+                     }
+                 }
+             }
+             Trace("$entryCount rows produced $featureCount features.") if T(2);
          }
      }
      # Finish the load.
-Line 1402
+Line 1473
      return $retVal;
  }
- =head3 LoadSynonymData
+ =head3 LoadFamilyData
- C<< my $stats = $spl->LoadSynonymData(); >>
+     my $stats = $spl->LoadFamilyData();
- Load the synonym groups into Sprout.
+ Load the protein families into Sprout.
  The following relations are loaded by this method.
-     SynonymGroup
+     Family
-     IsSynonymGroupFor
+     IsFamilyForFeature
- The source information for these relations is taken from the C<maps_to_id> method
+ The source information for these relations is taken from the C<families_for_protein>,
- of the B<FIG> object. The process starts from the features, so it is possible
+ C<family_function>, and C<sz_family> methods of the B<FIG> object.
- that there will be duplicates in the SynonymGroup load file, since the relationship
- is one-to-many toward the features. The automatic sort on primary entity relations
- will fix this for us.
  =over 4
-Line 1429
+Line 1497
  =cut
  #: Return Type $%;
- sub LoadSynonymData {
+ sub LoadFamilyData {
      # Get this object instance.
      my ($self) = @_;
      # Get the FIG object.
      my $fig = $self->{fig};
      # Get the genome hash.
      my $genomeHash = $self->{genomes};
-     # Create a load object for the table we're loading.
+     # Create load objects for the tables we're loading.
-     my $loadSynonymGroup = $self->_TableLoader('SynonymGroup');
+     my $loadFamily = $self->_TableLoader('Family');
-     my $loadIsSynonymGroupFor = $self->_TableLoader('IsSynonymGroupFor');
+     my $loadIsFamilyForFeature = $self->_TableLoader('IsFamilyForFeature');
      if ($self->{options}->{loadOnly}) {
          Trace("Loading from existing files.") if T(2);
      } else {
-         Trace("Generating synonym group data.") if T(2);
+         Trace("Generating family data.") if T(2);
+         # Create a hash for the family IDs.
+         my %familyHash = ();
          # Loop through the genomes.
          for my $genomeID (sort keys %{$genomeHash}) {
-             Trace("Processing $genomeID.") if T(3);
+             Trace("Processing features for $genomeID.") if T(2);
-             # Get all of the features for this genome. The only method that does this is
+             # Loop through this genome's PEGs.
-             # all_features_detailed, which returns extra baggage that we discard.
+             for my $fid ($fig->all_features($genomeID, "peg")) {
-             my $featureData = $fig->all_features_detailed($genomeID);
+                 $loadIsFamilyForFeature->Add("features", 1);
-             my @fids = map { $_->[0] } @{$featureData};
+                 # Get this feature's families.
-             Trace(scalar(@fids) . " features found for genome $genomeID.") if T(3);
+                 my @families = $fig->families_for_protein($fid);
-             # Loop through the feature IDs.
+                 # Loop through the families, connecting them to the feature.
-             for my $fid (@fids) {
+                 for my $family (@families) {
-                 # Get the group for this feature.
+                     $loadIsFamilyForFeature->Put($family, $fid);
-                 my $synonym = $fig->maps_to_id($fid);
+                     # If this is a new family, create a record for it.
-                 # Only proceed if the synonym is a real group.
+                     if (! exists $familyHash{$family}) {
-                 if ($synonym ne $fid) {
+                         $familyHash{$family} = 1;
-                     $loadSynonymGroup->Put($synonym);
+                         $loadFamily->Add("families", 1);
-                     $loadIsSynonymGroupFor->Put($synonym, $fid);
+                         my $size = $fig->sz_family($family);
+                         my $func = $fig->family_function($family);
+                         $loadFamily->Put($family, $size, $func);
+                     }
+                 }
+             }
+         }
+     }
+     # Finish the load.
+     my $retVal = $self->_FinishAll();
+     return $retVal;
+ }
+ =head3 LoadDrugData
+     my $stats = $spl->LoadDrugData();
+ Load the drug target data into Sprout.
+ The following relations are loaded by this method.
+     PDB
+     DocksWith
+     IsProteinForFeature
+     Ligand
+ The source information for these relations is taken from attributes. The
+ C<PDB> attribute links a PDB to a feature, and is used to build B<IsProteinForFeature>.
+ The C<zinc_name> attribute describes the ligands. The C<docking_results>
+ attribute contains the information for the B<DocksWith> relationship. It is
+ expected that additional attributes and tables will be added in the future.
+ =over 4
+ =item RETURNS
+ Returns a statistics object for the loads.
+ =back
+ =cut
+ #: Return Type $%;
+ sub LoadDrugData {
+     # Get this object instance.
+     my ($self) = @_;
+     # Get the FIG object.
+     my $fig = $self->{fig};
+     # Get the genome hash.
+     my $genomeHash = $self->{genomes};
+     # Create load objects for the tables we're loading.
+     my $loadPDB = $self->_TableLoader('PDB');
+     my $loadLigand = $self->_TableLoader('Ligand');
+     my $loadIsProteinForFeature = $self->_TableLoader('IsProteinForFeature');
+     my $loadDocksWith = $self->_TableLoader('DocksWith');
+     if ($self->{options}->{loadOnly}) {
+         Trace("Loading from existing files.") if T(2);
+     } else {
+         Trace("Generating drug target data.") if T(2);
+         # First comes the "DocksWith" relationship. This will give us a list of PDBs.
+         # We can also encounter PDBs when we process "IsProteinForFeature". To manage
+         # this process, PDB information is collected in a hash table and then
+         # unspooled after both relationships are created.
+         my %pdbHash = ();
+         Trace("Generating docking data.") if T(2);
+         # Get all the docking data. This may cause problems if there are too many PDBs,
+         # at which point we'll need another algorithm. The indicator that this is
+         # happening will be a timeout error in the next statement.
+         my @dockData = $fig->query_attributes('$key = ? AND $value < ?',
+                                               ['docking_results', $FIG_Config::dockLimit]);
+         Trace(scalar(@dockData) . " rows of docking data found.") if T(3);
+         for my $dockData (@dockData) {
+             # Get the docking data components.
+             my ($pdbID, $docking_key, @valueData) = @{$dockData};
+             # Fix the PDB ID. It's supposed to be lower-case, but this does not always happen.
+             $pdbID = lc $pdbID;
+             # Strip off the object type.
+             $pdbID =~ s/pdb://;
+             # Extract the ZINC ID from the docking key. Note that there are two possible
+             # formats.
+             my (undef, $zinc_id) = $docking_key =~ /^docking_results::(ZINC)?(\d+)$/;
+             if (! $zinc_id) {
+                 Trace("Invalid docking result key $docking_key for $pdbID.") if T(0);
+                 $loadDocksWith->Add("errors");
+             } else {
+                 # Get the pieces of the value and parse the energy.
+                 # Note that we don't care about the rank, since
+                 # we can sort on the energy level itself in our database.
+                 my ($energy, $tool, $type) = @valueData;
+                 my ($rank, $total, $vanderwaals, $electrostatic) = split /\s*;\s*/, $energy;
+                 # Ignore predicted results.
+                 if ($type ne "Predicted") {
+                     # Count this docking result.
+                     if (! exists $pdbHash{$pdbID}) {
+                         $pdbHash{$pdbID} = 1;
+                     } else {
+                         $pdbHash{$pdbID}++;
+                     }
+                     # Write the result to the output.
+                     $loadDocksWith->Put($pdbID, $zinc_id, $electrostatic, $type, $tool,
+                                         $total, $vanderwaals);
+                 }
+             }
+         }
+         Trace("Connecting features.") if T(2);
+         # Loop through the genomes.
+         for my $genome (sort keys %{$genomeHash}) {
+             Trace("Generating PDBs for $genome.") if T(3);
+             # Get all of the PDBs that BLAST against this genome's features.
+             my @attributeData = $fig->get_attributes("fig|$genome%", 'PDB::%');
+             for my $pdbData (@attributeData) {
+                 # The PDB ID is coded as a subkey.
+                 if ($pdbData->[1] !~ /PDB::(.+)/i) {
+                     Trace("Invalid PDB ID \"$pdbData->[1]\" in attribute table.") if T(0);
+                     $loadPDB->Add("errors");
+                 } else {
+                     my $pdbID = $1;
+                     # Insure the PDB is in the hash.
+                     if (! exists $pdbHash{$pdbID}) {
+                         $pdbHash{$pdbID} = 0;
+                     }
+                     # The score and locations are coded in the attribute value.
+                     if ($pdbData->[2] !~ /^([^;]+)(.*)$/) {
+                         Trace("Invalid PDB data for $pdbID and feature $pdbData->[0].") if T(0);
+                         $loadIsProteinForFeature->Add("errors");
+                     } else {
+                         my ($score, $locData) = ($1,$2);
+                         # The location data may not be present, so we have to start with some
+                         # defaults and then check.
+                         my ($start, $end) = (1, 0);
+                         if ($locData) {
+                             $locData =~ /(\d+)-(\d+)/;
+                             $start = $1;
+                             $end = $2;
+                         }
+                         # If we still don't have the end location, compute it from
+                         # the feature length.
+                         if (! $end) {
+                             # Most features have one location, but we do a list iteration
+                             # just in case.
+                             my @locations = $fig->feature_location($pdbData->[0]);
+                             $end = 0;
+                             for my $loc (@locations) {
+                                 my $locObject = BasicLocation->new($loc);
+                                 $end += $locObject->Length;
+                             }
+                         }
+                         # Decode the score.
+                         my $realScore = FIGRules::DecodeScore($score);
+                         # Connect the PDB to the feature.
+                         $loadIsProteinForFeature->Put($pdbID, $pdbData->[0], $start, $realScore, $end);
+                     }
+                 }
+             }
+         }
+         # We've got all our PDBs now, so we unspool them from the hash.
+         Trace("Generating PDBs. " . scalar(keys %pdbHash) . " found.") if T(2);
+         my $count = 0;
+         for my $pdbID (sort keys %pdbHash) {
+             $loadPDB->Put($pdbID, $pdbHash{$pdbID});
+             $count++;
+             Trace("$count PDBs processed.") if T(3) && ($count % 500 == 0);
+         }
+         # Finally we create the ligand table. This information can be found in the
+         # zinc_name attribute.
+         Trace("Loading ligands.") if T(2);
+         # The ligand list is huge, so we have to get it in pieces. We also have to check for duplicates.
+         my $last_zinc_id = "";
+         my $zinc_id = "";
+         my $done = 0;
+         while (! $done) {
+             # Get the next 10000 ligands. We insist that the object ID is greater than
+             # the last ID we processed.
+             Trace("Loading batch starting with ZINC:$zinc_id.") if T(3);
+             my @attributeData = $fig->query_attributes('$object > ? AND $key = ? ORDER BY $object LIMIT 10000',
+                                                        ["ZINC:$zinc_id", "zinc_name"]);
+             Trace(scalar(@attributeData) . " attribute rows returned.") if T(3);
+             if (! @attributeData) {
+                 # Here there are no attributes left, so we quit the loop.
+                 $done = 1;
+             } else {
+                 # Process the attribute data we've received.
+                 for my $zinc_data (@attributeData) {
+                     # The ZINC ID is found in the first return column, prefixed with the word ZINC.
+                     if ($zinc_data->[0] =~ /^ZINC:(\d+)$/) {
+                         $zinc_id = $1;
+                         # Check for a duplicate.
+                         if ($zinc_id eq $last_zinc_id) {
+                             $loadLigand->Add("duplicate");
+                         } else {
+                             # Here it's safe to output the ligand. The ligand name is the attribute value
+                             # (third column in the row).
+                             $loadLigand->Put($zinc_id, $zinc_data->[2]);
+                             # Insure we don't try to add this ID again.
+                             $last_zinc_id = $zinc_id;
+                         }
+                     } else {
+                         Trace("Invalid zinc ID \"$zinc_data->[0]\" in attribute table.") if T(0);
+                         $loadLigand->Add("errors");
                  }
              }
          }
      }
+         Trace("Ligands loaded.") if T(2);
+     }
      # Finish the load.
      my $retVal = $self->_FinishAll();
      return $retVal;
-Line 1471
+Line 1740
  =head2 Internal Utility Methods
+ =head3 SpecialAttribute
+     my $count = SproutLoad::SpecialAttribute($id, \@attributes, $idxMatch, \@idxValues, $pattern, $loader);
+ Look for special attributes of a given type. A special attribute is found by comparing one of
+ the columns of the incoming attribute list to a search pattern. If a match is found, then
+ a set of columns is put into an output table connected to the specified ID.
+ For example, when processing features, the attribute list we look at has three columns: attribute
+ name, attribute value, and attribute value HTML. The IEDB attribute exists if the attribute name
+ begins with C<iedb_>. The call signature is therefore
+     my $found = SpecialAttribute($fid, \@attributeList, 0, [0,2], '^iedb_', $loadFeatureIEDB);
+ The pattern is matched against column 0, and if we have a match, then column 2's value is put
+ to the output along with the specified feature ID.
+ =over 4
+ =item id
+ ID of the object whose special attributes are being loaded. This forms the first column of the
+ output.
+ =item attributes
+ Reference to a list of tuples.
+ =item idxMatch
+ Index in each tuple of the column to be matched against the pattern. If the match is
+ successful, an output record will be generated.
+ =item idxValues
+ Reference to a list containing the indexes in each tuple of the columns to be put as
+ the second column of the output.
+ =item pattern
+ Pattern to be matched against the specified column. The match will be case-insensitive.
+ =item loader
+ An object to which each output record will be put. Usually this is an B<ERDBLoad> object,
+ but technically it could be anything with a C<Put> method.
+ =item RETURN
+ Returns a count of the matches found.
+ =item
+ =back
+ =cut
+ sub SpecialAttribute {
+     # Get the parameters.
+     my ($id, $attributes, $idxMatch, $idxValues, $pattern, $loader) = @_;
+     # Declare the return variable.
+     my $retVal = 0;
+     # Loop through the attribute rows.
+     for my $row (@{$attributes}) {
+         # Check for a match.
+         if ($row->[$idxMatch] =~ m/$pattern/i) {
+             # We have a match, so output a row. This is a bit tricky, since we may
+             # be putting out multiple columns of data from the input.
+             my $value = join(" ", map { $row->[$_] } @{$idxValues});
+             $loader->Put($id, $value);
+             $retVal++;
+         }
+     }
+     Trace("$retVal special attributes found for $id and loader " . $loader->RelName() . ".") if T(4) && $retVal;
+     # Return the number of matches.
+     return $retVal;
+ }
  =head3 TableLoader
  Create an ERDBLoad object for the specified table. The object is also added to
-Line 1485
+Line 1832
  Name of the table (relation) being loaded.
- =item ignore
- TRUE if the table should be ignored entirely, else FALSE.
  =item RETURN
  Returns an ERDBLoad object for loading the specified table.
-Line 1499
+Line 1842
  sub _TableLoader {
      # Get the parameters.
-     my ($self, $tableName, $ignore) = @_;
+     my ($self, $tableName) = @_;
      # Create the load object.
-     my $retVal = ERDBLoad->new($self->{erdb}, $tableName, $self->{loadDirectory}, $self->LoadOnly,
+     my $retVal = ERDBLoad->new($self->{erdb}, $tableName, $self->{loadDirectory}, $self->LoadOnly);
-                                $ignore);
      # Cache it in the loader list.
      push @{$self->{loaders}}, $retVal;
      # Return it to the caller.
-Line 1570
+Line 1912
          $retVal->Accumulate($stats);
          Trace("Statistics for $relName:\n" . $stats->Show()) if T(2);
          }
-     }
      # Return the load statistics.
      return $retVal;
  }
+ =head3 GetGenomeAttributes
+     my $aHashRef = GetGenomeAttributes($fig, $genomeID, \@fids, \@propKeys);
+ Return a hash of attributes keyed on feature ID. This method gets all the NMPDR-related
+ attributes for all the features of a genome in a single call, then organizes them into
+ a hash.
+ =over 4
+ =item fig
+ FIG-like object for accessing attributes.
+ =item genomeID
+ ID of the genome who's attributes are desired.
+ =item fids
+ Reference to a list of the feature IDs whose attributes are to be kept.
+ =item propKeys
+ A list of the keys to retrieve.
+ =item RETURN
+ Returns a reference to a hash. The key of the hash is the feature ID. The value is the
+ reference to a list of the feature's attribute tuples. Each tuple contains the feature ID,
+ the attribute key, and one or more attribute values.
+ =back
+ =cut
+ sub GetGenomeAttributes {
+     # Get the parameters.
+     my ($fig, $genomeID, $fids, $propKeys) = @_;
+     # Declare the return variable.
+     my $retVal = {};
+     # Initialize the hash. This not only enables us to easily determine which FIDs to
+     # keep, it insures that the caller sees a list reference for every known fid,
+     # simplifying the logic.
+     for my $fid (@{$fids}) {
+         $retVal->{$fid} = [];
+     }
+     # Get the attributes. If ev_code_cron is running, we may get a timeout error, so
+     # an eval is used.
+     my @aList = ();
+     eval {
+         @aList = $fig->get_attributes("fig|$genomeID%", $propKeys);
+         Trace(scalar(@aList) . " attributes returned for genome $genomeID.") if T(3);
+     };
+     # Check for a problem.
+     if ($@) {
+         Trace("Retrying attributes for $genomeID due to error: $@") if T(1);
+         # Our fallback plan is to process the attributes in blocks of 100. This is much slower,
+         # but allows us to continue processing.
+         my $nFids = scalar @{$fids};
+         for (my $i = 0; $i < $nFids; $i += 100) {
+             # Determine the index of the last feature ID we'll be specifying on this pass.
+             # Normally it's $i + 99, but if we're close to the end it may be less.
+             my $end = ($i + 100 > $nFids ? $nFids - 1 : $i + 99);
+             # Get a slice of the fid list.
+             my @slice = @{$fids}[$i .. $end];
+             # Get the relevant attributes.
+             Trace("Retrieving attributes for fids $i to $end.") if T(3);
+             my @aShort = $fig->get_attributes(\@slice, $propKeys);
+             Trace(scalar(@aShort) . " attributes returned for fids $i to $end.") if T(3);
+             push @aList, @aShort;
+         }
+     }
+     # Now we should have all the interesting attributes in @aList. Populate the hash with
+     # them.
+     for my $aListEntry (@aList) {
+         my $fid = $aListEntry->[0];
+         if (exists $retVal->{$fid}) {
+             push @{$retVal->{$fid}}, $aListEntry;
+         }
+     }
+     # Return the result.
+     return $retVal;
+ }
 ;

 Legend:



Removed from v.1.48
 


changed lines


 
Added in v.1.90
 Legend:



Removed from v.1.48
 


changed lines


 
Added in v.1.90
-Removed from v.1.48
+Added in v.1.90

MCS Webmaster	ViewVC Help
Powered by ViewVC 1.0.3