[Bio] / Sprout / ERDB.pm Repository:
ViewVC logotype

Diff of /Sprout/ERDB.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.11, Thu Jun 23 21:24:49 2005 UTC revision 1.31, Thu Jan 19 09:28:11 2006 UTC
# Line 2  Line 2 
2    
3      use strict;      use strict;
4      use Tracer;      use Tracer;
5      use DBKernel;      use DBrtns;
6      use Data::Dumper;      use Data::Dumper;
7      use XML::Simple;      use XML::Simple;
8      use DBQuery;      use DBQuery;
9      use DBObject;      use DBObject;
10      use Stats;      use Stats;
11      use Time::HiRes qw(gettimeofday);      use Time::HiRes qw(gettimeofday);
12        use FIG;
13    
14  =head1 Entity-Relationship Database Package  =head1 Entity-Relationship Database Package
15    
# Line 300  Line 301 
301  # Table of information about our datatypes. "sqlType" is the corresponding SQL datatype string.  # Table of information about our datatypes. "sqlType" is the corresponding SQL datatype string.
302  # "maxLen" is the maximum permissible length of the incoming string data used to populate a field  # "maxLen" is the maximum permissible length of the incoming string data used to populate a field
303  # of the specified type. "dataGen" is PERL string that will be evaluated if no test data generation  # of the specified type. "dataGen" is PERL string that will be evaluated if no test data generation
304   #string is specified in the field definition.  # string is specified in the field definition. "avgLen" is the average byte length for estimating
305  my %TypeTable = ( char =>    { sqlType => 'CHAR(1)',            maxLen => 1,            dataGen => "StringGen('A')" },  # record sizes.
306                    int =>     { sqlType => 'INTEGER',            maxLen => 20,           dataGen => "IntGen(0, 99999999)" },  my %TypeTable = ( char =>    { sqlType => 'CHAR(1)',            maxLen => 1,            avgLen =>   1, dataGen => "StringGen('A')" },
307                    string =>  { sqlType => 'VARCHAR(255)',       maxLen => 255,          dataGen => "StringGen(IntGen(10,250))" },                    int =>     { sqlType => 'INTEGER',            maxLen => 20,           avgLen =>   4, dataGen => "IntGen(0, 99999999)" },
308                    text =>    { sqlType => 'TEXT',               maxLen => 1000000000,   dataGen => "StringGen(IntGen(80,1000))" },                    string =>  { sqlType => 'VARCHAR(255)',       maxLen => 255,          avgLen => 100, dataGen => "StringGen(IntGen(10,250))" },
309                    date =>    { sqlType => 'BIGINT',             maxLen => 80,           dataGen => "DateGen(-7, 7, IntGen(0,1400))" },                    text =>    { sqlType => 'TEXT',               maxLen => 1000000000,   avgLen => 500, dataGen => "StringGen(IntGen(80,1000))" },
310                    float =>   { sqlType => 'DOUBLE PRECISION',   maxLen => 40,           dataGen => "FloatGen(0.0, 100.0)" },                    date =>    { sqlType => 'BIGINT',             maxLen => 80,           avgLen =>   8, dataGen => "DateGen(-7, 7, IntGen(0,1400))" },
311                    boolean => { sqlType => 'SMALLINT',           maxLen => 1,            dataGen => "IntGen(0, 1)" },                    float =>   { sqlType => 'DOUBLE PRECISION',   maxLen => 40,           avgLen =>   8, dataGen => "FloatGen(0.0, 100.0)" },
312                      boolean => { sqlType => 'SMALLINT',           maxLen => 1,            avgLen =>   1, dataGen => "IntGen(0, 1)" },
313                   'key-string' =>                   'key-string' =>
314                               { sqlType => 'VARCHAR(40)',        maxLen => 40,           dataGen => "StringGen(IntGen(10,40))" },                               { sqlType => 'VARCHAR(40)',        maxLen => 40,           avgLen =>  10, dataGen => "StringGen(IntGen(10,40))" },
315                   'name-string' =>                   'name-string' =>
316                               { sqlType => 'VARCHAR(80)',        maxLen => 80,           dataGen => "StringGen(IntGen(10,80))" },                               { sqlType => 'VARCHAR(80)',        maxLen => 80,           avgLen =>  40, dataGen => "StringGen(IntGen(10,80))" },
317                   'medium-string' =>                   'medium-string' =>
318                               { sqlType => 'VARCHAR(160)',       maxLen => 160,          dataGen => "StringGen(IntGen(10,160))" },                               { sqlType => 'VARCHAR(160)',       maxLen => 160,          avgLen =>  40, dataGen => "StringGen(IntGen(10,160))" },
319                  );                  );
320    
321  # Table translating arities into natural language.  # Table translating arities into natural language.
# Line 369  Line 371 
371    
372  =head3 ShowMetaData  =head3 ShowMetaData
373    
374  C<< $database->ShowMetaData($fileName); >>  C<< $erdb->ShowMetaData($fileName); >>
375    
376  This method outputs a description of the database. This description can be used to help users create  This method outputs a description of the database. This description can be used to help users create
377  the data to be loaded into the relations.  the data to be loaded into the relations.
# Line 506  Line 508 
508          # Separate out the source, the target, and the join clause.          # Separate out the source, the target, and the join clause.
509          $joinKey =~ m!^([^/]+)/(.+)$!;          $joinKey =~ m!^([^/]+)/(.+)$!;
510          my ($sourceRelation, $targetRelation) = ($1, $2);          my ($sourceRelation, $targetRelation) = ($1, $2);
511          Trace("Join with key $joinKey is from $sourceRelation to $targetRelation.") if T(4);          Trace("Join with key $joinKey is from $sourceRelation to $targetRelation.") if T(Joins => 4);
512          my $source = $self->ComputeObjectSentence($sourceRelation);          my $source = $self->ComputeObjectSentence($sourceRelation);
513          my $target = $self->ComputeObjectSentence($targetRelation);          my $target = $self->ComputeObjectSentence($targetRelation);
514          my $clause = $joinTable->{$joinKey};          my $clause = $joinTable->{$joinKey};
# Line 524  Line 526 
526    
527  =head3 DumpMetaData  =head3 DumpMetaData
528    
529  C<< $database->DumpMetaData(); >>  C<< $erdb->DumpMetaData(); >>
530    
531  Return a dump of the metadata structure.  Return a dump of the metadata structure.
532    
# Line 539  Line 541 
541    
542  =head3 CreateTables  =head3 CreateTables
543    
544  C<< $datanase->CreateTables(); >>  C<< $erdb->CreateTables(); >>
545    
546  This method creates the tables for the database from the metadata structure loaded by the  This method creates the tables for the database from the metadata structure loaded by the
547  constructor. It is expected this function will only be used on rare occasions, when the  constructor. It is expected this function will only be used on rare occasions, when the
# Line 551  Line 553 
553  sub CreateTables {  sub CreateTables {
554      # Get the parameters.      # Get the parameters.
555      my ($self) = @_;      my ($self) = @_;
556      my $metadata = $self->{_metaData};      # Get the relation names.
557      my $dbh = $self->{_dbh};      my @relNames = $self->GetTableNames();
558      # Loop through the entities.      # Loop through the relations.
559      my $entityHash = $metadata->{Entities};      for my $relationName (@relNames) {
     for my $entityName (keys %{$entityHash}) {  
         my $entityData = $entityHash->{$entityName};  
         # Tell the user what we're doing.  
         Trace("Creating relations for entity $entityName.") if T(1);  
         # Loop through the entity's relations.  
         for my $relationName (keys %{$entityData->{Relations}}) {  
560              # Create a table for this relation.              # Create a table for this relation.
561              $self->CreateTable($relationName);              $self->CreateTable($relationName);
562              Trace("Relation $relationName created.") if T(1);          Trace("Relation $relationName created.") if T(2);
         }  
     }  
     # Loop through the relationships.  
     my $relationshipTable = $metadata->{Relationships};  
     for my $relationshipName (keys %{$metadata->{Relationships}}) {  
         # Create a table for this relationship.  
         Trace("Creating relationship $relationshipName.") if T(1);  
         $self->CreateTable($relationshipName);  
563      }      }
564  }  }
565    
566  =head3 CreateTable  =head3 CreateTable
567    
568  C<< $database->CreateTable($tableName, $indexFlag); >>  C<< $erdb->CreateTable($tableName, $indexFlag, $estimatedRows); >>
569    
570  Create the table for a relation and optionally create its indexes.  Create the table for a relation and optionally create its indexes.
571    
# Line 587  Line 575 
575    
576  Name of the relation (which will also be the table name).  Name of the relation (which will also be the table name).
577    
578  =item $indexFlag  =item indexFlag
579    
580  TRUE if the indexes for the relation should be created, else FALSE. If FALSE,  TRUE if the indexes for the relation should be created, else FALSE. If FALSE,
581  L</CreateIndexes> must be called later to bring the indexes into existence.  L</CreateIndexes> must be called later to bring the indexes into existence.
582    
583    =item estimatedRows (optional)
584    
585    If specified, the estimated maximum number of rows for the relation. This
586    information allows the creation of tables using storage engines that are
587    faster but require size estimates, such as MyISAM.
588    
589  =back  =back
590    
591  =cut  =cut
592    
593  sub CreateTable {  sub CreateTable {
594      # Get the parameters.      # Get the parameters.
595      my ($self, $relationName, $indexFlag) = @_;      my ($self, $relationName, $indexFlag, $estimatedRows) = @_;
596      # Get the database handle.      # Get the database handle.
597      my $dbh = $self->{_dbh};      my $dbh = $self->{_dbh};
598      # Get the relation data and determine whether or not the relation is primary.      # Get the relation data and determine whether or not the relation is primary.
# Line 622  Line 616 
616      # Insure the table is not already there.      # Insure the table is not already there.
617      $dbh->drop_table(tbl => $relationName);      $dbh->drop_table(tbl => $relationName);
618      Trace("Table $relationName dropped.") if T(2);      Trace("Table $relationName dropped.") if T(2);
619        # If there are estimated rows, create an estimate so we can take advantage of
620        # faster DB technologies.
621        my $estimation = undef;
622        if ($estimatedRows) {
623            $estimation = [$self->EstimateRowSize($relationName), $estimatedRows];
624        }
625      # Create the table.      # Create the table.
626      Trace("Creating table $relationName: $fieldThing") if T(2);      Trace("Creating table $relationName: $fieldThing") if T(2);
627      $dbh->create_table(tbl => $relationName, flds => $fieldThing);      $dbh->create_table(tbl => $relationName, flds => $fieldThing, estimates => $estimation);
628      Trace("Relation $relationName created in database.") if T(2);      Trace("Relation $relationName created in database.") if T(2);
629      # If we want to build the indexes, we do it here.      # If we want to build the indexes, we do it here.
630      if ($indexFlag) {      if ($indexFlag) {
# Line 632  Line 632 
632      }      }
633  }  }
634    
635    =head3 VerifyFields
636    
637    C<< my $count = $erdb->VerifyFields($relName, \@fieldList); >>
638    
639    Run through the list of proposed field values, insuring that all the character fields are
640    below the maximum length. If any fields are too long, they will be truncated in place.
641    
642    =over 4
643    
644    =item relName
645    
646    Name of the relation for which the specified fields are destined.
647    
648    =item fieldList
649    
650    Reference to a list, in order, of the fields to be put into the relation.
651    
652    =item RETURN
653    
654    Returns the number of fields truncated.
655    
656    =back
657    
658    =cut
659    
660    sub VerifyFields {
661        # Get the parameters.
662        my ($self, $relName, $fieldList) = @_;
663        # Initialize the return value.
664        my $retVal = 0;
665        # Get the relation definition.
666        my $relData = $self->_FindRelation($relName);
667        # Get the list of field descriptors.
668        my $fieldTypes = $relData->{Fields};
669        my $fieldCount = scalar @{$fieldTypes};
670        # Loop through the two lists.
671        for (my $i = 0; $i < $fieldCount; $i++) {
672            # Get the type of the current field.
673            my $fieldType = $fieldTypes->[$i]->{type};
674            # If it's a character field, verify the length.
675            if ($fieldType =~ /string/) {
676                my $maxLen = $TypeTable{$fieldType}->{maxLen};
677                my $oldString = $fieldList->[$i];
678                if (length($oldString) > $maxLen) {
679                    # Here it's too big, so we truncate it.
680                    Trace("Truncating field $i in relation $relName to $maxLen characters from \"$oldString\".") if T(1);
681                    $fieldList->[$i] = substr $oldString, 0, $maxLen;
682                    $retVal++;
683                }
684            }
685        }
686        # Return the truncation count.
687        return $retVal;
688    }
689    
690  =head3 CreateIndex  =head3 CreateIndex
691    
692  C<< $database->CreateIndex($relationName); >>  C<< $erdb->CreateIndex($relationName); >>
693    
694  Create the indexes for a relation. If a table is being loaded from a large source file (as  Create the indexes for a relation. If a table is being loaded from a large source file (as
695  is the case in L</LoadTable>), it is best to create the indexes after the load. If that is  is the case in L</LoadTable>), it is sometimes best to create the indexes after the load.
696  the case, then L</CreateTable> should be called with the index flag set to FALSE, and this  If that is the case, then L</CreateTable> should be called with the index flag set to
697  method used after the load to create the indexes for the table.  FALSE, and this method used after the load to create the indexes for the table.
698    
699  =cut  =cut
700    
# Line 660  Line 715 
715          # Get the index's uniqueness flag.          # Get the index's uniqueness flag.
716          my $unique = (exists $indexData->{Unique} ? $indexData->{Unique} : 'false');          my $unique = (exists $indexData->{Unique} ? $indexData->{Unique} : 'false');
717          # Create the index.          # Create the index.
718          $dbh->create_index(idx => $indexName, tbl => $relationName, flds => $flds, unique => $unique);          my $rv = $dbh->create_index(idx => $indexName, tbl => $relationName,
719                                        flds => $flds, unique => $unique);
720            if ($rv) {
721          Trace("Index created: $indexName for $relationName ($flds)") if T(1);          Trace("Index created: $indexName for $relationName ($flds)") if T(1);
722            } else {
723                Confess("Error creating index $indexName for $relationName using ($flds): " . $dbh->error_message());
724            }
725      }      }
726  }  }
727    
728  =head3 LoadTables  =head3 LoadTables
729    
730  C<< my $stats = $database->LoadTables($directoryName, $rebuild); >>  C<< my $stats = $erdb->LoadTables($directoryName, $rebuild); >>
731    
732  This method will load the database tables from a directory. The tables must already have been created  This method will load the database tables from a directory. The tables must already have been created
733  in the database. (This can be done by calling L</CreateTables>.) The caller passes in a directory name;  in the database. (This can be done by calling L</CreateTables>.) The caller passes in a directory name;
# Line 710  Line 770 
770      $directoryName =~ s!/\\$!!;      $directoryName =~ s!/\\$!!;
771      # Declare the return variable.      # Declare the return variable.
772      my $retVal = Stats->new();      my $retVal = Stats->new();
773      # Get the metadata structure.      # Get the relation names.
774      my $metaData = $self->{_metaData};      my @relNames = $self->GetTableNames();
775      # Loop through the entities.      for my $relationName (@relNames) {
     for my $entity (values %{$metaData->{Entities}}) {  
         # Loop through the entity's relations.  
         for my $relationName (keys %{$entity->{Relations}}) {  
776              # Try to load this relation.              # Try to load this relation.
777              my $result = $self->_LoadRelation($directoryName, $relationName, $rebuild);              my $result = $self->_LoadRelation($directoryName, $relationName, $rebuild);
778              # Accumulate the statistics.              # Accumulate the statistics.
779              $retVal->Accumulate($result);              $retVal->Accumulate($result);
780          }          }
     }  
     # Loop through the relationships.  
     for my $relationshipName (keys %{$metaData->{Relationships}}) {  
         # Try to load this relationship's relation.  
         my $result = $self->_LoadRelation($directoryName, $relationshipName, $rebuild);  
         # Accumulate the statistics.  
         $retVal->Accumulate($result);  
     }  
781      # Add the duration of the load to the statistical object.      # Add the duration of the load to the statistical object.
782      $retVal->Add('duration', gettimeofday - $startTime);      $retVal->Add('duration', gettimeofday - $startTime);
783      # Return the accumulated statistics.      # Return the accumulated statistics.
784      return $retVal;      return $retVal;
785  }  }
786    
787    
788  =head3 GetTableNames  =head3 GetTableNames
789    
790  C<< my @names = $database->GetTableNames; >>  C<< my @names = $erdb->GetTableNames; >>
791    
792  Return a list of the relations required to implement this database.  Return a list of the relations required to implement this database.
793    
# Line 754  Line 804 
804    
805  =head3 GetEntityTypes  =head3 GetEntityTypes
806    
807  C<< my @names = $database->GetEntityTypes; >>  C<< my @names = $erdb->GetEntityTypes; >>
808    
809  Return a list of the entity type names.  Return a list of the entity type names.
810    
# Line 769  Line 819 
819      return sort keys %{$entityList};      return sort keys %{$entityList};
820  }  }
821    
822    =head3 IsEntity
823    
824    C<< my $flag = $erdb->IsEntity($entityName); >>
825    
826    Return TRUE if the parameter is an entity name, else FALSE.
827    
828    =over 4
829    
830    =item entityName
831    
832    Object name to be tested.
833    
834    =item RETURN
835    
836    Returns TRUE if the specified string is an entity name, else FALSE.
837    
838    =back
839    
840    =cut
841    
842    sub IsEntity {
843        # Get the parameters.
844        my ($self, $entityName) = @_;
845        # Test to see if it's an entity.
846        return exists $self->{_metaData}->{Entities}->{$entityName};
847    }
848    
849  =head3 Get  =head3 Get
850    
851  C<< my $query = $database->Get(\@objectNames, $filterClause, $param1, $param2, ..., $paramN); >>  C<< my $query = $erdb->Get(\@objectNames, $filterClause, $param1, $param2, ..., $paramN); >>
852    
853  This method returns a query object for entities of a specified type using a specified filter.  This method returns a query object for entities of a specified type using a specified filter.
854  The filter is a standard WHERE/ORDER BY clause with question marks as parameter markers and each  The filter is a standard WHERE/ORDER BY clause with question marks as parameter markers and each
# Line 779  Line 856 
856  following call requests all B<Genome> objects for the genus specified in the variable  following call requests all B<Genome> objects for the genus specified in the variable
857  $genus.  $genus.
858    
859  C<< $query = $sprout->Get(['Genome'], "Genome(genus) = ?", $genus); >>  C<< $query = $erdb->Get(['Genome'], "Genome(genus) = ?", $genus); >>
860    
861  The WHERE clause contains a single question mark, so there is a single additional  The WHERE clause contains a single question mark, so there is a single additional
862  parameter representing the parameter value. It would also be possible to code  parameter representing the parameter value. It would also be possible to code
863    
864  C<< $query = $sprout->Get(['Genome'], "Genome(genus) = \'$genus\'"); >>  C<< $query = $erdb->Get(['Genome'], "Genome(genus) = \'$genus\'"); >>
865    
866  however, this version of the call would generate a syntax error if there were any quote  however, this version of the call would generate a syntax error if there were any quote
867  characters inside the variable C<$genus>.  characters inside the variable C<$genus>.
# Line 796  Line 873 
873  It is possible to specify multiple entity and relationship names in order to retrieve more than  It is possible to specify multiple entity and relationship names in order to retrieve more than
874  one object's data at the same time, which allows highly complex joined queries. For example,  one object's data at the same time, which allows highly complex joined queries. For example,
875    
876  C<< $query = $sprout->Get(['Genome', 'ComesFrom', 'Source'], "Genome(genus) = ?", $genus); >>  C<< $query = $erdb->Get(['Genome', 'ComesFrom', 'Source'], "Genome(genus) = ?", $genus); >>
877    
878  If multiple names are specified, then the query processor will automatically determine a  If multiple names are specified, then the query processor will automatically determine a
879  join path between the entities and relationships. The algorithm used is very simplistic.  join path between the entities and relationships. The algorithm used is very simplistic.
# Line 829  Line 906 
906    
907  C<< "Genome(genus) = ? ORDER BY Genome(species)" >>  C<< "Genome(genus) = ? ORDER BY Genome(species)" >>
908    
909    Note that the case is important. Only an uppercase "ORDER BY" with a single space will
910    be processed. The idea is to make it less likely to find the verb by accident.
911    
912  The rules for field references in a sort order are the same as those for field references in the  The rules for field references in a sort order are the same as those for field references in the
913  filter clause in general; however, odd things may happen if a sort field is from a secondary  filter clause in general; however, odd things may happen if a sort field is from a secondary
914  relation.  relation.
# Line 937  Line 1017 
1017                  $lastObject = $thisObject;                  $lastObject = $thisObject;
1018              }              }
1019          }          }
1020          # Now we need to handle the whole ORDER BY thing. We'll put the order by clause          # Now we need to handle the whole ORDER BY / LIMIT thing. The important part
1021          # in the following variable.          # here is we want the filter clause to be empty if there's no WHERE filter.
1022            # We'll put the ORDER BY / LIMIT clauses in the following variable.
1023          my $orderClause = "";          my $orderClause = "";
1024          # Locate the ORDER BY verb (if any).          # Locate the ORDER BY or LIMIT verbs (if any). We use a non-greedy
1025          if ($filterString =~ m/^(.*)ORDER BY/g) {          # operator so that we find the first occurrence of either verb.
1026              # Here we have an ORDER BY verb. Split it off of the filter string.          if ($filterString =~ m/^(.*?)\s*(ORDER BY|LIMIT)/g) {
1027                # Here we have an ORDER BY or LIMIT verb. Split it off of the filter string.
1028              my $pos = pos $filterString;              my $pos = pos $filterString;
1029              $orderClause = substr($filterString, $pos);              $orderClause = $2 . substr($filterString, $pos);
1030              $filterString = $1;              $filterString = $1;
1031          }          }
1032          # Add the filter and the join clauses (if any) to the SELECT command.          # Add the filter and the join clauses (if any) to the SELECT command.
# Line 954  Line 1036 
1036          if (@joinWhere) {          if (@joinWhere) {
1037              $command .= " WHERE " . join(' AND ', @joinWhere);              $command .= " WHERE " . join(' AND ', @joinWhere);
1038          }          }
1039          # Add the sort clause (if any) to the SELECT command.          # Add the sort or limit clause (if any) to the SELECT command.
1040          if ($orderClause) {          if ($orderClause) {
1041              $command .= " ORDER BY $orderClause";              $command .= " $orderClause";
1042          }          }
1043      }      }
1044      Trace("SQL query: $command") if T(2);      Trace("SQL query: $command") if T(SQL => 4);
1045      Trace("PARMS: '" . (join "', '", @params) . "'") if (T(3) && (@params > 0));      Trace("PARMS: '" . (join "', '", @params) . "'") if (T(SQL => 4) && (@params > 0));
1046      my $sth = $dbh->prepare_command($command);      my $sth = $dbh->prepare_command($command);
1047      # Execute it with the parameters bound in.      # Execute it with the parameters bound in.
1048      $sth->execute(@params) || Confess("SELECT error" . $sth->errstr());      $sth->execute(@params) || Confess("SELECT error" . $sth->errstr());
# Line 971  Line 1053 
1053    
1054  =head3 GetList  =head3 GetList
1055    
1056  C<< my @dbObjects = $database->GetList(\@objectNames, $filterClause, $param1, $param2, ..., $paramN); >>  C<< my @dbObjects = $erdb->GetList(\@objectNames, $filterClause, $param1, $param2, ..., $paramN); >>
1057    
1058  Return a list of object descriptors for the specified objects as determined by the  Return a list of object descriptors for the specified objects as determined by the
1059  specified filter clause.  specified filter clause.
# Line 1034  Line 1116 
1116    
1117  =head3 ComputeObjectSentence  =head3 ComputeObjectSentence
1118    
1119  C<< my $sentence = $database->ComputeObjectSentence($objectName); >>  C<< my $sentence = $erdb->ComputeObjectSentence($objectName); >>
1120    
1121  Check an object name, and if it is a relationship convert it to a relationship sentence.  Check an object name, and if it is a relationship convert it to a relationship sentence.
1122    
# Line 1069  Line 1151 
1151    
1152  =head3 DumpRelations  =head3 DumpRelations
1153    
1154  C<< $database->DumpRelations($outputDirectory); >>  C<< $erdb->DumpRelations($outputDirectory); >>
1155    
1156  Write the contents of all the relations to tab-delimited files in the specified directory.  Write the contents of all the relations to tab-delimited files in the specified directory.
1157  Each file will have the same name as the relation dumped, with an extension of DTX.  Each file will have the same name as the relation dumped, with an extension of DTX.
# Line 1111  Line 1193 
1193    
1194  =head3 InsertObject  =head3 InsertObject
1195    
1196  C<< my $ok = $database->InsertObject($objectType, \%fieldHash); >>  C<< my $ok = $erdb->InsertObject($objectType, \%fieldHash); >>
1197    
1198  Insert an object into the database. The object is defined by a type name and then a hash  Insert an object into the database. The object is defined by a type name and then a hash
1199  of field names to values. Field values in the primary relation are represented by scalars.  of field names to values. Field values in the primary relation are represented by scalars.
# Line 1120  Line 1202 
1202  example, the following line inserts an inactive PEG feature named C<fig|188.1.peg.1> with aliases  example, the following line inserts an inactive PEG feature named C<fig|188.1.peg.1> with aliases
1203  C<ZP_00210270.1> and C<gi|46206278>.  C<ZP_00210270.1> and C<gi|46206278>.
1204    
1205  C<< $database->InsertObject('Feature', { id => 'fig|188.1.peg.1', active => 0, feature-type => 'peg', alias => ['ZP_00210270.1', 'gi|46206278']}); >>  C<< $erdb->InsertObject('Feature', { id => 'fig|188.1.peg.1', active => 0, feature-type => 'peg', alias => ['ZP_00210270.1', 'gi|46206278']}); >>
1206    
1207  The next statement inserts a C<HasProperty> relationship between feature C<fig|158879.1.peg.1> and  The next statement inserts a C<HasProperty> relationship between feature C<fig|158879.1.peg.1> and
1208  property C<4> with an evidence URL of C<http://seedu.uchicago.edu/query.cgi?article_id=142>.  property C<4> with an evidence URL of C<http://seedu.uchicago.edu/query.cgi?article_id=142>.
1209    
1210  C<< $database->InsertObject('HasProperty', { 'from-link' => 'fig|158879.1.peg.1', 'to-link' => 4, evidence = 'http://seedu.uchicago.edu/query.cgi?article_id=142'}); >>  C<< $erdb->InsertObject('HasProperty', { 'from-link' => 'fig|158879.1.peg.1', 'to-link' => 4, evidence = 'http://seedu.uchicago.edu/query.cgi?article_id=142'}); >>
1211    
1212  =over 4  =over 4
1213    
# Line 1250  Line 1332 
1332    
1333  =head3 LoadTable  =head3 LoadTable
1334    
1335  C<< my %results = $database->LoadTable($fileName, $relationName, $truncateFlag); >>  C<< my %results = $erdb->LoadTable($fileName, $relationName, $truncateFlag); >>
1336    
1337  Load data from a tab-delimited file into a specified table, optionally re-creating the table  Load data from a tab-delimited file into a specified table, optionally re-creating the table
1338  first.  first.
# Line 1271  Line 1353 
1353    
1354  =item RETURN  =item RETURN
1355    
1356  Returns a statistical object containing the number of records read and a list of  Returns a statistical object containing a list of the error messages.
 the error messages.  
1357    
1358  =back  =back
1359    
# Line 1286  Line 1367 
1367      Trace("Loading table $relationName from $fileName") if T(2);      Trace("Loading table $relationName from $fileName") if T(2);
1368      # Get the database handle.      # Get the database handle.
1369      my $dbh = $self->{_dbh};      my $dbh = $self->{_dbh};
1370        # Get the input file size.
1371        my $fileSize = -s $fileName;
1372      # Get the relation data.      # Get the relation data.
1373      my $relation = $self->_FindRelation($relationName);      my $relation = $self->_FindRelation($relationName);
1374      # Check the truncation flag.      # Check the truncation flag.
1375      if ($truncateFlag) {      if ($truncateFlag) {
1376          Trace("Creating table $relationName") if T(2);          Trace("Creating table $relationName") if T(2);
1377            # Compute the row count estimate. We take the size of the load file,
1378            # divide it by the estimated row size, and then multiply by 1.5 to
1379            # leave extra room. We postulate a minimum row count of 1000 to
1380            # prevent problems with incoming empty load files.
1381            my $rowSize = $self->EstimateRowSize($relationName);
1382            my $estimate = FIG::max($fileSize * 1.5 / $rowSize, 1000);
1383          # Re-create the table without its index.          # Re-create the table without its index.
1384          $self->CreateTable($relationName, 0);          $self->CreateTable($relationName, 0, $estimate);
1385          # If this is a pre-index DBMS, create the index here.          # If this is a pre-index DBMS, create the index here.
1386          if ($dbh->{_preIndex}) {          if ($dbh->{_preIndex}) {
1387              eval {              eval {
# Line 1303  Line 1392 
1392              }              }
1393          }          }
1394      }      }
     # Determine whether or not this is a primary relation. Primary relations have an extra  
     # field indicating whether or not a given object is new or was loaded from the flat files.  
     my $primary = $self->_IsPrimary($relationName);  
     # Get the number of fields in this relation.  
     my @fieldList = @{$relation->{Fields}};  
     my $fieldCount = @fieldList;  
     # Start a database transaction.  
     $dbh->begin_tran;  
     # Open the relation file. We need to create a cleaned-up copy before loading.  
     open TABLEIN, '<', $fileName;  
     my $tempName = "$fileName.tbl";  
     open TABLEOUT, '>', $tempName;  
     my $inputCount = 0;  
     # Loop through the file.  
     while (<TABLEIN>) {  
         $inputCount++;  
         # Chop off the new-line character.  
         my $record = Tracer::Strip($_);  
         # Only proceed if the record is non-blank.  
         if ($record) {  
             # Escape all the backslashes found in the line.  
             $record =~ s/\\/\\\\/g;  
             # Insure the number of fields is correct.  
             my @fields = split /\t/, $record;  
             while (@fields > $fieldCount) {  
                 my $extraField = $fields[$#fields];  
                 delete $fields[$#fields];  
                 if ($extraField) {  
                     Trace("Nonblank extra field value \"$extraField\" deleted from record $inputCount of $fileName.") if T(1);  
                 }  
             }  
             while (@fields < $fieldCount) {  
                 push @fields, "";  
             }  
             # If this is a primary relation, add a 0 for the new-record flag (indicating that  
             # this record is not new, but part of the original load).  
             if ($primary) {  
                 push @fields, "0";  
             }  
             # Write the record.  
             $record = join "\t", @fields;  
             print TABLEOUT "$record\n";  
             # Count the record written.  
             my $count = $retVal->Add('records');  
             my $len = length $record;  
             Trace("Record $count written with $len characters.") if T(4);  
         } else {  
             # Here we have a blank record.  
             $retVal->Add('skipped');  
         }  
     }  
     # Close the files.  
     close TABLEIN;  
     close TABLEOUT;  
     Trace("Temporary file $tempName created.") if T(2);  
1395      # Load the table.      # Load the table.
1396      my $rv;      my $rv;
1397      eval {      eval {
1398          $rv = $dbh->load_table(file => $tempName, tbl => $relationName);          $rv = $dbh->load_table(file => $fileName, tbl => $relationName);
1399      };      };
1400      if (!defined $rv) {      if (!defined $rv) {
1401          $retVal->AddMessage($@) if ($@);          $retVal->AddMessage($@) if ($@);
1402          $retVal->AddMessage("Table load failed for $relationName using $tempName.");          $retVal->AddMessage("Table load failed for $relationName using $fileName.");
1403          Trace("Table load failed for $relationName.") if T(1);          Trace("Table load failed for $relationName.") if T(1);
1404      } else {      } else {
1405          # Here we successfully loaded the table. Trace the number of records loaded.          # Here we successfully loaded the table.
1406          Trace("$retVal->{records} records read for $relationName.") if T(2);          $retVal->Add("tables");
1407            my $size = -s $fileName;
1408            Trace("$size bytes loaded into $relationName.") if T(2);
1409          # If we're rebuilding, we need to create the table indexes.          # If we're rebuilding, we need to create the table indexes.
1410          if ($truncateFlag && ! $dbh->{_preIndex}) {          if ($truncateFlag && ! $dbh->{_preIndex}) {
1411              eval {              eval {
# Line 1380  Line 1416 
1416              }              }
1417          }          }
1418      }      }
1419      # Commit the database changes.      # Analyze the table to improve performance.
1420      $dbh->commit_tran;      $dbh->vacuum_it($relationName);
     # Delete the temporary file.  
     unlink $tempName;  
1421      # Return the statistics.      # Return the statistics.
1422      return $retVal;      return $retVal;
1423  }  }
1424    
1425  =head3 GenerateEntity  =head3 GenerateEntity
1426    
1427  C<< my $fieldHash = $database->GenerateEntity($id, $type, \%values); >>  C<< my $fieldHash = $erdb->GenerateEntity($id, $type, \%values); >>
1428    
1429  Generate the data for a new entity instance. This method creates a field hash suitable for  Generate the data for a new entity instance. This method creates a field hash suitable for
1430  passing as a parameter to L</InsertObject>. The ID is specified by the callr, but the rest  passing as a parameter to L</InsertObject>. The ID is specified by the callr, but the rest
# Line 1448  Line 1482 
1482    
1483  =head3 GetEntity  =head3 GetEntity
1484    
1485  C<< my $entityObject = $sprout->GetEntity($entityType, $ID); >>  C<< my $entityObject = $erdb->GetEntity($entityType, $ID); >>
1486    
1487  Return an object describing the entity instance with a specified ID.  Return an object describing the entity instance with a specified ID.
1488    
# Line 1484  Line 1518 
1518    
1519  =head3 GetEntityValues  =head3 GetEntityValues
1520    
1521  C<< my @values = GetEntityValues($entityType, $ID, \@fields); >>  C<< my @values = $erdb->GetEntityValues($entityType, $ID, \@fields); >>
1522    
1523  Return a list of values from a specified entity instance.  Return a list of values from a specified entity instance.
1524    
# Line 1527  Line 1561 
1561    
1562  =head3 GetAll  =head3 GetAll
1563    
1564  C<< my @list = $sprout->GetAll(\@objectNames, $filterClause, \@parameters, \@fields, $count); >>  C<< my @list = $erdb->GetAll(\@objectNames, $filterClause, \@parameters, \@fields, $count); >>
1565    
1566  Return a list of values taken from the objects returned by a query. The first three  Return a list of values taken from the objects returned by a query. The first three
1567  parameters correspond to the parameters of the L</Get> method. The final parameter is  parameters correspond to the parameters of the L</Get> method. The final parameter is
# Line 1543  Line 1577 
1577  spreadsheet cell, and each feature will be represented by a list containing the  spreadsheet cell, and each feature will be represented by a list containing the
1578  feature ID followed by all of its aliases.  feature ID followed by all of its aliases.
1579    
1580  C<< $query = $sprout->Get(['ContainsFeature', 'Feature'], "ContainsFeature(from-link) = ?", [$ssCellID], ['Feature(id)', 'Feature(alias)']); >>  C<< $query = $erdb->Get(['ContainsFeature', 'Feature'], "ContainsFeature(from-link) = ?", [$ssCellID], ['Feature(id)', 'Feature(alias)']); >>
1581    
1582  =over 4  =over 4
1583    
# Line 1592  Line 1626 
1626      } else {      } else {
1627          push @parmList, $parameterList;          push @parmList, $parameterList;
1628      }      }
     # Create the query.  
     my $query = $self->Get($objectNames, $filterClause, @parmList);  
     # Set up a counter of the number of records read.  
     my $fetched = 0;  
1629      # Insure the counter has a value.      # Insure the counter has a value.
1630      if (!defined $count) {      if (!defined $count) {
1631          $count = 0;          $count = 0;
1632      }      }
1633        # Add the row limit to the filter clause.
1634        if ($count > 0) {
1635            $filterClause .= " LIMIT $count";
1636        }
1637        # Create the query.
1638        my $query = $self->Get($objectNames, $filterClause, @parmList);
1639        # Set up a counter of the number of records read.
1640        my $fetched = 0;
1641      # Loop through the records returned, extracting the fields. Note that if the      # Loop through the records returned, extracting the fields. Note that if the
1642      # counter is non-zero, we stop when the number of records read hits the count.      # counter is non-zero, we stop when the number of records read hits the count.
1643      my @retVal = ();      my @retVal = ();
# Line 1612  Line 1650 
1650      return @retVal;      return @retVal;
1651  }  }
1652    
1653    =head3 EstimateRowSize
1654    
1655    C<< my $rowSize = $erdb->EstimateRowSize($relName); >>
1656    
1657    Estimate the row size of the specified relation. The estimated row size is computed by adding
1658    up the average length for each data type.
1659    
1660    =over 4
1661    
1662    =item relName
1663    
1664    Name of the relation whose estimated row size is desired.
1665    
1666    =item RETURN
1667    
1668    Returns an estimate of the row size for the specified relation.
1669    
1670    =back
1671    
1672    =cut
1673    #: Return Type $;
1674    sub EstimateRowSize {
1675        # Get the parameters.
1676        my ($self, $relName) = @_;
1677        # Declare the return variable.
1678        my $retVal = 0;
1679        # Find the relation descriptor.
1680        my $relation = $self->_FindRelation($relName);
1681        # Get the list of fields.
1682        for my $fieldData (@{$relation->{Fields}}) {
1683            # Get the field type and add its length.
1684            my $fieldLen = $TypeTable{$fieldData->{type}}->{avgLen};
1685            $retVal += $fieldLen;
1686        }
1687        # Return the result.
1688        return $retVal;
1689    }
1690    
1691  =head2 Internal Utility Methods  =head2 Internal Utility Methods
1692    
1693  =head3 GetLoadStats  =head3 GetLoadStats
# Line 1623  Line 1699 
1699  =cut  =cut
1700    
1701  sub _GetLoadStats {  sub _GetLoadStats {
1702      return Stats->new('records');      return Stats->new();
1703  }  }
1704    
1705  =head3 GenerateFields  =head3 GenerateFields
# Line 1987  Line 2063 
2063  sub _LoadMetaData {  sub _LoadMetaData {
2064      # Get the parameters.      # Get the parameters.
2065      my ($filename) = @_;      my ($filename) = @_;
2066        Trace("Reading Sprout DBD from $filename.") if T(2);
2067      # Slurp the XML file into a variable. Extensive use of options is used to insure we      # Slurp the XML file into a variable. Extensive use of options is used to insure we
2068      # get the exact structure we want.      # get the exact structure we want.
2069      my $metadata = XML::Simple::XMLin($filename,      my $metadata = XML::Simple::XMLin($filename,
# Line 2014  Line 2091 
2091      for my $entityName (keys %{$entityList}) {      for my $entityName (keys %{$entityList}) {
2092          my $entityStructure = $entityList->{$entityName};          my $entityStructure = $entityList->{$entityName};
2093          #          #
2094          # The first step is to run creating all the entity's default values. For C<Field> elements,          # The first step is to create all the entity's default values. For C<Field> elements,
2095          # the relation name must be added where it is not specified. For relationships,          # the relation name must be added where it is not specified. For relationships,
2096          # the B<from-link> and B<to-link> fields must be inserted, and for entities an B<id>          # the B<from-link> and B<to-link> fields must be inserted, and for entities an B<id>
2097          # field must be added to each relation. Finally, each field will have a C<PrettySort> attribute          # field must be added to each relation. Finally, each field will have a C<PrettySort> attribute
# Line 2193  Line 2270 
2270          my @fromList = ();          my @fromList = ();
2271          my @toList = ();          my @toList = ();
2272          my @bothList = ();          my @bothList = ();
2273          Trace("Join table build for $entityName.") if T(3);          Trace("Join table build for $entityName.") if T(metadata => 4);
2274          for my $relationshipName (keys %{$relationshipList}) {          for my $relationshipName (keys %{$relationshipList}) {
2275              my $relationship = $relationshipList->{$relationshipName};              my $relationship = $relationshipList->{$relationshipName};
2276              # Determine if this relationship has our entity in one of its link fields.              # Determine if this relationship has our entity in one of its link fields.
2277              my $fromEntity = $relationship->{from};              my $fromEntity = $relationship->{from};
2278              my $toEntity = $relationship->{to};              my $toEntity = $relationship->{to};
2279              Trace("Join check for relationship $relationshipName from $fromEntity to $toEntity.") if T(3);              Trace("Join check for relationship $relationshipName from $fromEntity to $toEntity.") if T(4);
2280              if ($fromEntity eq $entityName) {              if ($fromEntity eq $entityName) {
2281                  if ($toEntity eq $entityName) {                  if ($toEntity eq $entityName) {
2282                      # Here the relationship is recursive.                      # Here the relationship is recursive.
2283                      push @bothList, $relationshipName;                      push @bothList, $relationshipName;
2284                      Trace("Relationship $relationshipName put in both-list.") if T(3);                      Trace("Relationship $relationshipName put in both-list.") if T(metadata => 4);
2285                  } else {                  } else {
2286                      # Here the relationship comes from the entity.                      # Here the relationship comes from the entity.
2287                      push @fromList, $relationshipName;                      push @fromList, $relationshipName;
2288                      Trace("Relationship $relationshipName put in from-list.") if T(3);                      Trace("Relationship $relationshipName put in from-list.") if T(metadata => 4);
2289                  }                  }
2290              } elsif ($toEntity eq $entityName) {              } elsif ($toEntity eq $entityName) {
2291                  # Here the relationship goes to the entity.                  # Here the relationship goes to the entity.
2292                  push @toList, $relationshipName;                  push @toList, $relationshipName;
2293                  Trace("Relationship $relationshipName put in to-list.") if T(3);                  Trace("Relationship $relationshipName put in to-list.") if T(metadata => 4);
2294              }              }
2295          }          }
2296          # Create the nonrecursive joins. Note that we build two hashes for running          # Create the nonrecursive joins. Note that we build two hashes for running
# Line 2229  Line 2306 
2306                  # Create joins between the entity and this relationship.                  # Create joins between the entity and this relationship.
2307                  my $linkField = "$relationshipName.${linkType}_link";                  my $linkField = "$relationshipName.${linkType}_link";
2308                  my $joinClause = "$entityName.id = $linkField";                  my $joinClause = "$entityName.id = $linkField";
2309                  Trace("Entity join clause is $joinClause for $entityName and $relationshipName.") if T(4);                  Trace("Entity join clause is $joinClause for $entityName and $relationshipName.") if T(metadata => 4);
2310                  $joinTable{"$entityName/$relationshipName"} = $joinClause;                  $joinTable{"$entityName/$relationshipName"} = $joinClause;
2311                  $joinTable{"$relationshipName/$entityName"} = $joinClause;                  $joinTable{"$relationshipName/$entityName"} = $joinClause;
2312                  # Create joins between this relationship and the other relationships.                  # Create joins between this relationship and the other relationships.
# Line 2250  Line 2327 
2327                              # relationship and itself are prohibited.                              # relationship and itself are prohibited.
2328                              my $relJoinClause = "$otherName.${otherType}_link = $linkField";                              my $relJoinClause = "$otherName.${otherType}_link = $linkField";
2329                              $joinTable{$joinKey} = $relJoinClause;                              $joinTable{$joinKey} = $relJoinClause;
2330                              Trace("Relationship join clause is $relJoinClause for $joinKey.") if T(4);                              Trace("Relationship join clause is $relJoinClause for $joinKey.") if T(metadata => 4);
2331                          }                          }
2332                      }                      }
2333                  }                  }
# Line 2259  Line 2336 
2336                  # relationship can only be ambiguous with another recursive relationship,                  # relationship can only be ambiguous with another recursive relationship,
2337                  # and the incoming relationship from the outer loop is never recursive.                  # and the incoming relationship from the outer loop is never recursive.
2338                  for my $otherName (@bothList) {                  for my $otherName (@bothList) {
2339                      Trace("Setting up relationship joins to recursive relationship $otherName with $relationshipName.") if T(3);                      Trace("Setting up relationship joins to recursive relationship $otherName with $relationshipName.") if T(metadata => 4);
2340                      # Join from the left.                      # Join from the left.
2341                      $joinTable{"$relationshipName/$otherName"} =                      $joinTable{"$relationshipName/$otherName"} =
2342                          "$linkField = $otherName.from_link";                          "$linkField = $otherName.from_link";
# Line 2274  Line 2351 
2351          # rise to situations where we can't create the path we want; however, it is always          # rise to situations where we can't create the path we want; however, it is always
2352          # possible to get the same effect using multiple queries.          # possible to get the same effect using multiple queries.
2353          for my $relationshipName (@bothList) {          for my $relationshipName (@bothList) {
2354              Trace("Setting up entity joins to recursive relationship $relationshipName with $entityName.") if T(3);              Trace("Setting up entity joins to recursive relationship $relationshipName with $entityName.") if T(metadata => 4);
2355              # Join to the entity from each direction.              # Join to the entity from each direction.
2356              $joinTable{"$entityName/$relationshipName"} =              $joinTable{"$entityName/$relationshipName"} =
2357                  "$entityName.id = $relationshipName.from_link";                  "$entityName.id = $relationshipName.from_link";
# Line 2325  Line 2402 
2402      # index descriptor does not exist, it will be created automatically so we can add      # index descriptor does not exist, it will be created automatically so we can add
2403      # the field to it.      # the field to it.
2404      unshift @{$newIndex->{IndexFields}}, $firstField;      unshift @{$newIndex->{IndexFields}}, $firstField;
2405        # If this is a one-to-many relationship, the "To" index is unique.
2406        if ($relationshipStructure->{arity} eq "1M" && $indexKey eq "To") {
2407            $newIndex->{Unique} = 'true';
2408        }
2409      # Add the index to the relation.      # Add the index to the relation.
2410      _AddIndex("idx$relationshipName$indexKey", $relationStructure, $newIndex);      _AddIndex("idx$relationshipName$indexKey", $relationStructure, $newIndex);
2411  }  }
Legend:
Removed from v.1.11  
changed lines
  Added in v.1.31
MCS Webmaster
 ViewVC Help
Powered by ViewVC 1.0.3