[Bio] / Sprout / ERDBLoader.pl Repository:
ViewVC logotype

Diff of /Sprout/ERDBLoader.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.11, Thu May 28 18:06:58 2009 UTC revision 1.16, Mon Jan 3 18:08:10 2011 UTC
# Line 35  Line 35 
35    
36  =head2 Introduction  =head2 Introduction
37    
38  This script finishes the database load process begun by [[ERDBGeneratorPl]].  This script finishes the database load process begun by L<ERDBGenerator.pl>.
39    
40  [[ERDBGeneratorPl]] divides the source data into sections, and generates a  L<ERDBGenerator.pl> divides the source data into sections, and generates a
41  partial load file for each section of each table. To finish the load process, we  partial load file for each section of each table. To finish the load process, we
42  need to combine the partial files into single files and load the resulting  need to combine the partial files into single files and load the resulting
43  single files into the database tables.  single files into the database tables.
44    
45  Like [[ERDBGeneratorPl]], this script acts on load groups-- sets of related  Like L<ERDBGenerator.pl>, this script acts on load groups-- sets of related
46  tables that are loaded at the same time. For each table in a named group that  tables that are loaded at the same time. For each table in a named group that
47  does not exist in the database, the script first attempts to find a completed  does not exist in the database, the script first attempts to find a complete set
48  data file. If one does not exist, it attempts to create one by collating section  of section files that it will collate into a data file. If there are no sections,
49  files. Once the collated section files for a load group are finished, they are  then it will look for a data file that is already collated. Once the collated
50  loaded into the database.  section files for a load group are all verified, they are loaded into the database.
51    
52  =head2 Positional Parameters  =head2 Positional Parameters
53    
# Line 77  Line 77 
77    
78  If specified, turns on tracing of SQL activity.  If specified, turns on tracing of SQL activity.
79    
 =item clear  
   
 If specified, existing load files will be recreated from sections if the sections  
 are present.  
   
80  =item background  =item background
81    
82  Save the standard and error output to files. The files will be created  Save the standard and error output to files. The files will be created
# Line 96  Line 91 
91  =item keepSections  =item keepSections
92    
93  If specified, section files (the fragments of data load files created by  If specified, section files (the fragments of data load files created by
94  [[ERDBGeneratorPl]], will not be deleted after they are collated.  L<ERDBGenerator.pl>, will not be deleted after they are collated.
95    
96  =item warn  =item warn
97    
# Line 133  Line 128 
128  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],
129                                             {                                             {
130                                                dbName => ["", "if specified, the SQL name of the target database"],                                                dbName => ["", "if specified, the SQL name of the target database"],
                                               clear => ["", "overwrite existing load files if sections are present"],  
131                                                trace => ["2", "tracing level"],                                                trace => ["2", "tracing level"],
132                                                keepSections => ["", "if specified, section files will not be deleted after being collated"],                                                keepSections => ["", "if specified, section files will not be deleted after being collated"],
133                                                phone => ["", "phone number (international format) to call when load finishes"],                                                phone => ["", "phone number (international format) to call when load finishes"],
# Line 149  Line 143 
143      # Get the parameters.      # Get the parameters.
144      my ($database, @groups) = @parameters;      my ($database, @groups) = @parameters;
145      # Connect to the database and get its load directory.      # Connect to the database and get its load directory.
146      my $erdb = ERDB::GetDatabase($database, undef, %$options);      my $erdb = ERDB::GetDatabase($database, undef, %$options, externalDBD => 1);
147      # Fix the group list.      # Fix the group list.
148      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, \@groups);      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, \@groups);
149      # Get the source object and load directory for this database.      # Get the source object and load directory for this database.
# Line 181  Line 175 
175              # Get the data file name.              # Get the data file name.
176              my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);              my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
177              # Do we have it?              # Do we have it?
178              if (-f $dataFile && ! $options->{clear}) {              my $haveFile = -f $dataFile;
179                  # Yes. This is good news.              # See if we can build it. Verify that we have all the sections.
                 $stats->Add('tables-found' => 1);  
                 Trace("Table file found for $table.") if T(3);  
             } else {  
                 # No, we must build it. Verify that we have all the sections.  
180                  my @missingFiles = grep { ! -f $_ } @sectionFiles;                  my @missingFiles = grep { ! -f $_ } @sectionFiles;
181                  # Did we find everything?                  # Did we find everything?
182                  if (scalar @missingFiles) {              if (scalar(@missingFiles) && ! $haveFile) {
183                      # No! Denote that we have a missing table.                  # No, and there's no main file! Denote that we have a missing table.
184                      $missingTable++;                      $missingTable++;
185                      $stats->Add('tables-skipped' => 1);                      $stats->Add('tables-skipped' => 1);
186                      # Tell the user about all the missing files.                      # Tell the user about all the missing files.
# Line 198  Line 188 
188                          $stats->Add('sections-missing' => 1);                          $stats->Add('sections-missing' => 1);
189                          $stats->AddMessage("Data file $missingFile not found for table $table.");                          $stats->AddMessage("Data file $missingFile not found for table $table.");
190                      }                      }
191                  } else {              } elsif (! scalar @missingFiles) {
192                      # We have all the sections. Try to assemble them into a data file.                      # We have all the sections. Try to assemble them into a data file.
193                      my $sortStart = time();                      my $sortStart = time();
194                      my $sortCommand = $erdb->SortNeeded($table) . " >$dataFile";                      my $sortCommand = $erdb->SortNeeded($table) . " >$dataFile";
# Line 223  Line 213 
213                      close $oh;                      close $oh;
214                      $stats->Add('tables-collated' => 1);                      $stats->Add('tables-collated' => 1);
215                      $stats->Add('collate-time' => time() - $sortStart);                      $stats->Add('collate-time' => time() - $sortStart);
216                  }              } else {
217                    # We have a data file and no sections, so we use the data file.
218                    $stats->Add('tables-found' => 1);
219              }              }
220              # Now that we know we have a full data file, we can delete the              # Now that we know we have a full data file, we can delete the
221              # section files to make room in the data directory. The user can              # section files to make room in the data directory. The user can
# Line 276  Line 268 
268              Trace("$fileName loaded into $table.") if T(3);              Trace("$fileName loaded into $table.") if T(3);
269          }          }
270          $stats->Add("groups-loaded" => 1);          $stats->Add("groups-loaded" => 1);
271          $stats->Add('load-time' => 1);          $stats->Add('load-time' => (time() - $loadStart));
272      }      }
273        # Save the DBD.
274        Trace("Saving DBD.") if T(2);
275        $erdb->InternalizeDBD();
276      $stats->Add('total-time' => time() - $totalStart);      $stats->Add('total-time' => time() - $totalStart);
277      # Display the statistics from this run.      # Display the statistics from this run.
278      Trace("Statistics for load:\n" . $stats->Show()) if T(2);      Trace("Statistics for load:\n" . $stats->Show()) if T(2);

Legend:
Removed from v.1.11  
changed lines
  Added in v.1.16

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3