[Bio] / Sprout / ERDBLoader.pl Repository:
ViewVC logotype

Diff of /Sprout/ERDBLoader.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2, Wed Oct 1 03:07:08 2008 UTC revision 1.3, Thu Oct 2 16:32:42 2008 UTC
# Line 69  Line 69 
69  Specifies the tracing level. The higher the tracing level, the more messages  Specifies the tracing level. The higher the tracing level, the more messages
70  will appear in the trace log. Use E to specify emergency tracing.  will appear in the trace log. Use E to specify emergency tracing.
71    
 =item resume  
   
 If specified, then the group list must contain a single group. The specified  
 group and all groups after it in the group list will be processed.  
   
72  =item user  =item user
73    
74  Name suffix to be used for log files. If omitted, the PID is used.  Name suffix to be used for log files. If omitted, the PID is used.
# Line 93  Line 88 
88    
89  Display this command's parameters and options.  Display this command's parameters and options.
90    
91    =item keepSections
92    
93    If specified, section files (the fragments of data load files created by
94    [[ERDBGeneratorPl]], will not be deleted after they are collated.
95    
96  =item warn  =item warn
97    
98  Create an event in the RSS feed when an error occurs.  Create an event in the RSS feed when an error occurs.
# Line 109  Line 109 
109  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],
110                                             {                                             {
111                                                trace => ["", "tracing level"],                                                trace => ["", "tracing level"],
112                                                resume => ["", "if specified, the specified group and all groups that normally come after it will be processed"],                                                keepSections => ["", "if specified, section files will not be deleted after being collated"],
113                                                phone => ["", "phone number (international format) to call when load finishes"]                                                phone => ["", "phone number (international format) to call when load finishes"]
114                                             },                                             },
115                                             "<database> <group1> <group2> ...",                                             "<database> <group1> <group2> ...",
# Line 123  Line 123 
123      # Connect to the database.      # Connect to the database.
124      my $erdb = ERDB::GetDatabase($database);      my $erdb = ERDB::GetDatabase($database);
125      # Fix the group list.      # Fix the group list.
126      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, $options, \@groups);      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, \@groups);
127        Trace("Real groups are: " . join(" ", @realGroups)) if T(3);
128      # Get the source object and load directory for this database.      # Get the source object and load directory for this database.
129      my $source = $erdb->GetSourceObject();      my $source = $erdb->GetSourceObject();
130      my $directory = $erdb->LoadDirectory();      my $directory = $erdb->LoadDirectory();
# Line 131  Line 132 
132      my @sectionList = $erdb->SectionList($source);      my @sectionList = $erdb->SectionList($source);
133      # Create a statistics object to track our progress.      # Create a statistics object to track our progress.
134      my $stats = Stats->new();      my $stats = Stats->new();
     # Get the hash of group names to table names.  
     my $groupHash = ERDBLoadGroup::GetGroupHash($erdb);  
135      # Start a timer.      # Start a timer.
136      my $totalStart = time();      my $totalStart = time();
137      # Loop through the groups.      # Loop through the groups.
138      for my $group (@realGroups) {      for my $group (@realGroups) {
139          # Get the list of tables for this group.          # Get the list of tables for this group.
140          my $tableList = $groupHash->{$group};          my @tableList = ERDBLoadGroup::GetTables($erdb, $group);
141          # We need to insure there is a data file for every table. If we fail to find one,          # We need to insure there is a data file for every table. If we fail to find one,
142          # we set the following error flag, which prevents us from loading the database.          # we set the following error flag, which prevents us from loading the database.
143          my $missingTable = 0;          my $missingTable = 0;
144          # Loop through the tables in this group.          # Loop through the tables in this group.
145          for my $table (@$tableList) {          for my $table (@tableList) {
146                Trace("Processing table $table.") if T(2);
147              # Get the data file name.              # Get the data file name.
148              my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);              my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
149              # Do we have it?              # Do we have it?
# Line 163  Line 163 
163                  # Did we find everything?                  # Did we find everything?
164                  if (scalar @missingFiles) {                  if (scalar @missingFiles) {
165                      # No! Denote that we have a missing table.                      # No! Denote that we have a missing table.
166                      $missingTable = 1;                      $missingTable++;
167                      $stats->Add('tables-skipped' => 1);                      $stats->Add('tables-skipped' => 1);
168                  } else {                  } else {
169                      # Yes! Try to assemble the sections into a data file.                      # Yes! Try to assemble the sections into a data file.
# Line 182  Line 182 
182                      close $oh;                      close $oh;
183                      $stats->Add('tables-collated' => 1);                      $stats->Add('tables-collated' => 1);
184                      # Now that we've collated the section files, we can delete them                      # Now that we've collated the section files, we can delete them
185                      # to make room in the data directory.                      # to make room in the data directory. The user can turn this
186                        # behavior off with the keepSections option.
187                        if (! $options->{keepSections}) {
188                      for my $sectionFile (@sectionFiles) {                      for my $sectionFile (@sectionFiles) {
189                          unlink $sectionFile;                          unlink $sectionFile;
190                          $stats->Add('files-deleted' => 1);                          $stats->Add('files-deleted' => 1);
191                      }                      }
192                            Trace("Section files for $table deleted.") if T(3);
193                        }
194                      $stats->Add('collate-time' => time() - $sortStart);                      $stats->Add('collate-time' => time() - $sortStart);
195                  }                  }
196              }              }
# Line 195  Line 199 
199          if ($missingTable) {          if ($missingTable) {
200              # Yes, skip this group.              # Yes, skip this group.
201              $stats->Add('groups-skipped' => 1);              $stats->Add('groups-skipped' => 1);
202                Trace("Skipping $group group: $missingTable missing tables.") if T(3);
203          } else {          } else {
204              # No! Load this group into the database.              # No! Load this group into the database.
205                Trace("Loading $group group into database.") if T(2);
206              my $loadStart = time();              my $loadStart = time();
207              for my $table (@$tableList) {              for my $table (@tableList) {
208                  my $fileName = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);                  my $fileName = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
209                  my $newStats = $erdb->LoadTable($fileName, $table, truncate => 1, failOnError => 1);                  my $newStats = $erdb->LoadTable($fileName, $table, truncate => 1, failOnError => 1);
210                  $stats->Accumulate($newStats);                  $stats->Accumulate($newStats);
211                    Trace("$fileName loaded into $table.") if T(3);
212              }              }
213              $stats->Add("groups-loaded" => 1);              $stats->Add("groups-loaded" => 1);
214              $stats->Add('load-time' => 1);              $stats->Add('load-time' => 1);

Legend:
Removed from v.1.2  
changed lines
  Added in v.1.3

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3