[Bio] / Sprout / ERDBGenerator.pl Repository:
ViewVC logotype

Diff of /Sprout/ERDBGenerator.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1, Tue Sep 30 15:23:55 2008 UTC revision 1.2, Wed Oct 1 03:07:08 2008 UTC
# Line 50  Line 50 
50    
51  =item groups  =item groups
52    
53  List of the table groups to load, or C<*> to load all table groups.  List of the table groups to load. A C<+> at the end of the list indicates that
54    all groups that follow the last-named group in the standard order should
55    be loaded. A C<+> by itself loads all groups in standard order.
56    
57  =back  =back
58    
# Line 95  Line 97 
97    
98  If specified, then this prcoess will terminate after the specified number of  If specified, then this prcoess will terminate after the specified number of
99  section load errors; otherwise, the process will keep going after a section  section load errors; otherwise, the process will keep going after a section
100  error.  error. A value of C<0> means the process will ignore all errors. A value of
101    C<1> means it will stop after the first error.
102    
103  =item phone  =item phone
104    
105  Phone number to message when the script is complete.  Phone number to message when the script is complete.
106    
 =item resume  
   
 If specified, then the group list must contain a single group. The specified  
 group and all groups after it in the group list will be processed.  
   
107  =item sections  =item sections
108    
109  Name of a file containing a list of sections to process. If C<*> is specified (the  Name of a file containing a list of sections to process. If C<*> is specified (the
# Line 149  Line 147 
147                                                clear => ["", "if specified, the entire load directory will be cleared"],                                                clear => ["", "if specified, the entire load directory will be cleared"],
148                                                clean => ["", "if specified, temporary files in the load directory will be deleted"],                                                clean => ["", "if specified, temporary files in the load directory will be deleted"],
149                                                clearGroups => ["", "if specified, pre-exising load files from the groups processed will be deleted"],                                                clearGroups => ["", "if specified, pre-exising load files from the groups processed will be deleted"],
150                                                  maxErrors => ["0", "if non-zero, the maximum allowed number of section failures"],
151                                                  phone => ["", "phone number (international format) to call when load finishes"],
152                                                trace => ["2", "tracing level"],                                                trace => ["2", "tracing level"],
153                                                workers => ["1", "number of worker processes"],                                                workers => ["1", "number of worker processes"],
                                               phone => ["", "phone number (international format) to call when load finishes"],  
154                                                resume => ["", "if specified, the specified group and all groups that normally come after it will be loaded"],                                                resume => ["", "if specified, the specified group and all groups that normally come after it will be loaded"],
155                                                sections => ["*", "name of a file in the database's load directory containing a list of sections to process"],                                                sections => ["*", "name of a file in the database's load directory containing a list of sections to process"],
                                               maxErrors => ["", "if specified, the maximum allowed number of section failures"],  
156                                             },                                             },
157                                             "<database> <group1> <group2> ...",                                             "<database> <group1> <group2> ...",
158                                             @ARGV);                                             @ARGV);
# Line 166  Line 164 
164  eval {  eval {
165      # Get the parameters.      # Get the parameters.
166      my ($database, @groups) = @parameters;      my ($database, @groups) = @parameters;
167      # Connect to the database.      # Connect to the database and get its load directory.
168      my $erdb = ERDB::GetDatabase($database);      my $erdb = ERDB::GetDatabase($database);
169        my $directory = $erdb->LoadDirectory();
170      # Fix the group list.      # Fix the group list.
171      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, $options, \@groups);      my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, \@groups);
172      # Get the source object and load directory for this database.      # Get the source object.
173      my $source = $erdb->GetSourceObject();      my $source = $erdb->GetSourceObject();
     my $directory = $erdb->LoadDirectory();  
174      # Are we a worker or a manager?      # Are we a worker or a manager?
175      if ($options->{workers} == 0) {      if ($options->{workers} == 0) {
176          # Yes, we're a worker.          # Yes, we're a worker.
# Line 220  Line 218 
218              }              }
219              Trace("$deleteCount files deleted from load directory during ClearGroups.") if T(2);              Trace("$deleteCount files deleted from load directory during ClearGroups.") if T(2);
220          }          }
221            # Delete any leftover kill file if it exists.
222            my $killFileName = ERDBLoadGroup::KillFileName($erdb, $directory);
223            if (-f $killFileName) {
224                Trace("Deleting kill file $killFileName.") if T(2);
225                unlink $killFileName;
226            }
227          # Now we need to get our list of sections. Check to see if the user          # Now we need to get our list of sections. Check to see if the user
228          # supplied a section file.          # supplied a section file.
229          my $sectionFile = $options->{sections};          my $sectionFile = $options->{sections};
# Line 251  Line 255 
255          # Are we a sequential load or a multi-worker manager?          # Are we a sequential load or a multi-worker manager?
256          my $numWorkers = $options->{workers};          my $numWorkers = $options->{workers};
257          if ($numWorkers == 1) {          if ($numWorkers == 1) {
258              # We're sequential. We do all the work ourselves.              # We're sequential, so we do all the work ourselves.
259              Trace("Sequential load started.") if T(2);              Trace("Sequential load started.") if T(2);
260              LoadFromInput($ih, $erdb, $source, \@realGroups, \%workerOptions);              LoadFromInput($ih, $erdb, $source, \@realGroups, \%workerOptions);
261          } else {          } else {
# Line 342  Line 346 
346      # We'll count our errors in here.      # We'll count our errors in here.
347      my $errorCount = 0;      my $errorCount = 0;
348      my $maxErrors = $options->{maxErrors};      my $maxErrors = $options->{maxErrors};
349        # Compute the kill file name.
350        my $killFileName = ERDBLoadGroup::KillFileName($erdb, $erdb->LoadDirectory());
351        my $killed = 0;
352      # Loop through the sections.      # Loop through the sections.
353      while (! eof $ih) {      while (! $killed && ! eof $ih) {
354          # Get this section ID.          # Get this section ID.
355          my ($section) = Tracer::GetLine($ih);          my ($section) = Tracer::GetLine($ih);
356          # Process it for each load group.          # Process it for each load group.
357          for my $group (@$groups) {          for my $group (@$groups) {
358                # Only proceed if we haven't been killed.
359                if (! $killed) {
360                    # Check for a kill file.
361                    if (-f $killFileName) {
362                        # Found one, so kill ourselves.
363                        Trace("Worker $$ terminated by kill file.") if T(2);
364                        $killed = 1;
365                    } else {
366                        # No kill file, so we process the section.
367              Trace("Processing section $section for group $group in worker $$.") if T(3);              Trace("Processing section $section for group $group in worker $$.") if T(3);
368              my $ok = $loaders{$group}->ProcessSection($section);              my $ok = $loaders{$group}->ProcessSection($section);
369              # Check to see if we've exceeded the maximum error count.                      # Check to see if we've exceeded the maximum error count. We only care
370              if (! $ok && $maxErrors ne '' && $errorCount++ >= $maxErrors) {                      # if maxErrors is nonzero.
371                  Confess("Error limit exceeded in database loader.");                      if (! $ok && $maxErrors && ++$errorCount >= $maxErrors) {
372                            Trace("Error limit exceeded in database loader.") if T(0);
373                            $killed = 1;
374                        }
375                    }
376              }              }
377          }          }
378      }      }

Legend:
Removed from v.1.1  
changed lines
  Added in v.1.2

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3