[Bio] / Sprout / ERDBLoadGroup.pm Repository:
ViewVC logotype

Diff of /Sprout/ERDBLoadGroup.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.4, Thu Oct 9 17:23:54 2008 UTC revision 1.5, Mon Jan 19 21:46:21 2009 UTC
# Line 77  Line 77 
77    
78  name of this load group  name of this load group
79    
80    =item label
81    
82    name of this worker process
83    
84  =item lastKey  =item lastKey
85    
86  ID of the last major object processed  ID of the last major object processed
# Line 111  Line 115 
115    
116  =head3 new  =head3 new
117    
118      my $edbl = ERDBLoadGroup->new($source, $db, $directory, $options, @tables);      my $edbl = ERDBLoadGroup->new($db, $directory, $options, @tables);
119    
120  Construct a new ERDBLoadGroup object. The following parameters are expected:  Construct a new ERDBLoadGroup object. The following parameters are expected:
121    
122  =over 4  =over 4
123    
 =item source  
   
 The object to be used by the subclass to access the source data. If this parameter  
 is undefined, the source object will be retrieved from the database object as soon  
 as the client calls the L</source> method.  
   
124  =item db  =item db
125    
126  The [[ErdbPm]] object for the database being loaded.  The [[ErdbPm]] object for the database being loaded.
# Line 142  Line 140 
140    
141  sub new {  sub new {
142      # Get the parameters.      # Get the parameters.
143      my ($class, $source, $db, $options, @tables) = @_;      my ($class, $db, $options, @tables) = @_;
144      # Create a statistics object      # Create a statistics object
145      my $stats = Stats->new();      my $stats = Stats->new();
146      # Compute the group name from the class name. It is the first word in      # Compute the group name from the class name. It is the first word in
# Line 160  Line 158 
158                      directory => $directory,                      directory => $directory,
159                      group => $group,                      group => $group,
160                      stats => $stats,                      stats => $stats,
161                      source => $source,                      source => undef,
162                        label => ($options->{label} || $$),
163                      lastKey => undef,                      lastKey => undef,
164                      loaders => {},                      loaders => {},
165                      tables => \@tables,                      tables => \@tables,
# Line 375  Line 374 
374  =item statName  =item statName
375    
376  Name of the statistic to be incremented. This should be a plural noun  Name of the statistic to be incremented. This should be a plural noun
377  describing the object whose kep is coming in.  describing the object whose key is coming in.
378    
379  =item key  =item key
380    
# Line 400  Line 399 
399      # Do we need to output a progress message?      # Do we need to output a progress message?
400      if ($period && T(3) && ($newValue % $period == 0)) {      if ($period && T(3) && ($newValue % $period == 0)) {
401          # Yes.          # Yes.
402          Trace("$newValue $statName processed for $self->{group} group.");          Trace("$newValue $statName processed by $self->{label} for $self->{group} group.");
403      }      }
404  }  }
405    
# Line 487  Line 486 
486      $self->{section} = $section;      $self->{section} = $section;
487      # Get the database object.      # Get the database object.
488      my $db = $self->db();      my $db = $self->db();
     # Start a timer and protect ourselves from errors.  
     my $startTime = time();  
     eval {  
489          # Get the list of tables for this group.          # Get the list of tables for this group.
490          my @tables = @{$self->{tables}};          my @tables = @{$self->{tables}};
491        # Should we skip this section?
492        if ($self->SkipIndicated($section, \@tables)) {
493            Trace("Resume mode: section $section skipped for group $self->{group}.") if T(3);
494            $self->Add("section-skips" => 1);
495        } else {
496            # Not skipping. Start a timer and protect ourselves from errors.
497            my $startTime = time();
498            eval {
499          # Get the loader hash.          # Get the loader hash.
500          my $loaderHash = $self->{loaders};          my $loaderHash = $self->{loaders};
501          # Initialize the loaders for the necessary tables.          # Initialize the loaders for the necessary tables.
# Line 527  Line 531 
531          for my $loader (values %{$self->{loaders}}) {          for my $loader (values %{$self->{loaders}}) {
532              $loader->Finish();              $loader->Finish();
533          }          }
534          # Update the load count and the timer.              # Update the load count.
535          $self->Add("section-loads" => 1);          $self->Add("section-loads" => 1);
536            }
537            # Update the timer.
538          $self->Add(duration => (time() - $startTime));          $self->Add(duration => (time() - $startTime));
539      }      }
540  }  }
# Line 729  Line 735 
735      return $retVal;      return $retVal;
736  }  }
737    
738    =head3 SkipIndicated
739    
740        my $flag = $edbl->SkipIndicated($section, \@tables);
741    
742    Return FALSE if the current group should be run for the current section.
743    If the C<resume> option is not set, this method always returns FALSE;
744    otherwise, it will look at the files currently in the load directory and
745    if enough of them are present, it will return TRUE, indicating there's
746    no point in generating data for the indicated tables with respect to the
747    current section. In other words, it will return TRUE if, for every table,
748    there is either a load file for that table or a load file for the
749    specified section of that table.
750    
751    =over 4
752    
753    =item section
754    
755    ID of the relevant section.
756    
757    =item tables
758    
759    List of tables to check.
760    
761    =item RETURN
762    
763    Returns TRUE if load files are already generated for the specified section, else FALSE.
764    
765    =back
766    
767    =cut
768    
769    sub SkipIndicated {
770        # Get the parameters.
771        my ($self, $section, $tables) = @_;
772        # Declare the return variable. It's FALSE if there's no resume parameter.
773        my $retVal = $self->{options}->{resume};
774        # Loop through the table names while $retval is TRUE.
775        for my $table (@$tables) { last if ! $retVal;
776            # Compute the file names.
777            my @files = map { ERDBGenerate::CreateFileName($table, $_, data => $self->{directory}) }
778                (undef, $section);
779            # If neither is present, we can't skip. So, if the grep below returns an empty
780            # list, we set $retVal FALSE, which stops the loop.
781            if (scalar(grep { -f $_ } @files) == 0) {
782                $retVal = 0;
783                Trace("Section $section not found for $table in $self->{group}. Regeneration required.") if T(3);
784            }
785        }
786        # Return the result.
787        return $retVal;
788    }
789    
790    
791  =head2 Virtual Methods  =head2 Virtual Methods
792    

Legend:
Removed from v.1.4  
changed lines
  Added in v.1.5

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3