[Bio] / Sprout / ERDBLoader.pl Repository:
ViewVC logotype

Diff of /Sprout/ERDBLoader.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1, Tue Sep 30 15:23:55 2008 UTC revision 1.2, Wed Oct 1 03:07:08 2008 UTC
# Line 23  Line 23 
23  use ERDBLoadGroup;  use ERDBLoadGroup;
24  use ERDBGenerate;  use ERDBGenerate;
25  use Stats;  use Stats;
26    use Time::HiRes;
27    
28    
29  =head1 ERDBLoader Script  =head1 ERDBLoader Script
# Line 44  Line 45 
45  tables that are loaded at the same time. For each table in a named group that  tables that are loaded at the same time. For each table in a named group that
46  does not exist in the database, the script first attempts to find a completed  does not exist in the database, the script first attempts to find a completed
47  data file. If one does not exist, it attempts to create one by collating section  data file. If one does not exist, it attempts to create one by collating section
48  files. Once the collated section file is finished, it is loaded into the  files. Once the collated section files for a load group are finished, they are
49  database.  loaded into the database.
50    
51  =head2 Positional Parameters  =head2 Positional Parameters
52    
# Line 68  Line 69 
69  Specifies the tracing level. The higher the tracing level, the more messages  Specifies the tracing level. The higher the tracing level, the more messages
70  will appear in the trace log. Use E to specify emergency tracing.  will appear in the trace log. Use E to specify emergency tracing.
71    
 =item replace  
   
 Normally, if a table exists in the database, it will not be loaded. If this  
 option is specified, however, existing tables will be dropped and recreated.  
   
72  =item resume  =item resume
73    
74  If specified, then the group list must contain a single group. The specified  If specified, then the group list must contain a single group. The specified
# Line 113  Line 109 
109  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],  my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],
110                                             {                                             {
111                                                trace => ["", "tracing level"],                                                trace => ["", "tracing level"],
                                               replace => ["", "if specified, existing tables will be overwritten"],  
112                                                resume => ["", "if specified, the specified group and all groups that normally come after it will be processed"],                                                resume => ["", "if specified, the specified group and all groups that normally come after it will be processed"],
113                                                phone => ["", "phone number (international format) to call when load finishes"]                                                phone => ["", "phone number (international format) to call when load finishes"]
114                                             },                                             },
# Line 138  Line 133 
133      my $stats = Stats->new();      my $stats = Stats->new();
134      # Get the hash of group names to table names.      # Get the hash of group names to table names.
135      my $groupHash = ERDBLoadGroup::GetGroupHash($erdb);      my $groupHash = ERDBLoadGroup::GetGroupHash($erdb);
136        # Start a timer.
137        my $totalStart = time();
138      # Loop through the groups.      # Loop through the groups.
139      for my $group (@realGroups) {      for my $group (@realGroups) {
140          # Get the list of tables for this group.          # Get the list of tables for this group.
141          my $tableList = $groupHash->{$group};          my $tableList = $groupHash->{$group};
142            # We need to insure there is a data file for every table. If we fail to find one,
143            # we set the following error flag, which prevents us from loading the database.
144            my $missingTable = 0;
145            # Loop through the tables in this group.
146            for my $table (@$tableList) {
147                # Get the data file name.
148                my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
149                # Do we have it?
150                if (-f $dataFile) {
151                    # Yes. This is good news.
152                    $stats->Add('tables-found' => 1);
153                } else {
154                    # No, we must build it. Verify that we have all the sections.
155                    my @sectionFiles =
156                        map { ERDBGenerate::CreateFileName($table, $_, 'data', $directory) } @sectionList;
157                    my @missingFiles = grep { ! -f $_ } @sectionFiles;
158                    # Tell the user about all the missing files.
159                    for my $missingFile (@missingFiles) {
160                        $stats->Add('sections-missing' => 1);
161                        $stats->AddMessage("Data file $missingFile not found for table $table.");
162                    }
163                    # Did we find everything?
164                    if (scalar @missingFiles) {
165                        # No! Denote that we have a missing table.
166                        $missingTable = 1;
167                        $stats->Add('tables-skipped' => 1);
168                    } else {
169                        # Yes! Try to assemble the sections into a data file.
170                        my $sortStart = time();
171                        my $sortCommand = $erdb->SortNeeded($table);
172                        my $oh = Open(undef, "| $sortCommand >$dataFile");
173                        for my $sectionFile (@sectionFiles) {
174                            Trace("Collating $sectionFile.") if T(4);
175                            $stats->Add('sections-loaded' => 1);
176                            for my $line (Tracer::GetFile($sectionFile)) {
177                                print $oh "$line\n";
178                                $stats->Add('lines-collated' => 1);
179                            }
180                        }
181                        # Finish the sort step.
182                        close $oh;
183                        $stats->Add('tables-collated' => 1);
184                        # Now that we've collated the section files, we can delete them
185                        # to make room in the data directory.
186                        for my $sectionFile (@sectionFiles) {
187                            unlink $sectionFile;
188                            $stats->Add('files-deleted' => 1);
189                        }
190                        $stats->Add('collate-time' => time() - $sortStart);
191                    }
192                }
193            }
194            # Were any tables missing?
195            if ($missingTable) {
196                # Yes, skip this group.
197                $stats->Add('groups-skipped' => 1);
198            } else {
199                # No! Load this group into the database.
200                my $loadStart = time();
201                for my $table (@$tableList) {
202                    my $fileName = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
203                    my $newStats = $erdb->LoadTable($fileName, $table, truncate => 1, failOnError => 1);
204                    $stats->Accumulate($newStats);
205                }
206                $stats->Add("groups-loaded" => 1);
207                $stats->Add('load-time' => 1);
208            }
209      }      }
210        $stats->Add('total-time' => time() - $totalStart);
211        # Display the statistics from this run.
212        Trace("Statistics for load:\n" . $stats->Show()) if T(2);
213  };  };
214  if ($@) {  if ($@) {
215      Trace("Script failed with error: $@") if T(0);      Trace("Script failed with error: $@") if T(0);
     $rtype = "error";  
216  } else {  } else {
217      Trace("Script complete.") if T(2);      Trace("Script complete.") if T(2);
     $rtype = "no error";  
218  }  }
219  if ($options->{phone}) {  if ($options->{phone}) {
220      my $msgID = Tracer::SendSMS($options->{phone}, "ERDBLoader terminated with $rtype.");      my $msgID = Tracer::SendSMS($options->{phone}, "ERDBLoader completed.");
221      if ($msgID) {      if ($msgID) {
222          Trace("Phone message sent with ID $msgID.") if T(2);          Trace("Phone message sent with ID $msgID.") if T(2);
223      } else {      } else {

Legend:
Removed from v.1.1  
changed lines
  Added in v.1.2

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3