[Bio] / Sprout / ERDBLoader.pl Repository:
ViewVC logotype

View of /Sprout/ERDBLoader.pl

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1.3 - (download) (as text) (annotate)
Thu Oct 2 16:32:42 2008 UTC (11 years ago) by parrello
Branch: MAIN
Changes since 1.2: +24 -17 lines
General load technology fixes.

#!/usr/bin/perl -w

# Copyright (c) 2003-2006 University of Chicago and Fellowship
# for Interpretations of Genomes. All Rights Reserved.
# This file is part of the SEED Toolkit.
# The SEED Toolkit is free software. You can redistribute
# it and/or modify it under the terms of the SEED Toolkit
# Public License.
# You should have received a copy of the SEED Toolkit Public License
# along with this program; if not write to the University of Chicago
# at info@ci.uchicago.edu or the Fellowship for Interpretation of
# Genomes at veronika@thefig.info or download a copy from
# http://www.theseed.org/LICENSE.TXT.

use strict;
use Tracer;
use ERDB;
use ERDBLoadGroup;
use ERDBGenerate;
use Stats;
use Time::HiRes;

=head1 ERDBLoader Script

    ERDBLoader [options] <database> <group1> <group2> ...

ERDB Database Load Finisher

=head2 Introduction

This script finishes the database load process begun by [[ERDBGeneratorPl]].

[[ERDBGeneratorPl]] divides the source data into sections, and generates a
partial load file for each section of each table. To finish the load process, we
need to combine the partial files into single files and load the resulting
single files into the database tables.

Like [[ERDBGeneratorPl]], this script acts on load groups-- sets of related
tables that are loaded at the same time. For each table in a named group that
does not exist in the database, the script first attempts to find a completed
data file. If one does not exist, it attempts to create one by collating section
files. Once the collated section files for a load group are finished, they are
loaded into the database.

=head2 Positional Parameters

=over 4

=item database

Name of the ERDB database. This should be the class name for the subclass used
to access the database.


=head2 Command-Line Options

=over 4

=item trace

Specifies the tracing level. The higher the tracing level, the more messages
will appear in the trace log. Use E to specify emergency tracing.

=item user

Name suffix to be used for log files. If omitted, the PID is used.

=item sql

If specified, turns on tracing of SQL activity.

=item background

Save the standard and error output to files. The files will be created
in the FIG temporary directory and will be named C<err>I<User>C<.log> and
C<out>I<User>C<.log>, respectively, where I<User> is the value of the
B<user> option above.

=item help

Display this command's parameters and options.

=item keepSections

If specified, section files (the fragments of data load files created by
[[ERDBGeneratorPl]], will not be deleted after they are collated.

=item warn

Create an event in the RSS feed when an error occurs.

=item phone

Phone number to message when the script is complete.



# Get the command-line options and parameters.
my ($options, @parameters) = StandardSetup([qw(ERDBLoadGroup ERDB Stats) ],
                                              trace => ["", "tracing level"],
                                              keepSections => ["", "if specified, section files will not be deleted after being collated"],
                                              phone => ["", "phone number (international format) to call when load finishes"]
                                           "<database> <group1> <group2> ...",
# Set a variable to contain return type information.
my $rtype;
# Insure we catch errors.
eval {
    # Get the parameters.
    my ($database, @groups) = @parameters;
    # Connect to the database.
    my $erdb = ERDB::GetDatabase($database);
    # Fix the group list.
    my @realGroups = ERDBLoadGroup::ComputeGroups($erdb, \@groups);
    Trace("Real groups are: " . join(" ", @realGroups)) if T(3);
    # Get the source object and load directory for this database.
    my $source = $erdb->GetSourceObject();
    my $directory = $erdb->LoadDirectory();
    # Get the list of sections.
    my @sectionList = $erdb->SectionList($source);
    # Create a statistics object to track our progress.
    my $stats = Stats->new();
    # Start a timer.
    my $totalStart = time();
    # Loop through the groups.
    for my $group (@realGroups) {
        # Get the list of tables for this group.
        my @tableList = ERDBLoadGroup::GetTables($erdb, $group);
        # We need to insure there is a data file for every table. If we fail to find one,
        # we set the following error flag, which prevents us from loading the database.
        my $missingTable = 0;
        # Loop through the tables in this group.
        for my $table (@tableList) {
            Trace("Processing table $table.") if T(2);
            # Get the data file name.
            my $dataFile = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
            # Do we have it?
            if (-f $dataFile) {
                # Yes. This is good news.
                $stats->Add('tables-found' => 1);
            } else {
                # No, we must build it. Verify that we have all the sections.
                my @sectionFiles =
                    map { ERDBGenerate::CreateFileName($table, $_, 'data', $directory) } @sectionList;
                my @missingFiles = grep { ! -f $_ } @sectionFiles;
                # Tell the user about all the missing files.
                for my $missingFile (@missingFiles) {
                    $stats->Add('sections-missing' => 1);
                    $stats->AddMessage("Data file $missingFile not found for table $table.");
                # Did we find everything?
                if (scalar @missingFiles) {
                    # No! Denote that we have a missing table.
                    $stats->Add('tables-skipped' => 1);
                } else {
                    # Yes! Try to assemble the sections into a data file.
                    my $sortStart = time();
                    my $sortCommand = $erdb->SortNeeded($table);
                    my $oh = Open(undef, "| $sortCommand >$dataFile");
                    for my $sectionFile (@sectionFiles) {
                        Trace("Collating $sectionFile.") if T(4);
                        $stats->Add('sections-loaded' => 1);
                        for my $line (Tracer::GetFile($sectionFile)) {
                            print $oh "$line\n";
                            $stats->Add('lines-collated' => 1);
                    # Finish the sort step.
                    close $oh;
                    $stats->Add('tables-collated' => 1);
                    # Now that we've collated the section files, we can delete them
                    # to make room in the data directory. The user can turn this
                    # behavior off with the keepSections option.
                    if (! $options->{keepSections}) {
                        for my $sectionFile (@sectionFiles) {
                            unlink $sectionFile;
                            $stats->Add('files-deleted' => 1);
                        Trace("Section files for $table deleted.") if T(3);
                    $stats->Add('collate-time' => time() - $sortStart);
        # Were any tables missing?
        if ($missingTable) {
            # Yes, skip this group.
            $stats->Add('groups-skipped' => 1);
            Trace("Skipping $group group: $missingTable missing tables.") if T(3);
        } else {
            # No! Load this group into the database.
            Trace("Loading $group group into database.") if T(2);
            my $loadStart = time();
            for my $table (@tableList) {
                my $fileName = ERDBGenerate::CreateFileName($table, undef, 'data', $directory);
                my $newStats = $erdb->LoadTable($fileName, $table, truncate => 1, failOnError => 1);
                Trace("$fileName loaded into $table.") if T(3);
            $stats->Add("groups-loaded" => 1);
            $stats->Add('load-time' => 1);
    $stats->Add('total-time' => time() - $totalStart);
    # Display the statistics from this run.
    Trace("Statistics for load:\n" . $stats->Show()) if T(2);
if ($@) {
    Trace("Script failed with error: $@") if T(0);
} else {
    Trace("Script complete.") if T(2);
if ($options->{phone}) {
    my $msgID = Tracer::SendSMS($options->{phone}, "ERDBLoader completed.");
    if ($msgID) {
        Trace("Phone message sent with ID $msgID.") if T(2);
    } else {
        Trace("Phone message not sent.") if T(2);


MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3