[Bio] / Sprout / NMPDRSetup.pl Repository:
ViewVC logotype

View of /Sprout/NMPDRSetup.pl

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1.2 - (download) (as text) (annotate)
Thu Dec 6 14:58:03 2007 UTC (12 years, 4 months ago) by parrello
Branch: MAIN
Changes since 1.1: +54 -57 lines
Changed POD format for better compatability with Wiki.

#!/usr/bin/perl -w

=head1 NMPDR Setup Script

This script sets up a new version of the NMPDR web complex. It takes a single
positional parameter-- the numeric ID of the new version. So, for example

    NMPDRSetup 07

would set up version 7.

The script should run as part of the previous version. It will set up a new
version on the same machine. At some later point we may want to make it more

To run this script in the background, use something like

    nohup NMPDRSetup -background -user=You 07 >null &

This will cause the output, error, and trace files to be produced in the
FIG temporary directory. The output file will be C<outYou.log>, the
error file will be C<errYou.log>, and the trace file C<traceYou.log>.

Currently, the development NMPDR runs on the nmpdr-1 server, while
the production NMPDR runs on the nmpdr-3 server. When copying files
between servers for purposes of synchronization, it is important that
the rsync originate from the destination server so that the ownership is
not all messed up.

=over 4

=item trace

Numeric trace level. A higher trace level causes more messages to appear. The
default trace level is 2.

=item user

User name to append to trace and output file names. If none is specified, the
PID is used.

=item envDir

Fully-qualified name of the environment directory. The default is 

=item relName

Name to use for the release created. The default is C<nmpdr_XX>, where XX is the vesion
number. In general, any use of C<XX> in the release name will be replaced by the two-
digit version number in the positional parameters.

=item blastMAT

Location of the BLAST tar file. The default is C<$FIG_Config::nmpdr_base/BLASTMAT.tgz>. If
C<copy> is specified, the BLAST directory is copied from this version to the new version.

=item h

Display this command's parameters and options.

=item dataDir

Path for the SEED data directory. The default is C<0>, which indicates the most
recently-modified directory will be used.

=item force

Delete any existing SEED. If this option is not specified, the program will fail
if it finds a SEED already at the location of the new NMPDR.

=item test

If specified, information is displayed but nothing is changed.

=item stage

Specify the stage at which to start. The current stages are as follows

=over 4

=item 1 Set up directories and check out the source.

=item 2 Bootstrap and build the new FIG.

=item 3 Perform configuration fixes.

=item 4 Fix file permissions.

=item 5 Copy the cover pages.

=item 6 Copy the drug target files.


=item group

Name of the group to be given write permission for all the files. The default
is C<figadmin>.

=item server

Name of the server that contains the NMPDR cover pages.


=head2 Maintenance Notes

This script will need to be updated if the default directories change. In addition,
the CVS information is hard-coded.

Currently the FIG_Config is set up to use the NEXT pointer in the proxy URL system.
This means it has to be edited manually during the cutover.


use strict;
use Tracer;
use DocUtils;
use TestUtils;
use Cwd;
use File::Copy;
use File::Path;
use File::stat;
use POSIX;
use IO::Handle;

# Get the command-line options.
my ($options, @parameters) = StandardSetup(['Sprout', 'DocUtils'],
                                        { envDir =>   ["$FIG_Config::nmpdr_base/FIGdisk/env",
                                                       'environment directory containing pointers to applications and development tools'],
                                          blastMAT => ["copy",
                                                       'BLASTMAT TAR file or "copy" to copy the source BLASTMAT directory'],
                                          dataDir =>  ["0",
                                                       'fully-qualified name of the SEED data directory to use; default is most recent'],
                                          relName =>  ["nmpdr_XX",
                                                       'name to use for the release created (default "nmpdr_XX")'],
                                          test =>     [0,
                                                       'print some data but do not run'],
                                          stage =>    [0,
                                                       'stage at which to start'],
                                          force =>    [0,
                                                       'force an update over a possible existing SEED'],
                                          group =>    ['fig',
                                                       'group to be given write permission for the new files'],
                                          server =>   ['nmpdr-1.nmpdr.org',
                                                       'name of the server containing the development NMPDR cover pages'],

# Get the main directory names.
my $arch = $FIG_Config::arch;
my $env_dir = $options->{envDir};
my $blastmat_tarfile = $options->{blastMAT};
my $base_dir = $FIG_Config::nmpdr_base;

# Get the new version number.
my $version = $parameters[0];
# Compute the current version number.
$FIG_Config::nmpdr_version =~ /(\d+)/;
my $old_version = $1;

# Set up the CVS data. Note that we are using an anonymous login. You can
# update from the NMPDR server but you can't commit!
my $cvsroot = ":pserver:anonymous\@biocvs.mcs.anl.gov:/disks/cvs/bio";
my $cvs_packages = "seed_base GBrowse Sprout NmpdrConfigs FigSeedViewer PPO SubsystemEditor WebApplication";

# Check the parameters. Note that if the starting stage is 5 (display instructions), we don't need to do any of this.
if ($options->{stage} < 5) {
    if (! $FIG_Config::nmpdr_base) {
        Confess("Please specify a value for FIG_Config::nmpdr_base.");
    } elsif (! -d $FIG_Config::nmpdr_base) {
        Confess("NMPDR base directory \"$FIG_Config::nmpdr_base\" not found.");
    } elsif ($blastmat_tarfile ne 'copy' && ! -e $blastmat_tarfile) {
        Confess("BlastMAT TAR file not found at $blastmat_tarfile.");
    } elsif (! -d $env_dir) {
        Confess("Environment directory $env_dir not found.");
    } elsif (! $version) {
        Confess("No version number specified.");
    } elsif ($version !~ /^\d+$/) {
        Confess("Invalid (non-numeric) version number $version.");
# Make the version number pretty.
$version = sprintf("%02d", $version);
# Add the version number to the release name.
my $version_rel = $options->{relName};
$version_rel =~ s/XX/$version/e;
# Now we can start the setup.
Trace("Creating NMPDR version $version with name $version_rel.");
# Get the directory for the version-specific stuff.
my $version_dir = "$base_dir/v$version";
# This will be the directory of FIG stuff.
my $target_dir = "$version_dir/FIGdisk";
# Compute the Sprout database name.
my $sprout_db_name = "nmpdr_v${version}_sprout";
Trace("Starting at stage $options->{stage}.") if T(2);
# Stage 1: SEED directories and CVS source.
if ($options->{stage} <= 1) {
    # Insure we're not wiping out an old SEED.
    if (-e $target_dir) {
        if ($options->{force}) {
            # Here we have an old seed but we want to build over the
            # top of it.
            Trace("Deleting SEED already in place.") if T(2);
            if ($options->{test}) {
                Trace("Tree $target_dir would be deleted.") if T(2);
            } else {
        } else {
            Confess("A SEED was already found at $target_dir. Use \"-force\" to force an update anyway.");
    # At this point, if we're still alive, there's no target directory, so
    # we want to create it.
    # Build the main FIG directory.
    # Build the temporary directory.
    # Build the sprout data directory.
    # Now we create the code directories.
    # Finally, we create the directory that holds the cover pages.
    # Change to the target directory. This means we're inside FIGdisk and above
    # FIG.
    # Symbolically link to the environment directory.
    if ($options->{test}) {
        Trace("Symlink from env to $env_dir would have been created.") if T(2);
    } else {
        do_symlink($env_dir, "env");
    # Create the BLASTMat directory.
    if ($blastmat_tarfile eq 'copy') {
        # Here we are copying the BLAST data from the previous version.
        my $new_blastmat = "$target_dir/BLASTMAT";
        if ($options->{test}) {
            Trace("BLASTMAT data will be copied from $FIG_Config::blastmat to $target_dir/BLASTMAT.") if T(2);
        } else {
            Trace("Copying BLASTMAT directory from $FIG_Config::blastmat to $new_blastmat.") if T(2);
            for my $fileName (OpenDir($FIG_Config::blastmat)) {
                copy "$FIG_Config::blastmat/$fileName", "$new_blastmat/$fileName";
    } else {
        # Here we are extracting the BLAST data from a TAR file.
        if ($options->{test}) {
            Trace("BLASTMAT would have been pulled from $blastmat_tarfile.") if T(2);
        } else {
    # Next we copy the Sprout group file.
    my $newSproutDataDir = "$target_dir/FIG/SproutData";
    if ($options->{test}) {
        Trace("Sprout groups table would be copied from $FIG_Config::sproutData to $newSproutDataDir.") if T(2);
    } else {
        Trace("Copying Sprout groups table from $FIG_Config::sproutData to $newSproutDataDir.") if T(2);
        copy "$FIG_Config::sproutData/groups.tbl", "$newSproutDataDir/groups.tbl";
    # Now we need to symbolically link the data directory to the NMPDR master SEED.
    my $best_data = "";
    if ($options->{dataDir} ne "0") {
        # Here the user specified a particular directory.
        $best_data = $options->{dataDir};
    } else {
        # Here we want the most recently-modified data directory under
        # $base_dir/FIGdisk/FIG.
        my $data_base = "$base_dir/FIGdisk/FIG";
        my @possibleDatas = grep { $_ =~ /^Data/ } OpenDir("$data_base");
        my $best_time = 0;
        Trace(scalar(@possibleDatas) . " subdirectories found under $data_base.") if T(2);
        for my $possibleData (@possibleDatas) {
            # Note we're using the File::stat thing here, which returns a hash of the
            # file info, instead of the built-in function.
            my $this_data = "$data_base/$possibleData";
            my $fileInfo = stat $this_data;
            Trace("Directory $this_data has modify time " . $fileInfo->mtime . ".") if T(3);
            if ($fileInfo->mtime >= $best_time) {
                Trace("Directory $this_data selected.") if T(3);
                $best_data = $this_data;
                $best_time = $fileInfo->mtime;
        # If we didn't find any data, it's an error.
        if (! $best_data) {
            Confess("Could not find a data directory under $data_base.");
    # Verify that we have a directory.
    if (! -d $best_data) {
        Confess("Cannot use $best_data because it is not a directory.");
    # Symlink to the data directory we found. Now it will appear to be a part
    # of the normal FIG data structure tree.
    if ($options->{test}) {
        Trace("Directory $best_data will be used for FIG/Data.") if T(2);
    } else {
        do_symlink($best_data, "FIG/Data");
    # Change to the code directory.
    # Create a directory to hold the release.
    my $rel = $version_rel;
    # Set up the CVS log file. Note it will be in the same directory as
    # our other log files with the same suffix.
    my $lf = "$FIG_Config::temp/CVS$options->{user}.log";
    if ($options->{test}) {
        Trace("CVS checkout would be to " . getcwd() . ".") if T(2);
    } else {
        Trace("Checking out from CVS.") if T(2);
        my $rc = system("cvs -d $cvsroot co $cvs_packages >$lf 2>&1 ");
        if ($? != 0) {
            # Here the checkout failed.
            Confess("Error during CVS checkout for $cvsroot.");
        } else {
            Trace("CVS checkout log is in $lf.") if T(2);
        # Now copy the configuration script and make it executable.
        copy("FigCommon/configure", "$target_dir/configure") ||
            Confess("Could not copy configuration script.");
        chmod 0775, "$target_dir/configure";
        # Copy the make file.
        copy("ReleaseTools/Makefile.top", "Makefile") ||
            Confess("Could not copy master make file.");
        # Write out the release name.
        Open(\*RELFILE, ">$target_dir/DEFAULT_RELEASE");
        print RELFILE "$rel\n";
        # Hack out the PostGres initialization. NMPDR is MySQL-only.
    Trace("Stage 1 complete.") if T(2);
# STAGE 2: Bootstrap and build the new FIG.
if ($options->{stage} <= 2) {
    # Change to the target FIG directory. We'll run the configuration
    # script from here, and it requires another log file.
    if ($options->{test}) {
        Trace("Configuration script would be executed from $target_dir.") if T(2);
    } else {
        delete $ENV{PERL5LIB};
        my $lf = "$FIG_Config::temp/Config$options->{user}.log";
        Trace("Executing configuration script from $target_dir.") if T(2);
        system("./configure $arch >$lf 2>&1");
        if ($? != 0) {
            Trace("Error during configuration script.") if T(0);
    Trace("Stage 2 complete.") if T(2);
#   STAGE 3: Configure FIG_Config and the config script
#            and protect from updates.
if ($options->{stage} <= 3) {
    # Switch to the configuration directory.
    # Create a hash of the changes that need to be made.
    my %changes = (
                   temp_url          => "\$nmpdr_site_url/FIG-Tmp",
                   cgi_url           => "\$nmpdr_site_url/FIG",
                   template_url      => "\$nmpdr_site_url/templates",
                   java              => $FIG_Config::java,
                   dbms              => "mysql",
                   sproutDbms        => "mysql",
                   db                => $FIG_Config::db,
                   dbuser            => $FIG_Config::dbuser,
                   dbpass            => $FIG_Config::dbpass,
                   dbsock            => $FIG_Config::dbsock,
                   dbport            => $FIG_Config::dbport,
                   sproutDB          => $sprout_db_name,
                   oldSproutDB       => $FIG_Config::sproutDB,
                   oldSproutDBD      => "$FIG_Config::fig/SproutDBD.xml",
                   sproutUser        => "seed",
                   sproutPass        => "",
                   sproutSock        => "",
                   sproutPort        => 3306,
                   nmpdr_version     => $version_rel,
                   nmpdr_base        => $FIG_Config::nmpdr_base,
                   nmpdr_pages       => "$version_dir/html",
                   preIndex          => 1,
                   otherGroup        => "(supporting)",
                   feature_types     => "peg rna pp pi",
                   results_per_page  => 50,
                   blast_limit       => 1000,
                   prop_search_limit => 4,
                   drug_directory    => "\$var/DrugTargets",
                   dockLimit         => "00000030",
                   advanced_class    => "FidSearch BlastSearch",
                   splitter          => " : ",
                   attrURL           => "http://anno-3.nmpdr.org/attrib_server/AttribXMLRPC.cgi",
                   attrDbms          => "mysql",
                   attrDbName        => "fig_v5_attributes",
                   attrHost          => undef,
                   attrUser          => "seed",
                   attrPass          => "",
                   attrPort          => 3306,
                   attrSock          => undef,
                   attrDBD           => "/disks/nmpdr/v$version/FIGdisk/FIG/AttributesDBD.xml",
                   phone             => "{ user => 'BruceTheHumanPet', password => 'STARl00k', api_id => '2561022' }",
                   nmpdr_mode        => 1,
    # Pull forward the server values from the current version.
    for my $serverKey (qw(bbh_server_url sim_server_url pin_server_url pch_server_url use_pch_server)) {
        my $value = eval("\$FIG_Config::$serverKey");
        if (defined $value) {
            $changes{$serverKey} = $value;

    # Create a list of lines to be added to the front.
    my $inserts = [
                   "\$nmpdr_site_url = \"http://$options->{server}/next\";",
    if ($options->{test}) {
        Trace("Configuration file update would run from " . getcwd() . ".") if T(2);
    } else {
        Trace("Updating configuration file in " . getcwd() . ".") if T(2);
        DocUtils::ModifyConfigFile("FIG_Config.pm", \%changes, $inserts);
    # The next step is to update the shell script. We need to add CVS information and
    # the temp directory.
    my %elims = (CVSROOT => $cvsroot,
                 CVS_RSH => 'ssh',
                 TMPDIR => "$target_dir/FIG/Tmp",
    # Switch to the configuration directory.
    # Check to see if we're in test mode.
    if ($options->{test}) {
        Trace("File fig-user-env.sh would be updated.") if T(2);
        for my $elim (keys %elims) {
            Trace("export $elim = \"$elims{$elim}\"") if T(2);
    } else {
        # Slurp in the entire file.
        my @lines = Tracer::GetFile("fig-user-env.sh");
        # Loop through the variables.
        for my $elim (keys %elims) {
            # Remove any existing lines for the variable.
            @lines = grep { $_ !~ /(export $elim)|($elim=)/ } @lines;
            # Add the new lines.
            Trace("Updating $elim in fig-user-env.sh.");
            push @lines, "$elim=\"$elims{$elim}\"";
            push @lines, "export $elim";
        # Write the file back out.
        Open(\*FIGOUT, ">fig-user-env.sh~");
        for my $line (@lines) {
            print FIGOUT "$line\n";
        close FIGOUT;
        rename "fig-user-env.sh~", "fig-user-env.sh";
    # Copy the WebApplication configuration subdirectory from the
    # current version to the next version. First, we get our current
    # configuration directory.
    my $currentConfig = "$FIG_Config::fig_disk/config";
    # Now we create the web-application subdirectory in the target directory.
    mkdir "WebApplication";
    # Copy across the files inside.
    my @lines = `cp -r $currentConfig/WebApplication/* $target_dir/config/WebApplication`;
    Trace("WebApplication configuration files copied:\n" . join("\n", @lines)) if T(2);
    # Finally, create a marker file to protect this installation from
    # automatic updates.
    my $marker_file = "$target_dir/config/DISABLE_P2P";
    if ($options->{test}) {
        Trace("Marker file $marker_file would be created.") if T(2);
    } else {
        Trace("Creating marker file $marker_file.") if T(2);
        Open(\*FIGOUT, ">$marker_file");
        print FIGOUT "\n";
        close FIGOUT;
    # We're done.
    Trace("Stage 3 complete.") if T(2);
# STAGE 4: fix the file permissions.
if ($options->{stage} <= 4) {
    # Get the group name and ID.
    my $gname = $options->{group};
    my $gid = getgrnam($gname);
    if ($options->{test}) {
        Trace("Permission fix would occur in $version_dir. Group $gname has ID $gid.") if T(2);
    } else {
        # Set the permissions.
        Tracer::SetPermissions($version_dir, $options->{group}, 02664, tmp => 02777);
    Trace("Stage 4 complete.") if T(2);
# STAGE 5: Copy the cover pages.
if ($options->{stage} <= 5) {
    Trace("Copying cover pages.") if T(2);
    my @syncData = `rsync --archive --recursive /disks/nmpdr/$old_version/html /disks/nmpdr/$version_dir/`;
    Trace("Copy results\n" . join("\n", @syncData)) if T(3);
    # Create the version number file for the wiki.
    my $text = join("\n", "<?php",
                          "\$wgNmpdrVersion = $version;",
    my $fh = Tracer::Open(undef, ">/disks/nmpdr/$version_dir/html/wiki/extensions/NmpdrVersion.php");
    print "$fh", $text;
    close $fh;
    Trace("Version file created for wiki.") if T(2);
    Trace("Stage 5 complete.") if T(2);
# STAGE 6: Copy the drug target files.
if ($options->{stage} <= 6) {
    Trace("Copying drug target files.") if T(2);
    my @syncData = `  rsync --archive --recursive /disks/nmpdr/$old_version/FIGdisk/FIG/var/DrugTargetData /disks/nmpdr/$version_dir/`;
    Trace("Copy results\n" . join("\n", @syncData)) if T(3);
    Trace("Stage 6 complete.") if T(2);

Trace("Processing complete.") if T(0);

# Forge a symbolic Link.
sub do_symlink {
    my($from, $to) = @_;
    # Erase the old link if necessary.
    if (-e $to) {
        Trace("Erasing old link $to.") if T(3);
        unlink $to;
    symlink($from, $to) ||
        Confess("Symlink $from $to failed: $!");

# Untar a bunch of files.
sub do_untar {
    my($file) = @_;
    my $args = "-x -f $file";
    if ($file =~ /gz$/) {
        $args .= " -z";
    } elsif ($file =~ /bz2/) {
        $args .= " -j";
    my $cmd = "tar $args";
    my $rc = system($cmd);
    if ($rc != 0) {
        Confess("tar failed with rc=$rc: $cmd");


MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3