[Bio] / Sprout / FTPDownload.pl Repository:
ViewVC logotype

View of /Sprout/FTPDownload.pl

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1.3 - (download) (as text) (annotate)
Mon Feb 18 07:05:46 2008 UTC (12 years, 1 month ago) by parrello
Branch: MAIN
Changes since 1.2: +0 -0 lines
Removed some obsolete methods.

#!/usr/bin/perl -w

=head1 FTP Downloader

This script downloads the latest copy of the NMPDR cover pages to the development site.
The cover pages are expected to be found in the C<ncsa> subdirectory of C<ftp.figresearch.com>,
and the development site is located at C<$FIG_Config::nmpdr_base/next/html>. The modification
date of the files on the remote site will be compared to the dates for the files currently
in the directory. If the modification date on the remote is newer, a new copy is downloaded
over the top of the old one.

The currently-supported command-line options are as follows.

=over 4

=item binary

Comma-delimited list of the extensions for the files to be transferred in binary. The default
is C<png,gif,jpg,mp3,jar,zip,swf,pdf,tar>, which covers most everything expected to occur on a
web site.

=item timeZone

Time zone displacement in hours between the NMPDR server and the B<figresearch.com> server.
The default is 1, indicating that we have to add one hour to the NMPDR time stamps to make
them match the B<figresearch.com> time stamps.

=item force

Download the remote file even if the local file is newer.

=item user

Name suffix to be used for log files. If omitted, the PID is used.

=item trace

Numeric trace level. A higher trace level causes more messages to appear. The
default trace level is 2. Tracing will be directly to the standard output
as well as to a C<trace>I<User>C<.log> file in the FIG temporary directory,
where I<User> is the value of the B<user> option above.

=item sql

If specified, turns on tracing of SQL activity.

=item background

Save the standard and error output to files. The files will be created
in the FIG temporary directory and will be named C<err>I<User>C<.log> and
C<out>I<User>C<.log>, respectively, where I<User> is the value of the
B<user> option above.

=item h

Display this command's parameters and options.



use strict;
use Tracer;
use DocUtils;
use TestUtils;
use Cwd;
use File::Copy;
use File::Path;
use FIG;
use Net::FTP;
use File::stat;

# Get the command-line options and parameters.
my ($options, @parameters) = StandardSetup([],
                                            timeZone => [1, "time zone displacement in hours"],
                                            binary => ['png,gif,jpg,mp3,jar,zip,swf,pdf,tar',
                                                       "suffixes indicating binary files"],
                                            force => [0, "download even if remote file is older"],
                                            trace => [2, 'tracing level'],

# Compute the name of the target directory.
my $targetDir = "$FIG_Config::nmpdr_base/next/html";
# Set up the remote host configuration data. If the remote host changes, you will need
# to modify this section.
my $remoteHost = 'ftp.figresearch.com';
my $remoteUser = 'fig@figresearch.com';
my $remotePass = '1star*BIO';
my $remoteDir  = 'ncsa';
# Finally, save the time zone offset in seconds. This is a global variable, and it is
# subtracted from the remote time to get the comparable local time.
my $TimeZoneOffset = $options->{timeZone} * 3600;
# Create a hash that tells us which file extensions indicate binary files. This is also
# a global variable.
my %BinaryExtensions = map { $_ => 1 } split(/,/, $options->{binary});
# Finally, check the FORCE option. This is the last of the globals.
my $ForceDownload = $options->{force};
# Access the remote host. We use real FTP and connect passively to avoid firewall
# problems.
my $ftp = Net::FTP->new($remoteHost, Passive => 1);
if (! $ftp) {
    Confess("Cannot connect to $remoteHost: $@");
# Log in to the host.
$ftp->login($remoteUser, $remotePass) ||
    Confess("Login to $remoteHost failed for user $remoteUser.");
# Change to the appropriate directory.
$ftp->cwd($remoteDir) ||
    Confess("Error switching to $remoteDir on $remoteHost.");
# Process the directory.
ProcessDirectory($ftp, $targetDir);
# Tell the user we're done.
Trace("Processing complete.") if T(2);

# Process the current directory. This is a recursive method. When it finds a
# subdirectory, it changes to that directory, calls itself, then restores
# to the original directory. The effect is not unlike a depth-first tree
# traversal. The first parameter is the FTP object; the second is the
# corresponding target directory.
sub ProcessDirectory {
    # Get the parameters.
    my ($ftp, $targetDir) = @_;
    # Insure the target directory exists.
    Trace("Processing directory $targetDir.") if T(3);
    # Get a list of the local files. OpenDir will exclude files whose names begin
    # with a dot. We put the file names into a hash, and each file we find on the
    # remote will have its hash value set to 1. After we're done with this
    # directory, we go back and delete the files whose hash values are still 0.
    my %locals = map { $_ => 0 } OpenDir($targetDir, 1);
    # Get a list of the remote files. We request these in the long form so that
    # we can tell which are directories and which are files.
    my @remotes = $ftp->dir();
    # Loop through the file list.
    for my $remote (@remotes) {
        # Check to see if this is a directory.
        my $dir = (substr($remote, 0, 1) eq 'd');
        # Get the file name from the end of the line. If we don't find one, we ignore
        # the line. Note also the match will fail if the name begins with a dot.
        if ($remote =~ /\s+([^.]\S+)$/) {
            my $fileName = $1;
            # Now we have the file name and the directory flag. If we have a directory,
            # we process it recursively.
            if ($dir) {
                # Change to the subdirectory on the remote.
                $ftp->cwd($fileName) ||
                    Confess("Could not change to subdirectory $fileName on remote host.");
                # Process the directory.
                ProcessDirectory($ftp, "$targetDir/$fileName");
                # Change back to our directory.
                $ftp->cdup() ||
                    Confess("Could not change back to parent of subdirectory $fileName.");
            } else {
                # Here we have a real file. We need to check the dates. First we get the
                # remote file's modification date. Note that we convert it to local time
                # using the global time zone offset.
                my $remoteDate = $ftp->mdtm($fileName) - $TimeZoneOffset;
                # Get the local file name.
                my $localName = "$targetDir/$fileName";
                # The local file's modification date is 0 if the file does not exist.
                my $localDate = 0;
                if (exists $locals{$fileName}) {
                    # If the local file exists and we're not in force-download mode,
                    # we get the local time stamp.
                    $localDate = stat($localName)->mtime unless $ForceDownload;
                    # While we're here, denote that the file exists on the remote so
                    # that we won't delete it later.
                    $locals{$fileName} = 1;
                # Compare the file times.
                if ($ForceDownload || $remoteDate > $localDate) {
                    # We want to download. Check the file extension to see if this is
                    # a binary transfer.
                    my $type = "ascii";
                    if ($fileName =~ /\.(\S+)$/) {
                        if ($BinaryExtensions{lc $1}) {
                            $type = "binary";
                    Trace("Downloading $fileName to $targetDir in $type mode.") if T(3);
                    if ($type eq "ascii") {
                    } else {
                    # Do the download.
                    my $okFlag = $ftp->get($fileName, $localName);
                    # If the download failed, we delete the local file so it will
                    # be retrieved when we try again.
                    if (! $okFlag) {
                        unlink $localName;
                        Confess("Error downloading $fileName in $targetDir.");
    # All done downloading. Now we need to delete the files not found on the remote.
    for my $fileName (keys %locals) {
        if (! $locals{$fileName}) {
            # We have a file to delete. Get its real name.
            my $localName = "$targetDir/$fileName";
            # Check to see if it's a directory.
            if (-d $localName) {
                # It is a directory, so we delete the entire tree.
                Trace("Deleting directory $localName.") if T(3);
            } else {
                # It's a file, so delete it normally.
                Trace("Deleting file $localName.") if T(3);
                unlink $localName;
    # We're done with this directory.
    Trace("Directory $targetDir processed.") if T(3);


MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3