[Bio] / Sprout / FTPDownload.pl Repository:
ViewVC logotype

Annotation of /Sprout/FTPDownload.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.3 - (view) (download) (as text)

1 : parrello 1.1 #!/usr/bin/perl -w
2 :    
3 :     =head1 FTP Downloader
4 :    
5 :     This script downloads the latest copy of the NMPDR cover pages to the development site.
6 :     The cover pages are expected to be found in the C<ncsa> subdirectory of C<ftp.figresearch.com>,
7 :     and the development site is located at C<$FIG_Config::nmpdr_base/next/html>. The modification
8 :     date of the files on the remote site will be compared to the dates for the files currently
9 :     in the directory. If the modification date on the remote is newer, a new copy is downloaded
10 :     over the top of the old one.
11 :    
12 :     The currently-supported command-line options are as follows.
13 :    
14 :     =over 4
15 :    
16 :     =item binary
17 :    
18 :     Comma-delimited list of the extensions for the files to be transferred in binary. The default
19 :     is C<png,gif,jpg,mp3,jar,zip,swf,pdf,tar>, which covers most everything expected to occur on a
20 :     web site.
21 :    
22 :     =item timeZone
23 :    
24 :     Time zone displacement in hours between the NMPDR server and the B<figresearch.com> server.
25 :     The default is 1, indicating that we have to add one hour to the NMPDR time stamps to make
26 :     them match the B<figresearch.com> time stamps.
27 :    
28 :     =item force
29 :    
30 :     Download the remote file even if the local file is newer.
31 :    
32 :     =item user
33 :    
34 :     Name suffix to be used for log files. If omitted, the PID is used.
35 :    
36 :     =item trace
37 :    
38 :     Numeric trace level. A higher trace level causes more messages to appear. The
39 :     default trace level is 2. Tracing will be directly to the standard output
40 :     as well as to a C<trace>I<User>C<.log> file in the FIG temporary directory,
41 :     where I<User> is the value of the B<user> option above.
42 :    
43 :     =item sql
44 :    
45 :     If specified, turns on tracing of SQL activity.
46 :    
47 :     =item background
48 :    
49 :     Save the standard and error output to files. The files will be created
50 :     in the FIG temporary directory and will be named C<err>I<User>C<.log> and
51 :     C<out>I<User>C<.log>, respectively, where I<User> is the value of the
52 :     B<user> option above.
53 :    
54 :     =item h
55 :    
56 :     Display this command's parameters and options.
57 :    
58 :     =back
59 :    
60 :     =cut
61 :    
62 :     use strict;
63 :     use Tracer;
64 :     use DocUtils;
65 :     use TestUtils;
66 :     use Cwd;
67 :     use File::Copy;
68 :     use File::Path;
69 :     use FIG;
70 :     use Net::FTP;
71 :     use File::stat;
72 :    
73 :     # Get the command-line options and parameters.
74 :     my ($options, @parameters) = StandardSetup([],
75 :     {
76 :     timeZone => [1, "time zone displacement in hours"],
77 :     binary => ['png,gif,jpg,mp3,jar,zip,swf,pdf,tar',
78 :     "suffixes indicating binary files"],
79 :     force => [0, "download even if remote file is older"],
80 : parrello 1.2 trace => [2, 'tracing level'],
81 : parrello 1.1 },
82 :     "",
83 :     @ARGV);
84 :    
85 :     # Compute the name of the target directory.
86 :     my $targetDir = "$FIG_Config::nmpdr_base/next/html";
87 :     # Set up the remote host configuration data. If the remote host changes, you will need
88 :     # to modify this section.
89 :     my $remoteHost = 'ftp.figresearch.com';
90 :     my $remoteUser = 'fig@figresearch.com';
91 :     my $remotePass = '1star*BIO';
92 :     my $remoteDir = 'ncsa';
93 :     # Finally, save the time zone offset in seconds. This is a global variable, and it is
94 :     # subtracted from the remote time to get the comparable local time.
95 :     my $TimeZoneOffset = $options->{timeZone} * 3600;
96 :     # Create a hash that tells us which file extensions indicate binary files. This is also
97 :     # a global variable.
98 :     my %BinaryExtensions = map { $_ => 1 } split(/,/, $options->{binary});
99 :     # Finally, check the FORCE option. This is the last of the globals.
100 :     my $ForceDownload = $options->{force};
101 :     # Access the remote host. We use real FTP and connect passively to avoid firewall
102 :     # problems.
103 :     my $ftp = Net::FTP->new($remoteHost, Passive => 1);
104 :     if (! $ftp) {
105 :     Confess("Cannot connect to $remoteHost: $@");
106 :     }
107 :     # Log in to the host.
108 :     $ftp->login($remoteUser, $remotePass) ||
109 :     Confess("Login to $remoteHost failed for user $remoteUser.");
110 :     # Change to the appropriate directory.
111 :     $ftp->cwd($remoteDir) ||
112 :     Confess("Error switching to $remoteDir on $remoteHost.");
113 :     # Process the directory.
114 :     ProcessDirectory($ftp, $targetDir);
115 :     # Tell the user we're done.
116 :     Trace("Processing complete.") if T(2);
117 :    
118 :     # Process the current directory. This is a recursive method. When it finds a
119 :     # subdirectory, it changes to that directory, calls itself, then restores
120 :     # to the original directory. The effect is not unlike a depth-first tree
121 :     # traversal. The first parameter is the FTP object; the second is the
122 :     # corresponding target directory.
123 :     sub ProcessDirectory {
124 :     # Get the parameters.
125 :     my ($ftp, $targetDir) = @_;
126 :     # Insure the target directory exists.
127 :     Insure($targetDir);
128 :     Trace("Processing directory $targetDir.") if T(3);
129 :     # Get a list of the local files. OpenDir will exclude files whose names begin
130 :     # with a dot. We put the file names into a hash, and each file we find on the
131 :     # remote will have its hash value set to 1. After we're done with this
132 :     # directory, we go back and delete the files whose hash values are still 0.
133 :     my %locals = map { $_ => 0 } OpenDir($targetDir, 1);
134 :     # Get a list of the remote files. We request these in the long form so that
135 :     # we can tell which are directories and which are files.
136 :     my @remotes = $ftp->dir();
137 :     # Loop through the file list.
138 :     for my $remote (@remotes) {
139 :     # Check to see if this is a directory.
140 :     my $dir = (substr($remote, 0, 1) eq 'd');
141 :     # Get the file name from the end of the line. If we don't find one, we ignore
142 :     # the line. Note also the match will fail if the name begins with a dot.
143 :     if ($remote =~ /\s+([^.]\S+)$/) {
144 :     my $fileName = $1;
145 :     # Now we have the file name and the directory flag. If we have a directory,
146 :     # we process it recursively.
147 :     if ($dir) {
148 :     # Change to the subdirectory on the remote.
149 :     $ftp->cwd($fileName) ||
150 :     Confess("Could not change to subdirectory $fileName on remote host.");
151 :     # Process the directory.
152 :     ProcessDirectory($ftp, "$targetDir/$fileName");
153 :     # Change back to our directory.
154 :     $ftp->cdup() ||
155 :     Confess("Could not change back to parent of subdirectory $fileName.");
156 :     } else {
157 :     # Here we have a real file. We need to check the dates. First we get the
158 :     # remote file's modification date. Note that we convert it to local time
159 :     # using the global time zone offset.
160 :     my $remoteDate = $ftp->mdtm($fileName) - $TimeZoneOffset;
161 :     # Get the local file name.
162 :     my $localName = "$targetDir/$fileName";
163 :     # The local file's modification date is 0 if the file does not exist.
164 :     my $localDate = 0;
165 :     if (exists $locals{$fileName}) {
166 :     # If the local file exists and we're not in force-download mode,
167 :     # we get the local time stamp.
168 :     $localDate = stat($localName)->mtime unless $ForceDownload;
169 :     # While we're here, denote that the file exists on the remote so
170 :     # that we won't delete it later.
171 :     $locals{$fileName} = 1;
172 :     }
173 :     # Compare the file times.
174 :     if ($ForceDownload || $remoteDate > $localDate) {
175 :     # We want to download. Check the file extension to see if this is
176 :     # a binary transfer.
177 :     my $type = "ascii";
178 :     if ($fileName =~ /\.(\S+)$/) {
179 :     if ($BinaryExtensions{lc $1}) {
180 :     $type = "binary";
181 :     }
182 :     }
183 :     Trace("Downloading $fileName to $targetDir in $type mode.") if T(3);
184 :     if ($type eq "ascii") {
185 :     $ftp->ascii;
186 :     } else {
187 :     $ftp->binary;
188 :     }
189 :     # Do the download.
190 :     my $okFlag = $ftp->get($fileName, $localName);
191 :     # If the download failed, we delete the local file so it will
192 :     # be retrieved when we try again.
193 :     if (! $okFlag) {
194 :     unlink $localName;
195 :     Confess("Error downloading $fileName in $targetDir.");
196 :     }
197 :     }
198 :     }
199 :     }
200 :     }
201 :     # All done downloading. Now we need to delete the files not found on the remote.
202 :     for my $fileName (keys %locals) {
203 :     if (! $locals{$fileName}) {
204 :     # We have a file to delete. Get its real name.
205 :     my $localName = "$targetDir/$fileName";
206 :     # Check to see if it's a directory.
207 :     if (-d $localName) {
208 :     # It is a directory, so we delete the entire tree.
209 :     Trace("Deleting directory $localName.") if T(3);
210 :     rmtree($localName);
211 :     } else {
212 :     # It's a file, so delete it normally.
213 :     Trace("Deleting file $localName.") if T(3);
214 :     unlink $localName;
215 :     }
216 :     }
217 :     }
218 :     # We're done with this directory.
219 :     Trace("Directory $targetDir processed.") if T(3);
220 :     }
221 :    
222 :     1;

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3