]>
Commit | Line | Data |
---|---|---|
7ebf9245 | 1 | #!/usr/bin/perl -w |
56ea0f1e | 2 | use IPC::Run qw(run); |
3 | ||
631fa1b7 MM |
4 | use Sys::Syslog; |
5 | ||
6 | openlog("distfiles-ff", "pid", "user"); | |
7 | ||
7a0595ea | 8 | $commits_list = "pld-cvs-commit\@lists.pld-linux.org"; |
7ebf9245 | 9 | |
b5c674c5 | 10 | $spool_dir = "./spool"; |
01806192 MM |
11 | $copy_dir = "src"; # relative to ftp root |
12 | $no_url_dir = "./upload"; | |
13 | $df_server = "distfiles.pld-linux.org"; | |
40c35f1c | 14 | $df_scp = "plddist\@distfiles.pld-linux.org:ftp"; |
62f7358e | 15 | $user_agent = "PLD/distfiles"; |
7ebf9245 MM |
16 | |
17 | @md5 = (); | |
ccbaa374 | 18 | @url = (); |
7ebf9245 MM |
19 | $problems = ""; |
20 | $normal_out = ""; | |
21 | $requester = ""; | |
22 | $file = ""; | |
47f8fda7 MM |
23 | $fetched_count = 0; |
24 | $force_reply = 0; | |
bca91c1b | 25 | $req_login = "nobody"; |
cba859d3 | 26 | $spec = ""; |
44cd280f | 27 | @files = (); |
7ebf9245 | 28 | |
631fa1b7 MM |
29 | sub fatal($) |
30 | { | |
31 | my $msg = shift; | |
32 | ||
33 | syslog("err","FATAL: $msg"); | |
34 | die "$msg"; | |
35 | } | |
36 | ||
7ebf9245 MM |
37 | # try lookup some file in spool, exit if it cannot be done |
38 | sub find_file_in_spool() | |
39 | { | |
631fa1b7 | 40 | opendir(DIR, $spool_dir) || fatal("can't opendir $spool_dir: $!"); |
7ebf9245 MM |
41 | while (1) { |
42 | my $f = readdir(DIR); | |
43 | defined $f or last; | |
44 | -f "$spool_dir/$f" or next; | |
45 | $file = "$spool_dir/$f"; | |
46 | last; | |
47 | } | |
48 | closedir(DIR); | |
49 | ||
50 | exit 0 if ($file eq ""); | |
51 | } | |
52 | ||
53 | # read file from spool, and try unlink it. if cannot unlink -- exit | |
576dff03 | 54 | # sets $requester (email), $problems, @md5 (arrays of md5's) |
ccbaa374 | 55 | # and @url (map from md5 to urls) |
7ebf9245 MM |
56 | sub read_spool_file() |
57 | { | |
631fa1b7 | 58 | syslog("info","reading spool file $file"); |
7ebf9245 MM |
59 | open(F, "< $file") || exit 0; |
60 | $requester = <F>; | |
61 | chomp $requester; | |
576dff03 | 62 | $requester =~ /^[a-zA-Z_0-9@.-]+$/ |
631fa1b7 | 63 | or fatal("$file: evil requester: $requester"); |
bca91c1b MM |
64 | $req_login = ""; |
65 | $requester =~ /^([^@]+)\@/ and $req_login = $1; | |
631fa1b7 | 66 | $req_login =~ /^[a-z0-9A-Z_.]+$/ or fatal("$file: evil requester $requester"); |
cba859d3 | 67 | $spec = <F>; |
bf2219c8 | 68 | chomp $spec; |
cba859d3 | 69 | $spec =~ s/\.spec$//; |
bf2219c8 | 70 | |
47f8fda7 MM |
71 | my $flags = <F>; |
72 | $force_reply++ if ($flags =~ /force-reply/); | |
576dff03 | 73 | |
7ebf9245 MM |
74 | while (<F>) { |
75 | if (/^ERROR/) { | |
76 | s/^ERROR: //; | |
77 | $problems .= $_; | |
f2a640bf | 78 | next; |
7ebf9245 | 79 | } |
8ecb7fad | 80 | if (/^([a-f0-9]{32})\s+((ftp|http|https|no-url|no-url-copy):\/\/([a-z0-9A-Z;:\=\?&\@\+\~\.,\-\/_]|\%[0-9])+(#\/[a-zA-Z0-9\._-]+)?)\s*$/) { |
e4708e3e | 81 | if (/\/$/) { |
7a1a3819 | 82 | $problems .= "$file: cannot fetch dir"; |
e4708e3e | 83 | } else { |
7a1a3819 ER |
84 | push @md5, $1; |
85 | push @url, $2; | |
e4708e3e | 86 | } |
87 | } else { | |
7a1a3819 | 88 | $problems .= "FILE: $file: corrupted"; |
e4708e3e | 89 | } |
7ebf9245 MM |
90 | } |
91 | close(F); | |
92 | ||
93 | unlink($file) || exit 0; | |
94 | } | |
95 | ||
44cd280f MM |
96 | sub basename($) |
97 | { | |
98 | my $f = shift; | |
99 | $f =~ s|.*/||; | |
100 | return $f; | |
101 | } | |
102 | ||
01806192 | 103 | sub by_md5($$) |
7ebf9245 MM |
104 | { |
105 | my ($md5, $url) = @_; | |
7ebf9245 | 106 | $md5 =~ /^(.)(.)/; |
01806192 | 107 | return "/by-md5/$1/$2/$md5/" . basename($url); |
6ce39ee7 MM |
108 | } |
109 | ||
01806192 | 110 | sub got_on_distfiles($$) |
6ce39ee7 | 111 | { |
01806192 MM |
112 | my ($md5, $url) = @_; |
113 | my $p = by_md5($md5, $url); | |
114 | my $l = `lftp -c 'debug 0; open $df_server; quote size $p'`; | |
115 | return $l =~ /^213 /; | |
116 | } | |
117 | ||
118 | sub copy_to_df($$) | |
119 | { | |
120 | my ($from, $to) = @_; | |
92eba545 | 121 | my $cmd = "scp -pr -B -q $from $df_scp/$to"; |
631fa1b7 | 122 | open(E, "$cmd 2>&1 |") or fatal("$cmd failed"); |
01806192 MM |
123 | my $oops = ""; |
124 | while (<E>) { | |
125 | $oops .= $_; | |
126 | } | |
219df45c MM |
127 | $oops .= "\nThe command has exited with a non-zero status." |
128 | unless (close (E)); | |
01806192 MM |
129 | $problems .= "scp problems: $cmd:\n$oops\n" |
130 | if ($oops ne ""); | |
631fa1b7 | 131 | syslog("err","copy_to_df oops for '$cmd': $oops") if ($oops ne ""); |
01806192 | 132 | return ($oops ne ""); |
6ce39ee7 MM |
133 | } |
134 | ||
b5c674c5 | 135 | sub move_file($$$) |
6ce39ee7 | 136 | { |
b5c674c5 | 137 | my ($md5, $url, $local_copy) = @_; |
6ce39ee7 | 138 | |
01806192 | 139 | my $bn = basename($url); |
576dff03 | 140 | |
01806192 MM |
141 | if ($local_copy ne "$tmp_dir/$md5/$bn") { |
142 | if (system("mv -f \"$local_copy\" \"$tmp_dir/$md5/$bn\"")) { | |
143 | $problems .= "FATAL: cannot move $local_copy to $tmp_dir\n"; | |
144 | return; | |
145 | } | |
146 | $local_copy = "$tmp_dir/$md5/$bn"; | |
147 | } | |
025462c7 JR |
148 | # no-url local copy may have wrong permissions (it's created by ftp upload) |
149 | # always fix them | |
150 | chmod(0644, $local_copy); | |
576dff03 | 151 | |
01806192 MM |
152 | if (open(D, "> $tmp_dir/$md5/$bn.desc")) { |
153 | print D "URL: $url\n"; | |
d286a317 | 154 | print D "Login: $req_login\n"; |
01806192 MM |
155 | print D "MD5: $md5\n"; |
156 | print D 'Fetched-by: $Id$'."\n"; | |
157 | print D "Time: " . time . "\n"; | |
158 | close(D); | |
7ebf9245 | 159 | } else { |
01806192 MM |
160 | $problems .= "ERROR: cannot write $bn.desc\n"; |
161 | } | |
162 | ||
163 | my $dir = by_md5($md5, $url); | |
164 | $dir =~ s|/[^/]+/[^/]+$||; | |
165 | if (copy_to_df("$tmp_dir/$md5/", $dir) == 0) { | |
576dff03 | 166 | $normal_out .= |
44cd280f MM |
167 | "STORED: $url\n" . |
168 | "\t$md5 " . basename($url) . "\n" . | |
01806192 | 169 | "\tSize: " . (-s $local_copy) . " bytes\n"; |
47f8fda7 | 170 | $fetched_count++; |
7ebf9245 MM |
171 | } |
172 | } | |
173 | ||
b5c674c5 MM |
174 | sub make_src_symlink($$) |
175 | { | |
176 | my ($md5, $url) = @_; | |
576dff03 | 177 | |
01806192 | 178 | return unless ($url =~ /^no-url/); |
576dff03 | 179 | |
01806192 MM |
180 | my $b = basename($url); |
181 | if (open(S, "> $tmp_dir/$b.link")) { | |
b5c674c5 | 182 | if ($url =~ /^no-url-copy/) { |
01806192 | 183 | print S (".." . by_md5($md5, $url)); |
b5c674c5 | 184 | } else { |
01806192 | 185 | print S "REMOVE"; |
b5c674c5 | 186 | } |
01806192 MM |
187 | close(S); |
188 | copy_to_df("$tmp_dir/$b.link", "$copy_dir/$b.link"); | |
b5c674c5 | 189 | } else { |
01806192 | 190 | $problems .= "ERROR: cannot write $tmp_dir/$b.link\n"; |
b5c674c5 MM |
191 | } |
192 | } | |
193 | ||
194 | sub md5($) | |
195 | { | |
196 | my $file = shift; | |
56ea0f1e | 197 | my $in = ""; |
198 | my $md5 = ""; | |
199 | my $err = ""; | |
200 | my @cmd = ("md5sum", $file); | |
201 | ||
202 | run \@cmd, \$in, \$md5, \$err; | |
203 | if ($err ne "") { | |
204 | chomp($err); | |
a6b09732 | 205 | $problems .= "FATAL: " . $err . "\n"; |
206 | return "error"; | |
56ea0f1e | 207 | } |
208 | chomp $md5; | |
b5c674c5 MM |
209 | $md5 =~ /^([a-f0-9]{32})/ and $md5 = $1; |
210 | return $md5; | |
211 | } | |
212 | ||
213 | sub handle_no_url($$) | |
214 | { | |
215 | my ($md5, $url) = @_; | |
576dff03 | 216 | |
e4708e3e | 217 | unless ($url =~ m#://([^/]+)#) { |
218 | $problems .= "$url: corrupted! (no-url)"; | |
219 | return; | |
220 | } | |
b5c674c5 | 221 | my $basename = $1; |
bca91c1b | 222 | my $file = "$no_url_dir/$req_login/$basename"; |
b5c674c5 MM |
223 | |
224 | if (-f $file) { | |
225 | my $computed_md5 = md5($file); | |
226 | if ($computed_md5 ne $md5) { | |
227 | $problems .= "FATAL: $file md5 mismatch, needed $md5, got $computed_md5\n"; | |
228 | } else { | |
229 | move_file($md5, $url, $file); | |
230 | make_src_symlink($md5, $url); | |
231 | } | |
232 | } else { | |
bca91c1b | 233 | $problems .= "FATAL: $file was not uploaded\n"; |
b5c674c5 MM |
234 | } |
235 | } | |
236 | ||
7ebf9245 MM |
237 | sub fetch_file($$) |
238 | { | |
239 | my ($md5, $url) = @_; | |
240 | my $out = ""; | |
08ea99b9 | 241 | my $all_out = ""; |
01806192 MM |
242 | my $bn = basename($url); |
243 | my $local = "$tmp_dir/$md5/$bn"; | |
b1659ea9 KK |
244 | my @cmd = ("wget", "-nv", "--no-check-certificate", "--user-agent=$user_agent", "-O", $local, $url); |
245 | my $cmd_joined = join(' ', @cmd); | |
246 | my @cmd2 = ("wget", "-nv", "--no-check-certificate", "--user-agent=$user_agent", "--passive-ftp", "-O", $local, $url); | |
247 | my $cmd2_joined = join(' ', @cmd2); | |
b5c674c5 | 248 | |
631fa1b7 MM |
249 | syslog("info","fetch_file($md5,$url)"); |
250 | ||
01806192 | 251 | push @files, $bn; |
44cd280f | 252 | |
b38176ba | 253 | if ( $bn =~ m/(%[0-9a-f]{2})/i ) { |
254 | $problems .= "$bn: refusing to download file with uri escape codes ($1) in the name\n"; | |
255 | $bn =~ s/%[0-9a-f]{2}/_/g; | |
f3eeba03 | 256 | $problems .= "HINT: use $url?/$bn as source to rename the file\n\n"; |
b38176ba | 257 | return; |
258 | } | |
259 | ||
01806192 | 260 | if (got_on_distfiles($md5, $url)) { |
576dff03 | 261 | $normal_out .= |
7a1a3819 ER |
262 | "ALREADY GOT: $url\n" . |
263 | "\t$md5 " . basename($url) . "\n"; | |
b5c674c5 | 264 | make_src_symlink($md5, $url); |
6ce39ee7 MM |
265 | return; |
266 | } | |
01806192 | 267 | |
2ae8fdec | 268 | mkdir("$tmp_dir/$md5"); |
576dff03 | 269 | |
b5c674c5 MM |
270 | if ($url =~ /^no-url/) { |
271 | handle_no_url($md5, $url); | |
272 | return; | |
273 | } | |
576dff03 | 274 | |
b1659ea9 | 275 | my $pid = open(W, "-|"); |
631fa1b7 | 276 | fatal("Cannot fork $!") unless defined $pid; |
b1659ea9 | 277 | unless ( $pid ) { |
631fa1b7 MM |
278 | open STDERR, ">&", \*STDOUT or fatal("$0: open: $!"); |
279 | exec { $cmd[0] } @cmd or fatal("$0: exec: $!"); | |
b1659ea9 | 280 | } |
7ebf9245 | 281 | while (<W>) { |
08ea99b9 | 282 | $all_out .= $_; |
7ebf9245 MM |
283 | /URL:.*\s+\-\>\s+.*/ and next; |
284 | $out .= $_; | |
285 | } | |
286 | close(W); | |
287 | if ($out ne "") { | |
631fa1b7 | 288 | syslog("err","$cmd_joined: $out"); |
b1659ea9 | 289 | $problems .= "$cmd_joined:\n$out\n\n"; |
7ebf9245 | 290 | } |
d1c674a4 | 291 | if ( $? ) { |
b106d8c1 | 292 | $problems .= sprintf "%s:\nexited with code %d (0x%02x)\n\n", |
b1659ea9 | 293 | $cmd_joined, |
d1c674a4 | 294 | $? >> 8, |
295 | $? & 0xff; | |
296 | } | |
e23ae36a | 297 | if (-f $local && -s $local == 0 && $url =~ /^ftp:/) { |
f4f705aa | 298 | $out = ""; |
b1659ea9 | 299 | my $pid = open(W, "-|"); |
631fa1b7 | 300 | fatal("Cannot fork $!") unless defined $pid; |
b1659ea9 | 301 | unless ( $pid ) { |
631fa1b7 MM |
302 | open STDERR, ">&", \*STDOUT or fatal("$0: open: $!"); |
303 | exec { $cmd2[0] } @cmd2 or fatal("$0: exec: $!"); | |
b1659ea9 | 304 | } |
c0467b40 | 305 | while (<W>) { |
f4f705aa | 306 | $all_out .= "\n\t\t$_"; |
c0467b40 | 307 | /URL:.*\s+\-\>\s+.*/ and next; |
308 | $out .= $_; | |
309 | } | |
310 | close(W); | |
311 | if ($out ne "") { | |
b1659ea9 | 312 | $problems .= "$cmd2_joined:\n$out\n\n"; |
d1c674a4 | 313 | } |
314 | if ( $? ) { | |
b106d8c1 | 315 | $problems .= sprintf "%s:\nexited with code %d (0x%02x)\n\n", |
b1659ea9 | 316 | $cmd2_joined, |
d1c674a4 | 317 | $? >> 8, |
318 | $? & 0xff; | |
c0467b40 | 319 | } |
320 | } | |
4db63fc1 | 321 | if (-r $local && -s $local > 0) { |
01806192 | 322 | my $computed_md5 = md5($local); |
7ebf9245 MM |
323 | if ($computed_md5 ne $md5) { |
324 | $problems .= "FATAL: $url md5 mismatch, needed $md5, got $computed_md5\n"; | |
325 | } else { | |
d642cf7e | 326 | my $testcmd = "file \"$local\" |"; |
327 | my $testres = ""; | |
a14ed9b7 | 328 | if ($url =~ /^(http|https):/ && $local =~ /\.(tar\.(bz2|gz)|tgz|zip|jar|xpi)$/) { |
631fa1b7 | 329 | open(T, $testcmd) or fatal("$testcmd failed"); |
d642cf7e | 330 | $testres = <T>; |
331 | close(T); | |
332 | } | |
333 | if ($testres =~ /empty|(ASCII|HTML|SGML).*text/) { | |
334 | $testres =~ s/.*://; | |
335 | $problems .= "FATAL: data returned from $url: $testres"; | |
336 | } else { | |
337 | move_file($md5, $url, $local); | |
338 | } | |
7ebf9245 | 339 | } |
4db63fc1 | 340 | } elsif (-f $local && -s $local > 0) { |
b1659ea9 | 341 | $problems .= "FATAL: $url ($md5) was not fetched correctly ($cmd_joined: $all_out): file is not readable\n"; |
d1c674a4 | 342 | } elsif (-f $local && not -s $local) { |
b1659ea9 | 343 | $problems .= "FATAL: $url ($md5) was not fetched correctly ($cmd_joined: $all_out): file fetched but has 0 length\n"; |
7ebf9245 | 344 | } else { |
b1659ea9 | 345 | $problems .= "FATAL: $url ($md5) was not fetched correctly ($cmd_joined: $all_out)\n"; |
7ebf9245 | 346 | } |
01806192 MM |
347 | # save space |
348 | unlink($local); | |
7ebf9245 MM |
349 | } |
350 | ||
351 | sub fetch_files() | |
352 | { | |
353 | $problems .= "\n\n" if ($problems ne ""); | |
ccbaa374 | 354 | foreach $i (0..$#md5) { |
355 | fetch_file($md5[$i], $url[$i]); | |
7ebf9245 MM |
356 | } |
357 | } | |
358 | ||
359 | sub send_email() | |
360 | { | |
631fa1b7 | 361 | syslog("info","sending email to $requester"); |
9a1e36f3 MM |
362 | open(EMAIL, "| /usr/sbin/sendmail -t"); |
363 | #open(EMAIL, "| cat"); | |
44cd280f | 364 | my $marker = ""; |
7ebf9245 | 365 | if ($problems ne "") { |
44cd280f | 366 | $marker = "ERRORS: "; |
7ebf9245 | 367 | } |
04550b88 MM |
368 | my $req_login; |
369 | $requester =~ /^(.*)\@/ and $req_login = $1; | |
44cd280f MM |
370 | |
371 | splice(@files, 10, @files - 10, "...") | |
372 | if (@files > 10); | |
576dff03 ER |
373 | |
374 | print EMAIL | |
04550b88 | 375 | "From: $req_login <$requester> |
482655f9 MM |
376 | To: $commits_list |
377 | Cc: $requester | |
8408fdb0 | 378 | Subject: DISTFILES: ${spec}: ${marker}@{files} |
e157986d MM |
379 | Message-ID: <$$." . time . "\@distfiles.pld-linux.org> |
380 | X-distfiles-program: file-fetcher.pl | |
381 | X-distfiles-version: " . '$Id$' . " | |
7ebf9245 MM |
382 | |
383 | $problems | |
47f8fda7 MM |
384 | Files fetched: $fetched_count |
385 | ||
7ebf9245 | 386 | $normal_out |
e157986d MM |
387 | |
388 | -- | |
d8cd573c | 389 | Virtually Yours: distfiles. |
7ebf9245 | 390 | "; |
631fa1b7 | 391 | close(EMAIL) or fatal("close() failed"); |
7ebf9245 MM |
392 | } |
393 | ||
01806192 MM |
394 | sub make_tmp_dir() |
395 | { | |
396 | my $id = `uuidgen 2>/dev/null`; | |
397 | chomp $id; | |
398 | $id = rand if (!defined $id or $id eq ""); | |
399 | $tmp_dir = "./tmp/$id"; | |
631fa1b7 | 400 | mkdir($tmp_dir) or fatal("mkdir($tmp_dir) failed"); |
01806192 MM |
401 | } |
402 | ||
403 | sub clean_tmp_dir() | |
404 | { | |
405 | system("rm -rf $tmp_dir") | |
406 | if ($tmp_dir ne "" && -d $tmp_dir); | |
407 | } | |
408 | ||
2e33bbbc MM |
409 | umask(002); |
410 | ||
7ebf9245 MM |
411 | find_file_in_spool(); |
412 | read_spool_file(); | |
01806192 | 413 | make_tmp_dir(); |
7ebf9245 | 414 | fetch_files(); |
47f8fda7 | 415 | send_email() unless (!$force_reply and $problems eq "" and $fetched_count == 0); |
01806192 | 416 | clean_tmp_dir(); |
319fb6c1 ER |
417 | |
418 | # vim: ts=2:sw=2:et:fdm=marker |