]>
Commit | Line | Data |
---|---|---|
7ebf9245 MM |
1 | #!/usr/bin/perl -w |
2 | # | |
3 | ||
482655f9 | 4 | $commits_list = "pld-cvs-commit\@pld-linux.org"; |
7ebf9245 | 5 | |
b5c674c5 | 6 | $spool_dir = "./spool"; |
01806192 MM |
7 | $copy_dir = "src"; # relative to ftp root |
8 | $no_url_dir = "./upload"; | |
9 | $df_server = "distfiles.pld-linux.org"; | |
10 | $df_scp = "plddist\@$df_server:ftp"; | |
7ebf9245 MM |
11 | |
12 | @md5 = (); | |
13 | %url = (); | |
14 | $problems = ""; | |
15 | $normal_out = ""; | |
16 | $requester = ""; | |
17 | $file = ""; | |
47f8fda7 MM |
18 | $fetched_count = 0; |
19 | $force_reply = 0; | |
bca91c1b | 20 | $req_login = "nobody"; |
44cd280f | 21 | @files = (); |
7ebf9245 MM |
22 | |
23 | # try lookup some file in spool, exit if it cannot be done | |
24 | sub find_file_in_spool() | |
25 | { | |
26 | opendir(DIR, $spool_dir) || die "can't opendir $spool_dir: $!"; | |
27 | while (1) { | |
28 | my $f = readdir(DIR); | |
29 | defined $f or last; | |
30 | -f "$spool_dir/$f" or next; | |
31 | $file = "$spool_dir/$f"; | |
32 | last; | |
33 | } | |
34 | closedir(DIR); | |
35 | ||
36 | exit 0 if ($file eq ""); | |
37 | } | |
38 | ||
39 | # read file from spool, and try unlink it. if cannot unlink -- exit | |
40 | # sets $requester (email), $problems, @md5 (arrays of md5's) | |
41 | # and %url (map from md5 to urls) | |
42 | sub read_spool_file() | |
43 | { | |
44 | open(F, "< $file") || exit 0; | |
45 | $requester = <F>; | |
46 | chomp $requester; | |
47 | $requester =~ /^[a-zA-Z_\-0-9\@\.]+$/ | |
48 | or die "$file: evil requester: $requester"; | |
bca91c1b MM |
49 | $req_login = ""; |
50 | $requester =~ /^([^@]+)\@/ and $req_login = $1; | |
51 | $req_login =~ /^[a-z0-9A-Z_]+$/ or die "$file: evil requester $requester"; | |
47f8fda7 MM |
52 | my $flags = <F>; |
53 | $force_reply++ if ($flags =~ /force-reply/); | |
7ebf9245 MM |
54 | |
55 | while (<F>) { | |
56 | if (/^ERROR/) { | |
57 | s/^ERROR: //; | |
58 | $problems .= $_; | |
f2a640bf | 59 | next; |
7ebf9245 | 60 | } |
b5c674c5 | 61 | /^([a-f0-9]{32})\s+((ftp|http|no-url|no-url-copy):\/\/([a-z0-9A-Z:\+\~\.\-\/_]|\%[0-9])+)\s*$/ |
7ebf9245 MM |
62 | or die "$file: corrupted"; |
63 | push @md5, $1; | |
64 | $url{$1} = $2; | |
65 | /\/$/ and die "$file: cannot fetch dir"; | |
66 | } | |
67 | close(F); | |
68 | ||
69 | unlink($file) || exit 0; | |
70 | } | |
71 | ||
44cd280f MM |
72 | sub basename($) |
73 | { | |
74 | my $f = shift; | |
75 | $f =~ s|.*/||; | |
76 | return $f; | |
77 | } | |
78 | ||
01806192 | 79 | sub by_md5($$) |
7ebf9245 MM |
80 | { |
81 | my ($md5, $url) = @_; | |
7ebf9245 | 82 | $md5 =~ /^(.)(.)/; |
01806192 | 83 | return "/by-md5/$1/$2/$md5/" . basename($url); |
6ce39ee7 MM |
84 | } |
85 | ||
01806192 | 86 | sub got_on_distfiles($$) |
6ce39ee7 | 87 | { |
01806192 MM |
88 | my ($md5, $url) = @_; |
89 | my $p = by_md5($md5, $url); | |
90 | my $l = `lftp -c 'debug 0; open $df_server; quote size $p'`; | |
91 | return $l =~ /^213 /; | |
92 | } | |
93 | ||
94 | sub copy_to_df($$) | |
95 | { | |
96 | my ($from, $to) = @_; | |
97 | my $cmd = "scp -r -B -q $from $df_scp/$to"; | |
98 | open(E, "$cmd 2>&1 |") or die; | |
99 | my $oops = ""; | |
100 | while (<E>) { | |
101 | $oops .= $_; | |
102 | } | |
103 | close(E); | |
104 | $problems .= "scp problems: $cmd:\n$oops\n" | |
105 | if ($oops ne ""); | |
106 | return ($oops ne ""); | |
6ce39ee7 MM |
107 | } |
108 | ||
b5c674c5 | 109 | sub move_file($$$) |
6ce39ee7 | 110 | { |
b5c674c5 | 111 | my ($md5, $url, $local_copy) = @_; |
6ce39ee7 | 112 | |
01806192 | 113 | my $bn = basename($url); |
6ce39ee7 | 114 | |
01806192 MM |
115 | if ($local_copy ne "$tmp_dir/$md5/$bn") { |
116 | if (system("mv -f \"$local_copy\" \"$tmp_dir/$md5/$bn\"")) { | |
117 | $problems .= "FATAL: cannot move $local_copy to $tmp_dir\n"; | |
118 | return; | |
119 | } | |
120 | $local_copy = "$tmp_dir/$md5/$bn"; | |
121 | } | |
6ce39ee7 | 122 | |
01806192 MM |
123 | if (open(D, "> $tmp_dir/$md5/$bn.desc")) { |
124 | print D "URL: $url\n"; | |
125 | print D "MD5: $md5\n"; | |
126 | print D 'Fetched-by: $Id$'."\n"; | |
127 | print D "Time: " . time . "\n"; | |
128 | close(D); | |
7ebf9245 | 129 | } else { |
01806192 MM |
130 | $problems .= "ERROR: cannot write $bn.desc\n"; |
131 | } | |
132 | ||
133 | my $dir = by_md5($md5, $url); | |
134 | $dir =~ s|/[^/]+/[^/]+$||; | |
135 | if (copy_to_df("$tmp_dir/$md5/", $dir) == 0) { | |
44cd280f MM |
136 | $normal_out .= |
137 | "STORED: $url\n" . | |
138 | "\t$md5 " . basename($url) . "\n" . | |
01806192 | 139 | "\tSize: " . (-s $local_copy) . " bytes\n"; |
47f8fda7 | 140 | $fetched_count++; |
7ebf9245 MM |
141 | } |
142 | } | |
143 | ||
b5c674c5 MM |
144 | sub make_src_symlink($$) |
145 | { | |
146 | my ($md5, $url) = @_; | |
01806192 MM |
147 | |
148 | return unless ($url =~ /^no-url/); | |
149 | ||
150 | my $b = basename($url); | |
151 | if (open(S, "> $tmp_dir/$b.link")) { | |
b5c674c5 | 152 | if ($url =~ /^no-url-copy/) { |
01806192 | 153 | print S (".." . by_md5($md5, $url)); |
b5c674c5 | 154 | } else { |
01806192 | 155 | print S "REMOVE"; |
b5c674c5 | 156 | } |
01806192 MM |
157 | close(S); |
158 | copy_to_df("$tmp_dir/$b.link", "$copy_dir/$b.link"); | |
b5c674c5 | 159 | } else { |
01806192 | 160 | $problems .= "ERROR: cannot write $tmp_dir/$b.link\n"; |
b5c674c5 MM |
161 | } |
162 | } | |
163 | ||
164 | sub md5($) | |
165 | { | |
166 | my $file = shift; | |
167 | my $md5 = `md5sum "$file" 2>/dev/null`; | |
168 | $md5 =~ /^([a-f0-9]{32})/ and $md5 = $1; | |
169 | return $md5; | |
170 | } | |
171 | ||
172 | sub handle_no_url($$) | |
173 | { | |
174 | my ($md5, $url) = @_; | |
175 | ||
176 | $url =~ m|://([^/]+)| or die "corrupted! (no-url)"; | |
177 | my $basename = $1; | |
bca91c1b | 178 | my $file = "$no_url_dir/$req_login/$basename"; |
b5c674c5 MM |
179 | |
180 | if (-f $file) { | |
181 | my $computed_md5 = md5($file); | |
182 | if ($computed_md5 ne $md5) { | |
183 | $problems .= "FATAL: $file md5 mismatch, needed $md5, got $computed_md5\n"; | |
184 | } else { | |
185 | move_file($md5, $url, $file); | |
186 | make_src_symlink($md5, $url); | |
187 | } | |
188 | } else { | |
bca91c1b | 189 | $problems .= "FATAL: $file was not uploaded\n"; |
b5c674c5 MM |
190 | } |
191 | } | |
192 | ||
7ebf9245 MM |
193 | sub fetch_file($$) |
194 | { | |
195 | my ($md5, $url) = @_; | |
196 | my $out = ""; | |
01806192 MM |
197 | my $bn = basename($url); |
198 | my $local = "$tmp_dir/$md5/$bn"; | |
199 | my $cmd = "wget -nv -O $local \"$url\""; | |
b5c674c5 | 200 | |
01806192 | 201 | push @files, $bn; |
44cd280f | 202 | |
01806192 | 203 | if (got_on_distfiles($md5, $url)) { |
00eefa39 MM |
204 | $normal_out .= |
205 | "ALREADY GOT: $url\n" . | |
01806192 | 206 | "\t$md5 " . basename($url) . "\n"; |
b5c674c5 | 207 | make_src_symlink($md5, $url); |
6ce39ee7 MM |
208 | return; |
209 | } | |
01806192 MM |
210 | |
211 | mkdir("$tmp_dir/$md5") or die; | |
212 | ||
b5c674c5 MM |
213 | if ($url =~ /^no-url/) { |
214 | handle_no_url($md5, $url); | |
215 | return; | |
216 | } | |
217 | ||
7ebf9245 MM |
218 | open(W, "$cmd 2>&1 |"); |
219 | while (<W>) { | |
220 | /URL:.*\s+\-\>\s+.*/ and next; | |
221 | $out .= $_; | |
222 | } | |
223 | close(W); | |
224 | if ($out ne "") { | |
225 | $problems .= "$cmd:\n$out\n\n"; | |
226 | } | |
01806192 MM |
227 | if (-f $local && -s $local > 0) { |
228 | my $computed_md5 = md5($local); | |
7ebf9245 MM |
229 | if ($computed_md5 ne $md5) { |
230 | $problems .= "FATAL: $url md5 mismatch, needed $md5, got $computed_md5\n"; | |
231 | } else { | |
01806192 | 232 | move_file($md5, $url, $local); |
7ebf9245 MM |
233 | } |
234 | } else { | |
235 | $problems .= "FATAL: $url ($md5) was not fetched\n"; | |
236 | } | |
01806192 MM |
237 | # save space |
238 | unlink($local); | |
7ebf9245 MM |
239 | } |
240 | ||
241 | sub fetch_files() | |
242 | { | |
243 | $problems .= "\n\n" if ($problems ne ""); | |
244 | foreach $md5 (@md5) { | |
245 | fetch_file($md5, $url{$md5}); | |
246 | } | |
247 | } | |
248 | ||
249 | sub send_email() | |
250 | { | |
9a1e36f3 MM |
251 | open(EMAIL, "| /usr/sbin/sendmail -t"); |
252 | #open(EMAIL, "| cat"); | |
44cd280f | 253 | my $marker = ""; |
7ebf9245 | 254 | if ($problems ne "") { |
44cd280f | 255 | $marker = "ERRORS: "; |
7ebf9245 | 256 | } |
04550b88 MM |
257 | my $req_login; |
258 | $requester =~ /^(.*)\@/ and $req_login = $1; | |
44cd280f MM |
259 | |
260 | splice(@files, 10, @files - 10, "...") | |
261 | if (@files > 10); | |
262 | ||
e157986d | 263 | print EMAIL |
04550b88 | 264 | "From: $req_login <$requester> |
482655f9 MM |
265 | To: $commits_list |
266 | Cc: $requester | |
44cd280f | 267 | Subject: DISTFILES: ${marker}@{files} |
e157986d MM |
268 | Message-ID: <$$." . time . "\@distfiles.pld-linux.org> |
269 | X-distfiles-program: file-fetcher.pl | |
270 | X-distfiles-version: " . '$Id$' . " | |
7ebf9245 MM |
271 | |
272 | $problems | |
47f8fda7 MM |
273 | Files fetched: $fetched_count |
274 | ||
7ebf9245 | 275 | $normal_out |
e157986d MM |
276 | |
277 | -- | |
d8cd573c | 278 | Virtually Yours: distfiles. |
7ebf9245 MM |
279 | "; |
280 | close(EMAIL) or die; | |
281 | } | |
282 | ||
01806192 MM |
283 | sub make_tmp_dir() |
284 | { | |
285 | my $id = `uuidgen 2>/dev/null`; | |
286 | chomp $id; | |
287 | $id = rand if (!defined $id or $id eq ""); | |
288 | $tmp_dir = "./tmp/$id"; | |
289 | mkdir($tmp_dir) or die; | |
290 | } | |
291 | ||
292 | sub clean_tmp_dir() | |
293 | { | |
294 | system("rm -rf $tmp_dir") | |
295 | if ($tmp_dir ne "" && -d $tmp_dir); | |
296 | } | |
297 | ||
7ebf9245 MM |
298 | find_file_in_spool(); |
299 | read_spool_file(); | |
01806192 | 300 | make_tmp_dir(); |
7ebf9245 | 301 | fetch_files(); |
47f8fda7 | 302 | send_email() unless (!$force_reply and $problems eq "" and $fetched_count == 0); |
01806192 | 303 | clean_tmp_dir(); |