]>
Commit | Line | Data |
---|---|---|
7ebf9245 MM |
1 | #!/usr/bin/perl -w |
2 | # | |
3 | ||
4 | $spool_dir = "./spool"; | |
5 | $ftp_dir = "./ftp"; | |
6 | $email_cc = ""; | |
7 | ||
8 | ||
9 | @md5 = (); | |
10 | %url = (); | |
11 | $problems = ""; | |
12 | $normal_out = ""; | |
13 | $requester = ""; | |
14 | $file = ""; | |
47f8fda7 MM |
15 | $fetched_count = 0; |
16 | $force_reply = 0; | |
7ebf9245 MM |
17 | |
18 | # try lookup some file in spool, exit if it cannot be done | |
19 | sub find_file_in_spool() | |
20 | { | |
21 | opendir(DIR, $spool_dir) || die "can't opendir $spool_dir: $!"; | |
22 | while (1) { | |
23 | my $f = readdir(DIR); | |
24 | defined $f or last; | |
25 | -f "$spool_dir/$f" or next; | |
26 | $file = "$spool_dir/$f"; | |
27 | last; | |
28 | } | |
29 | closedir(DIR); | |
30 | ||
31 | exit 0 if ($file eq ""); | |
32 | } | |
33 | ||
34 | # read file from spool, and try unlink it. if cannot unlink -- exit | |
35 | # sets $requester (email), $problems, @md5 (arrays of md5's) | |
36 | # and %url (map from md5 to urls) | |
37 | sub read_spool_file() | |
38 | { | |
39 | open(F, "< $file") || exit 0; | |
40 | $requester = <F>; | |
41 | chomp $requester; | |
42 | $requester =~ /^[a-zA-Z_\-0-9\@\.]+$/ | |
43 | or die "$file: evil requester: $requester"; | |
47f8fda7 MM |
44 | my $flags = <F>; |
45 | $force_reply++ if ($flags =~ /force-reply/); | |
7ebf9245 MM |
46 | |
47 | while (<F>) { | |
48 | if (/^ERROR/) { | |
49 | s/^ERROR: //; | |
50 | $problems .= $_; | |
51 | } | |
52 | /^([a-f0-9]{32})\s+((ftp|http):\/\/([a-z0-9A-Z:\+\~\.\-\/_]|\%[0-9])+)\s*$/ | |
53 | or die "$file: corrupted"; | |
54 | push @md5, $1; | |
55 | $url{$1} = $2; | |
56 | /\/$/ and die "$file: cannot fetch dir"; | |
57 | } | |
58 | close(F); | |
59 | ||
60 | unlink($file) || exit 0; | |
61 | } | |
62 | ||
6ce39ee7 | 63 | sub file_path($$) |
7ebf9245 MM |
64 | { |
65 | my ($md5, $url) = @_; | |
66 | ||
67 | $md5 =~ /^(.)(.)/; | |
68 | my $md5_dir = "$ftp_dir/by-md5/$1/$2/$md5"; | |
7ebf9245 MM |
69 | $url =~ /\/([^\/]+)$/ or die; |
70 | my $basename = $1; | |
6ce39ee7 MM |
71 | return "$md5_dir/$basename"; |
72 | } | |
73 | ||
47f8fda7 | 74 | sub install_dir |
6ce39ee7 MM |
75 | { |
76 | my $dir = shift; | |
77 | return if (-e $dir); | |
78 | $dir =~ /(.*)\/[^\/]+$/ and install_dir($1); | |
79 | mkdir($dir); | |
80 | } | |
81 | ||
82 | sub move_file($$) | |
83 | { | |
84 | my ($md5, $url) = @_; | |
85 | ||
86 | my $path = file_path($md5, $url); | |
87 | ||
88 | $path =~ /(.*)\/[^\/]+$/ and install_dir($1); | |
89 | ||
90 | if (system("mv -f \"tmp/$md5\" \"$path\"")) { | |
7ebf9245 MM |
91 | $problems .= "FATAL: cannot mv file ($url)\n"; |
92 | } else { | |
032dd834 MM |
93 | if (open(D, "> $path.desc")) { |
94 | print D "URL: $url\n"; | |
95 | print D "MD5: $md5\n"; | |
2fe72b7b | 96 | print D 'Fetched-by: $Id$'."\n"; |
032dd834 MM |
97 | close(D); |
98 | } else { | |
99 | $problems .= "ERROR: cannot write $path.desc\n"; | |
100 | } | |
f0760d64 MM |
101 | |
102 | # Forg logs | |
103 | $md5 =~ /^(..)/ and my $m2 = $1; | |
104 | $md5 =~ /^(.)(.)/; | |
105 | if (open(L, ">> $ENV{HOME}/.lists/lista_$m2")) { | |
106 | print L "/home/ftp/pub/Linux/distfiles/by-md5/$1/$2/$md5/"; | |
107 | $url =~ /\/([^\/]+)$/; | |
108 | print L "$1\n"; | |
109 | close(L); | |
110 | } | |
111 | ||
6ce39ee7 | 112 | $normal_out .= "STORED: $url ($md5, " . (-s $path) . " bytes)\n"; |
47f8fda7 | 113 | $fetched_count++; |
7ebf9245 MM |
114 | } |
115 | } | |
116 | ||
117 | sub fetch_file($$) | |
118 | { | |
119 | my ($md5, $url) = @_; | |
120 | my $out = ""; | |
121 | my $cmd = "wget -nv -O tmp/$md5 \"$url\""; | |
6ce39ee7 MM |
122 | |
123 | my $path = file_path($md5, $url); | |
124 | if (-f $path) { | |
125 | $normal_out .= "ALREADY GOT: $url ($md5, " . (-s $path) . " bytes)\n"; | |
126 | return; | |
127 | } | |
128 | ||
7ebf9245 MM |
129 | open(W, "$cmd 2>&1 |"); |
130 | while (<W>) { | |
131 | /URL:.*\s+\-\>\s+.*/ and next; | |
132 | $out .= $_; | |
133 | } | |
134 | close(W); | |
135 | if ($out ne "") { | |
136 | $problems .= "$cmd:\n$out\n\n"; | |
137 | } | |
138 | if (-f "tmp/$md5" && -s "tmp/$md5" > 0) { | |
139 | my $computed_md5 = `md5sum tmp/$md5`; | |
140 | $computed_md5 =~ /^([a-f0-9]{32})/ and $computed_md5 = $1; | |
141 | if ($computed_md5 ne $md5) { | |
142 | $problems .= "FATAL: $url md5 mismatch, needed $md5, got $computed_md5\n"; | |
143 | } else { | |
144 | move_file($md5, $url); | |
145 | } | |
146 | } else { | |
147 | $problems .= "FATAL: $url ($md5) was not fetched\n"; | |
148 | } | |
ce5a46fa | 149 | unlink("tmp/$md5"); |
7ebf9245 MM |
150 | } |
151 | ||
152 | sub fetch_files() | |
153 | { | |
154 | $problems .= "\n\n" if ($problems ne ""); | |
155 | foreach $md5 (@md5) { | |
156 | fetch_file($md5, $url{$md5}); | |
157 | } | |
158 | } | |
159 | ||
160 | sub send_email() | |
161 | { | |
9a1e36f3 MM |
162 | open(EMAIL, "| /usr/sbin/sendmail -t"); |
163 | #open(EMAIL, "| cat"); | |
7ebf9245 MM |
164 | my $marker = "OK"; |
165 | if ($problems ne "") { | |
166 | $marker = "ERRORS"; | |
167 | } | |
e157986d MM |
168 | print EMAIL |
169 | "From: distfiles <feedback\@pld-linux.org> | |
170 | To: $requester | |
7ebf9245 MM |
171 | Cc: $email_cc |
172 | Subject: [distfiles] sources fetched $marker | |
e157986d MM |
173 | Message-ID: <$$." . time . "\@distfiles.pld-linux.org> |
174 | X-distfiles-program: file-fetcher.pl | |
175 | X-distfiles-version: " . '$Id$' . " | |
7ebf9245 MM |
176 | |
177 | $problems | |
47f8fda7 MM |
178 | Files fetched: $fetched_count |
179 | ||
7ebf9245 | 180 | $normal_out |
e157986d MM |
181 | |
182 | -- | |
183 | Virtually yours: distfiles. | |
7ebf9245 MM |
184 | "; |
185 | close(EMAIL) or die; | |
186 | } | |
187 | ||
188 | find_file_in_spool(); | |
189 | read_spool_file(); | |
190 | fetch_files(); | |
47f8fda7 | 191 | send_email() unless (!$force_reply and $problems eq "" and $fetched_count == 0); |