]>
Commit | Line | Data |
---|---|---|
b48e85a9 | 1 | #!/bin/awk -f |
b52a0a12 | 2 | # |
1b718db0 | 3 | # Copyright (C) 2000-2013 PLD-Team <feedback@pld-linux.org> |
b52a0a12 JB |
4 | # Authors: |
5 | # Sebastian Zagrodzki <zagrodzki@pld-linux.org> | |
6 | # Jacek Konieczny <jajcus@pld-linux.org> | |
7 | # Andrzej Krzysztofowicz <ankry@pld-linux.org> | |
8 | # Jakub Bogusz <qboosh@pld-linux.org> | |
9 | # Elan Ruusamäe <glen@pld-linux.org> | |
10 | # | |
11 | # See cvs log pldnotify.awk for list of contributors | |
12 | # | |
13 | # TODO: | |
14 | # - "SourceXDownload" support (use given URLs if present instead of cut-down SourceX URLs) | |
15 | # - "SourceXActiveFTP" support | |
86abc88a | 16 | # - support debian/watch http://wiki.debian.org/debian/watch/ |
3ca67c5b | 17 | |
1b718db0 ER |
18 | # NOTE: |
19 | # to test run this, run: | |
20 | # $ awk -vDEBUG=1 pldnotify.awk < specfile | |
f249f99a ER |
21 | # |
22 | # To get full out of it, you need to have following tools installed: | |
23 | # - perl, sed, wget, coreutils, util-linux | |
c6bffdd2 | 24 | # - perl-HTML-Tree (HTML::TreeBuilder module) for better links parser (-vUSE_PERL=0 to disable) |
b6784c6a | 25 | # - pear (php-pear-PEAR) for php-pear package updates |
f249f99a | 26 | # - npm for nodejs packages |
b6784c6a | 27 | # - gem (ruby-rubygems) for ruby/rubygem packages |
f249f99a ER |
28 | # |
29 | # Additionally "mirrors" file in current dir, controls local mirrors you prefer | |
72446964 ER |
30 | |
31 | function d(s) { | |
32 | if (!DEBUG) { | |
33 | return | |
34 | } | |
158cf497 | 35 | |
b48e85a9 ER |
36 | # print strftime("%Y-%m-%d %H:%M:%S ") s >> "/dev/stderr" |
37 | print s >> "/dev/stderr" | |
72446964 ER |
38 | } |
39 | ||
2aba2068 SZ |
40 | function fixedsub(s1,s2,t, ind) { |
41 | # substitutes fixed strings (not regexps) | |
42 | if (ind = index(t,s1)) { | |
43 | t = substr(t, 1, ind-1) s2 substr(t, ind+length(s1)) | |
44 | } | |
45 | return t | |
46 | } | |
6fb3305d | 47 | |
e3c5f324 JB |
48 | function ispre(s) { |
49 | if ((s~"pre")||(s~"PRE")||(s~"beta")||(s~"BETA")||(s~"alpha")||(s~"ALPHA")||(s~"rc")||(s~"RC")) { | |
72446964 | 50 | d("pre-version") |
e3c5f324 JB |
51 | return 1 |
52 | } else { | |
53 | return 0 | |
54 | } | |
55 | } | |
f0c5c231 | 56 | |
70a5e8d4 SZ |
57 | function compare_ver(v1,v2) { |
58 | # compares version numbers | |
59 | while (match(v1,/[a-zA-Z][0-9]|[0-9][a-zA-Z]/)) | |
60 | v1=(substr(v1,1,RSTART) "." substr(v1,RSTART+RLENGTH-1)) | |
61 | while (match(v2,/[a-zA-Z][0-9]|[0-9][a-zA-Z]/)) | |
62 | v2=(substr(v2,1,RSTART) "." substr(v2,RSTART+RLENGTH-1)) | |
63 | sub("^0*","",v1) | |
64 | sub("^0*","",v2) | |
e2ebf07c JB |
65 | gsub("\.0*",".",v1) |
66 | gsub("\.0*",".",v2) | |
72446964 ER |
67 | d("v1 == " v1) |
68 | d("v2 == " v2) | |
70a5e8d4 SZ |
69 | count=split(v1,v1a,"\.") |
70 | count2=split(v2,v2a,"\.") | |
f0c5c231 ER |
71 | |
72 | if (count<count2) mincount=count | |
70a5e8d4 | 73 | else mincount=count2 |
f0c5c231 | 74 | |
70a5e8d4 | 75 | for (i=1; i<=mincount; i++) { |
db83adc3 SZ |
76 | if (v1a[i]=="") v1a[i]=0 |
77 | if (v2a[i]=="") v2a[i]=0 | |
72446964 ER |
78 | d("i == " i) |
79 | d("v1[i] == " v1a[i]) | |
80 | d("v2[i] == " v2a[i]) | |
70a5e8d4 SZ |
81 | if ((v1a[i]~/[0-9]/)&&(v2a[i]~/[0-9]/)) { |
82 | if (length(v2a[i])>length(v1a[i])) | |
83 | return 1 | |
84 | else if (v2a[i]>v1a[i]) | |
85 | return 1 | |
86 | else if (length(v1a[i])>length(v2a[i])) | |
87 | return 0 | |
88 | else if (v1a[i]>v2a[i]) | |
89 | return 0 | |
90 | } else if ((v1a[i]~/[A-Za-z]/)&&(v2a[i]~/[A-Za-z]/)) { | |
91 | if (v2a[i]>v1a[i]) | |
92 | return 1 | |
93 | else if (v1a[i]>v2a[i]) | |
94 | return 0 | |
e3c5f324 | 95 | } else if (ispre(v1a[i]) == 1) |
70a5e8d4 SZ |
96 | return 1 |
97 | else | |
98 | return 0 | |
99 | } | |
100 | if ((count2==mincount)&&(count!=count2)) { | |
101 | for (i=count2+1; i<=count; i++) | |
e3c5f324 | 102 | if (ispre(v1a[i]) == 1) |
70a5e8d4 SZ |
103 | return 1 |
104 | return 0 | |
105 | } else if (count!=count2) { | |
106 | for (i=count+1; i<=count2; i++) | |
e3c5f324 | 107 | if (ispre(v2a[i]) == 1) |
70a5e8d4 SZ |
108 | return 0 |
109 | return 1 | |
110 | } | |
db4f445e | 111 | return 0 |
70a5e8d4 SZ |
112 | } |
113 | ||
f11b5a76 | 114 | function compare_ver_dec(v1,v2) { |
b8b1ac2a | 115 | # compares version numbers as decimal floats |
f11b5a76 | 116 | while (match(v1,/[0-9][a-zA-Z]/)) |
117 | v1=(substr(v1,1,RSTART) "." substr(v1,RSTART+RLENGTH-1)) | |
118 | while (match(v2,/[0-9][a-zA-Z]/)) | |
119 | v2=(substr(v2,1,RSTART) "." substr(v2,RSTART+RLENGTH-1)) | |
120 | sub("^0*","",v1) | |
121 | sub("^0*","",v2) | |
72446964 ER |
122 | d("v1 == " v1) |
123 | d("v2 == " v2) | |
f11b5a76 | 124 | count=split(v1,v1a,"\.") |
125 | count2=split(v2,v2a,"\.") | |
f0c5c231 ER |
126 | |
127 | if (count<count2) mincount=count | |
f11b5a76 | 128 | else mincount=count2 |
f0c5c231 | 129 | |
f11b5a76 | 130 | for (i=1; i<=mincount; i++) { |
131 | if (v1a[i]=="") v1a[i]=0 | |
132 | if (v2a[i]=="") v2a[i]=0 | |
72446964 ER |
133 | d("i == " i) |
134 | d("v1[i] == " v1a[i]) | |
135 | d("v2[i] == " v2a[i]) | |
f11b5a76 | 136 | if ((v1a[i]~/[0-9]/)&&(v2a[i]~/[0-9]/)) { |
137 | if (i==2) { | |
138 | if (0+("." v2a[i])>0+("." v1a[i])) | |
139 | return 1 | |
140 | else if (0+("." v1a[i])>0+("." v2a[i])) | |
141 | return 0 | |
142 | } else { | |
143 | if (length(v2a[i])>length(v1a[i])) | |
144 | return 1 | |
145 | else if (v2a[i]>v1a[i]) | |
146 | return 1 | |
147 | else if (length(v1a[i])>length(v2a[i])) | |
148 | return 0 | |
149 | else if (v1a[i]>v2a[i]) | |
150 | return 0 | |
151 | } | |
152 | } else if ((v1a[i]~/[A-Za-z]/)&&(v2a[i]~/[A-Za-z]/)) { | |
153 | if (v2a[i]>v1a[i]) | |
154 | return 1 | |
155 | else if (v1a[i]>v2a[i]) | |
156 | return 0 | |
e3c5f324 | 157 | } else if (ispre(v1a[i]) == 1) |
f11b5a76 | 158 | return 1 |
159 | else | |
160 | return 0 | |
161 | } | |
162 | if ((count2==mincount)&&(count!=count2)) { | |
163 | for (i=count2+1; i<=count; i++) | |
e3c5f324 | 164 | if (ispre(v1a[i]) == 1) |
f11b5a76 | 165 | return 1 |
166 | return 0 | |
167 | } else if (count!=count2) { | |
168 | for (i=count+1; i<=count2; i++) | |
e3c5f324 | 169 | if (ispre(v2a[i]) == 1) |
f11b5a76 | 170 | return 0 |
171 | return 1 | |
172 | } | |
173 | return 0 | |
174 | } | |
175 | ||
371e7443 AM |
176 | function link_seen(link) { |
177 | for (seenlink in frameseen) { | |
178 | if (seenlink == link) { | |
72446964 | 179 | d("Link: [" link "] seen already, skipping...") |
371e7443 AM |
180 | return 1 |
181 | } | |
182 | } | |
183 | frameseen[link]=1 | |
184 | return 0 | |
185 | } | |
186 | ||
deed2eef ER |
187 | function mktemp( _cmd, _tmpfile) { |
188 | _cmd = "mktemp /tmp/XXXXXX" | |
189 | _cmd | getline _tmpfile | |
190 | close(_cmd) | |
191 | return _tmpfile | |
192 | } | |
193 | ||
6ca0a958 | 194 | # fix link to artificial one that will be recognized rest of this script |
a458861e | 195 | function postfix_link(url, link, oldlink) { |
6ca0a958 AM |
196 | oldlink = link |
197 | if ((url ~/^(http|https):\/\/github.com\//) && (link ~ /.*\/tarball\//)) { | |
198 | gsub(".*\/tarball\/", "", link) | |
199 | link = link ".tar.gz" | |
200 | } | |
a458861e ER |
201 | if (oldlink != link) { |
202 | d("POST FIXED URL [ " oldlink " ] to [ " link " ]") | |
203 | } | |
6ca0a958 AM |
204 | return link |
205 | } | |
206 | ||
481f5179 ER |
207 | # use perl HTML::TreeBuilder module to extract links from html |
208 | # it returns TAGNAME LINK in output which is pretty stright forward to parse in awk | |
209 | function extract_links_cmd(tmpfile) { | |
210 | return "perl -MHTML::TreeBuilder -e ' \ | |
211 | my $content = join q//, <>; \ | |
212 | my $root = new HTML::TreeBuilder; \ | |
213 | $root->parse($content); \ | |
481f5179 | 214 | \ |
3c1b03e6 | 215 | my %links = (); \ |
481f5179 ER |
216 | for (@{$root->extract_links(qw(a iframe))}) { \ |
217 | my($link, $element, $attr, $tag) = @$_; \ | |
3c1b03e6 ER |
218 | $links{$link} = $tag; \ |
219 | } \ | |
220 | \ | |
221 | while (my($link, $tag) = each %links) { \ | |
481f5179 ER |
222 | print $tag, q/ /, $link, $/; \ |
223 | } \ | |
224 | ' " tmpfile | |
225 | } | |
226 | ||
70a5e8d4 | 227 | # get all <A HREF=..> tags from specified URL |
d5697af2 | 228 | function get_links(url,filename, errno,link,oneline,retval,odp,wholeodp,lowerodp,tmpfile,cmd) { |
8fef61cd AM |
229 | |
230 | wholeerr="" | |
231 | ||
deed2eef ER |
232 | tmpfile = mktemp() |
233 | tmpfileerr = mktemp() | |
8fef61cd | 234 | |
e68cb385 | 235 | if (url ~ /^http:\/\/(download|downloads|dl)\.(sf|sourceforge)\.net\//) { |
46ec2e93 AM |
236 | newurl = url |
237 | # http://dl.sourceforge.net/threestore/ | |
6e927244 | 238 | # http://downloads.sourceforge.net/project/mediainfo/source/mediainfo/ |
46ec2e93 AM |
239 | gsub("^http://(download|downloads|dl)\.(sf|sourceforge)\.net/", "", newurl) |
240 | gsub("^project/", "", newurl) | |
241 | gsub("/.*", "", newurl) | |
242 | newurl = sf_url(newurl) | |
243 | if (newurl ~ /^http/) { | |
244 | url = newurl | |
245 | d("sf url, mungled url to: " url) | |
246 | } | |
f0c5c231 | 247 | |
cbf1f6f1 | 248 | } else if (url ~ /^http:\/\/(.*)\.googlecode\.com\/files\//) { |
3836b919 AM |
249 | gsub("^http://", "", url) |
250 | gsub("\..*", "", url) | |
b3df8960 | 251 | url = "http://code.google.com/p/" url "/downloads/list" |
72446964 | 252 | d("googlecode url, mungled url to: " url) |
97dc82d4 | 253 | |
cbf1f6f1 | 254 | } else if (url ~ /^http:\/\/pecl.php.net\/get\//) { |
0ed4cd23 AM |
255 | gsub("-.*", "", filename) |
256 | url = "http://pecl.php.net/package/" filename | |
72446964 | 257 | d("pecl.php.net url, mungled url to: " url) |
0ed4cd23 | 258 | |
cbf1f6f1 | 259 | } else if (url ~ /^(http|ftp):\/\/mysql.*\/Downloads\/MySQL-5.1\//) { |
fbf290ff | 260 | url = "http://dev.mysql.com/downloads/mysql/5.1.html#source" |
cbf1f6f1 | 261 | d("mysql 5.1 url, mungled url to: " url) |
fbf290ff | 262 | |
97dc82d4 | 263 | } else if (url ~ /^(http|https):\/\/launchpad\.net\/(.*)\//) { |
eb6fb288 AM |
264 | gsub("^(http|https):\/\/launchpad\.net\/", "", url) |
265 | gsub("\/.*/", "", url) | |
266 | url = "https://code.launchpad.net/" url "/+download" | |
72446964 | 267 | d("main launchpad url, mungled url to: " url) |
eb6fb288 | 268 | |
97dc82d4 | 269 | } else if (url ~ /^(http|https):\/\/edge\.launchpad\.net\/(.*)\//) { |
eb6fb288 AM |
270 | gsub("^(http|https):\/\/edge\.launchpad\.net\/", "", url) |
271 | gsub("\/.*/", "", url) | |
272 | url = "https://edge.launchpad.net/" url "/+download" | |
72446964 | 273 | d("edge launchpad url, mungled url to: " url) |
eb6fb288 | 274 | |
97dc82d4 | 275 | } else if (url ~ /^(http|https):\/\/github.com\/.*\/(.*)\/tarball\//) { |
c12742eb | 276 | gsub("\/tarball\/.*", "/downloads", url) |
72446964 | 277 | d("github tarball url, mungled url to: " url) |
c12742eb | 278 | |
97dc82d4 | 279 | } else if (url ~ /^(http|https):\/\/github.com\/.*\/(.*)\/archive\//) { |
b8e0ec33 AM |
280 | gsub("\/archive\/.*", "/tags", url) |
281 | d("github archive url, mungled url to: " url) | |
b8e0ec33 | 282 | |
97dc82d4 | 283 | } else if (url ~ /^(http|https):\/\/bitbucket.org\/.*\/get\/.*/) { |
cbf1f6f1 | 284 | # https://bitbucket.org/logilab/pylint/get/tip.tar.bz2 -> https://bitbucket.org/logilab/pylint/downloads |
72aa4a20 AM |
285 | gsub("\/get\/.*", "/downloads", url) |
286 | d("github bitbucket url, mungled url to: " url) | |
72aa4a20 | 287 | |
97dc82d4 | 288 | } else if (url ~ /^(http|https):\/\/cgit\..*\/(.*)\/snapshot\//) { |
20878bad AM |
289 | gsub("\/snapshot\/.*", "/", url) |
290 | d("cgit snapshot tarball url, mungled url to: " url) | |
fbf290ff | 291 | |
97dc82d4 | 292 | } else if (url ~ /^(http|https):\/\/www2\.aquamaniac\.de\/sites\/download\//) { |
5904ac62 AM |
293 | url = "http://www2.aquamaniac.de/sites/download/packages.php" |
294 | d("aquamaniac.de tarball url, mungled url to: " url) | |
5904ac62 | 295 | |
97dc82d4 | 296 | } else if (url ~ /^(http|https):\/\/www.process-one.net\/downloads\/ejabberd\//) { |
afb5de4b AM |
297 | url = "http://www.process-one.net/en/ejabberd/archive/" |
298 | d("ejabberd tarball url, mungled url to: " url) | |
afb5de4b | 299 | |
97dc82d4 | 300 | } else if (url ~ /^(http|https):\/\/llvm.org\/releases\//) { |
9f57341e AM |
301 | url = "http://llvm.org/releases/download.html" |
302 | d("llvm tarball url, mungled url to: " url) | |
9f57341e | 303 | |
97dc82d4 | 304 | } else if (url ~ /^(http|https):\/\/download\.owncloud\.org\/community\//) { |
f6776a1b AF |
305 | url = "http://owncloud.org/changelog/" |
306 | d("owncloud tarball url, mungled url to: " url) | |
f6776a1b | 307 | |
97dc82d4 | 308 | } else if (url ~ /^(http|https):\/\/hackage\.haskell\.org\/packages\/archive\//) { |
bd37d7ae JP |
309 | gsub("\/packages\/archive","/package",url) |
310 | d("hackage haskell tarball url, mungled url to: " url) | |
bd37d7ae | 311 | |
97dc82d4 | 312 | } else if (url ~ /^http:\/\/www.taskwarrior.org\/download\//) { |
5a70128c AF |
313 | url = "http://taskwarrior.org/projects/taskwarrior/wiki/Download" |
314 | d("taskwarrior tarball url, mungled url to: " url) | |
2659deff AM |
315 | } else if (url ~/^http:\/\/www.rarlab.com\/rar\// && filename ~ /^unrarsrc/) { |
316 | url = "http://www.rarlab.com/rar_add.htm" | |
317 | d("unrar tarball url, mungled url to: " url) | |
318 | } else if (url ~/^http:\/\/www.rarlab.com\/rar\//) { | |
319 | url = "http://www.rarlab.com/download.htm" | |
320 | d("rar tarball url, mungled url to: " url) | |
5a70128c AF |
321 | } |
322 | ||
72446964 | 323 | d("Retrieving: " url) |
b21587b3 ER |
324 | user_agent = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2) Gecko/20100129 PLD/3.0 (Th) Iceweasel/3.6" |
325 | cmd = "wget -t 2 -T 45 --user-agent \"" user_agent "\" -nv -O - \"" url "\" --passive-ftp --no-check-certificate > " tmpfile " 2> " tmpfileerr | |
72446964 | 326 | d("Execute: " cmd) |
d5697af2 | 327 | errno = system(cmd) |
72446964 | 328 | d("Execute done") |
f0c5c231 | 329 | |
158cf497 | 330 | if (errno != 0) { |
72446964 | 331 | d("Reading failure response...") |
8fef61cd AM |
332 | wholeerr = "" |
333 | while (getline oneline < tmpfileerr) | |
158cf497 | 334 | wholeerr = (wholeerr " " oneline) |
72446964 | 335 | d("Error Response: " wholeerr) |
8fef61cd | 336 | |
158cf497 ER |
337 | system("rm -f " tmpfile) |
338 | system("rm -f " tmpfileerr) | |
6e927244 ER |
339 | retval = ("WGET ERROR: " errno ": " wholeerr) |
340 | return retval | |
341 | } | |
481f5179 ER |
342 | system("rm -f " tmpfileerr) |
343 | ||
344 | urldir = url; | |
345 | sub(/[^\/]+$/, "", urldir) | |
346 | ||
347 | if (USE_PERL) { | |
348 | cmd = extract_links_cmd(tmpfile) | |
349 | while (cmd | getline) { | |
350 | tag = $1 | |
351 | link = substr($0, length(tag) + 2) | |
352 | ||
353 | if (tag == "iframe") { | |
354 | d("Frame: " link) | |
355 | if (url !~ /\//) { | |
356 | link = (urldir link) | |
357 | d("Frame->: " link) | |
358 | } | |
359 | ||
360 | if (link_seen(link)) { | |
361 | continue | |
362 | } | |
363 | retval = (retval " " get_links(link)) | |
364 | } | |
365 | ||
366 | if (link_seen(link)) { | |
367 | continue | |
368 | } | |
369 | ||
370 | retval = (retval " " link) | |
371 | d("href(): " link) | |
372 | } | |
373 | close(cmd) | |
374 | system("rm -f " tmpfile) | |
375 | ||
376 | d("Returning: [" retval "]") | |
377 | return retval | |
378 | } | |
371e7443 | 379 | |
158cf497 ER |
380 | wholeodp = "" |
381 | d("Reading success response...") | |
382 | while (getline oneline < tmpfile) { | |
383 | wholeodp = (wholeodp " " oneline) | |
384 | # d("Response: " wholeodp) | |
385 | } | |
386 | d("Reponse read done...") | |
158cf497 | 387 | system("rm -f " tmpfile) |
6e927244 | 388 | |
e68cb385 AM |
389 | # MATCH one of these: |
390 | #while (match(wholeodp, /<([aA]|[fF][rR][aA][mM][eE])[ \t][^>]*>/) > 0) { | |
391 | #while (match(wholeodp, /<link>[^<]*<\/link>/) > 0) { | |
392 | ||
393 | while (match(wholeodp, /(<link>[^<]*<\/link>|<([aA]|[fF][rR][aA][mM][eE])[ \t][^>]*>)/) > 0) { | |
6e927244 ER |
394 | d("Processing links...") |
395 | odp = substr(wholeodp,RSTART,RLENGTH); | |
396 | wholeodp = substr(wholeodp,RSTART+RLENGTH); | |
397 | ||
398 | lowerodp = tolower(odp); | |
399 | if (lowerodp ~ /<frame[ \t]/) { | |
400 | sub(/[sS][rR][cC]=[ \t]*/, "src=", odp); | |
401 | match(odp, /src="[^"]+"/) | |
402 | newurl = substr(odp, RSTART+5, RLENGTH-6) | |
403 | d("Frame: " newurl) | |
404 | if (newurl !~ /\//) { | |
405 | newurl=(urldir newurl) | |
406 | d("Frame->: " newurl) | |
407 | } | |
371e7443 | 408 | |
6e927244 ER |
409 | if (link_seen(newurl)) { |
410 | newurl = "" | |
411 | continue | |
412 | } | |
371e7443 | 413 | |
6e927244 | 414 | retval = (retval " " get_links(newurl)) |
46ec2e93 | 415 | d("href('condition1': " newurl) |
6e927244 ER |
416 | } else if (lowerodp ~ /href=[ \t]*"[^"]*"/) { |
417 | sub(/[hH][rR][eE][fF]=[ \t]*"/,"href=\"",odp) | |
418 | match(odp,/href="[^"]*"/) | |
419 | link=substr(odp,RSTART,RLENGTH) | |
420 | odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH) | |
421 | link=substr(link,7,length(link)-7) | |
422 | link=postfix_link(url, link) | |
423 | ||
424 | if (link_seen(link)) { | |
425 | link="" | |
426 | continue | |
427 | } | |
371e7443 | 428 | |
9291db88 AM |
429 | # link ends with at least 2 digit version |
430 | mlink = "" | |
e8067adc | 431 | if (link ~ /^.*\/[v]*[0-9\.]+[0-9]\/$/) |
9291db88 AM |
432 | mlink = get_links(link) |
433 | ||
434 | retval = (retval " " link " " mlink) | |
46ec2e93 | 435 | d("href('condition2'): " link) |
6e927244 ER |
436 | } else if (lowerodp ~ /href=[ \t]*'[^']*'/) { |
437 | sub(/[hH][rR][eE][fF]=[ \t]*'/,"href='",odp) | |
438 | match(odp,/href='[^']*'/) | |
439 | link=substr(odp,RSTART,RLENGTH) | |
440 | odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH) | |
441 | link=substr(link,7,length(link)-7) | |
442 | link=postfix_link(url, link) | |
443 | ||
444 | if (link_seen(link)) { | |
445 | link="" | |
446 | continue | |
447 | } | |
371e7443 | 448 | |
6e927244 | 449 | retval = (retval " " link) |
46ec2e93 | 450 | d("href('condition3'): " link) |
6e927244 ER |
451 | } else if (lowerodp ~ /href=[ \t]*[^ \t>]*/) { |
452 | sub(/[hH][rR][eE][fF]=[ \t]*/,"href=",odp) | |
453 | match(odp,/href=[^ \t>]*/) | |
454 | link=substr(odp,RSTART,RLENGTH) | |
455 | odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH) | |
456 | link=substr(link,6,length(link)-5) | |
457 | ||
458 | if (link_seen(link)) { | |
459 | link="" | |
460 | continue | |
93c3b403 | 461 | } |
6e927244 ER |
462 | |
463 | retval = (retval " " link) | |
46ec2e93 | 464 | d("href('condition4'): " link) |
e68cb385 AM |
465 | } else if (lowerodp ~ /<link>/) { |
466 | link=lowerodp | |
467 | sub("/<link>/", link) | |
468 | sub("/\/download<\/link>/", link) | |
469 | ||
470 | if (link_seen(link)) { | |
471 | link="" | |
472 | continue | |
473 | } | |
474 | ||
475 | retval = (retval " " link) | |
46ec2e93 | 476 | d("href('condition5'): " link) |
6e927244 ER |
477 | } else { |
478 | # <a ...> but not href - skip | |
479 | d("skipping <a > without href: " odp) | |
3f63995d | 480 | } |
3f63995d | 481 | } |
f0c5c231 | 482 | |
481f5179 | 483 | d("Returning: [" retval "]") |
db4f445e | 484 | return retval |
70a5e8d4 SZ |
485 | } |
486 | ||
6fb3305d | 487 | function subst_defines(var,defs) { |
3255b4c7 | 488 | # substitute all possible RPM macros |
6fb3305d SZ |
489 | while ((var ~ /%{.*}/) || (var ~ /%[A-Za-z0-9_]+/)) { |
490 | oldvar=var | |
491 | for (j in defs) { | |
492 | gsub("%{" j "}", defs[j], var) | |
493 | gsub("%" j , defs[j], var) | |
7f70edcd AM |
494 | # conditional macros like %{?patchlevel:.5} - drop these for now |
495 | gsub("%{\?" j ":.*?}", "", var) | |
6fb3305d SZ |
496 | } |
497 | if (var==oldvar) { | |
72446964 ER |
498 | if (DEBUG) { |
499 | for (i in defs) { | |
500 | d(i " == " defs[i]) | |
501 | } | |
502 | } | |
6fb3305d SZ |
503 | return var |
504 | } | |
505 | } | |
506 | return var | |
507 | } | |
508 | ||
c0bee1bc JK |
509 | function find_mirror(url) { |
510 | ||
c54fbfc1 MK |
511 | while (succ = (getline line < "mirrors")) { |
512 | if (succ==-1) { return url } | |
c0bee1bc | 513 | nf=split(line,fields,"|") |
a6f2cc98 | 514 | if (nf>1){ |
c0bee1bc JK |
515 | origin=fields[1] |
516 | mirror=fields[2] | |
517 | mname=fields[3] | |
518 | prefix=substr(url,1,length(origin)) | |
519 | if (prefix==origin){ | |
718ae075 | 520 | d("Mirror found at " mname) |
c0bee1bc | 521 | close("mirrors") |
21b99dbc | 522 | return mirror substr(url,length(origin)+1) |
c0bee1bc JK |
523 | } |
524 | } | |
525 | } | |
526 | ||
527 | return url | |
528 | } | |
529 | ||
3255b4c7 | 530 | # fetches file list, and compares version numbers |
4c6cbc25 | 531 | function process_source(number, lurl, name, version) { |
72446964 | 532 | d("Processing " lurl) |
3f63995d | 533 | |
16a2ba0e ER |
534 | if (index(lurl, version) == 0) { |
535 | d("There is no version number ["version"] in ["lurl"]") | |
3f63995d SZ |
536 | return 0 |
537 | } | |
538 | ||
6fb3305d SZ |
539 | sub("://",":",lurl) |
540 | sub("/",":/",lurl) | |
541 | gsub("[^/]*$",":&",lurl) | |
542 | split(lurl,url,":") | |
543 | acc=url[1] | |
544 | host=url[2] | |
545 | dir=url[3] | |
70a5e8d4 SZ |
546 | filename=url[4] |
547 | ||
3f63995d | 548 | if (index(dir,version)) { |
60ded487 AM |
549 | # directory name as version maching mode: |
550 | # if /something/version/name-version.tarball then check | |
551 | # in /something/ looking for newer directory | |
70a5e8d4 SZ |
552 | dir=substr(dir,1,index(dir,version)-1) |
553 | sub("[^/]*$","",dir) | |
c396da00 | 554 | sub("(\.tar\.(bz|bz2|gz|lzma|xz)|zip)$","",filename) |
70a5e8d4 | 555 | } |
3f63995d | 556 | |
72446964 ER |
557 | d("Will check a directory: " dir) |
558 | d("and a file: " filename) | |
60ded487 | 559 | |
70a5e8d4 | 560 | filenameexp=filename |
158cf497 | 561 | gsub("[+]","\\+",filenameexp) |
0269da83 | 562 | sub(version,"[A-Za-z0-9.]+",filenameexp) |
158cf497 | 563 | gsub("[.]","\\.",filenameexp) |
d5537c5f | 564 | sub("\.(bz|bz2|gz|lzma|xz|zip)$",".(bz|bz2|gz|lzma|xz|zip)",filenameexp) |
72446964 | 565 | d("Expression: " filenameexp) |
70a5e8d4 SZ |
566 | match(filename,version) |
567 | prever=substr(filename,1,RSTART-1) | |
568 | postver=substr(filename,RSTART+RLENGTH) | |
72446964 ER |
569 | d("Before number: " prever) |
570 | d("and after: " postver) | |
f0c5c231 | 571 | newurl=find_mirror(acc "://" host dir) |
c54fbfc1 MK |
572 | #print acc "://" host dir |
573 | #newurl=url[1]"://"url[2]url[3]url[4] | |
574 | #newurl=acc "://" host dir filename | |
72446964 | 575 | d("Looking at " newurl) |
f0c5c231 | 576 | |
6fb3305d | 577 | references=0 |
db83adc3 | 578 | finished=0 |
db4f445e | 579 | oldversion=version |
6e927244 | 580 | odp = get_links(newurl, filename) |
3f63995d SZ |
581 | if( odp ~ "ERROR: ") { |
582 | print name "(" number ") " odp | |
188dc51f | 583 | } else { |
72446964 | 584 | d("WebPage downloaded") |
188dc51f SZ |
585 | c=split(odp,linki) |
586 | for (nr=1; nr<=c; nr++) { | |
587 | addr=linki[nr] | |
6ca0a958 | 588 | |
72446964 | 589 | d("Found link: " addr) |
6ca0a958 AM |
590 | |
591 | # github has very different tarball links that clash with this safe check | |
592 | if (!(newurl ~/^(http|https):\/\/github.com\/.*\/tarball/)) { | |
593 | if (addr ~ "[-_.0-9A-Za-z~]" filenameexp) { | |
594 | continue | |
595 | } | |
596 | } | |
597 | ||
598 | if (addr ~ filenameexp) { | |
188dc51f SZ |
599 | match(addr,filenameexp) |
600 | newfilename=substr(addr,RSTART,RLENGTH) | |
72446964 | 601 | d("Hypothetical new: " newfilename) |
2aba2068 SZ |
602 | newfilename=fixedsub(prever,"",newfilename) |
603 | newfilename=fixedsub(postver,"",newfilename) | |
72446964 | 604 | d("Version: " newfilename) |
eb6fb288 | 605 | if (newfilename ~ /\.(asc|sig|pkg|bin|binary|built)$/) continue |
60ded487 | 606 | # strip ending (happens when in directiory name as version matching mode) |
c396da00 | 607 | sub("(\.tar\.(bz|bz2|gz|lzma|xz)|zip)$","",newfilename) |
f11b5a76 | 608 | if (NUMERIC) { |
609 | if ( compare_ver_dec(version, newfilename)==1 ) { | |
72446964 | 610 | d("Yes, there is new one") |
f11b5a76 | 611 | version=newfilename |
612 | finished=1 | |
613 | } | |
614 | } else if ( compare_ver(version, newfilename)==1 ) { | |
72446964 | 615 | d("Yes, there is new one") |
188dc51f SZ |
616 | version=newfilename |
617 | finished=1 | |
618 | } | |
70a5e8d4 | 619 | } |
6fb3305d | 620 | } |
16a2ba0e | 621 | if (finished == 0) |
f1b8a975 | 622 | print name "(" number ") seems ok: " oldversion |
188dc51f SZ |
623 | else |
624 | print name "(" number ") [OLD] " oldversion " [NEW] " version | |
6fb3305d SZ |
625 | } |
626 | } | |
eae5334c | 627 | |
e68cb385 | 628 | function sf_url(sf_project) { |
46ec2e93 | 629 | sf_url_new = "" |
e68cb385 AM |
630 | sf_idurl="http://sourceforge.net/api/project/name/" sf_project "/json" |
631 | cmd = "wget -t 2 -T 45 -q -O - " sf_idurl " | awk -F: '/\"id\":/ { gsub(\",\", \"\", $2); print $2 } '" | |
632 | d("sf_url_cmd: " cmd) | |
633 | cmd | getline sf_id | |
634 | d("sf_url_id: " sf_id) | |
46ec2e93 AM |
635 | if (sf_id ~ /^[0-9]+$/) { |
636 | sf_url_new = "http://sourceforge.net/api/file/index/project-id/" sf_id "/mtime/desc/limit/20/rss" | |
637 | d("sf_url_new: " sf_url_new) | |
638 | } | |
639 | return sf_url_new | |
e68cb385 AM |
640 | } |
641 | ||
3445a27e | 642 | function rss_upgrade(name, ver, url, regex, cmd) { |
a51b9f08 | 643 | regex = "s/.*<title>" regex "<\/title>.*/\\1/p" |
b21587b3 | 644 | cmd = "wget -t 2 -T 45 -q -O - " url " | sed -nre '" regex "' | head -n1" |
41f7ded7 | 645 | |
a51b9f08 | 646 | d("rss_upgrade_cmd: " cmd) |
3445a27e | 647 | cmd | getline ver |
41f7ded7 ER |
648 | close(cmd) |
649 | ||
3445a27e | 650 | return ver |
a51b9f08 | 651 | } |
41f7ded7 | 652 | |
a51b9f08 ER |
653 | # check for ZF upgrade from rss |
654 | function zf_upgrade(name, ver) { | |
4c6cbc25 | 655 | return rss_upgrade(name, ver, \ |
a51b9f08 ER |
656 | "http://devzone.zend.com/tag/Zend_Framework_Management/format/rss2.0", \ |
657 | "Zend Framework ([^\\s]+) Released" \ | |
658 | ); | |
659 | } | |
660 | ||
661 | function hudson_upgrade(name, ver) { | |
4c6cbc25 | 662 | return rss_upgrade(name, ver, \ |
a51b9f08 ER |
663 | "https://hudson.dev.java.net/servlets/ProjectRSS?type=news", \ |
664 | "Hudson ([0-9.]+) released" \ | |
665 | ); | |
41f7ded7 ER |
666 | } |
667 | ||
eae5334c | 668 | # upgrade check for pear package using PEAR CLI |
3445a27e ER |
669 | function pear_upgrade(name, ver, cmd) { |
670 | sub(/^php-pear-/, "", name); | |
eae5334c | 671 | |
3445a27e ER |
672 | cmd = "pear remote-info " name " | awk '/^Latest/{print $NF}'" |
673 | d("PEAR: " cmd) | |
674 | cmd | getline ver | |
675 | close(cmd) | |
eae5334c | 676 | |
3445a27e | 677 | return ver |
eae5334c | 678 | } |
f0c5c231 | 679 | |
3445a27e | 680 | function vim_upgrade(name, ver, cmd) { |
3d29ff34 | 681 | # %patchset_source -f ftp://ftp.vim.org/pub/editors/vim/patches/7.2/7.2.%03g 1 %{patchlevel} |
3445a27e ER |
682 | cmd = "wget -q -O - ftp://ftp.vim.org/pub/editors/vim/patches/" DEFS["ver"] "/MD5SUMS|grep -vF .gz|tail -n1|awk '{print $2}'" |
683 | d("VIM: " cmd) | |
684 | cmd | getline ver | |
685 | close(cmd) | |
686 | return ver | |
3d29ff34 ER |
687 | } |
688 | ||
3445a27e | 689 | function nodejs_upgrade(name, ver, cmd) { |
1b718db0 ER |
690 | d("NODEJS " name " (as " DEFS["pkg"] ") " ver); |
691 | if (DEFS["pkg"]) { | |
692 | cmd = "npm info " DEFS["pkg"] " dist-tags.latest" | |
693 | } else { | |
694 | cmd = "npm info " name " dist-tags.latest" | |
695 | } | |
3445a27e | 696 | cmd | getline ver |
1b718db0 ER |
697 | close(cmd) |
698 | ||
3445a27e | 699 | return ver |
1b718db0 ER |
700 | } |
701 | ||
b6784c6a ER |
702 | function rubygem_upgrade(name, ver, cmd, pkg) { |
703 | if (DEFS["gem_name"]) { | |
704 | pkg = DEFS["gem_name"]; | |
705 | ||
706 | } else if (DEFS["gemname"]) { | |
707 | pkg = DEFS["gemname"]; | |
708 | ||
709 | } else if (DEFS["pkgname"]) { | |
710 | pkg = DEFS["pkgname"]; | |
711 | ||
712 | } else { | |
713 | pkg = name; | |
714 | gsub(/^ruby-/, "", pkg); | |
715 | } | |
716 | ||
717 | cmd = "gem list --remote '^" pkg "$' | awk '/" pkg "/ {v=$2; sub(/\(/, \"\", v); print v}'" | |
718 | d("RUBYGEM " name " (as " pkg ") " ver ": " cmd); | |
719 | cmd | getline ver | |
720 | ||
721 | close(cmd) | |
722 | ||
723 | return ver | |
724 | } | |
725 | ||
3c78605d ER |
726 | function google_linux_repo(name, ver, reponame, cmd, sourceurl) { |
727 | sourceurl = "http://dl.google.com/linux/" reponame "/rpm/stable/x86_64/repodata/primary.xml.gz" | |
728 | cmd = "curl -s " sourceurl " | zcat | perl -ne 'm{<name>" name "-" DEFS["state"] "</name>} and m{<version .*ver=.([\d.]+)} and print $1'" | |
729 | d("google repo: " cmd); | |
3445a27e | 730 | cmd | getline ver |
48371291 ER |
731 | close(cmd) |
732 | ||
3445a27e | 733 | return ver |
48371291 ER |
734 | } |
735 | ||
89734d5c ER |
736 | function jenkins_upgrade(name, ver, urls, url, i, c, chunks, nver) { |
737 | for (i in urls) { | |
738 | url = urls[i] | |
739 | # http://mirrors.jenkins-ci.org/war-stable/1.509.1/jenkins.war?/jenkins-1.509.1.war | |
740 | gsub("/" ver "/jenkins.war\?/jenkins-" ver ".war", "/", url); | |
741 | c = split(get_links(url), chunks, "/") | |
742 | # new version is second one from the bottom | |
743 | nver = chunks[c - 2] | |
744 | gsub(/ /, "", nver) | |
745 | return nver; | |
746 | } | |
747 | } | |
748 | ||
749 | function process_data(name, ver, rel, src, nver, i) { | |
eae5334c | 750 | if (name ~ /^php-pear-/) { |
4c6cbc25 | 751 | nver = pear_upgrade(name, ver); |
1b718db0 | 752 | } else if (name == "ZendFramework") { |
4c6cbc25 | 753 | nver = zf_upgrade(name, ver); |
1b718db0 | 754 | } else if (name == "hudson") { |
4c6cbc25 | 755 | nver = hudson_upgrade(name, ver); |
1b718db0 | 756 | } else if (name == "vim") { |
123a6771 | 757 | nver = vim_upgrade(name, ver); |
48371291 | 758 | } else if (name == "google-chrome") { |
3c78605d ER |
759 | nver = google_linux_repo(name, ver, "chrome"); |
760 | } else if (name == "google-talkplugin") { | |
761 | nver = google_linux_repo(name, ver, "talkplugin"); | |
1b718db0 | 762 | } else if (name ~ "^nodejs-") { |
4c6cbc25 | 763 | nver = nodejs_upgrade(name, ver); |
6fda96ee | 764 | } else if (name ~ "^ruby-" || name == "chef") { |
b6784c6a | 765 | nver = rubygem_upgrade(name, ver); |
89734d5c ER |
766 | } else if (name ~ "jenkins") { |
767 | nver = jenkins_upgrade(name, ver, src); | |
4c6cbc25 ER |
768 | } |
769 | ||
770 | if (nver) { | |
771 | if (compare_ver(ver, nver)) { | |
772 | print name " [OLD] " ver " [NEW] " nver | |
773 | } else { | |
774 | print name " seems ok: " ver | |
775 | } | |
776 | return; | |
777 | } | |
778 | ||
779 | if (name == "xulrunner") { | |
16a2ba0e ER |
780 | ver = subst_defines(DEFS["firefox_ver"], DEFS) |
781 | d("package xulrunner, change version to firefox ["ver"]") | |
782 | } | |
eae5334c | 783 | |
3255b4c7 SZ |
784 | # this function checks if substitutions were valid, and if true: |
785 | # processes each URL and tries to get current file list | |
6fb3305d | 786 | for (i in src) { |
16a2ba0e | 787 | if (src[i] ~ /%{nil}/) { |
254a5f0b AM |
788 | gsub(/\%\{nil\}/, "", src[i]) |
789 | } | |
6fb3305d | 790 | if ( src[i] !~ /%{.*}/ && src[i] !~ /%[A-Za-z0-9_]/ ) { |
72446964 | 791 | d("Source: " src[i]) |
16a2ba0e | 792 | process_source(i, src[i], name, ver) |
6fb3305d | 793 | } else { |
c4a1f100 | 794 | print FNAME ":" i ": impossible substitution: " src[i] |
6fb3305d SZ |
795 | } |
796 | } | |
797 | } | |
798 | ||
799 | BEGIN { | |
1b718db0 | 800 | # if you want to use DEBUG, run script with "-v DEBUG=1" |
6fb3305d | 801 | # or uncomment the line below |
70a5e8d4 | 802 | # DEBUG = 1 |
2e301ef5 SZ |
803 | |
804 | errno=system("wget --help > /dev/null 2>&1") | |
d44c273e | 805 | if (errno && errno != 3) { |
2e301ef5 SZ |
806 | print "No wget installed!" |
807 | exit 1 | |
808 | } | |
f11b5a76 | 809 | if (ARGC>=3 && ARGV[2]=="-n") { |
810 | NUMERIC=1 | |
f0c5c231 | 811 | for (i=3; i<ARGC; i++) ARGV[i-1]=ARGV[i] |
f11b5a76 | 812 | ARGC=ARGC-1 |
813 | } | |
6fb3305d SZ |
814 | } |
815 | ||
816 | FNR==1 { | |
817 | if ( ARGIND != 1 ) { | |
6aab34d4 AM |
818 | # clean frameseen for each ARG |
819 | for (i in frameseen) { | |
820 | delete frameseen[i] | |
821 | } | |
822 | frameseen[0] = 1 | |
823 | ||
3255b4c7 | 824 | process_data(NAME,VER,REL,SRC) |
6fb3305d SZ |
825 | NAME="" ; VER="" ; REL="" |
826 | for (i in DEFS) delete DEFS[i] | |
827 | for (i in SRC) delete SRC[i] | |
828 | } | |
829 | FNAME=FILENAME | |
f6c65bb7 | 830 | DEFS["_alt_kernel"]="" |
892e6dad | 831 | DEFS["20"]="\\ " |
9e3300e3 | 832 | DEFS["nil"]="" |
6fb3305d SZ |
833 | } |
834 | ||
3255b4c7 | 835 | /^[Uu][Rr][Ll]:/&&(URL=="") { URL=subst_defines($2,DEFS) ; DEFS["url"]=URL } |
6fb3305d SZ |
836 | /^[Nn]ame:/&&(NAME=="") { NAME=subst_defines($2,DEFS) ; DEFS["name"]=NAME } |
837 | /^[Vv]ersion:/&&(VER=="") { VER=subst_defines($2,DEFS) ; DEFS["version"]=VER } | |
838 | /^[Rr]elease:/&&(REL=="") { REL=subst_defines($2,DEFS) ; DEFS["release"]=REL } | |
a10d5bbe | 839 | /^[Ss]ource[0-9]*:/ { if (/(ftp|http|https):\/\//) SRC[FNR]=subst_defines($2,DEFS) } |
6fb3305d SZ |
840 | /%define/ { DEFS[$2]=subst_defines($3,DEFS) } |
841 | ||
842 | END { | |
3255b4c7 | 843 | process_data(NAME,VER,REL,SRC) |
6fb3305d | 844 | } |