pldnotify_.awk

   1 #!/bin/awk -f
   2 # $Revision$, $Date$
   3 #
   4 # Copyright (C) 2000-2010 PLD-Team <feedback@pld-linux.org>
   5 # Authors:
   6 #       Sebastian Zagrodzki <zagrodzki@pld-linux.org>
   7 #       Jacek Konieczny <jajcus@pld-linux.org>
   8 #       Andrzej Krzysztofowicz <ankry@pld-linux.org>
   9 #       Jakub Bogusz <qboosh@pld-linux.org>
  10 #       Elan Ruusamäe <glen@pld-linux.org>
  11 #
  12 # See cvs log pldnotify.awk for list of contributors
  13 #
  14 # TODO:
  15 # - "SourceXDownload" support (use given URLs if present instead of cut-down SourceX URLs)
  16 # - "SourceXActiveFTP" support
  17
  18
  19 function d(s) {
  20         if (!DEBUG) {
  21                 return
  22         }
  23         print s >> "/dev/stderr"
  24 }
  25
  26 function fixedsub(s1,s2,t,      ind) {
  27 # substitutes fixed strings (not regexps)
  28         if (ind = index(t,s1)) {
  29                 t = substr(t, 1, ind-1) s2 substr(t, ind+length(s1))
  30         }
  31         return t
  32 }
  33
  34 function ispre(s) {
  35         if ((s~"pre")||(s~"PRE")||(s~"beta")||(s~"BETA")||(s~"alpha")||(s~"ALPHA")||(s~"rc")||(s~"RC")) {
  36                 d("pre-version")
  37                 return 1
  38         } else {
  39                 return 0
  40         }
  41 }
  42
  43 function compare_ver(v1,v2) {
  44 # compares version numbers
  45         while (match(v1,/[a-zA-Z][0-9]|[0-9][a-zA-Z]/))
  46                 v1=(substr(v1,1,RSTART) "." substr(v1,RSTART+RLENGTH-1))
  47         while (match(v2,/[a-zA-Z][0-9]|[0-9][a-zA-Z]/))
  48                 v2=(substr(v2,1,RSTART) "." substr(v2,RSTART+RLENGTH-1))
  49         sub("^0*","",v1)
  50         sub("^0*","",v2)
  51         gsub("\.0*",".",v1)
  52         gsub("\.0*",".",v2)
  53         d("v1 == " v1)
  54         d("v2 == " v2)
  55         count=split(v1,v1a,"\.")
  56         count2=split(v2,v2a,"\.")
  57
  58         if (count<count2) mincount=count
  59         else mincount=count2
  60
  61         for (i=1; i<=mincount; i++) {
  62                 if (v1a[i]=="") v1a[i]=0
  63                 if (v2a[i]=="") v2a[i]=0
  64                 d("i == " i)
  65                 d("v1[i] == " v1a[i])
  66                 d("v2[i] == " v2a[i])
  67                 if ((v1a[i]~/[0-9]/)&&(v2a[i]~/[0-9]/)) {
  68                         if (length(v2a[i])>length(v1a[i]))
  69                                 return 1
  70                         else if (v2a[i]>v1a[i])
  71                                 return 1
  72                         else if (length(v1a[i])>length(v2a[i]))
  73                                 return 0
  74                         else if (v1a[i]>v2a[i])
  75                                 return 0
  76                 } else if ((v1a[i]~/[A-Za-z]/)&&(v2a[i]~/[A-Za-z]/)) {
  77                         if (v2a[i]>v1a[i])
  78                                 return 1
  79                         else if (v1a[i]>v2a[i])
  80                                 return 0
  81                 } else if (ispre(v1a[i]) == 1)
  82                         return 1
  83                 else
  84                         return 0
  85         }
  86         if ((count2==mincount)&&(count!=count2)) {
  87                 for (i=count2+1; i<=count; i++)
  88                         if (ispre(v1a[i]) == 1)
  89                                 return 1
  90                 return 0
  91         } else if (count!=count2) {
  92                 for (i=count+1; i<=count2; i++)
  93                         if (ispre(v2a[i]) == 1)
  94                                 return 0
  95                 return 1
  96         }
  97         return 0
  98 }
  99
 100 function compare_ver_dec(v1,v2) {
 101 # compares version numbers as decimal floats
 102         while (match(v1,/[0-9][a-zA-Z]/))
 103                 v1=(substr(v1,1,RSTART) "." substr(v1,RSTART+RLENGTH-1))
 104         while (match(v2,/[0-9][a-zA-Z]/))
 105                 v2=(substr(v2,1,RSTART) "." substr(v2,RSTART+RLENGTH-1))
 106         sub("^0*","",v1)
 107         sub("^0*","",v2)
 108         d("v1 == " v1)
 109         d("v2 == " v2)
 110         count=split(v1,v1a,"\.")
 111         count2=split(v2,v2a,"\.")
 112
 113         if (count<count2) mincount=count
 114         else mincount=count2
 115
 116         for (i=1; i<=mincount; i++) {
 117                 if (v1a[i]=="") v1a[i]=0
 118                 if (v2a[i]=="") v2a[i]=0
 119                 d("i == " i)
 120                 d("v1[i] == " v1a[i])
 121                 d("v2[i] == " v2a[i])
 122                 if ((v1a[i]~/[0-9]/)&&(v2a[i]~/[0-9]/)) {
 123                         if (i==2) {
 124                                 if (0+("." v2a[i])>0+("." v1a[i]))
 125                                         return 1
 126                                 else if (0+("." v1a[i])>0+("." v2a[i]))
 127                                         return 0
 128                         } else {
 129                                 if (length(v2a[i])>length(v1a[i]))
 130                                         return 1
 131                                 else if (v2a[i]>v1a[i])
 132                                         return 1
 133                                 else if (length(v1a[i])>length(v2a[i]))
 134                                         return 0
 135                                 else if (v1a[i]>v2a[i])
 136                                         return 0
 137                         }
 138                 } else if ((v1a[i]~/[A-Za-z]/)&&(v2a[i]~/[A-Za-z]/)) {
 139                         if (v2a[i]>v1a[i])
 140                                 return 1
 141                         else if (v1a[i]>v2a[i])
 142                                 return 0
 143                 } else if (ispre(v1a[i]) == 1)
 144                         return 1
 145                 else
 146                         return 0
 147         }
 148         if ((count2==mincount)&&(count!=count2)) {
 149                 for (i=count2+1; i<=count; i++)
 150                         if (ispre(v1a[i]) == 1)
 151                                 return 1
 152                 return 0
 153         } else if (count!=count2) {
 154                 for (i=count+1; i<=count2; i++)
 155                         if (ispre(v2a[i]) == 1)
 156                                 return 0
 157                 return 1
 158         }
 159         return 0
 160 }
 161
 162 function link_seen(link) {
 163         for (seenlink in frameseen) {
 164                 if (seenlink == link) {
 165                         d("Link: [" link "] seen already, skipping...")
 166                         return 1
 167                 }
 168         }
 169         frameseen[link]=1
 170         return 0
 171 }
 172
 173 function mktemp(   _cmd, _tmpfile) {
 174         _cmd = "mktemp /tmp/XXXXXX"
 175         _cmd | getline _tmpfile
 176         close(_cmd)
 177         return _tmpfile
 178 }
 179
 180 # fix link to artificial one that will be recognized rest of this script
 181 function postfix_link(url, link) {
 182         oldlink = link
 183         if ((url ~/^(http|https):\/\/github.com\//) && (link ~ /.*\/tarball\//)) {
 184                 gsub(".*\/tarball\/", "", link)
 185                 link = link ".tar.gz"
 186         }
 187         d("POST FIXING URL [ " oldlink " ] to [ " link " ]")
 188         return link
 189 }
 190
 191 # get all <A HREF=..> tags from specified URL
 192 function get_links(url,filename,   errno,link,oneline,retval,odp,wholeodp,lowerodp,tmpfile,cmd) {
 193
 194         wholeerr=""
 195
 196         tmpfile = mktemp()
 197         tmpfileerr = mktemp()
 198
 199         if (url ~ /^http:\/\/(download|dl).(sf|sourceforge).net\//) {
 200                 gsub("^http://(download|dl).(sf|sourceforge).net/", "", url)
 201                 gsub("/.*", "", url)
 202                 url = "http://sourceforge.net/projects/" url "/files/"
 203                 d("sf url, mungled url to: " url)
 204         }
 205
 206         if (url ~ /^http:\/\/(.*)\.googlecode\.com\/files\//) {
 207                 gsub("^http://", "", url)
 208                 gsub("\..*", "", url)
 209                 url = "http://code.google.com/p/" url "/downloads/list"
 210                 d("googlecode url, mungled url to: " url)
 211         }
 212
 213         if (url ~ /^http:\/\/pecl.php.net\/get\//) {
 214                 gsub("-.*", "", filename)
 215                 url = "http://pecl.php.net/package/" filename
 216                 d("pecl.php.net url, mungled url to: " url)
 217         }
 218
 219         if (url ~ /^(http|ftp):\/\/mysql.*\/Downloads\/MySQL-5.1\//) {
 220                 url = "http://dev.mysql.com/downloads/mysql/5.1.html#source"
 221                  d("mysql 5.1 url, mungled url to: " url)
 222         }
 223
 224         if (url ~/^(http|https):\/\/launchpad\.net\/(.*)\//) {
 225                 gsub("^(http|https):\/\/launchpad\.net\/", "", url)
 226                 gsub("\/.*/", "", url)
 227                 url = "https://code.launchpad.net/" url "/+download"
 228                 d("main launchpad url, mungled url to: " url)
 229         }
 230
 231         if (url ~/^(http|https):\/\/edge\.launchpad\.net\/(.*)\//) {
 232                 gsub("^(http|https):\/\/edge\.launchpad\.net\/", "", url)
 233                 gsub("\/.*/", "", url)
 234                 url = "https://edge.launchpad.net/" url "/+download"
 235                 d("edge launchpad url, mungled url to: " url)
 236         }
 237
 238         if (url ~/^(http|https):\/\/github.com\/.*\/(.*)\/tarball\//) {
 239                 gsub("\/tarball\/.*", "/downloads", url)
 240                 d("github tarball url, mungled url to: " url)
 241         }
 242
 243
 244         d("Retrieving: " url)
 245         cmd = "wget --user-agent \"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2) Gecko/20100129 PLD/3.0 (Th) Iceweasel/3.6\" -nv -O - \"" url "\" -t 2 -T 45 --passive-ftp --no-check-certificate > " tmpfile " 2> " tmpfileerr
 246         d("Execute: " cmd)
 247         errno = system(cmd)
 248         d("Execute done")
 249
 250         if (errno==0) {
 251                 wholeodp = ""
 252                 d("Reading succeess response...")
 253                 while (getline oneline < tmpfile)
 254                         wholeodp=(wholeodp " " oneline)
 255 #                       d("Response: " wholeodp)
 256         } else {
 257                 d("Reading failure response...")
 258                 wholeerr = ""
 259                 while (getline oneline < tmpfileerr)
 260                         wholeerr=(wholeerr " " oneline)
 261                 d("Error Response: " wholeerr)
 262         }
 263
 264         system("rm -f " tmpfile)
 265         system("rm -f " tmpfileerr)
 266
 267         urldir=url;
 268         sub(/[^\/]+$/,"",urldir)
 269
 270         if ( errno==0) {
 271                 while (match(wholeodp, /<([aA]|[fF][rR][aA][mM][eE])[ \t][^>]*>/) > 0) {
 272                         d("Processing links...")
 273                         odp=substr(wholeodp,RSTART,RLENGTH);
 274                         wholeodp=substr(wholeodp,RSTART+RLENGTH);
 275
 276                         lowerodp=tolower(odp);
 277                         if (lowerodp ~ /<frame[ \t]/) {
 278                                 sub(/[sS][rR][cC]=[ \t]*/,"src=",odp);
 279                                 match(odp,/src="[^"]+"/)
 280                                 newurl=substr(odp,RSTART+5,RLENGTH-6)
 281                                 d("Frame: " newurl)
 282                                 if (newurl !~ /\//) {
 283                                         newurl=(urldir newurl)
 284                                         d("Frame->: " newurl)
 285                                 }
 286
 287                                 if (link_seen(newurl)) {
 288                                         newurl=""
 289                                         continue
 290                                 }
 291
 292                                 retval=(retval " " get_links(newurl))
 293                         } else if (lowerodp ~ /href=[ \t]*"[^"]*"/) {
 294                                 sub(/[hH][rR][eE][fF]=[ \t]*"/,"href=\"",odp)
 295                                 match(odp,/href="[^"]*"/)
 296                                 link=substr(odp,RSTART,RLENGTH)
 297                                 odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH)
 298                                 link=substr(link,7,length(link)-7)
 299                                 link=postfix_link(url, link)
 300
 301                                 if (link_seen(link)) {
 302                                         link=""
 303                                         continue
 304                                 }
 305
 306                                 retval=(retval " " link)
 307                                 d("href(\"\"): " link)
 308                         } else if (lowerodp ~ /href=[ \t]*'[^']*'/) {
 309                                 sub(/[hH][rR][eE][fF]=[ \t]*'/,"href='",odp)
 310                                 match(odp,/href='[^']*'/)
 311                                 link=substr(odp,RSTART,RLENGTH)
 312                                 odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH)
 313                                 link=substr(link,7,length(link)-7)
 314                                 link=postfix_link(url, link)
 315
 316                                 if (link_seen(link)) {
 317                                         link=""
 318                                         continue
 319                                 }
 320
 321                                 retval=(retval " " link)
 322                                 d("href(''): " link)
 323                         } else if (lowerodp ~ /href=[ \t]*[^ \t>]*/) {
 324                                 sub(/[hH][rR][eE][fF]=[ \t]*/,"href=",odp)
 325                                 match(odp,/href=[^ \t>]*/)
 326                                 link=substr(odp,RSTART,RLENGTH)
 327                                 odp=substr(odp,1,RSTART) substr(odp,RSTART+RLENGTH)
 328                                 link=substr(link,6,length(link)-5)
 329
 330                                 if (link_seen(link)) {
 331                                         link=""
 332                                         continue
 333                                 }
 334
 335                                 retval=(retval " " link)
 336                                 d("href(): " link)
 337                         } else {
 338                                 # <a ...> but not href - skip
 339                                 d("skipping <a > without href: " odp)
 340                         }
 341                 }
 342         } else {
 343                 retval=("WGET ERROR: " errno ": " wholeerr)
 344         }
 345
 346
 347         d("Returning: " retval)
 348         return retval
 349 }
 350
 351 function subst_defines(var,defs) {
 352 # substitute all possible RPM macros
 353         while ((var ~ /%{.*}/) || (var ~ /%[A-Za-z0-9_]+/)) {
 354                 oldvar=var
 355                 for (j in defs) {
 356                         gsub("%{" j "}", defs[j], var)
 357                         gsub("%" j , defs[j], var)
 358                 }
 359                 if (var==oldvar) {
 360                         if (DEBUG) {
 361                                 for (i in defs) {
 362                                         d(i " == " defs[i])
 363                                 }
 364                         }
 365                         return var
 366                 }
 367         }
 368         return var
 369 }
 370
 371 function find_mirror(url) {
 372
 373         while (succ = (getline line < "mirrors")) {
 374             if (succ==-1) { return url }
 375                 nf=split(line,fields,"|")
 376                 if (nf>1){
 377                         origin=fields[1]
 378                         mirror=fields[2]
 379                         mname=fields[3]
 380                         prefix=substr(url,1,length(origin))
 381                         if (prefix==origin){
 382                                 d("Mirror fount at " mname)
 383                                 close("mirrors")
 384                                 return mirror substr(url,length(origin)+1)
 385                         }
 386                 }
 387         }
 388
 389         return url
 390 }
 391
 392 function process_source(number,lurl,name,version) {
 393 # fetches file list, and compares version numbers
 394         d("Processing " lurl)
 395
 396         if ( index(lurl,version)==0 ) {
 397                 d("There is no version number.")
 398                 return 0
 399         }
 400
 401         sub("://",":",lurl)
 402         sub("/",":/",lurl)
 403         gsub("[^/]*$",":&",lurl)
 404         split(lurl,url,":")
 405         acc=url[1]
 406         host=url[2]
 407         dir=url[3]
 408         filename=url[4]
 409
 410         if (index(dir,version)) {
 411                 # directory name as version maching mode:
 412                 # if /something/version/name-version.tarball then check
 413                 # in /something/ looking for newer directory
 414                 dir=substr(dir,1,index(dir,version)-1)
 415                 sub("[^/]*$","",dir)
 416                 sub("(\.tar\.(bz|bz2|gz)|zip)$","",filename)
 417         }
 418
 419         d("Will check a directory: " dir)
 420         d("and a file: " filename)
 421
 422         filenameexp=filename
 423         gsub("\+","\\+",filenameexp)
 424         sub(version,"[A-Za-z0-9.]+",filenameexp)
 425         gsub("\.","\\.",filenameexp)
 426         d("Expression: " filenameexp)
 427         match(filename,version)
 428         prever=substr(filename,1,RSTART-1)
 429         postver=substr(filename,RSTART+RLENGTH)
 430         d("Before number: " prever)
 431         d("and after: " postver)
 432         newurl=find_mirror(acc "://" host dir)
 433         #print acc "://" host dir
 434         #newurl=url[1]"://"url[2]url[3]url[4]
 435         #newurl=acc "://" host dir filename
 436         d("Looking at " newurl)
 437
 438         references=0
 439         finished=0
 440         oldversion=version
 441         odp=get_links(newurl,filename)
 442         if( odp ~ "ERROR: ") {
 443                 print name "(" number ") " odp
 444         } else {
 445                 d("WebPage downloaded")
 446                 c=split(odp,linki)
 447                 for (nr=1; nr<=c; nr++) {
 448                         addr=linki[nr]
 449
 450                         d("Found link: " addr)
 451
 452                         # github has very different tarball links that clash with this safe check
 453                         if (!(newurl ~/^(http|https):\/\/github.com\/.*\/tarball/)) {
 454                                 if (addr ~ "[-_.0-9A-Za-z~]" filenameexp) {
 455                                         continue
 456                                 }
 457                         }
 458
 459                         if (addr ~ filenameexp) {
 460                                 match(addr,filenameexp)
 461                                 newfilename=substr(addr,RSTART,RLENGTH)
 462                                 d("Hypothetical new: " newfilename)
 463                                 newfilename=fixedsub(prever,"",newfilename)
 464                                 newfilename=fixedsub(postver,"",newfilename)
 465                                 d("Version: " newfilename)
 466                                 if (newfilename ~ /\.(asc|sig|pkg|bin|binary|built)$/) continue
 467                                 # strip ending (happens when in directiory name as version matching mode)
 468                                 sub("(\.tar\.(bz|bz2|gz)|zip)$","",newfilename)
 469                                 if (NUMERIC) {
 470                                         if ( compare_ver_dec(version, newfilename)==1 ) {
 471                                                 d("Yes, there is new one")
 472                                                 version=newfilename
 473                                                 finished=1
 474                                         }
 475                                 } else if ( compare_ver(version, newfilename)==1 ) {
 476                                         d("Yes, there is new one")
 477                                         version=newfilename
 478                                         finished=1
 479                                 }
 480                         }
 481                 }
 482                 if (finished==0)
 483                         print name "(" number ") seems ok: " oldversion
 484                 else
 485                         print name "(" number ") [OLD] " oldversion " [NEW] " version
 486         }
 487 }
 488
 489 # upgrade check for pear package using PEAR CLI
 490 function pear_upgrade(name, ver,    pname, pearcmd, nver) {
 491         pname = name;
 492         sub(/^php-pear-/, "", pname);
 493
 494         pearcmd = "pear remote-info " pname " | awk '/^Latest/{print $NF}'"
 495         d("pearcmd: " pearcmd)
 496         pearcmd | getline nver
 497         close(pearcmd)
 498
 499         if (compare_ver(ver, nver)) {
 500                 print name " [OLD] " ver " [NEW] " nver
 501         } else {
 502                 print name " seems ok: " ver
 503         }
 504
 505         return
 506 }
 507
 508 function vim_upgrade(name, ver,     mver, nver, vimcmd) {
 509         # %patchset_source -f ftp://ftp.vim.org/pub/editors/vim/patches/7.2/7.2.%03g 1 %{patchlevel}
 510         mver = substr(ver, 0, 4)
 511         vimcmd = "wget -q -O - ftp://ftp.vim.org/pub/editors/vim/patches/"mver"/MD5SUMS|grep -vF .gz|tail -n1|awk '{print $2}'"
 512         d("vimcmd: " vimcmd)
 513         vimcmd | getline nver
 514         close(vimcmd)
 515
 516         if (compare_ver(ver, nver)) {
 517                 print name " [OLD] " ver " [NEW] " nver
 518         } else {
 519                 print name " seems ok: " ver
 520         }
 521 }
 522
 523 function process_data(name,ver,rel,src) {
 524         if (name ~ /^php-pear-/) {
 525                 return pear_upgrade(name, ver);
 526         }
 527         if (name == "vim") {
 528                 return vim_upgrade(name, ver);
 529         }
 530
 531 # this function checks if substitutions were valid, and if true:
 532 # processes each URL and tries to get current file list
 533         for (i in src) {
 534                 if ( src[i] ~ /%{nil}/ ) {
 535                         gsub(/\%\{nil\}/, "", src[i])
 536                 }
 537                 if ( src[i] !~ /%{.*}/ && src[i] !~ /%[A-Za-z0-9_]/ )  {
 538                         d("Source: " src[i])
 539                         process_source(i,src[i],name,ver)
 540                 } else {
 541                         print FNAME ":" i ": impossible substitution: " src[i]
 542                 }
 543         }
 544 }
 545
 546 BEGIN {
 547         # if U want to use DEBUG, run script with "-v DEBUG=1"
 548         # or uncomment the line below
 549         # DEBUG = 1
 550
 551         errno=system("wget --help > /dev/null 2>&1")
 552         if (errno) {
 553                 print "No wget installed!"
 554                 exit 1
 555         }
 556         if (ARGC>=3 && ARGV[2]=="-n") {
 557                 NUMERIC=1
 558                 for (i=3; i<ARGC; i++) ARGV[i-1]=ARGV[i]
 559                 ARGC=ARGC-1
 560         }
 561 }
 562
 563 FNR==1 {
 564         if ( ARGIND != 1 ) {
 565                 # clean frameseen for each ARG
 566                 for (i in frameseen) {
 567                         delete frameseen[i]
 568                 }
 569                 frameseen[0] = 1
 570
 571                 process_data(NAME,VER,REL,SRC)
 572                 NAME="" ; VER="" ; REL=""
 573                 for (i in DEFS) delete DEFS[i]
 574                 for (i in SRC) delete SRC[i]
 575         }
 576         FNAME=FILENAME
 577         DEFS["_alt_kernel"]=""
 578         DEFS["20"]="\\ "
 579 }
 580
 581 /^[Uu][Rr][Ll]:/&&(URL=="") { URL=subst_defines($2,DEFS) ; DEFS["url"]=URL }
 582 /^[Nn]ame:/&&(NAME=="") { NAME=subst_defines($2,DEFS) ; DEFS["name"]=NAME }
 583 /^[Vv]ersion:/&&(VER=="") { VER=subst_defines($2,DEFS) ; DEFS["version"]=VER }
 584 /^[Rr]elease:/&&(REL=="") { REL=subst_defines($2,DEFS) ; DEFS["release"]=REL }
 585 /^[Ss]ource[0-9]*:/ { if (/(ftp|http|https):\/\//) SRC[FNR]=subst_defines($2,DEFS) }
 586 /%define/ { DEFS[$2]=subst_defines($3,DEFS) }
 587
 588 END {
 589         process_data(NAME,VER,REL,SRC)
 590 }