- add alternative passive-ftp; untested

[projects/distfiles.git] / file-fetcher.pl

diff --git a/file-fetcher.pl b/file-fetcher.pl
index c7495237808eeeabcde5910f2a3cab5bb6ff0d02..8b299a25af999dce60aaf1eb8714d645497f2b7d 100644 (file)
--- a/file-fetcher.pl
+++ b/file-fetcher.pl
@@ -58,7 +58,7 @@ sub read_spool_file()
       $problems .= $_;
       next;
     }
-    /^([a-f0-9]{32})\s+((ftp|http|no-url|no-url-copy):\/\/([a-z0-9A-Z:\+\~\.\-\/_]|\%[0-9])+)\s*$/ 
+    /^([a-f0-9]{32})\s+((ftp|http|no-url|no-url-copy):\/\/([=\?a-z0-9A-Z:\+\~\.\-\/_]|\%[0-9])+)\s*$/ 
        or die "$file: corrupted";
     push @md5, $1;
     $url{$1} = $2;
@@ -195,9 +195,11 @@ sub fetch_file($$)
 {
   my ($md5, $url) = @_;
   my $out = "";
+  my $all_out = "";
   my $bn = basename($url);
   my $local = "$tmp_dir/$md5/$bn";
   my $cmd = "wget -nv -O $local \"$url\"";
+  my $cmd2 = "wget -nv --passive-ftp -O $local \"$url\"";
 
   push @files, $bn;
 
@@ -218,6 +220,7 @@ sub fetch_file($$)
   
   open(W, "$cmd 2>&1 |");
   while (<W>) {
+    $all_out .= $_;
     /URL:.*\s+\-\>\s+.*/ and next;
     $out .= $_;
   }
@@ -225,15 +228,39 @@ sub fetch_file($$)
   if ($out ne "") {
     $problems .= "$cmd:\n$out\n\n";
   }
+  if (-f $local && -s $local > 0 && $url =~ /^ftp:/) {
+    open(W, "$cmd2 2>&1 |");
+    while (<W>) {
+      $all_out .= $_;
+      /URL:.*\s+\-\>\s+.*/ and next;
+      $out .= $_;
+    }
+    close(W);
+    if ($out ne "") {
+      $problems .= "$cmd:\n$out\n\n";
+    }
+  }
   if (-f $local && -s $local > 0) {
     my $computed_md5 = md5($local);
     if ($computed_md5 ne $md5) {
       $problems .= "FATAL: $url md5 mismatch, needed $md5, got $computed_md5\n";
     } else {
-      move_file($md5, $url, $local);
+      my $testcmd = "file \"$local\" |";
+      my $testres = "";
+      if ($url =~ /^http:/ && $local =~ /\.(tar\.(bz2|gz)|tgz|zip|jar|xpi)$/) {
+        open(T, $testcmd) or die;
+        $testres = <T>;
+        close(T);
+      }
+      if ($testres =~ /empty|(ASCII|HTML|SGML).*text/) {
+        $testres =~ s/.*://;
+        $problems .= "FATAL: data returned from $url: $testres";
+      } else {
+        move_file($md5, $url, $local);
+      }
     }
   } else {
-    $problems .= "FATAL: $url ($md5) was not fetched\n";
+    $problems .= "FATAL: $url ($md5) was not fetched ($cmd: $all_out)\n";
   }
   # save space
   unlink($local);
@@ -296,6 +323,8 @@ sub clean_tmp_dir()
     if ($tmp_dir ne "" && -d $tmp_dir);
 }
 
+umask(002);
+
 find_file_in_spool();
 read_spool_file();
 make_tmp_dir();