diff -urN apt-0.3.19cnc52.org/apt-pkg/acquire-method.cc apt-0.3.19cnc52/apt-pkg/acquire-method.cc --- apt-0.3.19cnc52.org/apt-pkg/acquire-method.cc Thu Sep 6 22:20:42 2001 +++ apt-0.3.19cnc52/apt-pkg/acquire-method.cc Sat Sep 8 13:18:36 2001 @@ -97,7 +97,8 @@ if (Queue != 0) { snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" - "Message: %s\n",Queue->Uri.c_str(),Err.c_str()); + "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(), + FailExtra.c_str()); // Dequeue FetchItem *Tmp = Queue; @@ -108,7 +109,8 @@ } else snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" - "Message: %s\n",Err.c_str()); + "Message: %s %s\n",Err.c_str(), + FailExtra.c_str()); // Set the transient flag if (Transient == true) diff -urN apt-0.3.19cnc52.org/apt-pkg/acquire-method.h apt-0.3.19cnc52/apt-pkg/acquire-method.h --- apt-0.3.19cnc52.org/apt-pkg/acquire-method.h Thu Sep 6 22:20:42 2001 +++ apt-0.3.19cnc52/apt-pkg/acquire-method.h Sat Sep 8 13:15:34 2001 @@ -50,6 +50,7 @@ vector Messages; FetchItem *Queue; FetchItem *QueueBack; + string FailExtra; // Handlers for messages virtual bool Configuration(string Message); @@ -75,6 +76,7 @@ void Status(const char *Format,...); int Run(bool Single = false); + inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;}; pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff -urN apt-0.3.19cnc52.org/apt-pkg/contrib/strutl.cc apt-0.3.19cnc52/apt-pkg/contrib/strutl.cc --- apt-0.3.19cnc52.org/apt-pkg/contrib/strutl.cc Thu Sep 6 22:20:42 2001 +++ apt-0.3.19cnc52/apt-pkg/contrib/strutl.cc Sat Sep 8 12:54:40 2001 @@ -861,7 +861,29 @@ return Hits; } /*}}}*/ - +// CheckDomainList - See if Host is in a , seperate list /*{{{*/ +// --------------------------------------------------------------------- +/* The domain list is a comma seperate list of domains that are suffix + matched against the argument */ +bool CheckDomainList(string Host,string List) +{ + string::const_iterator Start = List.begin(); + for (string::const_iterator Cur = List.begin(); Cur <= List.end(); Cur++) + { + if (Cur < List.end() && *Cur != ',') + continue; + + // Match the end of the string.. + if ((Host.size() >= (unsigned)(Cur - Start)) && + Cur - Start != 0 && + stringcasecmp(Host.end() - (Cur - Start),Host.end(),Start,Cur) == 0) + return true; + + Start = Cur + 1; + } + return false; +} + /*}}}*/ // URI::CopyFrom - Copy from an object /*{{{*/ // --------------------------------------------------------------------- /* This parses the URI into all of its components */ diff -urN apt-0.3.19cnc52.org/apt-pkg/contrib/strutl.h apt-0.3.19cnc52/apt-pkg/contrib/strutl.h --- apt-0.3.19cnc52.org/apt-pkg/contrib/strutl.h Thu Sep 6 22:20:42 2001 +++ apt-0.3.19cnc52/apt-pkg/contrib/strutl.h Sat Sep 8 12:53:14 2001 @@ -45,6 +45,7 @@ unsigned int Length); bool TokSplitString(char Tok,char *Input,char **List, unsigned long ListMax); +bool CheckDomainList(string Host,string List); int stringcmp(const char *A,const char *AEnd,const char *B,const char *BEnd); inline int stringcmp(const char *A,const char *AEnd,const char *B) {return stringcmp(A,AEnd,B,B+strlen(B));}; diff -urN apt-0.3.19cnc52.org/methods/connect.cc apt-0.3.19cnc52/methods/connect.cc --- apt-0.3.19cnc52.org/methods/connect.cc Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/connect.cc Tue Feb 20 08:03:18 2001 @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### Connect - Replacement connect call @@ -30,6 +30,18 @@ static struct addrinfo *LastHostAddr = 0; static struct addrinfo *LastUsed = 0; +// RotateDNS - Select a new server from a DNS rotation /*{{{*/ +// --------------------------------------------------------------------- +/* This is called during certain errors in order to recover by selecting a + new server */ +void RotateDNS() +{ + if (LastUsed != 0 && LastUsed->ai_next != 0) + LastUsed = LastUsed->ai_next; + else + LastUsed = LastHostAddr; +} + /*}}}*/ // DoConnect - Attempt a connect operation /*{{{*/ // --------------------------------------------------------------------- /* This helper function attempts a connection to a single address. */ @@ -38,28 +50,44 @@ { // Show a status indicator char Name[NI_MAXHOST]; - Name[0] = 0; + char Service[NI_MAXSERV]; + + Name[0] = 0; + Service[0] = 0; getnameinfo(Addr->ai_addr,Addr->ai_addrlen, - Name,sizeof(Name),0,0,NI_NUMERICHOST); + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); Owner->Status("Connecting to %s (%s)",Host.c_str(),Name); - + + /* If this is an IP rotation store the IP we are using.. If something goes + wrong this will get tacked onto the end of the error message */ + if (LastHostAddr->ai_next != 0) + { + char Name2[NI_MAXHOST + NI_MAXSERV + 10]; + snprintf(Name2,sizeof(Name2),"[IP: %s %s]",Name,Service); + Owner->SetFailExtraMsg(string(Name2)); + } + else + Owner->SetFailExtraMsg(""); + // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, Addr->ai_protocol)) < 0) - return _error->Errno("socket","Could not create a socket"); + return _error->Errno("socket","Could not create a socket for %s (f=%u t=%u p=%u)", + Name,Addr->ai_family,Addr->ai_socktype,Addr->ai_protocol); SetNonBlock(Fd,true); if (connect(Fd,Addr->ai_addr,Addr->ai_addrlen) < 0 && errno != EINPROGRESS) return _error->Errno("connect","Cannot initiate the connection " - "to %s (%s).",Host.c_str(),Name); + "to %s:%s (%s).",Host.c_str(),Service,Name); /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) - return _error->Error("Could not connect to %s (%s), " - "connection timed out",Host.c_str(),Name); - + return _error->Error("Could not connect to %s:%s (%s), " + "connection timed out",Host.c_str(),Service,Name); + // Check the socket for an error condition unsigned int Err; unsigned int Len = sizeof(Err); @@ -67,8 +95,12 @@ return _error->Errno("getsockopt","Failed"); if (Err != 0) - return _error->Error("Could not connect to %s (%s).",Host.c_str(),Name); - + { + errno = Err; + return _error->Errno("connect","Could not connect to %s:%s (%s).",Host.c_str(), + Service,Name); + } + return true; } /*}}}*/ @@ -80,6 +112,13 @@ { if (_error->PendingError() == true) return false; + + // Convert the port name/number + char ServStr[300]; + if (Port != 0) + snprintf(ServStr,sizeof(ServStr),"%u",Port); + else + snprintf(ServStr,sizeof(ServStr),"%s",Service); /* We used a cached address record.. Yes this is against the spec but the way we have setup our rotating dns suggests that this is more @@ -88,59 +127,49 @@ { Owner->Status("Connecting to %s",Host.c_str()); - // Lookup the host - char S[300]; - if (Port != 0) - snprintf(S,sizeof(S),"%u",Port); - else - snprintf(S,sizeof(S),"%s",Service); - // Free the old address structure if (LastHostAddr != 0) { freeaddrinfo(LastHostAddr); LastHostAddr = 0; + LastUsed = 0; } // We only understand SOCK_STREAM sockets. struct addrinfo Hints; memset(&Hints,0,sizeof(Hints)); Hints.ai_socktype = SOCK_STREAM; - Hints.ai_protocol = IPPROTO_TCP; // Right? + Hints.ai_protocol = 0; // Resolve both the host and service simultaneously while (1) { int Res; - if ((Res = getaddrinfo(Host.c_str(),S,&Hints,&LastHostAddr)) != 0 || + if ((Res = getaddrinfo(Host.c_str(),ServStr,&Hints,&LastHostAddr)) != 0 || LastHostAddr == 0) { if (Res == EAI_NONAME || Res == EAI_SERVICE) { if (DefPort != 0) { - snprintf(S,sizeof(S),"%u",DefPort); + snprintf(ServStr,sizeof(ServStr),"%u",DefPort); DefPort = 0; continue; } return _error->Error("Could not resolve '%s'",Host.c_str()); } - return _error->Error("Something wicked happend resolving '%s/%s'", - Host.c_str(),S); + return _error->Error("Something wicked happened resolving '%s:%s' (%i)", + Host.c_str(),ServStr,Res); } break; } - if (LastHostAddr->ai_family == AF_UNIX) - return _error->Error("getaddrinfo returned a unix domain socket\n"); - LastHost = Host; LastPort = Port; - LastUsed = 0; } - // Get the printable IP address + // When we have an IP rotation stay with the last IP. struct addrinfo *CurHost = LastHostAddr; if (LastUsed != 0) CurHost = LastUsed; @@ -155,14 +184,28 @@ close(Fd); Fd = -1; - CurHost = CurHost->ai_next; - LastUsed = 0; + // Ignore UNIX domain sockets + do + { + CurHost = CurHost->ai_next; + } + while (CurHost != 0 && CurHost->ai_family == AF_UNIX); + + /* If we reached the end of the search list then wrap around to the + start */ + if (CurHost == 0 && LastUsed != 0) + CurHost = LastHostAddr; + + // Reached the end of the search cycle + if (CurHost == LastUsed) + break; + if (CurHost != 0) _error->Discard(); - } - + } + if (_error->PendingError() == true) - return false; - return _error->Error("Unable to connect to '%s'",Host.c_str()); + return false; + return _error->Error("Unable to connect to %s %s:",Host.c_str(),ServStr); } /*}}}*/ diff -urN apt-0.3.19cnc52.org/methods/connect.h apt-0.3.19cnc52/methods/connect.h --- apt-0.3.19cnc52.org/methods/connect.h Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/connect.h Tue Feb 20 08:03:18 2001 @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### Connect - Replacement connect call @@ -15,5 +15,6 @@ bool Connect(string To,int Port,const char *Service,int DefPort, int &Fd,unsigned long TimeOut,pkgAcqMethod *Owner); +void RotateDNS(); #endif diff -urN apt-0.3.19cnc52.org/methods/ftp.cc apt-0.3.19cnc52/methods/ftp.cc --- apt-0.3.19cnc52.org/methods/ftp.cc Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/ftp.cc Sat Sep 8 12:33:40 2001 @@ -1,9 +1,9 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### - HTTP Aquire Method - This is the FTP aquire method for APT. + FTP Aquire Method - This is the FTP aquire method for APT. This is a very simple implementation that does not try to optimize at all. Commands are sent syncronously with the FTP server (as the @@ -28,6 +28,7 @@ #include #include #include +#include // Internet stuff #include @@ -40,6 +41,22 @@ #include "ftp.h" /*}}}*/ +using namespace std; + +/* This table is for the EPRT and EPSV commands, it maps the OS address + family to the IETF address families */ +struct AFMap +{ + unsigned long Family; + unsigned long IETFFamily; +}; + +#ifndef AF_INET6 +struct AFMap AFMap[] = {{AF_INET,1},{}}; +#else +struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}}; +#endif + unsigned long TimeOut = 120; URI Proxy; string FtpMethod::FailFile; @@ -53,7 +70,7 @@ DataListenFd(-1), ServerName(Srv) { Debug = _config->FindB("Debug::Acquire::Ftp",false); - memset(&PasvAddr,0,sizeof(PasvAddr)); + PasvAddr = 0; } /*}}}*/ // FTPConn::~FTPConn - Destructor /*{{{*/ @@ -75,7 +92,10 @@ DataFd = -1; close(DataListenFd); DataListenFd = -1; - memset(&PasvAddr,0,sizeof(PasvAddr)); + + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; } /*}}}*/ // FTPConn::Open - Open a new connection /*{{{*/ @@ -89,7 +109,7 @@ return true; Close(); - + // Determine the proxy setting if (getenv("ftp_proxy") == 0) { @@ -108,32 +128,13 @@ else Proxy = getenv("ftp_proxy"); - // Parse no_proxy, a , seperated list of domains + // Parse no_proxy, a , separated list of domains if (getenv("no_proxy") != 0) { - const char *Start = getenv("no_proxy"); - const char *ServerEnd = ServerName.Host.end(); - - for (const char *Cur = Start; true ; Cur++) - { - if (*Cur != ',' && *Cur != 0) - continue; - - // match end of the string - if ((ServerName.Host.size() >= (Cur - Start)) - && stringcasecmp(ServerEnd - (Cur - Start), - ServerEnd,Start,Cur) == 0) - { - Proxy = ""; - break; - } - - Start = Cur + 1; - if (*Cur == 0) - break; - } + if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) + Proxy = ""; } - + // Determine what host and port to use based on the proxy settings int Port = 0; string Host; @@ -150,15 +151,27 @@ Host = Proxy.Host; } - // Connect to the remote server + /* Connect to the remote server. Since FTP is connection oriented we + want to make sure we get a new server every time we reconnect */ + RotateDNS(); if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false) return false; - socklen_t Len = sizeof(Peer); - if (getpeername(ServerFd,(sockaddr *)&Peer,&Len) != 0) + + // Login must be before getpeername otherwise dante won't work. + Owner->Status("Logging in"); + bool Res = Login(); + + // Get the remote server's address + PeerAddrLen = sizeof(PeerAddr); + if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0) return _error->Errno("getpeername","Unable to determine the peer name"); - Owner->Status("Logging in"); - return Login(); + // Get the local machine's address + ServerAddrLen = sizeof(ServerAddr); + if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0) + return _error->Errno("getsockname","Unable to determine the local name"); + + return Res; } /*}}}*/ // FTPConn::Login - Login to the remote server /*{{{*/ @@ -172,11 +185,7 @@ // Setup the variables needed for authentication string User = "anonymous"; - string Pass; - if (0) //akk - Pass = "apt_get_ftp_2.0@debian.linux.user"; - else - Pass = "apt_get_ftp_2.0@conectiva.linux.user"; + string Pass = "apt_get_ftp_2.0@pld.linux.user"; // Fill in the user/pass if (ServerName.User.empty() == false) @@ -209,7 +218,7 @@ if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true) TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true); else - TryPassive = _config->FindB("Acquire::FTP::Passive",true); + TryPassive = _config->FindB("Acquire::FTP::Passive",true); } else { @@ -266,6 +275,12 @@ } } + // Force the use of extended commands + if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true) + ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true); + else + ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false); + // Binary mode if (WriteMsg(Tag,Msg,"TYPE I") == false) return false; @@ -313,6 +328,8 @@ // Suck it back int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len); + if (Res == 0) + _error->Error("Server closed the connection"); if (Res <= 0) { _error->Errno("read","Read error"); @@ -439,10 +456,19 @@ // --------------------------------------------------------------------- /* Try to enter passive mode, the return code does not indicate if passive mode could or could not be established, only if there was a fatal error. - Borrowed mostly from lftp. We have to enter passive mode every time - we make a data connection :| */ + We have to enter passive mode every time we make a data connection :| */ bool FTPConn::GoPasv() { + /* The PASV command only works on IPv4 sockets, even though it could + in theory suppory IPv6 via an all zeros reply */ + if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET || + ForceExtended == true) + return ExtGoPasv(); + + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; + // Try to enable pasv mode unsigned int Tag; string Msg; @@ -452,41 +478,139 @@ // Unsupported function string::size_type Pos = Msg.find('('); if (Tag >= 400 || Pos == string::npos) - { - memset(&PasvAddr,0,sizeof(PasvAddr)); return true; - } // Scan it unsigned a0,a1,a2,a3,p0,p1; if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6) + return true; + + /* Some evil servers return 0 to mean their addr. We can actually speak + to these servers natively using IPv6 */ + if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0) { - memset(&PasvAddr,0,sizeof(PasvAddr)); + // Get the IP in text form + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family; + Hints.ai_flags |= AI_NUMERICHOST; + + // Get a new passive address. + char Port[100]; + snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1); + if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0) + return true; return true; } - // lftp used this horrid byte order manipulation.. Ik. - PasvAddr.sin_family = AF_INET; - unsigned char *a; - unsigned char *p; - a = (unsigned char *)&PasvAddr.sin_addr; - p = (unsigned char *)&PasvAddr.sin_port; + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_family = AF_INET; + Hints.ai_flags |= AI_NUMERICHOST; + + // Get a new passive address. + char Port[100]; + snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1); + char Name[100]; + snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3); + if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0) + return true; + return true; +} + /*}}}*/ +// FTPConn::ExtGoPasv - Enter Extended Passive mode /*{{{*/ +// --------------------------------------------------------------------- +/* Try to enter extended passive mode. See GoPasv above and RFC 2428 */ +bool FTPConn::ExtGoPasv() +{ + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; - // Some evil servers return 0 to mean their addr - if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0) + // Try to enable pasv mode + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"EPSV") == false) + return false; + + // Unsupported function + string::size_type Pos = Msg.find('('); + if (Tag >= 400 || Pos == string::npos) + return true; + + // Scan it + string::const_iterator List[4]; + unsigned Count = 0; + Pos++; + for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); I++) { - PasvAddr.sin_addr = Peer.sin_addr; + if (*I != Msg[Pos]) + continue; + if (Count >= 4) + return true; + List[Count++] = I; + } + if (Count != 4) + return true; + + // Break it up .. + unsigned long Proto = 0; + unsigned long Port = 0; + string IP; + IP = string(List[1]+1,List[2]); + Port = atoi(string(List[2]+1,List[3]).c_str()); + if (IP.empty() == false) + Proto = atoi(string(List[0]+1,List[1]).c_str()); + + if (Port == 0) + return false; + + // String version of the port + char PStr[100]; + snprintf(PStr,sizeof(PStr),"%lu",Port); + + // Get the IP in text form + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_flags |= AI_NUMERICHOST; + + /* The RFC defined case, connect to the old IP/protocol using the + new port. */ + if (IP.empty() == true) + { + // Get the IP in text form + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + IP = Name; + Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family; } else { - a[0] = a0; - a[1] = a1; - a[2] = a2; - a[3] = a3; + // Get the family.. + Hints.ai_family = 0; + for (unsigned J = 0; AFMap[J].Family != 0; J++) + if (AFMap[J].IETFFamily == Proto) + Hints.ai_family = AFMap[J].Family; + if (Hints.ai_family == 0) + return true; } - p[0] = p0; - p[1] = p1; + // Get a new passive address. + int Res; + if ((Res = getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr)) != 0) + return true; return true; } @@ -547,34 +671,35 @@ return false; // Oops, didn't work out, don't bother trying again. - if (PasvAddr.sin_port == 0) + if (PasvAddr == 0) TryPassive = false; } // Passive mode? - if (PasvAddr.sin_port != 0) + if (PasvAddr != 0) { // Get a socket - if ((DataFd = socket(AF_INET,SOCK_STREAM,0)) < 0) + if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype, + PasvAddr->ai_protocol)) < 0) return _error->Errno("socket","Could not create a socket"); // Connect to the server SetNonBlock(DataFd,true); - if (connect(DataFd,(sockaddr *)&PasvAddr,sizeof(PasvAddr)) < 0 && + if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 && errno != EINPROGRESS) return _error->Errno("socket","Could not create a socket"); /* This implements a timeout for connect by opening the connection nonblocking */ - if (WaitFd(ServerFd,true,TimeOut) == false) + if (WaitFd(DataFd,true,TimeOut) == false) return _error->Error("Could not connect data socket, connection timed out"); unsigned int Err; unsigned int Len = sizeof(Err); - if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0) + if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0) return _error->Errno("getsockopt","Failed"); if (Err != 0) - return _error->Error("Could not connect."); - + return _error->Error("Could not connect passive socket."); + return true; } @@ -582,43 +707,91 @@ close(DataListenFd); DataListenFd = -1; - // Get a socket - if ((DataListenFd = socket(AF_INET,SOCK_STREAM,0)) < 0) + // Get the information for a listening socket. + struct addrinfo *BindAddr = 0; + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_flags |= AI_PASSIVE; + Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family; + int Res; + if ((Res = getaddrinfo(0,"0",&Hints,&BindAddr)) != 0) + return _error->Error("getaddrinfo was unable to get a listening socket"); + + // Construct the socket + if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype, + BindAddr->ai_protocol)) < 0) + { + freeaddrinfo(BindAddr); return _error->Errno("socket","Could not create a socket"); + } // Bind and listen - sockaddr_in Addr; - memset(&Addr,0,sizeof(Addr)); - if (bind(DataListenFd,(sockaddr *)&Addr,sizeof(Addr)) < 0) + if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0) + { + freeaddrinfo(BindAddr); return _error->Errno("bind","Could not bind a socket"); + } + freeaddrinfo(BindAddr); if (listen(DataListenFd,1) < 0) return _error->Errno("listen","Could not listen on the socket"); SetNonBlock(DataListenFd,true); // Determine the name to send to the remote - sockaddr_in Addr2; - socklen_t Jnk = sizeof(Addr); - if (getsockname(DataListenFd,(sockaddr *)&Addr,&Jnk) < 0) - return _error->Errno("getsockname","Could not determine the socket's name"); - Jnk = sizeof(Addr2); - if (getsockname(ServerFd,(sockaddr *)&Addr2,&Jnk) < 0) + struct sockaddr_storage Addr; + socklen_t AddrLen = sizeof(Addr); + if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0) return _error->Errno("getsockname","Could not determine the socket's name"); - - // This bit ripped from qftp - unsigned long badr = ntohl(*(unsigned long *)&Addr2.sin_addr); - unsigned long bp = ntohs(Addr.sin_port); - // Send the port command + // Reverse the address. We need the server address and the data port. + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + char Service2[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&Addr,AddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen, + Name,sizeof(Name),Service2,sizeof(Service2), + NI_NUMERICHOST|NI_NUMERICSERV); + + // Send off an IPv4 address in the old port format + if (((struct sockaddr *)&Addr)->sa_family == AF_INET && + ForceExtended == false) + { + // Convert the dots in the quad into commas + for (char *I = Name; *I != 0; I++) + if (*I == '.') + *I = ','; + unsigned long Port = atoi(Service); + + // Send the port command + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"PORT %s,%d,%d", + Name, + (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false) + return false; + if (Tag >= 400) + return _error->Error("Unable to send PORT command"); + return true; + } + + // Construct an EPRT command + unsigned Proto = 0; + for (unsigned J = 0; AFMap[J].Family != 0; J++) + if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family) + Proto = AFMap[J].IETFFamily; + if (Proto == 0) + return _error->Error("Unkonwn address family %u (AF_*)", + ((struct sockaddr *)&Addr)->sa_family); + + // Send the EPRT command unsigned int Tag; string Msg; - if (WriteMsg(Tag,Msg,"PORT %d,%d,%d,%d,%d,%d", - (int) (badr >> 24) & 0xff, (int) (badr >> 16) & 0xff, - (int) (badr >> 8) & 0xff, (int) badr & 0xff, - (int) (bp >> 8) & 0xff, (int) bp & 0xff) == false) + if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false) return false; if (Tag >= 400) - return _error->Error("Unable to send port command"); - + return _error->Error("EPRT failed, server said: %s",Msg.c_str()); return true; } /*}}}*/ @@ -629,7 +802,7 @@ bool FTPConn::Finalize() { // Passive mode? Do nothing - if (PasvAddr.sin_port != 0) + if (PasvAddr != 0) return true; // Close any old socket.. @@ -906,18 +1079,20 @@ int main(int argc,const char *argv[]) { - /* See if we should become the http client - we do this for http + /* See if we should be come the http client - we do this for http proxy urls */ if (getenv("ftp_proxy") != 0) { URI Proxy = string(getenv("ftp_proxy")); + // Run the HTTP method if (Proxy.Access == "http") { // Copy over the environment setting char S[300]; snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy")); putenv(S); + putenv("no_proxy="); // Run the http method string Path = flNotFile(argv[0]) + "/http"; diff -urN apt-0.3.19cnc52.org/methods/ftp.h apt-0.3.19cnc52/methods/ftp.h --- apt-0.3.19cnc52.org/methods/ftp.h Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/ftp.h Sat Sep 8 12:35:01 2001 @@ -1,5 +1,6 @@ // -*- mode: cpp; mode: fold -*- -// Description /*{{{*/// $Id$ +// Description /*{{{*/// $Id$ +// $Id$ /* ###################################################################### FTP Aquire Method - This is the FTP aquire method for APT. @@ -17,12 +18,20 @@ int DataFd; int DataListenFd; URI ServerName; + bool ForceExtended; bool TryPassive; bool Debug; - - struct sockaddr_in PasvAddr; - struct sockaddr_in Peer; + struct addrinfo *PasvAddr; + + // Generic Peer Address + struct sockaddr_storage PeerAddr; + socklen_t PeerAddrLen; + + // Generic Server Address (us) + struct sockaddr_storage ServerAddr; + socklen_t ServerAddrLen; + // Private helper functions bool ReadLine(string &Text); bool Login(); @@ -41,6 +50,7 @@ bool Open(pkgAcqMethod *Owner); void Close(); bool GoPasv(); + bool ExtGoPasv(); // Query bool Size(const char *Path,unsigned long &Size); diff -urN apt-0.3.19cnc52.org/methods/http.cc apt-0.3.19cnc52/methods/http.cc --- apt-0.3.19cnc52.org/methods/http.cc Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/http.cc Sat Sep 8 12:41:18 2001 @@ -1,20 +1,18 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### HTTP Aquire Method - This is the HTTP aquire method for APT. It uses HTTP/1.1 and many of the fancy options there-in, such as - pipelining, range, if-range and so on. It accepts on the command line - a list of url destination pairs and writes to stdout the status of the - operation as defined in the APT method spec. - - It is based on a doubly buffered select loop. All the requests are + pipelining, range, if-range and so on. + + It is based on a doubly buffered select loop. A groupe of requests are fed into a single output buffer that is constantly fed out the socket. This provides ideal pipelining as in many cases all of the requests will fit into a single packet. The input socket is buffered - the same way and fed into the fd for the file. + the same way and fed into the fd for the file (may be a pipe in future). This double buffering provides fairly substantial transfer rates, compared to wget the http method is about 4% faster. Most importantly, @@ -39,6 +37,8 @@ #include #include #include +#include +#include // Internet stuff #include @@ -48,6 +48,7 @@ #include "http.h" /*}}}*/ +using namespace std; string HttpMethod::FailFile; int HttpMethod::FailFd = -1; @@ -140,7 +141,7 @@ unsigned long Sz = LeftRead(); if (OutQueue.length() - StrPos < Sz) Sz = OutQueue.length() - StrPos; - memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz); + memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz); // Advance StrPos += Sz; @@ -258,9 +259,6 @@ // ServerState::Open - Open a connection to the server /*{{{*/ // --------------------------------------------------------------------- /* This opens a connection to the server. */ -string LastHost; -int LastPort = 0; -struct addrinfo *LastHostAddr = 0; bool ServerState::Open() { // Use the already open connection if possible. @@ -270,7 +268,8 @@ Close(); In.Reset(); Out.Reset(); - + Persistent = true; + // Determine the proxy setting if (getenv("http_proxy") == 0) { @@ -289,32 +288,13 @@ else Proxy = getenv("http_proxy"); - // Parse no_proxy, a , seperated list of domains + // Parse no_proxy, a , separated list of domains if (getenv("no_proxy") != 0) { - const char *Start = getenv("no_proxy"); - const char *ServerEnd = ServerName.Host.end(); - - for (const char *Cur = Start; true ; Cur++) - { - if (*Cur != ',' && *Cur != 0) - continue; - - // match end of the string - if ((ServerName.Host.size() >= (Cur - Start)) - && stringcasecmp(ServerEnd - (Cur - Start), - ServerEnd,Start,Cur) == 0) - { - Proxy = ""; - break; - } - - Start = Cur + 1; - if (*Cur == 0) - break; - } + if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) + Proxy = ""; } - + // Determine what host and port to use based on the proxy settings int Port = 0; string Host; @@ -380,14 +360,23 @@ { string::const_iterator J = I; for (; J != Data.end() && *J != '\n' && *J != '\r';J++); - if (HeaderLine(string(I,J-I)) == false) + if (HeaderLine(string(I,J)) == false) return 2; I = J; } + + // 100 Continue is a Nop... + if (Result == 100) + continue; + + // Tidy up the connection persistance state. + if (Encoding == Closes && HaveContent == true) + Persistent = false; + return 0; } while (Owner->Go(false,this) == true); - + return 1; } /*}}}*/ @@ -513,7 +502,7 @@ string Tag = string(Line,0,Pos); string Val = string(Line,Pos2); - if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0) + if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0) { // Evil servers return no version if (Line[4] == '/') @@ -529,7 +518,19 @@ if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2) return _error->Error("The http server sent an invalid reply header"); } - + + /* Check the HTTP response header to get the default persistance + state. */ + if (Major < 1) + Persistent = false; + else + { + if (Major == 1 && Minor <= 0) + Persistent = false; + else + Persistent = true; + } + return true; } @@ -569,11 +570,19 @@ { HaveContent = true; if (stringcasecmp(Val,"chunked") == 0) - Encoding = Chunked; - + Encoding = Chunked; return true; } + if (stringcasecmp(Tag,"Connection:") == 0) + { + if (stringcasecmp(Val,"close") == 0) + Persistent = false; + if (stringcasecmp(Val,"keep-alive") == 0) + Persistent = true; + return true; + } + if (stringcasecmp(Tag,"Last-Modified:") == 0) { if (StrToTime(Val,Date) == false) @@ -660,10 +669,11 @@ Req += string("Proxy-Authorization: Basic ") + Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n"; - if (0)//akk - Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n"; - else - Req += "User-Agent: Conectiva APT-HTTP/1.2\r\n\r\n"; + if (Uri.User.empty() == false || Uri.Password.empty() == false) + Req += string("Authorization: Basic ") + + Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n"; + + Req += "User-Agent: PLD APT-HTTP/1.2\r\n\r\n"; if (Debug == true) cerr << Req << endl; @@ -686,10 +696,12 @@ FD_ZERO(&rfds); FD_ZERO(&wfds); - // Add the server - if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1) + /* Add the server. We only send more requests if the connection will + be persisting */ + if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1 + && Srv->Persistent == true) FD_SET(Srv->ServerFd,&wfds); - if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1) + if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1) FD_SET(Srv->ServerFd,&rfds); // Add the file @@ -714,7 +726,11 @@ tv.tv_usec = 0; int Res = 0; if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0) + { + if (errno == EINTR) + return true; return _error->Errno("select","Select failed"); + } if (Res == 0) { @@ -1007,7 +1023,15 @@ delete Server; Server = new ServerState(Queue->Uri,this); } - + + /* If the server has explicitly said this is the last connection + then we pre-emptively shut down the pipeline and tear down + the connection. This will speed up HTTP/1.0 servers a tad + since we don't have to wait for the close sequence to + complete */ + if (Server->Persistent == false) + Server->Close(); + // Reset the pipeline if (Server->ServerFd == -1) QueueBack = Queue; @@ -1035,6 +1059,7 @@ { _error->Error("Bad header Data"); Fail(true); + RotateDNS(); continue; } @@ -1053,6 +1078,7 @@ FailCounter = 0; } + RotateDNS(); continue; } }; @@ -1070,6 +1096,11 @@ // Run the data bool Result = Server->RunData(); + /* If the server is sending back sizeless responses then fill in + the size now */ + if (Res.Size == 0) + Res.Size = File->Size(); + // Close the file, destroy the FD object and timestamp it FailFd = -1; delete File; @@ -1090,7 +1121,7 @@ } else Fail(true); - + break; } @@ -1112,6 +1143,7 @@ case 5: { Fail(); + RotateDNS(); Server->Close(); break; } diff -urN apt-0.3.19cnc52.org/methods/http.h apt-0.3.19cnc52/methods/http.h --- apt-0.3.19cnc52.org/methods/http.h Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/http.h Sat Sep 8 12:42:54 2001 @@ -1,5 +1,6 @@ // -*- mode: cpp; mode: fold -*- -// Description /*{{{*/// $Id$ +// Description /*{{{*/// $Id$ +// $Id$ /* ###################################################################### HTTP Aquire Method - This is the HTTP aquire method for APT. @@ -12,6 +13,11 @@ #define MAXLEN 360 +#include + +using std::cout; +using std::endl; + class HttpMethod; class CircleBuf @@ -87,6 +93,9 @@ bool HaveContent; enum {Chunked,Stream,Closes} Encoding; enum {Header, Data} State; + bool Persistent; + + // This is a Persistent attribute of the server itself. bool Pipeline; HttpMethod *Owner; @@ -130,7 +139,7 @@ static void SigTerm(int); public: - friend ServerState; + friend class ServerState; FileFd *File; ServerState *Server; diff -urN apt-0.3.19cnc52.org/methods/rfc2553emu.cc apt-0.3.19cnc52/methods/rfc2553emu.cc --- apt-0.3.19cnc52.org/methods/rfc2553emu.cc Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/rfc2553emu.cc Tue Feb 20 08:03:18 2001 @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo, @@ -36,20 +36,6 @@ const char *End; char **CurAddr; - Addr = gethostbyname(nodename); - if (Addr == 0) - { - if (h_errno == TRY_AGAIN) - return EAI_AGAIN; - if (h_errno == NO_RECOVERY) - return EAI_FAIL; - return EAI_NONAME; - } - - // No A records - if (Addr->h_addr_list[0] == 0) - return EAI_NONAME; - // Try to convert the service as a number Port = htons(strtol(servname,(char **)&End,0)); Proto = SOCK_STREAM; @@ -86,10 +72,32 @@ hints->ai_socktype != 0) return EAI_SERVICE; } + + // Hostname lookup, only if this is not a listening socket + if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE) + { + Addr = gethostbyname(nodename); + if (Addr == 0) + { + if (h_errno == TRY_AGAIN) + return EAI_AGAIN; + if (h_errno == NO_RECOVERY) + return EAI_FAIL; + return EAI_NONAME; + } + + // No A records + if (Addr->h_addr_list[0] == 0) + return EAI_NONAME; + + CurAddr = Addr->h_addr_list; + } + else + CurAddr = (char **)&End; // Fake! // Start constructing the linked list *res = 0; - for (CurAddr = Addr->h_addr_list; *CurAddr != 0; CurAddr++) + for (; *CurAddr != 0; CurAddr++) { // New result structure *Result = (struct addrinfo *)calloc(sizeof(**Result),1); @@ -124,8 +132,15 @@ // Set the address ((struct sockaddr_in *)(*Result)->ai_addr)->sin_family = AF_INET; ((struct sockaddr_in *)(*Result)->ai_addr)->sin_port = Port; - ((struct sockaddr_in *)(*Result)->ai_addr)->sin_addr = *(in_addr *)(*CurAddr); - + + if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE) + ((struct sockaddr_in *)(*Result)->ai_addr)->sin_addr = *(in_addr *)(*CurAddr); + else + { + // Already zerod by calloc. + break; + } + Result = &(*Result)->ai_next; } @@ -202,9 +217,9 @@ { struct servent *Ent; if ((flags & NI_DATAGRAM) == NI_DATAGRAM) - Ent = getservbyport(sin->sin_port,"udp"); + Ent = getservbyport(ntohs(sin->sin_port),"udp"); else - Ent = getservbyport(sin->sin_port,"tcp"); + Ent = getservbyport(ntohs(sin->sin_port),"tcp"); if (Ent != 0) strncpy(serv,Ent->s_name,servlen); @@ -220,7 +235,7 @@ // Resolve as a plain numberic if ((flags & NI_NUMERICSERV) == NI_NUMERICSERV) { - snprintf(serv,servlen,"%u",sin->sin_port); + snprintf(serv,servlen,"%u",ntohs(sin->sin_port)); } } diff -urN apt-0.3.19cnc52.org/methods/rfc2553emu.h apt-0.3.19cnc52/methods/rfc2553emu.h --- apt-0.3.19cnc52.org/methods/rfc2553emu.h Thu Sep 6 22:20:43 2001 +++ apt-0.3.19cnc52/methods/rfc2553emu.h Sun Jun 18 08:04:45 2000 @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id$ +// $Id$ /* ###################################################################### RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo, @@ -74,6 +74,11 @@ #define EAI_MEMORY -11 #endif + /* If we don't have getaddrinfo then we probably don't have + sockaddr_storage either (same RFC) so we definately will not be + doing any IPv6 stuff. Do not use the members of this structure to + retain portability, cast to a sockaddr. */ + #define sockaddr_storage sockaddr_in #endif // getnameinfo support (glibc2.0 has getaddrinfo only) @@ -97,6 +102,12 @@ #define NI_DATAGRAM (1<<4) #endif + #define sockaddr_storage sockaddr_in +#endif + +// Glibc 2.0.7 misses this one +#ifndef AI_NUMERICHOST +#define AI_NUMERICHOST 0 #endif #endif