- rel 3; use exim-4_86+fixes branch instead of master

[packages/exim.git] / exim-git.patch

commit da1aecfce80fba32b9abb328cf78088d95b7700a
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Wed Aug 19 15:22:41 2015 +0200

    Fix post-transport-crash.
    
    The crash probably was introduced in a39bd74d3e94 and
    needs 'split_spool_directory=yes' to expose.
    
    Thanks to Wolfgang Breyha, who found the same fix.
    
    (cherry picked from commit 6b51df8340eacc95e3def9a4376506610e91996c)

diff --git a/src/src/transport.c b/src/src/transport.c
index fa6f869..a6ad3ed 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -1752,7 +1752,7 @@ while (1)
     {
     if (split_spool_directory)
 	sprintf(CS spool_file, "%s%c/%s-D",
-		      spool_dir, new_message_id[5], msgq[i].message_id);
+		      spool_dir, msgq[i].message_id[5], msgq[i].message_id);
     else
 	sprintf(CS spool_file, "%s%s-D", spool_dir, msgq[i].message_id);
 

commit d7168d8b112d3ba642a25ed04de36fb76d4a847d
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Thu Aug 20 13:58:06 2015 +0200

    Fix post-transport-crash: safeguard for missing spool BUG 1671
    
    Based on a proposal from Wolfgang Breyha.
    
    (cherry picked from commit dadff1d47e54962b0fdf98e8ce5cef42b6cb7fb5)

diff --git a/src/src/deliver.c b/src/src/deliver.c
index 78f8f4b..4154ff7 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -9,6 +9,7 @@
 
 
 #include "exim.h"
+#include <assert.h>
 
 
 /* Data block for keeping track of subprocesses for parallel remote
@@ -7904,17 +7905,36 @@ if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA =
 uschar *
 deliver_get_sender_address (uschar * id)
 {
+int rc;
+uschar * new_sender_address,
+       * save_sender_address;
+
 if (!spool_open_datafile(id))
   return NULL;
 
+/* Save and restore the global sender_address.  I'm not sure if we should
+not save/restore all the other global variables too, because
+spool_read_header() may change all of them. But OTOH, when this
+deliver_get_sender_address() gets called, the current message is done
+already and nobody needs the globals anymore. (HS12, 2015-08-21) */
+
 sprintf(CS spoolname, "%s-H", id);
-if (spool_read_header(spoolname, TRUE, TRUE) != spool_read_OK)
+save_sender_address = sender_address;
+
+rc = spool_read_header(spoolname, TRUE, TRUE);
+
+new_sender_address = sender_address;
+sender_address = save_sender_address;
+
+if (rc != spool_read_OK)
   return NULL;
 
+assert(new_sender_address);
+
 (void)close(deliver_datafile);
 deliver_datafile = -1;
 
-return sender_address;
+return new_sender_address;
 }
 
 /* vi: aw ai sw=2
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index a952413..5ac5533 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1274,14 +1274,19 @@ we will veto this new message.  */
 static BOOL
 smtp_are_same_identities(uschar * message_id, smtp_compare_t * s_compare)
 {
-uschar * save_sender_address = sender_address;
-uschar * current_local_identity =
+
+uschar * message_local_identity,
+       * current_local_identity,
+       * new_sender_address;
+
+current_local_identity =
   smtp_local_identity(s_compare->current_sender_address, s_compare->tblock);
-uschar * new_sender_address = deliver_get_sender_address(message_id);
-uschar * message_local_identity =
-  smtp_local_identity(new_sender_address, s_compare->tblock);
 
-sender_address = save_sender_address;
+if (!(new_sender_address = deliver_get_sender_address(message_id)))
+    return 0;
+
+message_local_identity =
+  smtp_local_identity(new_sender_address, s_compare->tblock);
 
 return Ustrcmp(current_local_identity, message_local_identity) == 0;
 }

commit 9e1acebc3fbb288da804ab9031c7c448dffd841a
Author: Heiko Schlittermann (HS12-RIPE) <hs@schlittermann.de>
Date:   Tue Aug 11 17:36:29 2015 +0200

    Fix ESMTP MAIL command option processing
    
    If the address containes spaces, the option processing
    was confused.
    
    (cherry picked from commit 2ef7ed082481b2dccd3c2e0eae849b24bf0b172a)

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index cf0a5d6..980d54b 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -9,6 +9,7 @@
 
 
 #include "exim.h"
+#include <assert.h>
 
 
 /* Initialize for TCP wrappers if so configured. It appears that the macro
@@ -232,6 +233,7 @@ static uschar *protocols[] = {
 
 /* Sanity check and validate optional args to MAIL FROM: envelope */
 enum {
+  ENV_MAIL_OPT_NULL,
   ENV_MAIL_OPT_SIZE, ENV_MAIL_OPT_BODY, ENV_MAIL_OPT_AUTH,
 #ifndef DISABLE_PRDR
   ENV_MAIL_OPT_PRDR,
@@ -240,7 +242,6 @@ enum {
 #ifdef EXPERIMENTAL_INTERNATIONAL
   ENV_MAIL_OPT_UTF8,
 #endif
-  ENV_MAIL_OPT_NULL
   };
 typedef struct {
   uschar *   name;  /* option requested during MAIL cmd */
@@ -260,7 +261,8 @@ static env_mail_type_t env_mail_type_list[] = {
 #ifdef EXPERIMENTAL_INTERNATIONAL
     { US"SMTPUTF8",ENV_MAIL_OPT_UTF8,  FALSE },		/* rfc6531 */
 #endif
-    { US"NULL",   ENV_MAIL_OPT_NULL,   FALSE }
+    /* keep this the last entry */
+    { US"NULL",   ENV_MAIL_OPT_NULL,   FALSE },
   };
 
 /* When reading SMTP from a remote host, we have to use our own versions of the
@@ -3887,7 +3889,7 @@ while (done <= 0)
       if (!extract_option(&name, &value)) break;
 
       for (mail_args = env_mail_type_list;
-           (char *)mail_args < (char *)env_mail_type_list + sizeof(env_mail_type_list);
+           mail_args->value != ENV_MAIL_OPT_NULL;
            mail_args++
           )
         if (strcmpic(name, mail_args->name) == 0)
@@ -4066,15 +4068,17 @@ while (done <= 0)
 	    }
 	  break;
 #endif
-        /* Unknown option. Stick back the terminator characters and break
+        /* No valid option. Stick back the terminator characters and break
         the loop.  Do the name-terminator second as extract_option sets
-	value==name when it found no equal-sign.
-	An error for a malformed address will occur. */
-        default:
+        value==name when it found no equal-sign.
+        An error for a malformed address will occur. */
+        case ENV_MAIL_OPT_NULL:
           value[-1] = '=';
           name[-1] = ' ';
           arg_error = TRUE;
           break;
+
+        default:  assert(0);
         }
       /* Break out of for loop if switch() had bad argument or
          when start of the email address is reached */

commit 22c82c48aa5d23c49d38e3581810ea54587df61b
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Aug 25 10:36:27 2015 +0100

    Close logs after daemon-process exceptional write.  Bug 728
    
    (cherry picked from commit c8899c20aa08c9ae6a4c291aad23ba90512bebe4)

diff --git a/src/src/daemon.c b/src/src/daemon.c
index 894a96f..a7a49f0 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -735,6 +735,7 @@ else (void)close(dup_accept_socket);
 /* Release any store used in this process, including the store used for holding
 the incoming host address and an expanded active_hostname. */
 
+log_close_all();
 store_reset(reset_point);
 sender_host_address = NULL;
 }

commit 4d3b8805796ffa39889060d9f216bfaf7391c542
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Sep 17 13:35:16 2015 +0100

    DNS: time-limit cached returns, using TTL.  Bug 1395
    
    This can matter for fast-changing data such as DNSBLs.
    
    (cherry picked from commit 14b3c5bc64a16df07583fe4b5ef2e0129d063893)
    
    DNS: avoid overflow in cache TTL for negative entries.  Bug 1395
    
    (cherry picked from commit e162fc9757d4b8cb41aca74214e968622d6c3dee)

diff --git a/src/src/dns.c b/src/src/dns.c
index 64958d9..abed126 100644
--- a/src/src/dns.c
+++ b/src/src/dns.c
@@ -390,7 +390,8 @@ from the following bytes. */
 
 dnss->aptr += namelen;
 GETSHORT(dnss->srr.type, dnss->aptr); /* Record type */
-dnss->aptr += 6;                      /* Don't want class or TTL */
+dnss->aptr += 2;                      /* Don't want class */
+GETLONG(dnss->srr.ttl, dnss->aptr);   /* TTL */
 GETSHORT(dnss->srr.size, dnss->aptr); /* Size of data portion */
 dnss->srr.data = dnss->aptr;          /* The record's data follows */
 dnss->aptr += dnss->srr.size;         /* Advance to next RR */
diff --git a/src/src/lookupapi.h b/src/src/lookupapi.h
index cdd1c85..03de8f6 100644
--- a/src/src/lookupapi.h
+++ b/src/src/lookupapi.h
@@ -34,7 +34,7 @@ typedef struct lookup_info {
     int,                          /* length of key or query */
     uschar **,                    /* for returning answer */
     uschar **,                    /* for error message */
-    BOOL *);                      /* to request cache cleanup */
+    uint *);                      /* cache TTL, sconds */
   void (*close)(                  /* close function */
     void *);                      /* handle */
   void (*tidy)(void);             /* tidy function */
@@ -46,9 +46,10 @@ typedef struct lookup_info {
 } lookup_info;
 
 /* This magic number is used by the following lookup_module_info structure
-   for checking API compatibility. It's equivalent to the string"LMM2" */
-#define LOOKUP_MODULE_INFO_MAGIC 0x4c4d4d32
+   for checking API compatibility. It used to be equivalent to the string"LMM3" */
+#define LOOKUP_MODULE_INFO_MAGIC 0x4c4d4933
 /* Version 2 adds: version_report */
+/* Version 3 change: non/cache becomes TTL in seconds */
 
 typedef struct lookup_module_info {
   uint magic;
diff --git a/src/src/lookups/README b/src/src/lookups/README
index 98905dc..31fea64 100644
--- a/src/src/lookups/README
+++ b/src/src/lookups/README
@@ -122,12 +122,15 @@ DEFER. The arguments are:
   uschar **errmsg     where to put an error message on failure;
                       this is initially set to "", and should be left
                       as that for a standard "entry not found" error
-  BOOL *do_cache      the lookup should set this to FALSE when it changes data.
-                      This is TRUE by default. When set to FALSE the cache tree
+  uint *do_cache      the lookup should set this to 0 when it changes data.
+                      This is MAXINT by default. When set to 0 the cache tree
                       of the current search handle will be cleaned and the
                       current result will NOT be cached. Currently the mysql
                       and pgsql lookups use this when UPDATE/INSERT queries are
                       executed.
+                      If set to a nonzero number of seconds, the cached value
+                      becomes unusable after this time. Currently the dnsdb
+                      lookup uses this to support the TTL value.
 
 Even though the key is zero-terminated, the length is passed because in the
 common case it has been computed already and is often needed.
diff --git a/src/src/lookups/cdb.c b/src/src/lookups/cdb.c
index ea017de..ba925dc 100644
--- a/src/src/lookups/cdb.c
+++ b/src/src/lookups/cdb.c
@@ -279,7 +279,7 @@ cdb_find(void *handle,
         int  key_len,
         uschar **result,
         uschar **errmsg,
-        BOOL *do_cache)
+        uint *do_cache)
 {
   struct cdb_state * cdbp = handle;
   uint32 item_key_len,
diff --git a/src/src/lookups/dbmdb.c b/src/src/lookups/dbmdb.c
index 03248e4..b8c42d5 100644
--- a/src/src/lookups/dbmdb.c
+++ b/src/src/lookups/dbmdb.c
@@ -87,7 +87,7 @@ the keylength in order to include the terminating zero. */
 
 static int
 dbmdb_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 EXIM_DB *d = (EXIM_DB *)handle;
 EXIM_DATUM key, data;
@@ -120,7 +120,7 @@ return FAIL;
 
 int
 static dbmnz_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return dbmdb_find(handle, filename, keystring, length-1, result, errmsg,
   do_cache);
@@ -140,7 +140,7 @@ return dbmdb_find(handle, filename, keystring, length-1, result, errmsg,
 
 static int
 dbmjz_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 uschar *key_item, *key_buffer, *key_p;
 const uschar *key_elems = keystring;
diff --git a/src/src/lookups/dnsdb.c b/src/src/lookups/dnsdb.c
index e3de279..70e6c8c 100644
--- a/src/src/lookups/dnsdb.c
+++ b/src/src/lookups/dnsdb.c
@@ -131,7 +131,7 @@ separator, as always, is colon. */
 
 static int
 dnsdb_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 int size = 256;
@@ -388,6 +388,9 @@ while ((domain = string_nextinlist(&keystring, &sep, NULL, 0)))
       {
       if (rr->type != searchtype) continue;
 
+      if (*do_cache > rr->ttl)
+	*do_cache = rr->ttl;
+
       if (type == T_A || type == T_AAAA || type == T_ADDRESSES)
         {
         dns_address *da;
diff --git a/src/src/lookups/dsearch.c b/src/src/lookups/dsearch.c
index f8c592a..9f7dd8d 100644
--- a/src/src/lookups/dsearch.c
+++ b/src/src/lookups/dsearch.c
@@ -67,7 +67,7 @@ for us. */
 
 int
 static dsearch_find(void *handle, uschar *dirname, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 struct stat statbuf;
 int save_errno;
diff --git a/src/src/lookups/ibase.c b/src/src/lookups/ibase.c
index 23e1dea..7fd53d0 100644
--- a/src/src/lookups/ibase.c
+++ b/src/src/lookups/ibase.c
@@ -451,7 +451,7 @@ deferred with a retryable error. */
 
 static int
 ibase_find(void *handle, uschar * filename, uschar * query, int length,
-           uschar ** result, uschar ** errmsg, BOOL *do_cache)
+           uschar ** result, uschar ** errmsg, uint *do_cache)
 {
     int sep = 0;
     uschar *server;
diff --git a/src/src/lookups/ldap.c b/src/src/lookups/ldap.c
index a56eff3..b870df1 100644
--- a/src/src/lookups/ldap.c
+++ b/src/src/lookups/ldap.c
@@ -1339,7 +1339,7 @@ The handle and filename arguments are not used. */
 
 static int
 eldap_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1348,7 +1348,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_SINGLE, result, errmsg));
 
 static int
 eldapm_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1357,7 +1357,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_MULTIPLE, result, errmsg));
 
 static int
 eldapdn_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
@@ -1366,7 +1366,7 @@ return(control_ldap_search(ldap_url, SEARCH_LDAP_DN, result, errmsg));
 
 int
 eldapauth_find(void *handle, uschar *filename, const uschar *ldap_url, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 /* Keep picky compilers happy */
 do_cache = do_cache;
diff --git a/src/src/lookups/lf_functions.h b/src/src/lookups/lf_functions.h
index 73e9303..d2487d3 100644
--- a/src/src/lookups/lf_functions.h
+++ b/src/src/lookups/lf_functions.h
@@ -12,7 +12,7 @@ extern int     lf_check_file(int, uschar *, int, int, uid_t *, gid_t *,
 extern uschar *lf_quote(uschar *, uschar *, int, uschar *, int *, int *);
 extern int     lf_sqlperform(const uschar *, const uschar *, const uschar *,
 		 const uschar *, uschar **,
-                 uschar **, BOOL *, int(*)(const uschar *, uschar *, uschar **,
-                 uschar **, BOOL *, BOOL *));
+                 uschar **, uint *, int(*)(const uschar *, uschar *, uschar **,
+                 uschar **, BOOL *, uint *));
 
 /* End of lf_functions.h */
diff --git a/src/src/lookups/lf_sqlperform.c b/src/src/lookups/lf_sqlperform.c
index 2d7f326..6d4f7a7 100644
--- a/src/src/lookups/lf_sqlperform.c
+++ b/src/src/lookups/lf_sqlperform.c
@@ -27,7 +27,7 @@ Arguments:
   query          the query
   result         where to pass back the result
   errmsg         where to pass back an error message
-  do_cache       to be set FALSE if data is changed
+  do_cache       to be set zero if data is changed
   func           the lookup function to call
 
 Returns:         the return from the lookup function, or DEFER
@@ -36,8 +36,8 @@ Returns:         the return from the lookup function, or DEFER
 int
 lf_sqlperform(const uschar *name, const uschar *optionname,
   const uschar *optserverlist, const uschar *query,
-  uschar **result, uschar **errmsg, BOOL *do_cache,
-  int(*fn)(const uschar *, uschar *, uschar **, uschar **, BOOL *, BOOL *))
+  uschar **result, uschar **errmsg, uint *do_cache,
+  int(*fn)(const uschar *, uschar *, uschar **, uschar **, BOOL *, uint *))
 {
 int sep, rc;
 uschar *server;
diff --git a/src/src/lookups/lsearch.c b/src/src/lookups/lsearch.c
index 3883d4b..eb70a45 100644
--- a/src/src/lookups/lsearch.c
+++ b/src/src/lookups/lsearch.c
@@ -323,7 +323,7 @@ return FAIL;
 
 static int
 lsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -340,7 +340,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 wildlsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -357,7 +357,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 nwildlsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 return internal_lsearch_find(handle, filename, keystring, length, result,
@@ -375,7 +375,7 @@ return internal_lsearch_find(handle, filename, keystring, length, result,
 
 static int
 iplsearch_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 do_cache = do_cache;  /* Keep picky compilers happy */
 if ((length == 1 && keystring[0] == '*') ||
diff --git a/src/src/lookups/mysql.c b/src/src/lookups/mysql.c
index 8dff86a..1ce8831 100644
--- a/src/src/lookups/mysql.c
+++ b/src/src/lookups/mysql.c
@@ -74,7 +74,7 @@ Arguments:
   resultptr    where to store the result
   errmsg       where to point an error message
   defer_break  TRUE if no more servers are to be tried after DEFER
-  do_cache     set false if data is changed
+  do_cache     set zero if data is changed
 
 The server string is of the form "host/dbname/user/password". The host can be
 host:port. This string is in a nextinlist temporary buffer, so can be
@@ -85,7 +85,7 @@ Returns:       OK, FAIL, or DEFER
 
 static int
 perform_mysql_search(const uschar *query, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
 MYSQL *mysql_handle = NULL;        /* Keep compilers happy */
 MYSQL_RES *mysql_result = NULL;
@@ -225,7 +225,7 @@ can be detected by calling mysql_field_count(). If its result is zero, no data
 was expected (this is all explained clearly in the MySQL manual). In this case,
 we return the number of rows affected by the command. In this event, we do NOT
 want to cache the result; also the whole cache for the handle must be cleaned
-up. Setting do_cache FALSE requests this. */
+up. Setting do_cache zero requests this. */
 
 if ((mysql_result = mysql_use_result(mysql_handle)) == NULL)
   {
@@ -233,7 +233,7 @@ if ((mysql_result = mysql_use_result(mysql_handle)) == NULL)
     {
     DEBUG(D_lookup) debug_printf("MYSQL: query was not one that returns data\n");
     result = string_sprintf("%d", mysql_affected_rows(mysql_handle));
-    *do_cache = FALSE;
+    *do_cache = 0;
     goto MYSQL_EXIT;
     }
   *errmsg = string_sprintf("MYSQL: lookup result failed: %s\n",
@@ -341,7 +341,7 @@ shared with other SQL lookups. */
 
 static int
 mysql_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return lf_sqlperform(US"MySQL", US"mysql_servers", mysql_servers, query,
   result, errmsg, do_cache, perform_mysql_search);
diff --git a/src/src/lookups/nis.c b/src/src/lookups/nis.c
index 7b012b1..1faa884 100644
--- a/src/src/lookups/nis.c
+++ b/src/src/lookups/nis.c
@@ -42,7 +42,7 @@ code. */
 
 static int
 nis_find(void *handle, uschar *filename, uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 uschar *nis_data;
@@ -68,7 +68,7 @@ return (rc == YPERR_KEY || rc == YPERR_MAP)? FAIL : DEFER;
 
 static int
 nis0_find(void *handle, uschar *filename, uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int rc;
 uschar *nis_data;
diff --git a/src/src/lookups/nisplus.c b/src/src/lookups/nisplus.c
index 8895cee..a4a7a2d 100644
--- a/src/src/lookups/nisplus.c
+++ b/src/src/lookups/nisplus.c
@@ -43,7 +43,7 @@ equals sign. */
 
 static int
 nisplus_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int i;
 int ssize = 0;
diff --git a/src/src/lookups/oracle.c b/src/src/lookups/oracle.c
index 1f2520a..adb17b4 100644
--- a/src/src/lookups/oracle.c
+++ b/src/src/lookups/oracle.c
@@ -517,7 +517,7 @@ deferred with a retryable error. */
 
 static int
 oracle_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int sep = 0;
 uschar *server;
diff --git a/src/src/lookups/passwd.c b/src/src/lookups/passwd.c
index e726f3e..315677f 100644
--- a/src/src/lookups/passwd.c
+++ b/src/src/lookups/passwd.c
@@ -34,7 +34,7 @@ return (void *)(-1);     /* Just return something non-null */
 
 static int
 passwd_find(void *handle, uschar *filename, const uschar *keystring, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 struct passwd *pw;
 
diff --git a/src/src/lookups/pgsql.c b/src/src/lookups/pgsql.c
index c86ac23..4be3d98 100644
--- a/src/src/lookups/pgsql.c
+++ b/src/src/lookups/pgsql.c
@@ -119,7 +119,7 @@ Returns:       OK, FAIL, or DEFER
 
 static int
 perform_pgsql_search(const uschar *query, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
 PGconn *pg_conn = NULL;
 PGresult *pg_result = NULL;
@@ -290,10 +290,10 @@ else
     /* The command was successful but did not return any data since it was
      * not SELECT but either an INSERT, UPDATE or DELETE statement. Tell the
      * high level code to not cache this query, and clean the current cache for
-     * this handle by setting *do_cache FALSE. */
+     * this handle by setting *do_cache zero. */
     result = string_copy(US PQcmdTuples(pg_result));
     offset = Ustrlen(result);
-    *do_cache = FALSE;
+    *do_cache = 0;
     DEBUG(D_lookup) debug_printf("PGSQL: command does not return any data "
       "but was successful. Rows affected: %s\n", result);
 
@@ -399,7 +399,7 @@ shared with other SQL lookups. */
 
 static int
 pgsql_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 return lf_sqlperform(US"PostgreSQL", US"pgsql_servers", pgsql_servers, query,
   result, errmsg, do_cache, perform_pgsql_search);
diff --git a/src/src/lookups/redis.c b/src/src/lookups/redis.c
index ac4d0ec..18cd3a0 100644
--- a/src/src/lookups/redis.c
+++ b/src/src/lookups/redis.c
@@ -65,7 +65,7 @@ redis_tidy(void)
  */
 static int
 perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
-  uschar **errmsg, BOOL *defer_break, BOOL *do_cache)
+  uschar **errmsg, BOOL *defer_break, uint *do_cache)
 {
        redisContext *redis_handle = NULL;        /* Keep compilers happy */
        redisReply *redis_reply = NULL;
@@ -197,7 +197,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
        case REDIS_REPLY_ERROR:
                *errmsg = string_sprintf("REDIS: lookup result failed: %s\n", redis_reply->str);
                *defer_break = FALSE;
-               *do_cache = FALSE;
+               *do_cache = 0;
                goto REDIS_EXIT;
                /* NOTREACHED */
 
@@ -205,7 +205,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
        case REDIS_REPLY_NIL:
                DEBUG(D_lookup) debug_printf("REDIS: query was not one that returned any data\n");
                result = string_sprintf("");
-               *do_cache = FALSE;
+               *do_cache = 0;
                goto REDIS_EXIT;
                /* NOTREACHED */
 
@@ -304,7 +304,7 @@ perform_redis_search(uschar *command, uschar *server, uschar **resultptr,
 
 static int
 redis_find(void *handle __attribute__((unused)), uschar *filename __attribute__((unused)),
-           uschar *command, int length, uschar **result, uschar **errmsg, BOOL *do_cache)
+           uschar *command, int length, uschar **result, uschar **errmsg, uint *do_cache)
 {
        return lf_sqlperform(US"Redis", US"redis_servers", redis_servers, command,
          result, errmsg, do_cache, perform_redis_search);
diff --git a/src/src/lookups/spf.c b/src/src/lookups/spf.c
index 23ad2ad..2671fc9 100644
--- a/src/src/lookups/spf.c
+++ b/src/src/lookups/spf.c
@@ -31,7 +31,9 @@ static void dummy(int x) { dummy2(x-1); }
 #include <spf2/spf_dns_resolv.h>
 #include <spf2/spf_dns_cache.h>
 
-static void *spf_open(uschar *filename, uschar **errmsg) {
+static void *
+spf_open(uschar *filename, uschar **errmsg)
+{
   SPF_server_t *spf_server = NULL;
   spf_server = SPF_server_new(SPF_DNS_CACHE, 0);
   if (spf_server == NULL) {
@@ -41,13 +43,17 @@ static void *spf_open(uschar *filename, uschar **errmsg) {
   return (void *) spf_server;
 }
 
-static void spf_close(void *handle) {
+static void
+spf_close(void *handle)
+{
   SPF_server_t *spf_server = handle;
   if (spf_server) SPF_server_free(spf_server);
 }
 
-static int spf_find(void *handle, uschar *filename, uschar *keystring, int key_len,
-             uschar **result, uschar **errmsg, BOOL *do_cache) {
+static int
+spf_find(void *handle, uschar *filename, uschar *keystring, int key_len,
+             uschar **result, uschar **errmsg, uint *do_cache)
+{
   SPF_server_t *spf_server = handle;
   SPF_request_t *spf_request = NULL;
   SPF_response_t *spf_response = NULL;
diff --git a/src/src/lookups/sqlite.c b/src/src/lookups/sqlite.c
index bb92c8c..e2330f9 100644
--- a/src/src/lookups/sqlite.c
+++ b/src/src/lookups/sqlite.c
@@ -81,7 +81,7 @@ return 0;
 
 static int
 sqlite_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 int ret;
 struct strbuf res = { NULL, 0, 0 };
@@ -93,7 +93,7 @@ if (ret != SQLITE_OK)
   return FAIL;
   }
 
-if (res.string == NULL) *do_cache = FALSE;
+if (res.string == NULL) *do_cache = 0;
 
 *result = res.string;
 return OK;
diff --git a/src/src/lookups/testdb.c b/src/src/lookups/testdb.c
index c82fa7f..401f7c8 100644
--- a/src/src/lookups/testdb.c
+++ b/src/src/lookups/testdb.c
@@ -38,7 +38,7 @@ return (void *)(1);    /* Just return something non-null */
 
 static int
 testdb_find(void *handle, uschar *filename, const uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 handle = handle;          /* Keep picky compilers happy */
 filename = filename;
@@ -57,7 +57,7 @@ if (Ustrcmp(query, "defer") == 0)
   return DEFER;
   }
 
-if (Ustrcmp(query, "nocache") == 0) *do_cache = FALSE;
+if (Ustrcmp(query, "nocache") == 0) *do_cache = 0;
 
 *result = string_copy(query);
 return OK;
diff --git a/src/src/lookups/whoson.c b/src/src/lookups/whoson.c
index 4166089..9ac5a3a 100644
--- a/src/src/lookups/whoson.c
+++ b/src/src/lookups/whoson.c
@@ -36,7 +36,7 @@ return (void *)(1);    /* Just return something non-null */
 
 static int
 whoson_find(void *handle, uschar *filename, uschar *query, int length,
-  uschar **result, uschar **errmsg, BOOL *do_cache)
+  uschar **result, uschar **errmsg, uint *do_cache)
 {
 uschar buffer[80];
 handle = handle;          /* Keep picky compilers happy */
diff --git a/src/src/search.c b/src/src/search.c
index a055291..cd522da 100644
--- a/src/src/search.c
+++ b/src/src/search.c
@@ -466,6 +466,7 @@ internal_search_find(void *handle, uschar *filename, uschar *keystring)
 {
 tree_node *t = (tree_node *)handle;
 search_cache *c = (search_cache *)(t->data.ptr);
+expiring_data *e;
 uschar *data = NULL;
 int search_type = t->name[0] - '0';
 int old_pool = store_pool;
@@ -491,18 +492,27 @@ store_pool = POOL_SEARCH;
 /* Look up the data for the key, unless it is already in the cache for this
 file. No need to check c->item_cache for NULL, tree_search will do so. */
 
-if ((t = tree_search(c->item_cache, keystring)) == NULL)
+if (  (t = tree_search(c->item_cache, keystring))
+   && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL))
+   )
+  { /* Data was in the cache already; set the pointer from the tree node */
+  data = e->ptr;
+  DEBUG(D_lookup) debug_printf("cached data used for lookup of %s%s%s\n",
+    keystring,
+    filename ? US"\n  in " : US"", filename ? filename : US"");
+  }
+else
   {
-  BOOL do_cache = TRUE;
+  uint do_cache = UINT_MAX;
   int keylength = Ustrlen(keystring);
 
   DEBUG(D_lookup)
     {
-    if (filename != NULL)
-      debug_printf("file lookup required for %s\n  in %s\n",
-        keystring, filename);
-    else
-      debug_printf("database lookup required for %s\n", keystring);
+    if (t) debug_printf("cached data found but past valid time; ");
+    debug_printf("%s lookup required for %s%s%s\n",
+      filename ? US"file" : US"database",
+      keystring,
+      filename ? US"\n  in " : US"", filename ? filename : US"");
     }
 
   /* Call the code for the different kinds of search. DEFER is handled
@@ -511,9 +521,7 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
 
   if (lookup_list[search_type]->find(c->handle, filename, keystring, keylength,
       &data, &search_error_message, &do_cache) == DEFER)
-    {
     search_find_defer = TRUE;
-    }
 
   /* A record that has been found is now in data, which is either NULL
   or points to a bit of dynamic store. Cache the result of the lookup if
@@ -524,10 +532,22 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
   else if (do_cache)
     {
     int len = keylength + 1;
-    t = store_get(sizeof(tree_node) + len);
-    memcpy(t->name, keystring, len);
-    t->data.ptr = data;
-    tree_insertnode(&c->item_cache, t);
+
+    if (t)	/* Previous, out-of-date cache entry.  Update with the */
+      { 	/* new result and forget the old one */
+      e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
+      e->ptr = data;
+      }
+    else
+      {
+      t = store_get(sizeof(tree_node) + len + sizeof(expiring_data));
+      e = (expiring_data *)((char *)t + sizeof(tree_node) + len);
+      e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
+      e->ptr = data;
+      memcpy(t->name, keystring, len);
+      t->data.ptr = e;
+      tree_insertnode(&c->item_cache, t);
+      }
     }
 
   /* If caching was disabled, empty the cache tree. We just set the cache
@@ -540,34 +560,19 @@ if ((t = tree_search(c->item_cache, keystring)) == NULL)
     }
   }
 
-/* Data was in the cache already; set the pointer from the tree node */
-
-else
-  {
-  data = US t->data.ptr;
-  DEBUG(D_lookup) debug_printf("cached data used for lookup of %s%s%s\n",
-    keystring,
-    (filename == NULL)? US"" : US"\n  in ",
-    (filename == NULL)? US"" : filename);
-  }
-
-/* Debug: output the answer */
-
 DEBUG(D_lookup)
   {
-  if (data == NULL)
-    {
-    if (search_find_defer) debug_printf("lookup deferred: %s\n",
-      search_error_message);
-    else debug_printf("lookup failed\n");
-    }
-  else debug_printf("lookup yielded: %s\n", data);
+  if (data)
+    debug_printf("lookup yielded: %s\n", data);
+  else if (search_find_defer)
+    debug_printf("lookup deferred: %s\n", search_error_message);
+  else debug_printf("lookup failed\n");
   }
 
 /* Return it in new dynamic store in the regular pool */
 
 store_pool = old_pool;
-return (data == NULL)? NULL : string_copy(data);
+return data ? string_copy(data) : NULL;
 }
 
 
diff --git a/src/src/structs.h b/src/src/structs.h
index 6f143d6..d9da38b 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -657,6 +657,16 @@ typedef struct tree_node {
   uschar  name[1];                /* node name - variable length */
 } tree_node;
 
+/* Structure for holding time-limited data such as DNS returns.
+We use this rather than extending tree_node to avoid wasting
+space for most tree use (variables...) at the cost of complexity
+for the lookups cache */
+
+typedef struct expiring_data {
+  time_t expiry;		  /* if nonzero, data invalid after this time */
+  void   *ptr;			  /* pointer to data */
+} expiring_data;
+
 /* Structure for holding the handle and the cached last lookup for searches.
 This block is pointed to by the tree entry for the file. The file can get
 closed if too many are opened at once. There is a LRU chain for deciding which
@@ -676,6 +686,7 @@ uncompressed, but the data pointer is into the raw data. */
 typedef struct {
   uschar  name[DNS_MAXNAME];      /* domain name */
   int     type;                   /* record type */
+  unsigned short ttl;		  /* time-to-live, seconds */
   int     size;                   /* size of data */
   uschar *data;                   /* pointer to data */
 } dns_record;
diff --git a/src/src/verify.c b/src/src/verify.c
index e00e7b9..d392fda 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -21,6 +21,7 @@ uschar ctbuffer[8192];
 /* Structure for caching DNSBL lookups */
 
 typedef struct dnsbl_cache_block {
+  time_t expiry;
   dns_address *rhs;
   uschar *text;
   int rc;
@@ -3584,21 +3585,37 @@ if (!string_format(query, sizeof(query), "%s.%s", prepend, domain))
 
 /* Look for this query in the cache. */
 
-t = tree_search(dnsbl_cache, query);
+if (  (t = tree_search(dnsbl_cache, query))
+   && (cb = t->data.ptr)->expiry > time(NULL)
+   )
+
+/* Previous lookup was cached */
+
+  {
+  HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
+  }
 
 /* If not cached from a previous lookup, we must do a DNS lookup, and
 cache the result in permanent memory. */
 
-if (t == NULL)
+else
   {
+  uint ttl = 3600;
+
   store_pool = POOL_PERM;
 
-  /* Set up a tree entry to cache the lookup */
+  if (t)
+    {
+    HDEBUG(D_dnsbl) debug_printf("cached data found but past valid time; ");
+    }
 
-  t = store_get(sizeof(tree_node) + Ustrlen(query));
-  Ustrcpy(t->name, query);
-  t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
-  (void)tree_insertnode(&dnsbl_cache, t);
+  else
+    {	/* Set up a tree entry to cache the lookup */
+    t = store_get(sizeof(tree_node) + Ustrlen(query));
+    Ustrcpy(t->name, query);
+    t->data.ptr = cb = store_get(sizeof(dnsbl_cache_block));
+    (void)tree_insertnode(&dnsbl_cache, t);
+    }
 
   /* Do the DNS loopup . */
 
@@ -3616,7 +3633,10 @@ if (t == NULL)
 
   Quite apart from one A6 RR generating multiple addresses, there are DNS
   lists that return more than one A record, so we must handle multiple
-  addresses generated in that way as well. */
+  addresses generated in that way as well.
+
+  Mark the cache entry with the "now" plus the minimum of the address TTLs,
+  or some suitably far-future time if none were found. */
 
   if (cb->rc == DNS_SUCCEED)
     {
@@ -3634,6 +3654,7 @@ if (t == NULL)
           *addrp = da;
           while (da->next != NULL) da = da->next;
           addrp = &(da->next);
+	  if (ttl > rr->ttl) ttl = rr->ttl;
           }
         }
       }
@@ -3645,17 +3666,10 @@ if (t == NULL)
     if (cb->rhs == NULL) cb->rc = DNS_NODATA;
     }
 
+  cb->expiry = time(NULL)+ttl;
   store_pool = old_pool;
   }
 
-/* Previous lookup was cached */
-
-else
-  {
-  HDEBUG(D_dnsbl) debug_printf("using result of previous DNS lookup\n");
-  cb = t->data.ptr;
-  }
-
 /* We now have the result of the DNS lookup, either newly done, or cached
 from a previous call. If the lookup succeeded, check against the address
 list if there is one. This may be a positive equality list (introduced by

commit c4dcf906ceb3a45c6b30f76476d73ca836b262cd
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Sep 19 13:59:22 2015 +0100

    Retry: always use interface, if set, for retry DB key.  Bug 1678
    
    Even constant values must be used, as multiple transports with
    different values may be in play and should be kept distinct.
    
    (cherry picked from commit 6f6dedccb47f231a0712d882da20feffbac8d0bc)

diff --git a/src/src/functions.h b/src/src/functions.h
index 0257904..94e3f5f 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -374,7 +374,7 @@ extern int     smtp_sock_connect(host_item *, int, int, uschar *,
 extern int     smtp_feof(void);
 extern int     smtp_ferror(void);
 extern uschar *smtp_get_connection_info(void);
-extern BOOL    smtp_get_interface(uschar *, int, address_item *, BOOL *,
+extern BOOL    smtp_get_interface(uschar *, int, address_item *,
                  uschar **, uschar *);
 extern BOOL    smtp_get_port(uschar *, address_item *, int *, uschar *);
 extern int     smtp_getc(void);
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index c704a0b..2fdf38b 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -26,7 +26,6 @@ Arguments:
                which case the function does nothing
   host_af    AF_INET or AF_INET6 for the outgoing IP address
   addr       the mail address being handled (for setting errors)
-  changed    if not NULL, set TRUE if expansion actually changed istring
   interface  point this to the interface
   msg        to add to any error message
 
@@ -36,7 +35,7 @@ Returns:     TRUE on success, FALSE on failure, with error message
 
 BOOL
 smtp_get_interface(uschar *istring, int host_af, address_item *addr,
-  BOOL *changed, uschar **interface, uschar *msg)
+  uschar **interface, uschar *msg)
 {
 const uschar * expint;
 uschar *iface;
@@ -54,8 +53,6 @@ if (expint == NULL)
   return FALSE;
   }
 
-if (changed != NULL) *changed = expint != istring;
-
 while (isspace(*expint)) expint++;
 if (*expint == 0) return TRUE;
 
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 5ac5533..ba7fb5e 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -3174,7 +3174,6 @@ for (cutoff_retry = 0; expired &&
     BOOL serialized = FALSE;
     BOOL host_is_expired = FALSE;
     BOOL message_defer = FALSE;
-    BOOL ifchanges = FALSE;
     BOOL some_deferred = FALSE;
     address_item *first_addr = NULL;
     uschar *interface = NULL;
@@ -3350,15 +3349,18 @@ for (cutoff_retry = 0; expired &&
     if (Ustrcmp(pistring, ":25") == 0) pistring = US"";
 
     /* Select IPv4 or IPv6, and choose an outgoing interface. If the interface
-    string changes upon expansion, we must add it to the key that is used for
-    retries, because connections to the same host from a different interface
-    should be treated separately. */
+    string is set, even if constant (as different transports can have different
+    constant settings), we must add it to the key that is used for retries,
+    because connections to the same host from a different interface should be
+    treated separately. */
 
     host_af = (Ustrchr(host->address, ':') == NULL)? AF_INET : AF_INET6;
-    if (!smtp_get_interface(ob->interface, host_af, addrlist, &ifchanges,
-         &interface, tid))
-      return FALSE;
-    if (ifchanges) pistring = string_sprintf("%s/%s", pistring, interface);
+    if ((rs = ob->interface) && *rs)
+      {
+      if (!smtp_get_interface(rs, host_af, addrlist, &interface, tid))
+	return FALSE;
+      pistring = string_sprintf("%s/%s", pistring, interface);
+      }
 
     /* The first time round the outer loop, check the status of the host by
     inspecting the retry data. The second time round, we are interested only
diff --git a/src/src/verify.c b/src/src/verify.c
index d392fda..6411c7e 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -444,7 +444,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
 
 	  host_af = (Ustrchr(host->address, ':') == NULL)? AF_INET:AF_INET6;
 
-	  if (!smtp_get_interface(tf->interface, host_af, addr, NULL, &interface,
+	  if (!smtp_get_interface(tf->interface, host_af, addr, &interface,
 		  US"callout") ||
 	      !smtp_get_port(tf->port, addr, &port, US"callout"))
 	    log_write(0, LOG_MAIN|LOG_PANIC, "<%s>: %s", addr->address,
@@ -579,7 +579,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
     deliver_domain = addr->domain;
     transport_name = addr->transport->name;
 
-    if (  !smtp_get_interface(tf->interface, host_af, addr, NULL, &interface,
+    if (  !smtp_get_interface(tf->interface, host_af, addr, &interface,
             US"callout")
        || !smtp_get_port(tf->port, addr, &port, US"callout")
        )