-
-# HG changeset patch
-# User Arkadiusz Miskiewicz <arekm@maven.pl>
-# Date 1535035719 -7200
-# Node ID 74ed2e4ba2456c97a4266b4c58faf4bf24b4d21e
-# Parent 3d2c10e918d29353b29506c635c26e2e5f3cd2bb
-Bug 678322 - process all duplicate headers when searching. r=jorgk
-
-Search in all headers even if header with the same name occured multiple times.
-Stop treating Received in special way as now it's not needed - we will search
-all Received occurences (just like any other header) now.
-
-Add tests for for the problem.
-
-diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
---- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
-+++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
-@@ -736,127 +736,129 @@ nsresult nsMsgSearchTerm::DeStreamNew (c
- CopyUTF8toUTF16(mozilla::MakeStringSpan(m_value.string), m_value.utf16String);
- }
- return NS_OK;
- }
-
-
- // Looks in the MessageDB for the user specified arbitrary header, if it finds the header, it then looks for a match against
- // the value for the header.
--nsresult nsMsgSearchTerm::MatchArbitraryHeader (nsIMsgSearchScopeTerm *scope,
-- uint32_t length /* in lines*/,
-- const char *charset,
-- bool charsetOverride,
-- nsIMsgDBHdr *msg,
-- nsIMsgDatabase* db,
-- const char * headers,
-- uint32_t headersSize,
-- bool ForFiltering,
-- bool *pResult)
-+nsresult nsMsgSearchTerm::MatchArbitraryHeader(nsIMsgSearchScopeTerm *scope,
-+ uint32_t length /* in lines*/,
-+ const char *charset,
-+ bool charsetOverride,
-+ nsIMsgDBHdr *msg,
-+ nsIMsgDatabase* db,
-+ const char * headers,
-+ uint32_t headersSize,
-+ bool ForFiltering,
-+ bool *pResult)
- {
- NS_ENSURE_ARG_POINTER(pResult);
-
- *pResult = false;
- nsresult rv = NS_OK;
- bool matchExpected = m_operator == nsMsgSearchOp::Contains ||
- m_operator == nsMsgSearchOp::Is ||
- m_operator == nsMsgSearchOp::BeginsWith ||
- m_operator == nsMsgSearchOp::EndsWith;
-- // init result to what we want if we don't find the header at all
-+ // Initialize result to what we want if we don't find the header at all.
- bool result = !matchExpected;
-
- nsCString dbHdrValue;
- msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
-- if (!dbHdrValue.IsEmpty())
-- // match value with the other info.
-- return MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
-+ if (!dbHdrValue.IsEmpty()) {
-+ // Match value with the other info. It doesn't check all header occurences,
-+ // so we use it only if we match and do line by line headers parsing otherwise.
-+ rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
-+ if (*pResult)
-+ return rv;
-
-- nsMsgBodyHandler * bodyHandler =
-- new nsMsgBodyHandler (scope, length, msg, db, headers, headersSize,
-- ForFiltering);
-- NS_ENSURE_TRUE(bodyHandler, NS_ERROR_OUT_OF_MEMORY);
-+ rv = NS_OK;
-+ }
-
-+ nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
-+ headers, headersSize,
-+ ForFiltering);
- bodyHandler->SetStripHeaders (false);
-
-- nsCString headerFullValue; // contains matched header value accumulated over multiple lines.
-+ nsCString headerFullValue; // Contains matched header value accumulated over multiple lines.
- nsAutoCString buf;
- nsAutoCString curMsgHeader;
-- bool searchingHeaders = true;
-+ bool processingHeaders = true;
-
-- // We will allow accumulation of received headers;
-- bool isReceivedHeader = m_arbitraryHeader.EqualsLiteral("received");
--
-- while (searchingHeaders)
-+ while (processingHeaders)
- {
- nsCString charsetIgnored;
- if (bodyHandler->GetNextLine(buf, charsetIgnored) < 0 || EMPTY_MESSAGE_LINE(buf))
-- searchingHeaders = false;
-- bool isContinuationHeader = searchingHeaders ?
-+ processingHeaders = false; // No more lines or emtpy line teminating headers.
-+
-+ bool isContinuationHeader = processingHeaders ?
- NS_IsAsciiWhitespace(buf.CharAt(0)) : false;
-
-- // We try to match the header from the last time through the loop, which should now
-- // have accumulated over possible multiple lines. For all headers except received,
-- // we process a single accumulation, but process accumulated received at the end.
-- if (!searchingHeaders || (!isContinuationHeader &&
-- (!headerFullValue.IsEmpty() && !isReceivedHeader)))
-+ // If we're not on a continuation header the header value is not empty,
-+ // we have finished accumulating the header value by iterating over all
-+ // header lines. Now we need to check whether the value is a match.
-+ if (!isContinuationHeader && !headerFullValue.IsEmpty())
- {
-- // Make sure buf has info besides just the header.
-- // Otherwise, it's either an empty header, or header not found.
-- if (!headerFullValue.IsEmpty())
-+ bool stringMatches;
-+ // Match value with the other info.
-+ rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
-+ if (matchExpected == stringMatches) // if we found a match
- {
-- bool stringMatches;
-- // match value with the other info.
-- rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
-- if (matchExpected == stringMatches) // if we found a match
-- {
-- searchingHeaders = false; // then stop examining the headers
-- result = stringMatches;
-- }
-+ // If we found a match, stop examining the headers.
-+ processingHeaders = false;
-+ result = stringMatches;
- }
-+ // Prepare for repeated header of the same type.
-+ headerFullValue.Truncate();
-+ }
-+
-+ // We got result or finished processing all lines.
-+ if (!processingHeaders)
- break;
-- }
-
- char * buf_end = (char *) (buf.get() + buf.Length());
- int headerLength = m_arbitraryHeader.Length();
-
- // If the line starts with whitespace, then we use the current header.
- if (!isContinuationHeader)
- {
-- // here we start a new header
-+ // Here we start a new header.
- uint32_t colonPos = buf.FindChar(':');
- curMsgHeader = StringHead(buf, colonPos);
- }
-
- if (curMsgHeader.Equals(m_arbitraryHeader, nsCaseInsensitiveCStringComparator()))
- {
-- // process the value
-- // value occurs after the header name or whitespace continuation char.
-- const char * headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
-+ // Process the value:
-+ // Value occurs after the header name or whitespace continuation char.
-+ const char *headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
- if (headerValue < buf_end && headerValue[0] == ':') // + 1 to account for the colon which is MANDATORY
- headerValue++;
-
-- // strip leading white space
-+ // Strip leading white space.
- while (headerValue < buf_end && isspace(*headerValue))
-- headerValue++; // advance to next character
-+ headerValue++;
-
-- // strip trailing white space
-+ // Strip trailing white space.
- char * end = buf_end - 1;
-- while (end > headerValue && isspace(*end)) // while we haven't gone back past the start and we are white space....
-+ while (headerValue < end && isspace(*end))
- {
-- *end = '\0'; // eat up the white space
-- end--; // move back and examine the previous character....
-+ *end = '\0';
-+ end--;
- }
-
-- // any continuation whitespace is converted to a single space. This includes both a continuation line, or a
-- // second value of the same header (eg the received header)
-+ // Any continuation whitespace is converted to a single space.
- if (!headerFullValue.IsEmpty())
- headerFullValue.Append(' ');
- headerFullValue.Append(nsDependentCString(headerValue));
- }
- }
-+
- delete bodyHandler;
- *pResult = result;
- return rv;
- }
-
- NS_IMETHODIMP nsMsgSearchTerm::MatchHdrProperty(nsIMsgDBHdr *aHdr, bool *aResult)
- {
- NS_ENSURE_ARG_POINTER(aResult);
-diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
---- a/mailnews/base/test/unit/test_search.js
-+++ b/mailnews/base/test/unit/test_search.js
-@@ -104,16 +104,110 @@ var Tests =
- testAttribute: OtherHeader,
- op: Contains,
- count: 1},
- { testString: "foobar",
- testAttribute: OtherHeader,
- op: Contains,
- count: 0},
-
-+ // test header with multiple occurences
-+ { testString: "one value",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "second",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "third value for test purposes",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "multiline value that needs to be handled.",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+
-+ { testString: "one",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "second",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "purposes",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "value",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "that needs to be",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "fifth",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "is the end my",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "the end",
-+ testAttribute: OtherHeader,
-+ op: EndsWith,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-+ { testString: "handled.",
-+ testAttribute: OtherHeader,
-+ op: EndsWith,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "one value",
-+ testAttribute: OtherHeader,
-+ op: EndsWith,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "third",
-+ testAttribute: OtherHeader,
-+ op: BeginsWith,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+ { testString: "This is",
-+ testAttribute: OtherHeader,
-+ op: BeginsWith,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+
-+ { testString: "nothing",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-+ { testString: "nothing",
-+ testAttribute: OtherHeader,
-+ op: DoesntContain,
-+ customHeader: "X-Duplicated-Header",
-+ count: 1},
-+
- // test accumulation of received header
- // only in first received
- { testString: "caspiaco",
- testAttribute: OtherHeader,
- op: Contains,
- customHeader: "Received",
- count: 1},
- // only in second
-@@ -123,16 +217,22 @@ var Tests =
- customHeader: "received",
- count: 1},
- // in neither
- { testString: "not there",
- testAttribute: OtherHeader,
- op: Contains,
- customHeader: "received",
- count: 0},
-+ // not on first line of received
-+ { testString: "m47LtAFJ007547",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "received",
-+ count: 1},
-
- // test multiple line arbitrary headers
- // in the first line
- { testString: "SpamAssassin 3.2.3",
- testAttribute: OtherHeader,
- op: Contains,
- customHeader: "X-Spam-Checker-Version",
- count: 1},
-diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
---- a/mailnews/test/data/bugmail12
-+++ b/mailnews/test/data/bugmail12
-@@ -49,16 +49,28 @@ X-Bugzilla-Component: MailNews: Filters
- X-Bugzilla-Keywords:
- X-Bugzilla-Severity: enhancement
- X-Bugzilla-Who: bugmail@example.org
- X-Bugzilla-Status: NEW
- X-Bugzilla-Priority: --
- X-Bugzilla-Assigned-To: nobody@mozilla.org
- X-Bugzilla-Target-Milestone: ---
- X-Bugzilla-Changed-Fields: Blocks
-+X-Duplicated-Header: one value
-+X-Duplicated-Header: second
-+X-Duplicated-Header: third value for test purposes
-+X-Duplicated-Header: multiline value
-+ that needs to be
-+ handled.
-+X-Duplicated-Header: here comes a continuation line
-+ that contains the word 'fifth' we're looking for
-+ and then another one
-+X-Duplicated-Header: This is
-+ the end
-+ my friend
- In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
- References: <bug-397009-254728@https.bugzilla.mozilla.org/>
- X-Priority: 4
- Content-Type: text/plain; charset="UTF-8"
- MIME-Version: 1.0
- X-user: ::::63.245.208.146:host29.hostmonster.com::::::
- DomainKey-Status: no signature
-
-
-
-# HG changeset patch
-# User Arkadiusz Miskiewicz <arekm@maven.pl>
-# Date 1535318767 -7200
-# Node ID 9289d8a37eb2dae82f2ecc0076ec8396126a5526
-# Parent ce828f6cf25697596ec928ee99dfc07bdf0ce006
-Bug 678322 - Follow-up: fix logic error and cater for IMAP case where headers are not available. r=jorgk
-
-diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
---- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
-+++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
-@@ -764,20 +764,21 @@ nsresult nsMsgSearchTerm::MatchArbitrary
- bool result = !matchExpected;
-
- nsCString dbHdrValue;
- msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
- if (!dbHdrValue.IsEmpty()) {
- // Match value with the other info. It doesn't check all header occurences,
- // so we use it only if we match and do line by line headers parsing otherwise.
- rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
-- if (*pResult)
-+ if (matchExpected == *pResult)
- return rv;
-
-- rv = NS_OK;
-+ // Preset result in case we don't have access to the headers, like for IMAP.
-+ result = *pResult;
- }
-
- nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
- headers, headersSize,
- ForFiltering);
- bodyHandler->SetStripHeaders (false);
-
- nsCString headerFullValue; // Contains matched header value accumulated over multiple lines.
-
-
-# HG changeset patch
-# User Arkadiusz Miskiewicz <arekm@maven.pl>
-# Date 1535451947 -7200
-# Node ID a09a76a92abfcfa36c00727cef84293175ade116
-# Parent ff6f03d6611e158e2d8e951d9fed621dfd90821f
-Bug 678322 - Follow-up: more tests including matching based on message string properties. r=jorgk
-
-nsMsgSearchTerm::MatchArbitraryHeader() can do matching using message string
-properties and by matching headers line by line.
-
-test_search.js was only testing second case for duplicated headers. Now we also test
-first case. Also added more tests for second case.
-
-diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
---- a/mailnews/base/test/unit/test_search.js
-+++ b/mailnews/base/test/unit/test_search.js
-@@ -125,16 +125,36 @@ var Tests =
- op: Is,
- customHeader: "X-Duplicated-Header",
- count: 1},
- { testString: "multiline value that needs to be handled.",
- testAttribute: OtherHeader,
- op: Is,
- customHeader: "X-Duplicated-Header",
- count: 1},
-+ { testString: "one value",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-+ { testString: "second",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-+ { testString: "third value for test purposes",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-+ { testString: "multiline value that needs to be handled.",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header",
-+ count: 0},
-
- { testString: "one",
- testAttribute: OtherHeader,
- op: Contains,
- customHeader: "X-Duplicated-Header",
- count: 1},
- { testString: "second",
- testAttribute: OtherHeader,
-@@ -198,16 +218,57 @@ var Tests =
- customHeader: "X-Duplicated-Header",
- count: 0},
- { testString: "nothing",
- testAttribute: OtherHeader,
- op: DoesntContain,
- customHeader: "X-Duplicated-Header",
- count: 1},
-
-+ { testString: "this header tests DB string properties",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 1},
-+ { testString: "which can be handled",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 1},
-+ { testString: "differently than X-Duplicated-Header, so better test it",
-+ testAttribute: OtherHeader,
-+ op: Is,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 1},
-+ { testString: "this header tests DB string properties",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 0},
-+ { testString: "which can be handled",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 0},
-+ { testString: "differently than X-Duplicated-Header, so better test it",
-+ testAttribute: OtherHeader,
-+ op: Isnt,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 0},
-+ { testString: "than X-Duplicated-Header,",
-+ testAttribute: OtherHeader,
-+ op: Contains,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 1},
-+ { testString: "than X-Duplicated-Header, so",
-+ testAttribute: OtherHeader,
-+ op: DoesntContain,
-+ customHeader: "X-Duplicated-Header-DB",
-+ count: 0},
-+
- // test accumulation of received header
- // only in first received
- { testString: "caspiaco",
- testAttribute: OtherHeader,
- op: Contains,
- customHeader: "Received",
- count: 1},
- // only in second
-@@ -372,17 +433,17 @@ function run_test()
- OnProgress: function(aProgress, aProgressMax) {},
- SetMessageKey: function(aKey) {},
- SetMessageId: function(aMessageId) {},
- OnStopCopy: function(aStatus) { testSearch();}
- };
-
- // set value of headers we want parsed into the db
- Services.prefs.setCharPref("mailnews.customDBHeaders",
-- "oneLiner twoLiner threeLiner noSpace withSpace");
-+ "oneLiner twoLiner threeLiner noSpace withSpace X-Duplicated-Header-DB");
- // Get a message into the local filestore. function testSearch() continues
- // the testing after the copy.
- var bugmail12 = do_get_file("../../../data/bugmail12");
- do_test_pending();
- MailServices.copy.CopyFileMessage(bugmail12, localAccountUtils.inboxFolder, null,
- false, 0, "", copyListener, null);
- }
-
-diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
---- a/mailnews/test/data/bugmail12
-+++ b/mailnews/test/data/bugmail12
-@@ -61,16 +61,22 @@ X-Duplicated-Header: multiline value
- that needs to be
- handled.
- X-Duplicated-Header: here comes a continuation line
- that contains the word 'fifth' we're looking for
- and then another one
- X-Duplicated-Header: This is
- the end
- my friend
-+X-Duplicated-Header-DB: this header
-+ tests DB
-+ string properties
-+X-Duplicated-Header-DB: which can be handled
-+X-Duplicated-Header-DB: differently than
-+ X-Duplicated-Header, so better test it
- In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
- References: <bug-397009-254728@https.bugzilla.mozilla.org/>
- X-Priority: 4
- Content-Type: text/plain; charset="UTF-8"
- MIME-Version: 1.0
- X-user: ::::63.245.208.146:host29.hostmonster.com::::::
- DomainKey-Status: no signature
-
-