3 # User Arkadiusz Miskiewicz <arekm@maven.pl>
4 # Date 1535035719 -7200
5 # Node ID 74ed2e4ba2456c97a4266b4c58faf4bf24b4d21e
6 # Parent 3d2c10e918d29353b29506c635c26e2e5f3cd2bb
7 Bug 678322 - process all duplicate headers when searching. r=jorgk
9 Search in all headers even if header with the same name occured multiple times.
10 Stop treating Received in special way as now it's not needed - we will search
11 all Received occurences (just like any other header) now.
13 Add tests for for the problem.
15 diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
16 --- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
17 +++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
18 @@ -736,127 +736,129 @@ nsresult nsMsgSearchTerm::DeStreamNew (c
19 CopyUTF8toUTF16(mozilla::MakeStringSpan(m_value.string), m_value.utf16String);
25 // Looks in the MessageDB for the user specified arbitrary header, if it finds the header, it then looks for a match against
26 // the value for the header.
27 -nsresult nsMsgSearchTerm::MatchArbitraryHeader (nsIMsgSearchScopeTerm *scope,
28 - uint32_t length /* in lines*/,
29 - const char *charset,
30 - bool charsetOverride,
33 - const char * headers,
34 - uint32_t headersSize,
37 +nsresult nsMsgSearchTerm::MatchArbitraryHeader(nsIMsgSearchScopeTerm *scope,
38 + uint32_t length /* in lines*/,
39 + const char *charset,
40 + bool charsetOverride,
43 + const char * headers,
44 + uint32_t headersSize,
48 NS_ENSURE_ARG_POINTER(pResult);
52 bool matchExpected = m_operator == nsMsgSearchOp::Contains ||
53 m_operator == nsMsgSearchOp::Is ||
54 m_operator == nsMsgSearchOp::BeginsWith ||
55 m_operator == nsMsgSearchOp::EndsWith;
56 - // init result to what we want if we don't find the header at all
57 + // Initialize result to what we want if we don't find the header at all.
58 bool result = !matchExpected;
61 msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
62 - if (!dbHdrValue.IsEmpty())
63 - // match value with the other info.
64 - return MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
65 + if (!dbHdrValue.IsEmpty()) {
66 + // Match value with the other info. It doesn't check all header occurences,
67 + // so we use it only if we match and do line by line headers parsing otherwise.
68 + rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
72 - nsMsgBodyHandler * bodyHandler =
73 - new nsMsgBodyHandler (scope, length, msg, db, headers, headersSize,
75 - NS_ENSURE_TRUE(bodyHandler, NS_ERROR_OUT_OF_MEMORY);
79 + nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
80 + headers, headersSize,
82 bodyHandler->SetStripHeaders (false);
84 - nsCString headerFullValue; // contains matched header value accumulated over multiple lines.
85 + nsCString headerFullValue; // Contains matched header value accumulated over multiple lines.
87 nsAutoCString curMsgHeader;
88 - bool searchingHeaders = true;
89 + bool processingHeaders = true;
91 - // We will allow accumulation of received headers;
92 - bool isReceivedHeader = m_arbitraryHeader.EqualsLiteral("received");
94 - while (searchingHeaders)
95 + while (processingHeaders)
97 nsCString charsetIgnored;
98 if (bodyHandler->GetNextLine(buf, charsetIgnored) < 0 || EMPTY_MESSAGE_LINE(buf))
99 - searchingHeaders = false;
100 - bool isContinuationHeader = searchingHeaders ?
101 + processingHeaders = false; // No more lines or emtpy line teminating headers.
103 + bool isContinuationHeader = processingHeaders ?
104 NS_IsAsciiWhitespace(buf.CharAt(0)) : false;
106 - // We try to match the header from the last time through the loop, which should now
107 - // have accumulated over possible multiple lines. For all headers except received,
108 - // we process a single accumulation, but process accumulated received at the end.
109 - if (!searchingHeaders || (!isContinuationHeader &&
110 - (!headerFullValue.IsEmpty() && !isReceivedHeader)))
111 + // If we're not on a continuation header the header value is not empty,
112 + // we have finished accumulating the header value by iterating over all
113 + // header lines. Now we need to check whether the value is a match.
114 + if (!isContinuationHeader && !headerFullValue.IsEmpty())
116 - // Make sure buf has info besides just the header.
117 - // Otherwise, it's either an empty header, or header not found.
118 - if (!headerFullValue.IsEmpty())
119 + bool stringMatches;
120 + // Match value with the other info.
121 + rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
122 + if (matchExpected == stringMatches) // if we found a match
124 - bool stringMatches;
125 - // match value with the other info.
126 - rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
127 - if (matchExpected == stringMatches) // if we found a match
129 - searchingHeaders = false; // then stop examining the headers
130 - result = stringMatches;
132 + // If we found a match, stop examining the headers.
133 + processingHeaders = false;
134 + result = stringMatches;
136 + // Prepare for repeated header of the same type.
137 + headerFullValue.Truncate();
140 + // We got result or finished processing all lines.
141 + if (!processingHeaders)
145 char * buf_end = (char *) (buf.get() + buf.Length());
146 int headerLength = m_arbitraryHeader.Length();
148 // If the line starts with whitespace, then we use the current header.
149 if (!isContinuationHeader)
151 - // here we start a new header
152 + // Here we start a new header.
153 uint32_t colonPos = buf.FindChar(':');
154 curMsgHeader = StringHead(buf, colonPos);
157 if (curMsgHeader.Equals(m_arbitraryHeader, nsCaseInsensitiveCStringComparator()))
159 - // process the value
160 - // value occurs after the header name or whitespace continuation char.
161 - const char * headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
162 + // Process the value:
163 + // Value occurs after the header name or whitespace continuation char.
164 + const char *headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
165 if (headerValue < buf_end && headerValue[0] == ':') // + 1 to account for the colon which is MANDATORY
168 - // strip leading white space
169 + // Strip leading white space.
170 while (headerValue < buf_end && isspace(*headerValue))
171 - headerValue++; // advance to next character
174 - // strip trailing white space
175 + // Strip trailing white space.
176 char * end = buf_end - 1;
177 - while (end > headerValue && isspace(*end)) // while we haven't gone back past the start and we are white space....
178 + while (headerValue < end && isspace(*end))
180 - *end = '\0'; // eat up the white space
181 - end--; // move back and examine the previous character....
186 - // any continuation whitespace is converted to a single space. This includes both a continuation line, or a
187 - // second value of the same header (eg the received header)
188 + // Any continuation whitespace is converted to a single space.
189 if (!headerFullValue.IsEmpty())
190 headerFullValue.Append(' ');
191 headerFullValue.Append(nsDependentCString(headerValue));
200 NS_IMETHODIMP nsMsgSearchTerm::MatchHdrProperty(nsIMsgDBHdr *aHdr, bool *aResult)
202 NS_ENSURE_ARG_POINTER(aResult);
203 diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
204 --- a/mailnews/base/test/unit/test_search.js
205 +++ b/mailnews/base/test/unit/test_search.js
206 @@ -104,16 +104,110 @@ var Tests =
207 testAttribute: OtherHeader,
210 { testString: "foobar",
211 testAttribute: OtherHeader,
215 + // test header with multiple occurences
216 + { testString: "one value",
217 + testAttribute: OtherHeader,
219 + customHeader: "X-Duplicated-Header",
221 + { testString: "second",
222 + testAttribute: OtherHeader,
224 + customHeader: "X-Duplicated-Header",
226 + { testString: "third value for test purposes",
227 + testAttribute: OtherHeader,
229 + customHeader: "X-Duplicated-Header",
231 + { testString: "multiline value that needs to be handled.",
232 + testAttribute: OtherHeader,
234 + customHeader: "X-Duplicated-Header",
237 + { testString: "one",
238 + testAttribute: OtherHeader,
240 + customHeader: "X-Duplicated-Header",
242 + { testString: "second",
243 + testAttribute: OtherHeader,
245 + customHeader: "X-Duplicated-Header",
247 + { testString: "purposes",
248 + testAttribute: OtherHeader,
250 + customHeader: "X-Duplicated-Header",
252 + { testString: "value",
253 + testAttribute: OtherHeader,
255 + customHeader: "X-Duplicated-Header",
257 + { testString: "that needs to be",
258 + testAttribute: OtherHeader,
260 + customHeader: "X-Duplicated-Header",
262 + { testString: "fifth",
263 + testAttribute: OtherHeader,
265 + customHeader: "X-Duplicated-Header",
267 + { testString: "is the end my",
268 + testAttribute: OtherHeader,
270 + customHeader: "X-Duplicated-Header",
272 + { testString: "the end",
273 + testAttribute: OtherHeader,
275 + customHeader: "X-Duplicated-Header",
277 + { testString: "handled.",
278 + testAttribute: OtherHeader,
280 + customHeader: "X-Duplicated-Header",
282 + { testString: "one value",
283 + testAttribute: OtherHeader,
285 + customHeader: "X-Duplicated-Header",
287 + { testString: "third",
288 + testAttribute: OtherHeader,
290 + customHeader: "X-Duplicated-Header",
292 + { testString: "This is",
293 + testAttribute: OtherHeader,
295 + customHeader: "X-Duplicated-Header",
298 + { testString: "nothing",
299 + testAttribute: OtherHeader,
301 + customHeader: "X-Duplicated-Header",
303 + { testString: "nothing",
304 + testAttribute: OtherHeader,
306 + customHeader: "X-Duplicated-Header",
309 // test accumulation of received header
310 // only in first received
311 { testString: "caspiaco",
312 testAttribute: OtherHeader,
314 customHeader: "Received",
317 @@ -123,16 +217,22 @@ var Tests =
318 customHeader: "received",
321 { testString: "not there",
322 testAttribute: OtherHeader,
324 customHeader: "received",
326 + // not on first line of received
327 + { testString: "m47LtAFJ007547",
328 + testAttribute: OtherHeader,
330 + customHeader: "received",
333 // test multiple line arbitrary headers
335 { testString: "SpamAssassin 3.2.3",
336 testAttribute: OtherHeader,
338 customHeader: "X-Spam-Checker-Version",
340 diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
341 --- a/mailnews/test/data/bugmail12
342 +++ b/mailnews/test/data/bugmail12
343 @@ -49,16 +49,28 @@ X-Bugzilla-Component: MailNews: Filters
345 X-Bugzilla-Severity: enhancement
346 X-Bugzilla-Who: bugmail@example.org
347 X-Bugzilla-Status: NEW
348 X-Bugzilla-Priority: --
349 X-Bugzilla-Assigned-To: nobody@mozilla.org
350 X-Bugzilla-Target-Milestone: ---
351 X-Bugzilla-Changed-Fields: Blocks
352 +X-Duplicated-Header: one value
353 +X-Duplicated-Header: second
354 +X-Duplicated-Header: third value for test purposes
355 +X-Duplicated-Header: multiline value
358 +X-Duplicated-Header: here comes a continuation line
359 + that contains the word 'fifth' we're looking for
360 + and then another one
361 +X-Duplicated-Header: This is
364 In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
365 References: <bug-397009-254728@https.bugzilla.mozilla.org/>
367 Content-Type: text/plain; charset="UTF-8"
369 X-user: ::::63.245.208.146:host29.hostmonster.com::::::
370 DomainKey-Status: no signature
375 # User Arkadiusz Miskiewicz <arekm@maven.pl>
376 # Date 1535318767 -7200
377 # Node ID 9289d8a37eb2dae82f2ecc0076ec8396126a5526
378 # Parent ce828f6cf25697596ec928ee99dfc07bdf0ce006
379 Bug 678322 - Follow-up: fix logic error and cater for IMAP case where headers are not available. r=jorgk
381 diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
382 --- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
383 +++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
384 @@ -764,20 +764,21 @@ nsresult nsMsgSearchTerm::MatchArbitrary
385 bool result = !matchExpected;
387 nsCString dbHdrValue;
388 msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
389 if (!dbHdrValue.IsEmpty()) {
390 // Match value with the other info. It doesn't check all header occurences,
391 // so we use it only if we match and do line by line headers parsing otherwise.
392 rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
394 + if (matchExpected == *pResult)
398 + // Preset result in case we don't have access to the headers, like for IMAP.
402 nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
403 headers, headersSize,
405 bodyHandler->SetStripHeaders (false);
407 nsCString headerFullValue; // Contains matched header value accumulated over multiple lines.
411 # User Arkadiusz Miskiewicz <arekm@maven.pl>
412 # Date 1535451947 -7200
413 # Node ID a09a76a92abfcfa36c00727cef84293175ade116
414 # Parent ff6f03d6611e158e2d8e951d9fed621dfd90821f
415 Bug 678322 - Follow-up: more tests including matching based on message string properties. r=jorgk
417 nsMsgSearchTerm::MatchArbitraryHeader() can do matching using message string
418 properties and by matching headers line by line.
420 test_search.js was only testing second case for duplicated headers. Now we also test
421 first case. Also added more tests for second case.
423 diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
424 --- a/mailnews/base/test/unit/test_search.js
425 +++ b/mailnews/base/test/unit/test_search.js
426 @@ -125,16 +125,36 @@ var Tests =
428 customHeader: "X-Duplicated-Header",
430 { testString: "multiline value that needs to be handled.",
431 testAttribute: OtherHeader,
433 customHeader: "X-Duplicated-Header",
435 + { testString: "one value",
436 + testAttribute: OtherHeader,
438 + customHeader: "X-Duplicated-Header",
440 + { testString: "second",
441 + testAttribute: OtherHeader,
443 + customHeader: "X-Duplicated-Header",
445 + { testString: "third value for test purposes",
446 + testAttribute: OtherHeader,
448 + customHeader: "X-Duplicated-Header",
450 + { testString: "multiline value that needs to be handled.",
451 + testAttribute: OtherHeader,
453 + customHeader: "X-Duplicated-Header",
457 testAttribute: OtherHeader,
459 customHeader: "X-Duplicated-Header",
461 { testString: "second",
462 testAttribute: OtherHeader,
463 @@ -198,16 +218,57 @@ var Tests =
464 customHeader: "X-Duplicated-Header",
466 { testString: "nothing",
467 testAttribute: OtherHeader,
469 customHeader: "X-Duplicated-Header",
472 + { testString: "this header tests DB string properties",
473 + testAttribute: OtherHeader,
475 + customHeader: "X-Duplicated-Header-DB",
477 + { testString: "which can be handled",
478 + testAttribute: OtherHeader,
480 + customHeader: "X-Duplicated-Header-DB",
482 + { testString: "differently than X-Duplicated-Header, so better test it",
483 + testAttribute: OtherHeader,
485 + customHeader: "X-Duplicated-Header-DB",
487 + { testString: "this header tests DB string properties",
488 + testAttribute: OtherHeader,
490 + customHeader: "X-Duplicated-Header-DB",
492 + { testString: "which can be handled",
493 + testAttribute: OtherHeader,
495 + customHeader: "X-Duplicated-Header-DB",
497 + { testString: "differently than X-Duplicated-Header, so better test it",
498 + testAttribute: OtherHeader,
500 + customHeader: "X-Duplicated-Header-DB",
502 + { testString: "than X-Duplicated-Header,",
503 + testAttribute: OtherHeader,
505 + customHeader: "X-Duplicated-Header-DB",
507 + { testString: "than X-Duplicated-Header, so",
508 + testAttribute: OtherHeader,
510 + customHeader: "X-Duplicated-Header-DB",
513 // test accumulation of received header
514 // only in first received
515 { testString: "caspiaco",
516 testAttribute: OtherHeader,
518 customHeader: "Received",
521 @@ -372,17 +433,17 @@ function run_test()
522 OnProgress: function(aProgress, aProgressMax) {},
523 SetMessageKey: function(aKey) {},
524 SetMessageId: function(aMessageId) {},
525 OnStopCopy: function(aStatus) { testSearch();}
528 // set value of headers we want parsed into the db
529 Services.prefs.setCharPref("mailnews.customDBHeaders",
530 - "oneLiner twoLiner threeLiner noSpace withSpace");
531 + "oneLiner twoLiner threeLiner noSpace withSpace X-Duplicated-Header-DB");
532 // Get a message into the local filestore. function testSearch() continues
533 // the testing after the copy.
534 var bugmail12 = do_get_file("../../../data/bugmail12");
536 MailServices.copy.CopyFileMessage(bugmail12, localAccountUtils.inboxFolder, null,
537 false, 0, "", copyListener, null);
540 diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
541 --- a/mailnews/test/data/bugmail12
542 +++ b/mailnews/test/data/bugmail12
543 @@ -61,16 +61,22 @@ X-Duplicated-Header: multiline value
546 X-Duplicated-Header: here comes a continuation line
547 that contains the word 'fifth' we're looking for
549 X-Duplicated-Header: This is
552 +X-Duplicated-Header-DB: this header
555 +X-Duplicated-Header-DB: which can be handled
556 +X-Duplicated-Header-DB: differently than
557 + X-Duplicated-Header, so better test it
558 In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
559 References: <bug-397009-254728@https.bugzilla.mozilla.org/>
561 Content-Type: text/plain; charset="UTF-8"
563 X-user: ::::63.245.208.146:host29.hostmonster.com::::::
564 DomainKey-Status: no signature