]> git.pld-linux.org Git - packages/thunderbird.git/blob - bug-678322.patch
up to 60.2.1
[packages/thunderbird.git] / bug-678322.patch
1
2 # HG changeset patch
3 # User Arkadiusz Miskiewicz <arekm@maven.pl>
4 # Date 1535035719 -7200
5 # Node ID 74ed2e4ba2456c97a4266b4c58faf4bf24b4d21e
6 # Parent  3d2c10e918d29353b29506c635c26e2e5f3cd2bb
7 Bug 678322 - process all duplicate headers when searching. r=jorgk
8
9 Search in all headers even if header with the same name occured multiple times.
10 Stop treating Received in special way as now it's not needed - we will search
11 all Received occurences (just like any other header) now.
12
13 Add tests for for the problem.
14
15 diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
16 --- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
17 +++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
18 @@ -736,127 +736,129 @@ nsresult nsMsgSearchTerm::DeStreamNew (c
19      CopyUTF8toUTF16(mozilla::MakeStringSpan(m_value.string), m_value.utf16String);
20    }
21    return NS_OK;
22  }
23  
24  
25  // Looks in the MessageDB for the user specified arbitrary header, if it finds the header, it then looks for a match against
26  // the value for the header.
27 -nsresult nsMsgSearchTerm::MatchArbitraryHeader (nsIMsgSearchScopeTerm *scope,
28 -                                                uint32_t length /* in lines*/,
29 -                                                const char *charset,
30 -                                                bool charsetOverride,
31 -                                                nsIMsgDBHdr *msg,
32 -                                                nsIMsgDatabase* db,
33 -                                                const char * headers,
34 -                                                uint32_t headersSize,
35 -                                                bool ForFiltering,
36 -                                                bool *pResult)
37 +nsresult nsMsgSearchTerm::MatchArbitraryHeader(nsIMsgSearchScopeTerm *scope,
38 +                                               uint32_t length /* in lines*/,
39 +                                               const char *charset,
40 +                                               bool charsetOverride,
41 +                                               nsIMsgDBHdr *msg,
42 +                                               nsIMsgDatabase* db,
43 +                                               const char * headers,
44 +                                               uint32_t headersSize,
45 +                                               bool ForFiltering,
46 +                                               bool *pResult)
47  {
48    NS_ENSURE_ARG_POINTER(pResult);
49  
50    *pResult = false;
51    nsresult rv = NS_OK;
52    bool matchExpected = m_operator == nsMsgSearchOp::Contains ||
53                         m_operator == nsMsgSearchOp::Is ||
54                         m_operator == nsMsgSearchOp::BeginsWith ||
55                         m_operator == nsMsgSearchOp::EndsWith;
56 -  // init result to what we want if we don't find the header at all
57 +  // Initialize result to what we want if we don't find the header at all.
58    bool result = !matchExpected;
59  
60    nsCString dbHdrValue;
61    msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
62 -  if (!dbHdrValue.IsEmpty())
63 -    // match value with the other info.
64 -    return MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
65 +  if (!dbHdrValue.IsEmpty()) {
66 +    // Match value with the other info. It doesn't check all header occurences,
67 +    // so we use it only if we match and do line by line headers parsing otherwise.
68 +    rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
69 +    if (*pResult)
70 +      return rv;
71  
72 -  nsMsgBodyHandler * bodyHandler =
73 -    new nsMsgBodyHandler (scope, length, msg, db, headers, headersSize,
74 -                          ForFiltering);
75 -  NS_ENSURE_TRUE(bodyHandler, NS_ERROR_OUT_OF_MEMORY);
76 +    rv = NS_OK;
77 +  }
78  
79 +  nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
80 +                                                       headers, headersSize,
81 +                                                       ForFiltering);
82    bodyHandler->SetStripHeaders (false);
83  
84 -  nsCString headerFullValue; // contains matched header value accumulated over multiple lines.
85 +  nsCString headerFullValue;  // Contains matched header value accumulated over multiple lines.
86    nsAutoCString buf;
87    nsAutoCString curMsgHeader;
88 -  bool searchingHeaders = true;
89 +  bool processingHeaders = true;
90  
91 -  // We will allow accumulation of received headers;
92 -  bool isReceivedHeader = m_arbitraryHeader.EqualsLiteral("received");
93 -
94 -  while (searchingHeaders)
95 +  while (processingHeaders)
96    {
97      nsCString charsetIgnored;
98      if (bodyHandler->GetNextLine(buf, charsetIgnored) < 0 || EMPTY_MESSAGE_LINE(buf))
99 -      searchingHeaders = false;
100 -    bool isContinuationHeader = searchingHeaders ?
101 +      processingHeaders = false;  // No more lines or emtpy line teminating headers.
102 +
103 +    bool isContinuationHeader = processingHeaders ?
104        NS_IsAsciiWhitespace(buf.CharAt(0)) : false;
105  
106 -    // We try to match the header from the last time through the loop, which should now
107 -    //  have accumulated over possible multiple lines. For all headers except received,
108 -    //  we process a single accumulation, but process accumulated received at the end.
109 -    if (!searchingHeaders || (!isContinuationHeader &&
110 -         (!headerFullValue.IsEmpty() && !isReceivedHeader)))
111 +    // If we're not on a continuation header the header value is not empty,
112 +    // we have finished accumulating the header value by iterating over all
113 +    // header lines. Now we need to check whether the value is a match.
114 +    if (!isContinuationHeader && !headerFullValue.IsEmpty())
115      {
116 -      // Make sure buf has info besides just the header.
117 -      // Otherwise, it's either an empty header, or header not found.
118 -      if (!headerFullValue.IsEmpty())
119 +      bool stringMatches;
120 +      // Match value with the other info.
121 +      rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
122 +      if (matchExpected == stringMatches) // if we found a match
123        {
124 -        bool stringMatches;
125 -        // match value with the other info.
126 -        rv = MatchRfc2047String(headerFullValue, charset, charsetOverride, &stringMatches);
127 -        if (matchExpected == stringMatches) // if we found a match
128 -        {
129 -          searchingHeaders = false;   // then stop examining the headers
130 -          result = stringMatches;
131 -        }
132 +        // If we found a match, stop examining the headers.
133 +        processingHeaders = false;
134 +        result = stringMatches;
135        }
136 +      // Prepare for repeated header of the same type.
137 +      headerFullValue.Truncate();
138 +    }
139 +
140 +    // We got result or finished processing all lines.
141 +    if (!processingHeaders)
142        break;
143 -    }
144  
145      char * buf_end = (char *) (buf.get() + buf.Length());
146      int headerLength = m_arbitraryHeader.Length();
147  
148      // If the line starts with whitespace, then we use the current header.
149      if (!isContinuationHeader)
150      {
151 -      // here we start a new header
152 +      // Here we start a new header.
153        uint32_t colonPos = buf.FindChar(':');
154        curMsgHeader = StringHead(buf, colonPos);
155      }
156  
157      if (curMsgHeader.Equals(m_arbitraryHeader, nsCaseInsensitiveCStringComparator()))
158      {
159 -      // process the value
160 -      // value occurs after the header name or whitespace continuation char.
161 -      const char * headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
162 +      // Process the value:
163 +      // Value occurs after the header name or whitespace continuation char.
164 +      const char *headerValue = buf.get() + (isContinuationHeader ? 1 : headerLength);
165        if (headerValue < buf_end && headerValue[0] == ':')  // + 1 to account for the colon which is MANDATORY
166          headerValue++;
167  
168 -      // strip leading white space
169 +      // Strip leading white space.
170        while (headerValue < buf_end && isspace(*headerValue))
171 -        headerValue++; // advance to next character
172 +        headerValue++;
173  
174 -      // strip trailing white space
175 +      // Strip trailing white space.
176        char * end = buf_end - 1;
177 -      while (end > headerValue && isspace(*end)) // while we haven't gone back past the start and we are white space....
178 +      while (headerValue < end && isspace(*end))
179        {
180 -        *end = '\0';  // eat up the white space
181 -        end--;      // move back and examine the previous character....
182 +        *end = '\0';
183 +        end--;
184        }
185  
186 -      // any continuation whitespace is converted to a single space. This includes both a continuation line, or a
187 -      //  second value of the same header (eg the received header)
188 +      // Any continuation whitespace is converted to a single space.
189        if (!headerFullValue.IsEmpty())
190          headerFullValue.Append(' ');
191        headerFullValue.Append(nsDependentCString(headerValue));
192      }
193    }
194 +
195    delete bodyHandler;
196    *pResult = result;
197    return rv;
198  }
199  
200  NS_IMETHODIMP nsMsgSearchTerm::MatchHdrProperty(nsIMsgDBHdr *aHdr, bool *aResult)
201  {
202    NS_ENSURE_ARG_POINTER(aResult);
203 diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
204 --- a/mailnews/base/test/unit/test_search.js
205 +++ b/mailnews/base/test/unit/test_search.js
206 @@ -104,16 +104,110 @@ var Tests =
207      testAttribute: OtherHeader,
208      op: Contains,
209      count: 1},
210    { testString: "foobar",
211      testAttribute: OtherHeader,
212      op: Contains,
213      count: 0},
214  
215 +  // test header with multiple occurences
216 +  { testString: "one value",
217 +    testAttribute: OtherHeader,
218 +    op: Is,
219 +    customHeader: "X-Duplicated-Header",
220 +    count: 1},
221 +  { testString: "second",
222 +    testAttribute: OtherHeader,
223 +    op: Is,
224 +    customHeader: "X-Duplicated-Header",
225 +    count: 1},
226 +  { testString: "third value for test purposes",
227 +    testAttribute: OtherHeader,
228 +    op: Is,
229 +    customHeader: "X-Duplicated-Header",
230 +    count: 1},
231 +  { testString: "multiline value that needs to be handled.",
232 +    testAttribute: OtherHeader,
233 +    op: Is,
234 +    customHeader: "X-Duplicated-Header",
235 +    count: 1},
236 +
237 +  { testString: "one",
238 +    testAttribute: OtherHeader,
239 +    op: Contains,
240 +    customHeader: "X-Duplicated-Header",
241 +    count: 1},
242 +  { testString: "second",
243 +    testAttribute: OtherHeader,
244 +    op: Contains,
245 +    customHeader: "X-Duplicated-Header",
246 +    count: 1},
247 +  { testString: "purposes",
248 +    testAttribute: OtherHeader,
249 +    op: Contains,
250 +    customHeader: "X-Duplicated-Header",
251 +    count: 1},
252 +  { testString: "value",
253 +    testAttribute: OtherHeader,
254 +    op: Contains,
255 +    customHeader: "X-Duplicated-Header",
256 +    count: 1},
257 +  { testString: "that needs to be",
258 +    testAttribute: OtherHeader,
259 +    op: Contains,
260 +    customHeader: "X-Duplicated-Header",
261 +    count: 1},
262 +  { testString: "fifth",
263 +    testAttribute: OtherHeader,
264 +    op: Contains,
265 +    customHeader: "X-Duplicated-Header",
266 +    count: 1},
267 +    { testString: "is the end my",
268 +    testAttribute: OtherHeader,
269 +    op: Contains,
270 +    customHeader: "X-Duplicated-Header",
271 +    count: 1},
272 +  { testString: "the end",
273 +    testAttribute: OtherHeader,
274 +    op: EndsWith,
275 +    customHeader: "X-Duplicated-Header",
276 +    count: 0},
277 +  { testString: "handled.",
278 +    testAttribute: OtherHeader,
279 +    op: EndsWith,
280 +    customHeader: "X-Duplicated-Header",
281 +    count: 1},
282 +  { testString: "one value",
283 +    testAttribute: OtherHeader,
284 +    op: EndsWith,
285 +    customHeader: "X-Duplicated-Header",
286 +    count: 1},
287 +  { testString: "third",
288 +    testAttribute: OtherHeader,
289 +    op: BeginsWith,
290 +    customHeader: "X-Duplicated-Header",
291 +    count: 1},
292 +  { testString: "This is",
293 +    testAttribute: OtherHeader,
294 +    op: BeginsWith,
295 +    customHeader: "X-Duplicated-Header",
296 +    count: 1},
297 +
298 +  { testString: "nothing",
299 +    testAttribute: OtherHeader,
300 +    op: Contains,
301 +    customHeader: "X-Duplicated-Header",
302 +    count: 0},
303 +  { testString: "nothing",
304 +    testAttribute: OtherHeader,
305 +    op: DoesntContain,
306 +    customHeader: "X-Duplicated-Header",
307 +    count: 1},
308 +
309    // test accumulation of received header
310    // only in first received
311    { testString: "caspiaco",
312      testAttribute: OtherHeader,
313      op: Contains,
314      customHeader: "Received",
315      count: 1},
316    // only in second
317 @@ -123,16 +217,22 @@ var Tests =
318      customHeader: "received",
319      count: 1},
320    // in neither
321    { testString: "not there",
322      testAttribute: OtherHeader,
323      op: Contains,
324      customHeader: "received",
325      count: 0},
326 +  // not on first line of received
327 +  { testString: "m47LtAFJ007547",
328 +    testAttribute: OtherHeader,
329 +    op: Contains,
330 +    customHeader: "received",
331 +    count: 1},
332  
333    // test multiple line arbitrary headers
334    // in the first line
335    { testString: "SpamAssassin 3.2.3",
336      testAttribute: OtherHeader,
337      op: Contains,
338      customHeader: "X-Spam-Checker-Version",
339      count: 1},
340 diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
341 --- a/mailnews/test/data/bugmail12
342 +++ b/mailnews/test/data/bugmail12
343 @@ -49,16 +49,28 @@ X-Bugzilla-Component: MailNews: Filters
344  X-Bugzilla-Keywords: 
345  X-Bugzilla-Severity: enhancement
346  X-Bugzilla-Who: bugmail@example.org
347  X-Bugzilla-Status: NEW
348  X-Bugzilla-Priority: --
349  X-Bugzilla-Assigned-To: nobody@mozilla.org
350  X-Bugzilla-Target-Milestone: ---
351  X-Bugzilla-Changed-Fields: Blocks
352 +X-Duplicated-Header: one value
353 +X-Duplicated-Header: second
354 +X-Duplicated-Header: third value for test purposes
355 +X-Duplicated-Header: multiline value  
356 +       that needs to be   
357 +       handled.
358 +X-Duplicated-Header: here comes a continuation line
359 + that contains the word 'fifth' we're looking for  
360 + and then another one
361 +X-Duplicated-Header: This is
362 +   the end
363 +      my friend
364  In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
365  References: <bug-397009-254728@https.bugzilla.mozilla.org/>
366  X-Priority: 4
367  Content-Type: text/plain; charset="UTF-8"
368  MIME-Version: 1.0
369  X-user: ::::63.245.208.146:host29.hostmonster.com::::::
370  DomainKey-Status: no signature
371  
372
373
374 # HG changeset patch
375 # User Arkadiusz Miskiewicz <arekm@maven.pl>
376 # Date 1535318767 -7200
377 # Node ID 9289d8a37eb2dae82f2ecc0076ec8396126a5526
378 # Parent  ce828f6cf25697596ec928ee99dfc07bdf0ce006
379 Bug 678322 - Follow-up: fix logic error and cater for IMAP case where headers are not available. r=jorgk
380
381 diff --git a/mailnews/base/search/src/nsMsgSearchTerm.cpp b/mailnews/base/search/src/nsMsgSearchTerm.cpp
382 --- a/mailnews/base/search/src/nsMsgSearchTerm.cpp
383 +++ b/mailnews/base/search/src/nsMsgSearchTerm.cpp
384 @@ -764,20 +764,21 @@ nsresult nsMsgSearchTerm::MatchArbitrary
385    bool result = !matchExpected;
386  
387    nsCString dbHdrValue;
388    msg->GetStringProperty(m_arbitraryHeader.get(), getter_Copies(dbHdrValue));
389    if (!dbHdrValue.IsEmpty()) {
390      // Match value with the other info. It doesn't check all header occurences,
391      // so we use it only if we match and do line by line headers parsing otherwise.
392      rv = MatchRfc2047String(dbHdrValue, charset, charsetOverride, pResult);
393 -    if (*pResult)
394 +    if (matchExpected == *pResult)
395        return rv;
396  
397 -    rv = NS_OK;
398 +    // Preset result in case we don't have access to the headers, like for IMAP.
399 +    result = *pResult;
400    }
401  
402    nsMsgBodyHandler *bodyHandler = new nsMsgBodyHandler(scope, length, msg, db,
403                                                         headers, headersSize,
404                                                         ForFiltering);
405    bodyHandler->SetStripHeaders (false);
406  
407    nsCString headerFullValue;  // Contains matched header value accumulated over multiple lines.
408
409
410 # HG changeset patch
411 # User Arkadiusz Miskiewicz <arekm@maven.pl>
412 # Date 1535451947 -7200
413 # Node ID a09a76a92abfcfa36c00727cef84293175ade116
414 # Parent  ff6f03d6611e158e2d8e951d9fed621dfd90821f
415 Bug 678322 - Follow-up: more tests including matching based on message string properties. r=jorgk
416
417 nsMsgSearchTerm::MatchArbitraryHeader() can do matching using message string
418 properties and by matching headers line by line.
419
420 test_search.js was only testing second case for duplicated headers. Now we also test
421 first case. Also added more tests for second case.
422
423 diff --git a/mailnews/base/test/unit/test_search.js b/mailnews/base/test/unit/test_search.js
424 --- a/mailnews/base/test/unit/test_search.js
425 +++ b/mailnews/base/test/unit/test_search.js
426 @@ -125,16 +125,36 @@ var Tests =
427      op: Is,
428      customHeader: "X-Duplicated-Header",
429      count: 1},
430    { testString: "multiline value that needs to be handled.",
431      testAttribute: OtherHeader,
432      op: Is,
433      customHeader: "X-Duplicated-Header",
434      count: 1},
435 +  { testString: "one value",
436 +    testAttribute: OtherHeader,
437 +    op: Isnt,
438 +    customHeader: "X-Duplicated-Header",
439 +    count: 0},
440 +  { testString: "second",
441 +    testAttribute: OtherHeader,
442 +    op: Isnt,
443 +    customHeader: "X-Duplicated-Header",
444 +    count: 0},
445 +  { testString: "third value for test purposes",
446 +    testAttribute: OtherHeader,
447 +    op: Isnt,
448 +    customHeader: "X-Duplicated-Header",
449 +    count: 0},
450 +  { testString: "multiline value that needs to be handled.",
451 +    testAttribute: OtherHeader,
452 +    op: Isnt,
453 +    customHeader: "X-Duplicated-Header",
454 +    count: 0},
455  
456    { testString: "one",
457      testAttribute: OtherHeader,
458      op: Contains,
459      customHeader: "X-Duplicated-Header",
460      count: 1},
461    { testString: "second",
462      testAttribute: OtherHeader,
463 @@ -198,16 +218,57 @@ var Tests =
464      customHeader: "X-Duplicated-Header",
465      count: 0},
466    { testString: "nothing",
467      testAttribute: OtherHeader,
468      op: DoesntContain,
469      customHeader: "X-Duplicated-Header",
470      count: 1},
471  
472 +  { testString: "this header tests DB string properties",
473 +    testAttribute: OtherHeader,
474 +    op: Is,
475 +    customHeader: "X-Duplicated-Header-DB",
476 +    count: 1},
477 +  { testString: "which can be handled",
478 +    testAttribute: OtherHeader,
479 +    op: Is,
480 +    customHeader: "X-Duplicated-Header-DB",
481 +    count: 1},
482 +  { testString: "differently than X-Duplicated-Header, so better test it",
483 +    testAttribute: OtherHeader,
484 +    op: Is,
485 +    customHeader: "X-Duplicated-Header-DB",
486 +    count: 1},
487 +  { testString: "this header tests DB string properties",
488 +    testAttribute: OtherHeader,
489 +    op: Isnt,
490 +    customHeader: "X-Duplicated-Header-DB",
491 +    count: 0},
492 +  { testString: "which can be handled",
493 +    testAttribute: OtherHeader,
494 +    op: Isnt,
495 +    customHeader: "X-Duplicated-Header-DB",
496 +    count: 0},
497 +  { testString: "differently than X-Duplicated-Header, so better test it",
498 +    testAttribute: OtherHeader,
499 +    op: Isnt,
500 +    customHeader: "X-Duplicated-Header-DB",
501 +    count: 0},
502 +  { testString: "than X-Duplicated-Header,",
503 +    testAttribute: OtherHeader,
504 +    op: Contains,
505 +    customHeader: "X-Duplicated-Header-DB",
506 +    count: 1},
507 +  { testString: "than X-Duplicated-Header, so",
508 +    testAttribute: OtherHeader,
509 +    op: DoesntContain,
510 +    customHeader: "X-Duplicated-Header-DB",
511 +    count: 0},
512 +
513    // test accumulation of received header
514    // only in first received
515    { testString: "caspiaco",
516      testAttribute: OtherHeader,
517      op: Contains,
518      customHeader: "Received",
519      count: 1},
520    // only in second
521 @@ -372,17 +433,17 @@ function run_test()
522      OnProgress: function(aProgress, aProgressMax) {},
523      SetMessageKey: function(aKey) {},
524      SetMessageId: function(aMessageId) {},
525      OnStopCopy: function(aStatus) { testSearch();}
526    };
527  
528    // set value of headers we want parsed into the db
529    Services.prefs.setCharPref("mailnews.customDBHeaders",
530 -                             "oneLiner twoLiner threeLiner noSpace withSpace");
531 +          "oneLiner twoLiner threeLiner noSpace withSpace X-Duplicated-Header-DB");
532    // Get a message into the local filestore. function testSearch() continues
533    // the testing after the copy.
534    var bugmail12 = do_get_file("../../../data/bugmail12");
535    do_test_pending();
536    MailServices.copy.CopyFileMessage(bugmail12, localAccountUtils.inboxFolder, null,
537                                      false, 0, "", copyListener, null);
538  }
539  
540 diff --git a/mailnews/test/data/bugmail12 b/mailnews/test/data/bugmail12
541 --- a/mailnews/test/data/bugmail12
542 +++ b/mailnews/test/data/bugmail12
543 @@ -61,16 +61,22 @@ X-Duplicated-Header: multiline value
544         that needs to be   
545         handled.
546  X-Duplicated-Header: here comes a continuation line
547   that contains the word 'fifth' we're looking for  
548   and then another one
549  X-Duplicated-Header: This is
550     the end
551        my friend
552 +X-Duplicated-Header-DB: this header  
553 +   tests DB   
554 +      string properties
555 +X-Duplicated-Header-DB: which can be handled
556 +X-Duplicated-Header-DB: differently than
557 +   X-Duplicated-Header, so better test it
558  In-Reply-To: <bug-397009-254728@https.bugzilla.mozilla.org/>
559  References: <bug-397009-254728@https.bugzilla.mozilla.org/>
560  X-Priority: 4
561  Content-Type: text/plain; charset="UTF-8"
562  MIME-Version: 1.0
563  X-user: ::::63.245.208.146:host29.hostmonster.com::::::
564  DomainKey-Status: no signature
565  
566
This page took 3.051697 seconds and 4 git commands to generate.