← Back to branch summary

~james-page/ubuntu/raring/dovecot/autopkgtest

« back to all changes in this revision

Viewing changes to src/lib-storage/index/index-search.c

  • Committer: Package Import Robot
  • Author(s): James Page
  • Date: 2012-06-11 11:11:54 UTC
  • mfrom: (1.15.2) (4.1.27 sid)
  • Revision ID: package-import@ubuntu.com-20120611111154-678cwbdj6ktgsv1h
Tags: 1:2.1.7-1ubuntu1
* Merge from Debian unstable, remaining changes:
  + Add mail-stack-delivery package:
    - Update d/rules
    - d/control: convert existing dovecot-postfix package to a dummy
      package and add new mail-stack-delivery package.
    - Update maintainer scripts.
    - Rename d/dovecot-postfix.* to debian/mail-stack-delivery.*
    - d/mail-stack-delivery.preinst: Move previously installed backups and
      config files to a new package namespace.
    - d/mail-stack-delivery.prerm: Added to handle downgrades.
  + Use Snakeoil SSL certificates by default:
    - d/control: Depend on ssl-cert.
    - d/dovecot-core.postinst: Relax grep for SSL_* a bit.
  + Add autopkgtest to debian/tests/*.
  + Add ufw integration:
    - d/dovecot-core.ufw.profile: new ufw profile.
    - d/rules: install profile in dovecot-core.
    - d/control: dovecot-core - suggest ufw.
  + d/{control,rules}: enable PIE hardening.
  + d/dovecot-core.dirs: Added usr/share/doc/dovecot-core
  + Add apport hook:
    - d/rules, d/source_dovecot.py
  + Add upstart job:
    - d/rules, d/dovecot-core.dovecot.upstart, d/control,
      d/dovecot-core.dirs, dovecot-imapd.{postrm, postinst, prerm},
      d/dovecot-pop3d.{postinst, postrm, prerm}.
      d/mail-stack-deliver.postinst: Convert init script to upstart.
  + d/control: Added Pre-Depends: dpkg (>= 1.15.6) to dovecot-dbg to support
    xz compression in Ubuntu.
  + d/control: Demote dovecot-common Recommends: to Suggests: to prevent
    install of extra packages on upgrade.
  + d/patches/dovecot-drac.patch: Updated with version for dovecot >= 2.0.0.
  + d/control: Drop B-D on systemd.
* Dropped changes:
  + d/patches/fix-racey-restart.patch: part of 2.1.x, no longer required.

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/lib-storage/index/index-search.c
1
 
/* Copyright (c) 2002-2011 Dovecot authors, see the included COPYING file */
 
1
/* Copyright (c) 2002-2012 Dovecot authors, see the included COPYING file */
2
2
 
3
3
#include "lib.h"
4
4
#include "ioloop.h"
7
7
#include "utc-offset.h"
8
8
#include "str.h"
9
9
#include "time-util.h"
 
10
#include "unichar.h"
10
11
#include "imap-match.h"
11
12
#include "message-address.h"
12
13
#include "message-date.h"
18
19
#include "index-sort.h"
19
20
#include "mail-search.h"
20
21
#include "mailbox-search-result-private.h"
 
22
#include "index-search-private.h"
21
23
 
22
24
#include <stdlib.h>
23
25
#include <ctype.h>
24
26
 
25
 
#define TXT_UNKNOWN_CHARSET "[BADCHARSET] Unknown charset"
26
 
#define TXT_INVALID_SEARCH_KEY "Invalid search key"
27
 
 
28
27
#define SEARCH_NOTIFY_INTERVAL_SECS 10
29
28
 
30
29
#define SEARCH_COST_DENTRY 3ULL
38
37
#define SEARCH_INITIAL_MAX_COST 30000
39
38
#define SEARCH_RECALC_MIN_USECS 50000
40
39
 
41
 
struct index_search_context {
42
 
        struct mail_search_context mail_ctx;
43
 
        struct mail_index_view *view;
44
 
        struct mailbox *box;
45
 
 
46
 
        uint32_t seq1, seq2;
47
 
        struct mail *mail;
48
 
        struct index_mail *imail;
49
 
        struct mail_thread_context *thread_ctx;
50
 
 
51
 
        const char *error;
52
 
 
53
 
        struct timeval search_start_time, last_notify;
54
 
        struct timeval last_nonblock_timeval;
55
 
        unsigned long long cost, next_time_check_cost;
56
 
 
57
 
        unsigned int failed:1;
58
 
        unsigned int sorted:1;
59
 
        unsigned int have_seqsets:1;
60
 
        unsigned int have_index_args:1;
61
 
        unsigned int have_mailbox_args:1;
62
 
};
63
 
 
64
40
struct search_header_context {
65
 
        struct index_search_context *index_context;
 
41
        struct index_mail *imail;
66
42
        struct mail_search_arg *args;
67
43
 
68
44
        struct message_header_line *hdr;
78
54
        struct message_part *part;
79
55
};
80
56
 
81
 
static const enum message_header_parser_flags hdr_parser_flags =
82
 
        MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE;
83
 
 
84
57
static void search_parse_msgset_args(unsigned int messages_count,
85
58
                                     struct mail_search_arg *args,
86
59
                                     uint32_t *seq1_r, uint32_t *seq2_r);
87
60
 
 
61
static void search_none(struct mail_search_arg *arg ATTR_UNUSED,
 
62
                        struct search_body_context *ctx ATTR_UNUSED)
 
63
{
 
64
}
 
65
 
88
66
static void search_init_arg(struct mail_search_arg *arg,
89
67
                            struct index_search_context *ctx)
90
68
{
91
 
        uint8_t guid[MAIL_GUID_128_SIZE];
 
69
        struct mailbox_metadata metadata;
92
70
        bool match;
93
71
 
94
72
        switch (arg->type) {
105
83
                ctx->have_index_args = TRUE;
106
84
                break;
107
85
        case SEARCH_MAILBOX_GUID:
108
 
                if (mailbox_get_guid(ctx->box, guid) < 0) {
 
86
                if (mailbox_get_metadata(ctx->box, MAILBOX_METADATA_GUID,
 
87
                                         &metadata) < 0) {
109
88
                        /* result will be unknown */
110
89
                        break;
111
90
                }
112
91
 
113
 
                match = strcmp(mail_guid_128_to_string(guid),
 
92
                match = strcmp(guid_128_to_string(metadata.guid),
114
93
                               arg->value.str) == 0;
115
 
                if (match != arg->not)
 
94
                if (match != arg->match_not)
116
95
                        arg->match_always = TRUE;
117
96
                else
118
97
                        arg->nonmatch_always = TRUE;
122
101
                ctx->have_mailbox_args = TRUE;
123
102
                break;
124
103
        case SEARCH_ALL:
125
 
                if (!arg->not)
 
104
                if (!arg->match_not)
126
105
                        arg->match_always = TRUE;
127
106
                else
128
107
                        arg->nonmatch_always = TRUE;
239
218
 
240
219
        switch (arg->type) {
241
220
        case SEARCH_MAILBOX:
242
 
                if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME,
 
221
                if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
243
222
                                     &str) < 0)
244
223
                        return -1;
245
224
 
247
226
                        return strcasecmp(arg->value.str, "INBOX") == 0;
248
227
                return strcmp(str, arg->value.str) == 0;
249
228
        case SEARCH_MAILBOX_GLOB:
250
 
                if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME,
 
229
                if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
251
230
                                     &str) < 0)
252
231
                        return -1;
253
232
                return imap_match(arg->value.mailbox_glob, str) == IMAP_MATCH_YES;
291
270
                have_tz_offset = FALSE; tz_offset = 0; date = (time_t)-1;
292
271
                switch (arg->value.date_type) {
293
272
                case MAIL_SEARCH_DATE_TYPE_SENT:
294
 
                        if (mail_get_date(ctx->mail, &date, &tz_offset) < 0)
 
273
                        if (mail_get_date(ctx->cur_mail, &date, &tz_offset) < 0)
295
274
                                return -1;
296
275
                        have_tz_offset = TRUE;
297
276
                        break;
298
277
                case MAIL_SEARCH_DATE_TYPE_RECEIVED:
299
 
                        if (mail_get_received_date(ctx->mail, &date) < 0)
 
278
                        if (mail_get_received_date(ctx->cur_mail, &date) < 0)
300
279
                                return -1;
301
280
                        break;
302
281
                case MAIL_SEARCH_DATE_TYPE_SAVED:
303
 
                        if (mail_get_save_date(ctx->mail, &date) < 0)
 
282
                        if (mail_get_save_date(ctx->cur_mail, &date) < 0)
304
283
                                return -1;
305
284
                        break;
306
285
                }
330
309
        /* sizes */
331
310
        case SEARCH_SMALLER:
332
311
        case SEARCH_LARGER:
333
 
                if (mail_get_virtual_size(ctx->mail, &virtual_size) < 0)
 
312
                if (mail_get_virtual_size(ctx->cur_mail, &virtual_size) < 0)
334
313
                        return -1;
335
314
 
336
315
                if (arg->type == SEARCH_SMALLER)
339
318
                        return virtual_size > arg->value.size;
340
319
 
341
320
        case SEARCH_GUID:
342
 
                if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, &str) < 0)
 
321
                if (mail_get_special(ctx->cur_mail, MAIL_FETCH_GUID, &str) < 0)
343
322
                        return -1;
344
323
                return strcmp(str, arg->value.str) == 0;
345
324
        default:
393
372
}
394
373
 
395
374
static struct message_search_context *
396
 
msg_search_arg_context(struct index_search_context *ctx,
397
 
                       struct mail_search_arg *arg)
 
375
msg_search_arg_context(struct mail_search_arg *arg)
398
376
{
399
 
        struct message_search_context *arg_ctx = arg->context;
400
 
        enum message_search_flags flags;
401
 
        int ret;
402
 
 
403
 
        if (arg_ctx != NULL)
404
 
                return arg_ctx;
405
 
 
406
 
        flags = (arg->type == SEARCH_BODY || arg->type == SEARCH_BODY_FAST) ?
407
 
                MESSAGE_SEARCH_FLAG_SKIP_HEADERS : 0;
408
 
 
409
 
        ret = message_search_init(arg->value.str,
410
 
                                  ctx->mail_ctx.args->charset, flags,
411
 
                                  &arg_ctx);
412
 
        if (ret > 0) {
413
 
                arg->context = arg_ctx;
414
 
                return arg_ctx;
415
 
        }
416
 
        if (ret == 0)
417
 
                ctx->error = TXT_UNKNOWN_CHARSET;
418
 
        else
419
 
                ctx->error = TXT_INVALID_SEARCH_KEY;
420
 
        return NULL;
 
377
        enum message_search_flags flags = MESSAGE_SEARCH_FLAG_DTCASE;
 
378
 
 
379
        if (arg->context == NULL) T_BEGIN {
 
380
                string_t *dtc = t_str_new(128);
 
381
 
 
382
                if (uni_utf8_to_decomposed_titlecase(arg->value.str,
 
383
                                                     strlen(arg->value.str),
 
384
                                                     dtc) < 0)
 
385
                        i_panic("search key not utf8: %s", arg->value.str);
 
386
 
 
387
                if (arg->type == SEARCH_BODY)
 
388
                        flags |= MESSAGE_SEARCH_FLAG_SKIP_HEADERS;
 
389
                /* we don't get here if arg is "", but dtc can be "" if it
 
390
                   only contains characters that we need to ignore. handle
 
391
                   those searches by returning them as non-matched. */
 
392
                if (str_len(dtc) > 0)
 
393
                        arg->context = message_search_init(str_c(dtc), flags);
 
394
        } T_END;
 
395
        return arg->context;
421
396
}
422
397
 
423
398
static void compress_lwsp(string_t *dest, const unsigned char *src,
500
475
        hdr.middle_len = 0;
501
476
        block.hdr = &hdr;
502
477
 
503
 
        msg_search_ctx = msg_search_arg_context(ctx->index_context, arg);
 
478
        msg_search_ctx = msg_search_arg_context(arg);
504
479
        if (msg_search_ctx == NULL)
505
480
                return;
506
481
 
551
526
                if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SENT)
552
527
                        break;
553
528
 
554
 
                if (arg->not) {
 
529
                if (arg->match_not) {
555
530
                        /* date header not found, so we match only for
556
531
                           NOT searches */
557
532
                        ARG_SET_RESULT(arg, 0);
580
555
                return;
581
556
 
582
557
        if (ctx->parse_headers)
583
 
                index_mail_parse_header(NULL, hdr, ctx->index_context->imail);
 
558
                index_mail_parse_header(NULL, hdr, ctx->imail);
584
559
 
585
560
        if (ctx->custom_header || strcasecmp(hdr->name, "Date") == 0) {
586
561
                ctx->hdr = hdr;
596
571
        struct message_search_context *msg_search_ctx;
597
572
        int ret;
598
573
 
599
 
        if (ctx->index_ctx->error != NULL)
600
 
                return;
601
 
 
602
574
        switch (arg->type) {
603
575
        case SEARCH_BODY:
604
 
        case SEARCH_BODY_FAST:
605
576
        case SEARCH_TEXT:
606
 
        case SEARCH_TEXT_FAST:
607
577
                break;
608
578
        default:
609
579
                return;
610
580
        }
611
581
 
612
 
        msg_search_ctx = msg_search_arg_context(ctx->index_ctx, arg);
 
582
        msg_search_ctx = msg_search_arg_context(arg);
613
583
        if (msg_search_ctx == NULL) {
614
584
                ARG_SET_RESULT(arg, 0);
615
585
                return;
619
589
        ret = message_search_msg(msg_search_ctx, ctx->input, ctx->part);
620
590
        if (ret < 0 && ctx->input->stream_errno == 0) {
621
591
                /* try again without cached parts */
622
 
                mail_set_cache_corrupted(ctx->index_ctx->mail,
 
592
                mail_set_cache_corrupted(ctx->index_ctx->cur_mail,
623
593
                                         MAIL_FETCH_MESSAGE_PARTS);
624
594
 
625
595
                i_stream_seek(ctx->input, 0);
631
601
}
632
602
 
633
603
static int search_arg_match_text(struct mail_search_arg *args,
634
 
                                 struct index_search_context *ctx, int ret)
 
604
                                 struct index_search_context *ctx)
635
605
{
636
 
        struct istream *input;
 
606
        const enum message_header_parser_flags hdr_parser_flags =
 
607
                MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE;
 
608
        struct index_mail *imail = (struct index_mail *)ctx->cur_mail;
 
609
        struct istream *input = NULL;
637
610
        struct mailbox_header_lookup_ctx *headers_ctx;
638
 
        struct mail_search_arg *arg;
 
611
        struct search_header_context hdr_ctx;
 
612
        struct search_body_context body_ctx;
639
613
        const char *const *headers;
640
 
        bool have_headers, have_body;
 
614
        bool have_headers, have_body, failed = FALSE;
 
615
        int ret;
641
616
 
642
617
        /* first check what we need to use */
643
618
        headers = mail_search_args_analyze(args, &have_headers, &have_body);
644
619
        if (!have_headers && !have_body)
645
 
                return ret;
 
620
                return -1;
 
621
 
 
622
        memset(&hdr_ctx, 0, sizeof(hdr_ctx));
 
623
        /* hdr_ctx.imail is different from imail for mails in
 
624
           virtual mailboxes */
 
625
        hdr_ctx.imail = (struct index_mail *)mail_get_real_mail(ctx->cur_mail);
 
626
        hdr_ctx.custom_header = TRUE;
 
627
        hdr_ctx.args = args;
 
628
 
 
629
        headers_ctx = headers == NULL ? NULL :
 
630
                mailbox_header_lookup_init(ctx->box, headers);
 
631
        if (headers != NULL &&
 
632
            (!have_body ||
 
633
             ctx->cur_mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER)) {
 
634
                /* try to look up the specified headers from cache */
 
635
                i_assert(*headers != NULL);
 
636
 
 
637
                if (mail_get_header_stream(ctx->cur_mail, headers_ctx,
 
638
                                           &input) < 0)
 
639
                        failed = TRUE;
 
640
                else {
 
641
                        message_parse_header(input, NULL, hdr_parser_flags,
 
642
                                             search_header, &hdr_ctx);
 
643
                }
 
644
                input = NULL;
 
645
        } else if (have_headers) {
 
646
                /* we need to read the entire header */
 
647
                if (mail_get_hdr_stream(ctx->cur_mail, NULL, &input) < 0)
 
648
                        failed = TRUE;
 
649
                else {
 
650
                        hdr_ctx.parse_headers =
 
651
                                index_mail_want_parse_headers(hdr_ctx.imail);
 
652
                        if (hdr_ctx.parse_headers) {
 
653
                                index_mail_parse_header_init(hdr_ctx.imail,
 
654
                                                             headers_ctx);
 
655
                        }
 
656
                        message_parse_header(input, NULL, hdr_parser_flags,
 
657
                                             search_header, &hdr_ctx);
 
658
                }
 
659
        }
 
660
        if (headers_ctx != NULL)
 
661
                mailbox_header_lookup_unref(&headers_ctx);
 
662
 
 
663
        if (failed) {
 
664
                /* opening mail failed. maybe because of lookup_abort.
 
665
                   update access_parts for prefetching */
 
666
                if (have_body)
 
667
                        imail->data.access_part |= READ_HDR | READ_BODY;
 
668
                else 
 
669
                        imail->data.access_part |= READ_HDR;
 
670
                return -1;
 
671
        }
646
672
 
647
673
        if (have_headers) {
648
 
                struct search_header_context hdr_ctx;
649
 
 
650
 
                if (have_body &&
651
 
                    ctx->mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER) {
652
 
                        /* just open the mail bypassing any caching, since
653
 
                           we're going to read through the body anyway */
654
 
                        headers = NULL;
655
 
                }
656
 
 
657
 
                if (headers == NULL) {
658
 
                        headers_ctx = NULL;
659
 
                        if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0)
660
 
                                return -1;
661
 
                } else {
662
 
                        /* FIXME: do this once in init */
663
 
                        i_assert(*headers != NULL);
664
 
                        headers_ctx =
665
 
                                mailbox_header_lookup_init(ctx->box, headers);
666
 
                        if (mail_get_header_stream(ctx->mail, headers_ctx,
667
 
                                                   &input) < 0) {
668
 
                                mailbox_header_lookup_unref(&headers_ctx);
669
 
                                return -1;
670
 
                        }
671
 
                }
672
 
 
673
 
                memset(&hdr_ctx, 0, sizeof(hdr_ctx));
674
 
                hdr_ctx.index_context = ctx;
675
 
                hdr_ctx.custom_header = TRUE;
676
 
                hdr_ctx.args = args;
677
 
                hdr_ctx.parse_headers = headers == NULL &&
678
 
                        index_mail_want_parse_headers(ctx->imail);
679
 
 
680
 
                if (hdr_ctx.parse_headers)
681
 
                        index_mail_parse_header_init(ctx->imail, headers_ctx);
682
 
                message_parse_header(input, NULL, hdr_parser_flags,
683
 
                                     search_header, &hdr_ctx);
684
 
                if (headers_ctx != NULL)
685
 
                        mailbox_header_lookup_unref(&headers_ctx);
686
 
        } else {
 
674
                /* see if the header search succeeded in finishing the search */
 
675
                ret = mail_search_args_foreach(args, search_none, NULL);
 
676
                if (ret >= 0 || !have_body)
 
677
                        return ret;
 
678
        }
 
679
 
 
680
        i_assert(have_body);
 
681
 
 
682
        if (ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
 
683
                imail->data.access_part |= READ_HDR | READ_BODY;
 
684
                return -1;
 
685
        }
 
686
 
 
687
        if (input == NULL) {
 
688
                /* we didn't search headers. */
687
689
                struct message_size hdr_size;
688
690
 
689
 
                if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0)
 
691
                if (mail_get_stream(ctx->cur_mail, &hdr_size, NULL, &input) < 0)
690
692
                        return -1;
691
 
 
692
693
                i_stream_seek(input, hdr_size.physical_size);
693
694
        }
694
695
 
695
 
        if (have_body) {
696
 
                struct search_body_context body_ctx;
697
 
 
698
 
                if (ctx->mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)
699
 
                        return -1;
700
 
 
701
 
                memset(&body_ctx, 0, sizeof(body_ctx));
702
 
                body_ctx.index_ctx = ctx;
703
 
                body_ctx.input = input;
704
 
                (void)mail_get_parts(ctx->mail, &body_ctx.part);
705
 
 
706
 
                ret = mail_search_args_foreach(args, search_body, &body_ctx);
707
 
        } else {
708
 
                /* see if we have a decision */
709
 
                ret = 1;
710
 
                arg = ctx->mail_ctx.args->args;
711
 
                for (; arg != NULL; arg = arg->next) {
712
 
                        if (arg->result == 0) {
713
 
                                ret = 0;
714
 
                                break;
715
 
                        }
716
 
                        if (arg->result < 0)
717
 
                                ret = -1;
718
 
                }
719
 
        }
720
 
        return ret;
 
696
        memset(&body_ctx, 0, sizeof(body_ctx));
 
697
        body_ctx.index_ctx = ctx;
 
698
        body_ctx.input = input;
 
699
        (void)mail_get_parts(ctx->cur_mail, &body_ctx.part);
 
700
 
 
701
        return mail_search_args_foreach(args, search_body, &body_ctx);
721
702
}
722
703
 
723
 
static bool search_msgset_fix_limits(unsigned int messages_count,
724
 
                                     ARRAY_TYPE(seq_range) *seqset, bool not)
 
704
static bool
 
705
search_msgset_fix_limits(unsigned int messages_count,
 
706
                         ARRAY_TYPE(seq_range) *seqset, bool match_not)
725
707
{
726
708
        struct seq_range *range;
727
709
        unsigned int count;
740
722
                seq_range_array_remove_range(seqset, messages_count + 1,
741
723
                                             (uint32_t)-1);
742
724
        }
743
 
        if (!not)
 
725
        if (!match_not)
744
726
                return array_count(seqset) > 0;
745
727
        else {
746
728
                /* if all messages are in the range, it can't match */
750
732
        }
751
733
}
752
734
 
753
 
static void search_msgset_fix(unsigned int messages_count,
754
 
                              ARRAY_TYPE(seq_range) *seqset,
755
 
                              uint32_t *seq1_r, uint32_t *seq2_r, bool not)
 
735
static void
 
736
search_msgset_fix(unsigned int messages_count,
 
737
                  ARRAY_TYPE(seq_range) *seqset,
 
738
                  uint32_t *seq1_r, uint32_t *seq2_r, bool match_not)
756
739
{
757
740
        const struct seq_range *range;
758
741
        unsigned int count;
759
742
        uint32_t min_seq, max_seq;
760
743
 
761
 
        if (!search_msgset_fix_limits(messages_count, seqset, not)) {
 
744
        if (!search_msgset_fix_limits(messages_count, seqset, match_not)) {
762
745
                *seq1_r = (uint32_t)-1;
763
746
                *seq2_r = 0;
764
747
                return;
765
748
        }
766
749
 
767
750
        range = array_get(seqset, &count);
768
 
        if (!not) {
 
751
        if (!match_not) {
769
752
                min_seq = range[0].seq1;
770
753
                max_seq = range[count-1].seq2;
771
754
        } else if (count == 0) {
800
783
 
801
784
                switch (args->type) {
802
785
                case SEARCH_SUB:
803
 
                        i_assert(!args->not);
 
786
                        i_assert(!args->match_not);
804
787
                        search_parse_msgset_args(messages_count,
805
788
                                                 args->value.subargs,
806
789
                                                 &seq1, &seq2);
807
790
                        break;
808
791
                case SEARCH_OR:
809
 
                        i_assert(!args->not);
 
792
                        i_assert(!args->match_not);
810
793
                        search_or_parse_msgset_args(messages_count,
811
794
                                                    args->value.subargs,
812
795
                                                    &seq1, &seq2);
813
796
                        break;
814
797
                case SEARCH_SEQSET:
815
798
                        search_msgset_fix(messages_count, &args->value.seqset,
816
 
                                          &seq1, &seq2, args->not);
 
799
                                          &seq1, &seq2, args->match_not);
817
800
                        break;
818
801
                default:
819
802
                        break;
844
827
        for (; args != NULL; args = args->next) {
845
828
                switch (args->type) {
846
829
                case SEARCH_SUB:
847
 
                        i_assert(!args->not);
 
830
                        i_assert(!args->match_not);
848
831
                        search_parse_msgset_args(messages_count,
849
832
                                                 args->value.subargs,
850
833
                                                 seq1_r, seq2_r);
852
835
                case SEARCH_OR:
853
836
                        /* go through our children and use the widest seqset
854
837
                           range */
855
 
                        i_assert(!args->not);
 
838
                        i_assert(!args->match_not);
856
839
                        search_or_parse_msgset_args(messages_count,
857
840
                                                    args->value.subargs,
858
841
                                                    seq1_r, seq2_r);
859
842
                        break;
860
843
                case SEARCH_SEQSET:
861
844
                        search_msgset_fix(messages_count, &args->value.seqset,
862
 
                                          seq1_r, seq2_r, args->not);
 
845
                                          seq1_r, seq2_r, args->match_not);
863
846
                        break;
864
847
                default:
865
848
                        break;
891
874
        for (; args != NULL; args = args->next) {
892
875
                if (args->type != SEARCH_FLAGS) {
893
876
                        if (args->type == SEARCH_ALL) {
894
 
                                if (args->not)
 
877
                                if (args->match_not)
895
878
                                        return FALSE;
896
879
                        }
897
880
                        continue;
898
881
                }
899
882
                if ((args->value.flags & MAIL_SEEN) != 0) {
900
883
                        /* SEEN with 0 seen? */
901
 
                        if (!args->not && hdr->seen_messages_count == 0)
 
884
                        if (!args->match_not && hdr->seen_messages_count == 0)
902
885
                                return FALSE;
903
886
 
904
887
                        if (hdr->seen_messages_count == hdr->messages_count) {
905
888
                                /* UNSEEN with all seen? */
906
 
                                if (args->not)
 
889
                                if (args->match_not)
907
890
                                        return FALSE;
908
891
 
909
892
                                /* SEEN with all seen */
910
893
                                args->match_always = TRUE;
911
 
                        } else if (args->not) {
 
894
                        } else if (args->match_not) {
912
895
                                /* UNSEEN with lowwater limiting */
913
896
                                search_limit_lowwater(ctx,
914
897
                                        hdr->first_unseen_uid_lowwater, seq1);
916
899
                }
917
900
                if ((args->value.flags & MAIL_DELETED) != 0) {
918
901
                        /* DELETED with 0 deleted? */
919
 
                        if (!args->not && hdr->deleted_messages_count == 0)
 
902
                        if (!args->match_not &&
 
903
                            hdr->deleted_messages_count == 0)
920
904
                                return FALSE;
921
905
 
922
906
                        if (hdr->deleted_messages_count ==
923
907
                            hdr->messages_count) {
924
908
                                /* UNDELETED with all deleted? */
925
 
                                if (args->not)
 
909
                                if (args->match_not)
926
910
                                        return FALSE;
927
911
 
928
912
                                /* DELETED with all deleted */
929
913
                                args->match_always = TRUE;
930
 
                        } else if (!args->not) {
 
914
                        } else if (!args->match_not) {
931
915
                                /* DELETED with lowwater limiting */
932
916
                                search_limit_lowwater(ctx,
933
917
                                        hdr->first_deleted_uid_lowwater, seq1);
1065
1049
        return ret;
1066
1050
}
1067
1051
 
 
1052
static void
 
1053
wanted_sort_fields_get(struct mailbox *box,
 
1054
                       const enum mail_sort_type *sort_program,
 
1055
                       struct mailbox_header_lookup_ctx *wanted_headers,
 
1056
                       enum mail_fetch_field *wanted_fields_r,
 
1057
                       struct mailbox_header_lookup_ctx **headers_ctx_r)
 
1058
{
 
1059
        ARRAY_TYPE(const_string) headers;
 
1060
        const char *header;
 
1061
        unsigned int i;
 
1062
 
 
1063
        *wanted_fields_r = 0;
 
1064
        *headers_ctx_r = NULL;
 
1065
 
 
1066
        t_array_init(&headers, 8);
 
1067
        for (i = 0; sort_program[i] != MAIL_SORT_END; i++) {
 
1068
                header = NULL;
 
1069
 
 
1070
                switch (sort_program[i] & MAIL_SORT_MASK) {
 
1071
                case MAIL_SORT_ARRIVAL:
 
1072
                        *wanted_fields_r |= MAIL_FETCH_RECEIVED_DATE;
 
1073
                        break;
 
1074
                case MAIL_SORT_CC:
 
1075
                        header = "Cc";
 
1076
                        break;
 
1077
                case MAIL_SORT_DATE:
 
1078
                        *wanted_fields_r |= MAIL_FETCH_DATE;
 
1079
                        break;
 
1080
                case MAIL_SORT_FROM:
 
1081
                        header = "From";
 
1082
                        break;
 
1083
                case MAIL_SORT_SIZE:
 
1084
                        *wanted_fields_r |= MAIL_FETCH_VIRTUAL_SIZE;
 
1085
                        break;
 
1086
                case MAIL_SORT_SUBJECT:
 
1087
                        header = "Subject";
 
1088
                        break;
 
1089
                case MAIL_SORT_TO:
 
1090
                        header = "To";
 
1091
                        break;
 
1092
                }
 
1093
                if (header != NULL)
 
1094
                        array_append(&headers, &header, 1);
 
1095
        }
 
1096
 
 
1097
        if (wanted_headers != NULL) {
 
1098
                for (i = 0; wanted_headers->name[i] != NULL; i++)
 
1099
                        array_append(&headers, &wanted_headers->name[i], 1);
 
1100
        }
 
1101
 
 
1102
        if (array_count(&headers) > 0) {
 
1103
                (void)array_append_space(&headers);
 
1104
                *headers_ctx_r = mailbox_header_lookup_init(box,
 
1105
                                                        array_idx(&headers, 0));
 
1106
        }
 
1107
}
 
1108
 
1068
1109
struct mail_search_context *
1069
1110
index_storage_search_init(struct mailbox_transaction_context *t,
1070
1111
                          struct mail_search_args *args,
1071
 
                          const enum mail_sort_type *sort_program)
 
1112
                          const enum mail_sort_type *sort_program,
 
1113
                          enum mail_fetch_field wanted_fields,
 
1114
                          struct mailbox_header_lookup_ctx *wanted_headers)
1072
1115
{
1073
1116
        struct index_search_context *ctx;
1074
1117
        struct mailbox_status status;
1079
1122
        ctx->view = t->view;
1080
1123
        ctx->mail_ctx.args = args;
1081
1124
        ctx->mail_ctx.sort_program = index_sort_program_init(t, sort_program);
 
1125
 
 
1126
        ctx->max_mails = t->box->storage->set->mail_prefetch_count + 1;
 
1127
        if (ctx->max_mails == 0)
 
1128
                ctx->max_mails = -1U;
1082
1129
        ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST;
1083
1130
        if (gettimeofday(&ctx->last_nonblock_timeval, NULL) < 0)
1084
1131
                i_fatal("gettimeofday() failed: %m");
1085
1132
 
1086
 
        mailbox_get_status(t->box, STATUS_MESSAGES, &status);
 
1133
        mailbox_get_open_status(t->box, STATUS_MESSAGES, &status);
1087
1134
        ctx->mail_ctx.progress_max = status.messages;
1088
1135
 
1089
1136
        i_array_init(&ctx->mail_ctx.results, 5);
1090
1137
        array_create(&ctx->mail_ctx.module_contexts, default_pool,
1091
1138
                     sizeof(void *), 5);
 
1139
        i_array_init(&ctx->mails, ctx->max_mails);
1092
1140
 
1093
1141
        mail_search_args_reset(ctx->mail_ctx.args->args, TRUE);
1094
1142
        if (args->have_inthreads) {
1098
1146
                        ctx->failed = TRUE;
1099
1147
        }
1100
1148
 
 
1149
        if (sort_program != NULL) {
 
1150
                wanted_sort_fields_get(ctx->box, sort_program, wanted_headers,
 
1151
                                       &ctx->mail_ctx.wanted_fields,
 
1152
                                       &ctx->mail_ctx.wanted_headers);
 
1153
        } else if (wanted_headers != NULL) {
 
1154
                ctx->mail_ctx.wanted_headers = wanted_headers;
 
1155
                mailbox_header_lookup_ref(wanted_headers);
 
1156
        }
 
1157
        ctx->mail_ctx.wanted_fields |= wanted_fields;
 
1158
 
1101
1159
        search_get_seqset(ctx, status.messages, args->args);
1102
1160
        (void)mail_search_args_foreach(args->args, search_init_arg, ctx);
1103
1161
 
1120
1178
int index_storage_search_deinit(struct mail_search_context *_ctx)
1121
1179
{
1122
1180
        struct index_search_context *ctx = (struct index_search_context *)_ctx;
 
1181
        struct mail **mailp;
1123
1182
        int ret;
1124
1183
 
1125
 
        ret = ctx->failed || ctx->error != NULL ? -1 : 0;
1126
 
 
1127
 
        if (ctx->error != NULL) {
1128
 
                mail_storage_set_error(ctx->box->storage,
1129
 
                                       MAIL_ERROR_PARAMS, ctx->error);
1130
 
        }
 
1184
        ret = ctx->failed ? -1 : 0;
1131
1185
 
1132
1186
        mail_search_args_reset(ctx->mail_ctx.args->args, FALSE);
1133
1187
        (void)mail_search_args_foreach(ctx->mail_ctx.args->args,
1134
1188
                                       search_arg_deinit, NULL);
1135
1189
 
 
1190
        if (ctx->mail_ctx.wanted_headers != NULL)
 
1191
                mailbox_header_lookup_unref(&ctx->mail_ctx.wanted_headers);
1136
1192
        if (ctx->mail_ctx.sort_program != NULL)
1137
1193
                index_sort_program_deinit(&ctx->mail_ctx.sort_program);
1138
1194
        if (ctx->thread_ctx != NULL)
1139
1195
                mail_thread_deinit(&ctx->thread_ctx);
1140
1196
        array_free(&ctx->mail_ctx.results);
1141
1197
        array_free(&ctx->mail_ctx.module_contexts);
 
1198
 
 
1199
        array_foreach_modifiable(&ctx->mails, mailp) {
 
1200
                struct index_mail *imail = (struct index_mail *)*mailp;
 
1201
 
 
1202
                imail->search_mail = FALSE;
 
1203
                mail_free(mailp);
 
1204
        }
 
1205
        array_free(&ctx->mails);
1142
1206
        i_free(ctx);
1143
1207
        return ret;
1144
1208
}
1145
1209
 
1146
 
static bool search_match_next(struct index_search_context *ctx)
 
1210
static unsigned long long
 
1211
search_get_cost(struct mailbox_transaction_context *trans)
 
1212
{
 
1213
        return trans->stats.open_lookup_count * SEARCH_COST_DENTRY +
 
1214
                trans->stats.stat_lookup_count * SEARCH_COST_DENTRY +
 
1215
                trans->stats.fstat_lookup_count * SEARCH_COST_ATTR +
 
1216
                trans->stats.cache_hit_count * SEARCH_COST_CACHE +
 
1217
                trans->stats.files_read_count * SEARCH_COST_FILES_READ +
 
1218
                (trans->stats.files_read_bytes/1024) * SEARCH_COST_KBYTE;
 
1219
}
 
1220
 
 
1221
static int search_match_once(struct index_search_context *ctx)
 
1222
{
 
1223
        int ret;
 
1224
 
 
1225
        ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
 
1226
                                       search_cached_arg, ctx);
 
1227
        if (ret < 0)
 
1228
                ret = search_arg_match_text(ctx->mail_ctx.args->args, ctx);
 
1229
        return ret;
 
1230
}
 
1231
 
 
1232
static bool search_arg_is_static(struct mail_search_arg *arg)
 
1233
{
 
1234
        struct mail_search_arg *subarg;
 
1235
 
 
1236
        switch (arg->type) {
 
1237
        case SEARCH_OR:
 
1238
        case SEARCH_SUB:
 
1239
                /* they're static only if all subargs are static */
 
1240
                subarg = arg->value.subargs;
 
1241
                for (; subarg != NULL; subarg = subarg->next) {
 
1242
                        if (!search_arg_is_static(subarg))
 
1243
                                return FALSE;
 
1244
                }
 
1245
                return TRUE;
 
1246
        case SEARCH_SEQSET:
 
1247
                /* changes between syncs, but we can't really handle this
 
1248
                   currently. seqsets should be converted to uidsets first. */
 
1249
        case SEARCH_FLAGS:
 
1250
        case SEARCH_KEYWORDS:
 
1251
        case SEARCH_MODSEQ:
 
1252
        case SEARCH_INTHREAD:
 
1253
                break;
 
1254
        case SEARCH_ALL:
 
1255
        case SEARCH_UIDSET:
 
1256
        case SEARCH_BEFORE:
 
1257
        case SEARCH_ON:
 
1258
        case SEARCH_SINCE:
 
1259
        case SEARCH_SMALLER:
 
1260
        case SEARCH_LARGER:
 
1261
        case SEARCH_HEADER:
 
1262
        case SEARCH_HEADER_ADDRESS:
 
1263
        case SEARCH_HEADER_COMPRESS_LWSP:
 
1264
        case SEARCH_BODY:
 
1265
        case SEARCH_TEXT:
 
1266
        case SEARCH_GUID:
 
1267
        case SEARCH_MAILBOX:
 
1268
        case SEARCH_MAILBOX_GUID:
 
1269
        case SEARCH_MAILBOX_GLOB:
 
1270
                return TRUE;
 
1271
        }
 
1272
        return FALSE;
 
1273
}
 
1274
 
 
1275
static void search_set_static_matches(struct mail_search_arg *arg)
 
1276
{
 
1277
        for (; arg != NULL; arg = arg->next) {
 
1278
                if (search_arg_is_static(arg))
 
1279
                        arg->result = 1;
 
1280
        }
 
1281
}
 
1282
 
 
1283
static bool search_has_static_nonmatches(struct mail_search_arg *arg)
 
1284
{
 
1285
        for (; arg != NULL; arg = arg->next) {
 
1286
                if (arg->result == 0 && search_arg_is_static(arg))
 
1287
                        return TRUE;
 
1288
        }
 
1289
        return FALSE;
 
1290
}
 
1291
 
 
1292
static void search_match_finish(struct index_search_context *ctx, int match)
 
1293
{
 
1294
        if (ctx->cur_mail->expunged)
 
1295
                ctx->mail_ctx.seen_lost_data = TRUE;
 
1296
 
 
1297
        if (match == 0 &&
 
1298
            search_has_static_nonmatches(ctx->mail_ctx.args->args)) {
 
1299
                /* if there are saved search results remember
 
1300
                   that this message never matches */
 
1301
                mailbox_search_results_never(&ctx->mail_ctx,
 
1302
                                             ctx->cur_mail->uid);
 
1303
        }
 
1304
}
 
1305
 
 
1306
static int search_match_next(struct index_search_context *ctx)
1147
1307
{
1148
1308
        static enum mail_lookup_abort cache_lookups[] = {
1149
1309
                MAIL_LOOKUP_ABORT_NOT_IN_CACHE,
1150
1310
                MAIL_LOOKUP_ABORT_READ_MAIL,
1151
1311
                MAIL_LOOKUP_ABORT_NEVER
1152
1312
        };
1153
 
        unsigned int i;
 
1313
        unsigned int i, n = N_ELEMENTS(cache_lookups);
1154
1314
        int ret = -1;
1155
1315
 
1156
1316
        if (ctx->have_mailbox_args) {
 
1317
                /* check that the mailbox name matches.
 
1318
                   this makes sense only with virtual mailboxes. */
1157
1319
                ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
1158
1320
                                               search_mailbox_arg, ctx);
1159
 
                if (ret >= 0)
1160
 
                        return ret > 0;
1161
 
        }
1162
 
 
1163
 
        /* try to avoid doing extra work for as long as possible */
1164
 
        for (i = 0; i < N_ELEMENTS(cache_lookups) && ret < 0; i++) {
1165
 
                ctx->mail->lookup_abort = cache_lookups[i];
1166
 
                ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
1167
 
                                               search_cached_arg, ctx);
1168
 
                if (ret >= 0)
1169
 
                        break;
1170
 
 
1171
 
                ret = search_arg_match_text(ctx->mail_ctx.args->args, ctx, ret);
1172
 
                if (ret >= 0)
1173
 
                        break;
1174
 
        }
1175
 
        ctx->mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
1176
 
        return ret > 0;
 
1321
        }
 
1322
 
 
1323
        /* avoid doing extra work for as long as possible */
 
1324
        if (ctx->max_mails > 1) {
 
1325
                /* we're doing prefetching. if we have to read the mail,
 
1326
                   do a prefetch first and the final search later */
 
1327
                n--;
 
1328
        }
 
1329
        for (i = 0; i < n && ret < 0; i++) {
 
1330
                ctx->cur_mail->lookup_abort = cache_lookups[i];
 
1331
                ret = search_match_once(ctx);
 
1332
        }
 
1333
        ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
 
1334
        search_match_finish(ctx, ret);
 
1335
        return ret;
1177
1336
}
1178
1337
 
1179
1338
static void index_storage_search_notify(struct mailbox *box,
1208
1367
        ctx->last_notify = ioloop_timeval;
1209
1368
}
1210
1369
 
1211
 
static bool search_arg_is_static(struct mail_search_arg *arg)
1212
 
{
1213
 
        struct mail_search_arg *subarg;
1214
 
 
1215
 
        switch (arg->type) {
1216
 
        case SEARCH_OR:
1217
 
        case SEARCH_SUB:
1218
 
                /* they're static only if all subargs are static */
1219
 
                subarg = arg->value.subargs;
1220
 
                for (; subarg != NULL; subarg = subarg->next) {
1221
 
                        if (!search_arg_is_static(subarg))
1222
 
                                return FALSE;
1223
 
                }
1224
 
                return TRUE;
1225
 
        case SEARCH_SEQSET:
1226
 
                /* changes between syncs, but we can't really handle this
1227
 
                   currently. seqsets should be converted to uidsets first. */
1228
 
        case SEARCH_FLAGS:
1229
 
        case SEARCH_KEYWORDS:
1230
 
        case SEARCH_MODSEQ:
1231
 
        case SEARCH_INTHREAD:
1232
 
                break;
1233
 
        case SEARCH_ALL:
1234
 
        case SEARCH_UIDSET:
1235
 
        case SEARCH_BEFORE:
1236
 
        case SEARCH_ON:
1237
 
        case SEARCH_SINCE:
1238
 
        case SEARCH_SMALLER:
1239
 
        case SEARCH_LARGER:
1240
 
        case SEARCH_HEADER:
1241
 
        case SEARCH_HEADER_ADDRESS:
1242
 
        case SEARCH_HEADER_COMPRESS_LWSP:
1243
 
        case SEARCH_BODY:
1244
 
        case SEARCH_TEXT:
1245
 
        case SEARCH_BODY_FAST:
1246
 
        case SEARCH_TEXT_FAST:
1247
 
        case SEARCH_GUID:
1248
 
        case SEARCH_MAILBOX:
1249
 
        case SEARCH_MAILBOX_GUID:
1250
 
        case SEARCH_MAILBOX_GLOB:
1251
 
                return TRUE;
1252
 
        }
1253
 
        return FALSE;
1254
 
}
1255
 
 
1256
 
static void search_set_static_matches(struct mail_search_arg *arg)
1257
 
{
1258
 
        for (; arg != NULL; arg = arg->next) {
1259
 
                if (search_arg_is_static(arg))
1260
 
                        arg->result = 1;
1261
 
        }
1262
 
}
1263
 
 
1264
 
static bool search_has_static_nonmatches(struct mail_search_arg *arg)
1265
 
{
1266
 
        for (; arg != NULL; arg = arg->next) {
1267
 
                if (arg->result == 0 && search_arg_is_static(arg))
1268
 
                        return TRUE;
1269
 
        }
1270
 
        return FALSE;
1271
 
}
1272
 
 
1273
 
static unsigned long long search_mail_get_cost(struct mail_private *mail)
1274
 
{
1275
 
        return mail->stats_open_lookup_count * SEARCH_COST_DENTRY +
1276
 
                mail->stats_stat_lookup_count * SEARCH_COST_DENTRY +
1277
 
                mail->stats_fstat_lookup_count * SEARCH_COST_ATTR +
1278
 
                mail->stats_cache_hit_count * SEARCH_COST_CACHE +
1279
 
                mail->stats_files_read_count * SEARCH_COST_FILES_READ +
1280
 
                (mail->stats_files_read_bytes/1024) * SEARCH_COST_KBYTE;
1281
 
}
1282
 
 
1283
1370
static bool search_would_block(struct index_search_context *ctx)
1284
1371
{
1285
1372
        struct timeval now;
1325
1412
        return ret;
1326
1413
}
1327
1414
 
1328
 
bool index_storage_search_next_nonblock(struct mail_search_context *_ctx,
1329
 
                                        struct mail *mail, bool *tryagain_r)
 
1415
static int search_more_with_mail(struct index_search_context *ctx,
 
1416
                                 struct mail *mail)
1330
1417
{
1331
 
        struct index_search_context *ctx = (struct index_search_context *)_ctx;
 
1418
        struct mail_search_context *_ctx = &ctx->mail_ctx;
1332
1419
        struct mailbox *box = _ctx->transaction->box;
1333
 
        struct mail_private *mail_private = (struct mail_private *)mail;
 
1420
        struct index_mail *imail = (struct index_mail *)mail;
1334
1421
        unsigned long long cost1, cost2;
1335
 
        bool old_stats_track, match = FALSE;
1336
 
 
1337
 
        *tryagain_r = FALSE;
1338
 
 
1339
 
        if (ctx->sorted) {
1340
 
                /* everything searched at this point already. just returning
1341
 
                   matches from sort list */
1342
 
                if (!index_sort_list_next(ctx->mail_ctx.sort_program, mail))
1343
 
                        return FALSE;
1344
 
                return TRUE;
1345
 
        }
 
1422
        int match, ret;
1346
1423
 
1347
1424
        if (search_would_block(ctx)) {
1348
1425
                /* this lookup is useful when a large number of
1349
1426
                   messages match */
1350
 
                *tryagain_r = TRUE;
1351
 
                return FALSE;
 
1427
                return 0;
1352
1428
        }
1353
1429
 
1354
 
        ctx->mail = mail;
1355
 
 
1356
1430
        if (ioloop_time - ctx->last_notify.tv_sec >=
1357
1431
            SEARCH_NOTIFY_INTERVAL_SECS)
1358
1432
                index_storage_search_notify(box, ctx);
1359
1433
 
1360
 
        old_stats_track = mail_private->stats_track;
1361
 
        mail_private->stats_track = TRUE;
1362
 
        cost1 = search_mail_get_cost(mail_private);
 
1434
        mail_search_args_reset(_ctx->args->args, FALSE);
 
1435
 
 
1436
        cost1 = search_get_cost(mail->transaction);
 
1437
        ret = -1;
1363
1438
        while (box->v.search_next_update_seq(_ctx)) {
1364
1439
                mail_set_seq(mail, _ctx->seq);
1365
 
                ctx->imail = (struct index_mail *)mail_get_real_mail(mail);
1366
1440
 
 
1441
                ctx->cur_mail = mail;
1367
1442
                T_BEGIN {
1368
1443
                        match = search_match_next(ctx);
1369
 
 
1370
 
                        if (ctx->mail->expunged)
1371
 
                                _ctx->seen_lost_data = TRUE;
1372
 
 
1373
 
                        if (!match &&
1374
 
                            search_has_static_nonmatches(_ctx->args->args)) {
1375
 
                                /* if there are saved search results remember
1376
 
                                   that this message never matches */
1377
 
                                mailbox_search_results_never(_ctx, mail->uid);
1378
 
                        }
1379
1444
                } T_END;
1380
 
                cost2 = search_mail_get_cost(mail_private);
 
1445
                ctx->cur_mail = NULL;
 
1446
 
 
1447
                i_assert(imail->data.search_results == NULL);
 
1448
                if (match < 0) {
 
1449
                        /* result isn't known yet, do a prefetch and
 
1450
                           finish later */
 
1451
                        imail->data.search_results =
 
1452
                                buffer_create_dynamic(imail->data_pool, 64);
 
1453
                        mail_search_args_result_serialize(_ctx->args,
 
1454
                                imail->data.search_results);
 
1455
                }
 
1456
 
 
1457
                mail_search_args_reset(_ctx->args->args, FALSE);
 
1458
 
 
1459
                if (match != 0) {
 
1460
                        ret = 1;
 
1461
                        break;
 
1462
                }
 
1463
 
 
1464
                cost2 = search_get_cost(mail->transaction);
1381
1465
                ctx->cost += cost2 - cost1;
1382
1466
                cost1 = cost2;
1383
1467
 
1384
 
                mail_search_args_reset(_ctx->args->args, FALSE);
1385
 
 
1386
 
                if (ctx->error != NULL)
1387
 
                        ctx->failed = TRUE;
1388
 
                else if (match) {
1389
 
                        if (_ctx->sort_program == NULL)
1390
 
                                break;
1391
 
 
 
1468
                if (search_would_block(ctx)) {
 
1469
                        ret = 0;
 
1470
                        break;
 
1471
                }
 
1472
        }
 
1473
        cost2 = search_get_cost(mail->transaction);
 
1474
        ctx->cost += cost2 - cost1;
 
1475
        return ret;
 
1476
}
 
1477
 
 
1478
struct mail *index_search_get_mail(struct index_search_context *ctx)
 
1479
{
 
1480
        struct index_mail *imail;
 
1481
        struct mail *const *mails, *mail;
 
1482
        unsigned int count;
 
1483
 
 
1484
        if (ctx->unused_mail_idx == ctx->max_mails)
 
1485
                return NULL;
 
1486
 
 
1487
        mails = array_get(&ctx->mails, &count);
 
1488
        if (ctx->unused_mail_idx < count)
 
1489
                return mails[ctx->unused_mail_idx];
 
1490
 
 
1491
        mail = mail_alloc(ctx->mail_ctx.transaction,
 
1492
                          ctx->mail_ctx.wanted_fields,
 
1493
                          ctx->mail_ctx.wanted_headers);
 
1494
        imail = (struct index_mail *)mail;
 
1495
        imail->search_mail = TRUE;
 
1496
        ctx->mail_ctx.transaction->stats_track = TRUE;
 
1497
 
 
1498
        array_append(&ctx->mails, &mail, 1);
 
1499
        return mail;
 
1500
}
 
1501
 
 
1502
static int search_more_with_prefetching(struct index_search_context *ctx,
 
1503
                                        struct mail **mail_r)
 
1504
{
 
1505
        struct mail *mail, *const *mails;
 
1506
        unsigned int count;
 
1507
        int ret = 0;
 
1508
 
 
1509
        while ((mail = index_search_get_mail(ctx)) != NULL) {
 
1510
                ret = search_more_with_mail(ctx, mail);
 
1511
                if (ret <= 0)
 
1512
                        break;
 
1513
 
 
1514
                if (ctx->mail_ctx.sort_program != NULL) {
 
1515
                        /* don't prefetch when using a sort program,
 
1516
                           since the mails' access order will change */
 
1517
                        i_assert(ctx->unused_mail_idx == 0);
 
1518
                        *mail_r = mail;
 
1519
                        return 1;
 
1520
                }
 
1521
                if (mail_prefetch(mail) && ctx->unused_mail_idx == 0) {
 
1522
                        /* no prefetching done, return it immediately */
 
1523
                        *mail_r = mail;
 
1524
                        return 1;
 
1525
                }
 
1526
                ctx->unused_mail_idx++;
 
1527
        }
 
1528
 
 
1529
        if (mail != NULL) {
 
1530
                if (ret == 0) {
 
1531
                        /* wait */
 
1532
                        return 0;
 
1533
                }
 
1534
                i_assert(ret < 0);
 
1535
                if (ctx->unused_mail_idx == 0) {
 
1536
                        /* finished */
 
1537
                        return -1;
 
1538
                }
 
1539
        } else {
 
1540
                /* prefetch buffer is full. */
 
1541
        }
 
1542
 
 
1543
        /* return the next message */
 
1544
        i_assert(ctx->unused_mail_idx > 0);
 
1545
 
 
1546
        mails = array_get(&ctx->mails, &count);
 
1547
        *mail_r = mails[0];
 
1548
        if (--ctx->unused_mail_idx > 0) {
 
1549
                array_delete(&ctx->mails, 0, 1);
 
1550
                array_append(&ctx->mails, mail_r, 1);
 
1551
        }
 
1552
        return 1;
 
1553
}
 
1554
 
 
1555
static bool search_finish_prefetch(struct index_search_context *ctx,
 
1556
                                   struct index_mail *imail)
 
1557
{
 
1558
        int ret;
 
1559
 
 
1560
        i_assert(imail->mail.mail.lookup_abort == MAIL_LOOKUP_ABORT_NEVER);
 
1561
 
 
1562
        ctx->cur_mail = &imail->mail.mail;
 
1563
        mail_search_args_result_deserialize(ctx->mail_ctx.args,
 
1564
                                            imail->data.search_results->data,
 
1565
                                            imail->data.search_results->used);
 
1566
        ret = search_match_once(ctx);
 
1567
        search_match_finish(ctx, ret);
 
1568
        ctx->cur_mail = NULL;
 
1569
        return ret > 0;
 
1570
}
 
1571
 
 
1572
static int search_more(struct index_search_context *ctx,
 
1573
                       struct mail **mail_r)
 
1574
{
 
1575
        struct index_mail *imail;
 
1576
        int ret;
 
1577
 
 
1578
        while ((ret = search_more_with_prefetching(ctx, mail_r)) > 0) {
 
1579
                imail = (struct index_mail *)*mail_r;
 
1580
                if (imail->data.search_results == NULL)
 
1581
                        break;
 
1582
 
 
1583
                /* searching wasn't finished yet */
 
1584
                if (search_finish_prefetch(ctx, imail))
 
1585
                        break;
 
1586
                /* search finished as non-match */
 
1587
        }
 
1588
        return ret;
 
1589
}
 
1590
 
 
1591
bool index_storage_search_next_nonblock(struct mail_search_context *_ctx,
 
1592
                                        struct mail **mail_r, bool *tryagain_r)
 
1593
{
 
1594
        struct index_search_context *ctx = (struct index_search_context *)_ctx;
 
1595
        struct mail *mail, *const *mailp;
 
1596
        uint32_t seq;
 
1597
        int ret;
 
1598
 
 
1599
        *tryagain_r = FALSE;
 
1600
 
 
1601
        if (_ctx->sort_program == NULL) {
 
1602
                ret = search_more(ctx, &mail);
 
1603
                if (ret == 0) {
 
1604
                        *tryagain_r = TRUE;
 
1605
                        return FALSE;
 
1606
                }
 
1607
                if (ret < 0)
 
1608
                        return FALSE;
 
1609
                *mail_r = mail;
 
1610
                return TRUE;
 
1611
        }
 
1612
 
 
1613
        if (!ctx->sorted) {
 
1614
                while ((ret = search_more(ctx, &mail)) > 0)
1392
1615
                        index_sort_list_add(_ctx->sort_program, mail);
1393
 
                }
1394
 
                match = FALSE;
1395
1616
 
1396
 
                if (search_would_block(ctx)) {
 
1617
                if (ret == 0) {
1397
1618
                        *tryagain_r = TRUE;
1398
 
                        break;
 
1619
                        return FALSE;
1399
1620
                }
1400
 
        }
1401
 
        ctx->mail = NULL;
1402
 
        ctx->imail = NULL;
1403
 
        mail_private->stats_track = old_stats_track;
1404
 
 
1405
 
        if (!match && _ctx->sort_program != NULL &&
1406
 
            !*tryagain_r && !ctx->failed) {
1407
1621
                /* finished searching the messages. now sort them and start
1408
1622
                   returning the messages. */
1409
1623
                ctx->sorted = TRUE;
1410
1624
                index_sort_list_finish(_ctx->sort_program);
1411
 
                return index_storage_search_next_nonblock(_ctx, mail,
1412
 
                                                          tryagain_r);
 
1625
                if (ctx->failed)
 
1626
                        return FALSE;
1413
1627
        }
1414
 
        return !ctx->failed && match;
 
1628
 
 
1629
        /* everything searched at this point already. just returning
 
1630
           matches from sort list */
 
1631
        if (!index_sort_list_next(_ctx->sort_program, &seq))
 
1632
                return FALSE;
 
1633
 
 
1634
        mailp = array_idx(&ctx->mails, 0);
 
1635
        mail_set_seq(*mailp, seq);
 
1636
        *mail_r = *mailp;
 
1637
        return TRUE;
1415
1638
}
1416
1639
 
1417
1640
bool index_storage_search_next_update_seq(struct mail_search_context *_ctx)
1429
1652
 
1430
1653
        if (!ctx->have_seqsets && !ctx->have_index_args &&
1431
1654
            _ctx->update_result == NULL) {
1432
 
                ctx->mail_ctx.progress_cur = _ctx->seq;
 
1655
                _ctx->progress_cur = _ctx->seq;
1433
1656
                return _ctx->seq <= ctx->seq2;
1434
1657
        }
1435
1658