← Back to branch summary

~ubuntu-branches/ubuntu/wily/dovecot/wily

« back to all changes in this revision

Viewing changes to src/lib-storage/index/maildir/maildir-uidlist.c

  • Committer: Bazaar Package Importer
  • Author(s): CHuck Short, Chuck Short
  • Date: 2009-11-06 00:47:29 UTC
  • mfrom: (4.1.9 squeeze)
  • Revision ID: james.westby@ubuntu.com-20091106004729-i39n7v9e7d4h51f6
Tags: 1:1.2.6-1ubuntu1
https://launchpad.net/bugs/164837
https://launchpad.net/bugs/329878
https://launchpad.net/bugs/254721
* Merge from debian testing, remaining changes:
  Add new binary pkg dovecot-postfix that integrates postfix and dovecot
  automatically: (LP: #164837)
  + debian/control:
    - add new binary with short description
    - set Architecture all for dovecot-postfix (LP: #329878)
  + debian/dovecot-postfix.postinst:
    - create initial certificate symlinks to snakeoil.
    - set up postfix with postconf to:
      - use Maildir/ as the default mailbox.
      - use dovecot as the sasl authentication server.
      - use dovecot LDA (deliver).
      - use tls for smtp{d} services.
    - fix certificates paths in postfix' main.cf
    - add reject_unauth_destination to postfix' recipient restrictions
    - add reject_unknown_sender_domain to postfix' sender restrictions
    - rename configuration name on remove, delete on purge
    - restart dovecot after linking certificates
    - handle use case when postfix is unconfigurated
   + debian/dovecot-postfix.dirs: create backup directory for postfix's configuration
   + restart postfix and dovecot.
   + debian/dovecot-postfix.postrm:
     - remove all dovecot related configuration from postfix.
     - restart postfix and dovecot.
   + debian/dovecot-common.init:
     - check if /etc/dovecot/dovecot-postfix.conf exists and use it
       as the configuration file if so.
   + debian/patches/warning-ubuntu-postfix.dpatch
     - add warning about dovecot-postfix.conf in dovecot default 
       configuration file
   + debian/patches/dovecot-postfix.conf.diff:
     - Ubuntu server custom changes to the default dovecot configuration for
       better interfation with postfix
     - enable sieve plugin
   + debian/patches/dovecot-postfix.conf.diff:
     + Ubuntu server custom changes to the default dovecot configuration for
       better integration with postfix:
       - enable imap, pop3, imaps, pop3s and managesieve by default.
       - enable dovecot LDA (deliver).
       - enable SASL auth socket in postfix private directory.
   + debian/rules:
     - copy, patch and install dovecot-postfix.conf in /etc/dovecot/.
     - build architecure independent packages too
   + Use Snakeoil SSL certificates by default.
     - debian/control: Depend on ssl-cert.
     - debian/patches/ssl-cert-snakeoil.dpatch: Change default SSL cert
       paths to snakeoil.
     - debian/dovecot-common.postinst: Relax grep for SSL_* a bit.
   + Add autopkgtest to debian/tests/*.
   + Fast TearDown: Update the lsb init header to not stop in level 6.
   + Add ufw integration:
     - Created debian/dovecot-common.ufw.profile.
     - debian/rules:
       + install profile
     - debian/control:
       + Suggest ufw
   + debian/{control,rules}: enable PIE hardening.
   + dovecot-imapd, dovecot-pop3: Replaces dovecot-common (<< 1:1.1). LP: #254721
   + debian/control:
     - Update Vcs-* headers.
   + debian/rules:
     - Create emtpy stamp.h.in files in dovecot-sieve/ and dovecot-managesieve/
       if they're not there since empty files are not included in the diff.gz 
       file.
   + Add SMTP-AUTH support for Outlook (login auth mechanism)
   + Dropped:
     - debian/patches/security-CVE-2009-3235: Applied upstream.
     - debian/patches/fix-pop3-assertion.dpatch: Applied upstream.
     - dovecot-sieve and dovecot-managesieve: Use the debian patches instead.

  [Chuck Short]
  - Updated dovecot-sieve to 0.1.13.
  - Updated dovecot-managesieve to 0.11.9.

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/lib-storage/index/maildir/maildir-uidlist.c
33
33
#include "file-dotlock.h"
34
34
#include "close-keep-errno.h"
35
35
#include "nfs-workarounds.h"
 
36
#include "eacces-error.h"
36
37
#include "maildir-storage.h"
37
38
#include "maildir-sync.h"
38
39
#include "maildir-filename.h"
82
83
 
83
84
        unsigned int version;
84
85
        unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid;
 
86
        unsigned int hdr_next_uid;
85
87
        unsigned int read_records_count, read_line_count;
86
88
        uoff_t last_read_offset;
87
89
        string_t *hdr_extensions;
89
91
        unsigned int recreate:1;
90
92
        unsigned int initial_read:1;
91
93
        unsigned int initial_hdr_read:1;
92
 
        unsigned int initial_sync:1;
93
94
        unsigned int retry_rewind:1;
 
95
        unsigned int locked_refresh:1;
 
96
        unsigned int unsorted:1;
94
97
};
95
98
 
96
99
struct maildir_uidlist_sync_ctx {
124
127
                                          struct maildir_uidlist_rec **rec_r);
125
128
 
126
129
static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist,
127
 
                                        bool nonblock, bool refresh)
 
130
                                        bool nonblock, bool refresh,
 
131
                                        bool refresh_when_locked)
128
132
{
129
133
        struct mailbox *box = &uidlist->ibox->box;
130
134
        const char *control_dir, *path;
134
138
        int i, ret;
135
139
 
136
140
        if (uidlist->lock_count > 0) {
 
141
                if (!uidlist->locked_refresh && refresh_when_locked) {
 
142
                        if (maildir_uidlist_refresh(uidlist) < 0)
 
143
                                return -1;
 
144
                }
137
145
                uidlist->lock_count++;
138
146
                return 1;
139
147
        }
140
148
 
 
149
        index_storage_lock_notify_reset(uidlist->ibox);
 
150
 
141
151
        control_dir = mailbox_list_get_path(box->storage->list, box->name,
142
152
                                            MAILBOX_LIST_PATH_TYPE_CONTROL);
143
153
        path = t_strconcat(control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);
158
168
                }
159
169
                if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT ||
160
170
                    uidlist->mbox == NULL) {
161
 
                        mail_storage_set_critical(box->storage,
162
 
                                "file_dotlock_create(%s) failed: %m", path);
 
171
                        if (errno == EACCES) {
 
172
                                mail_storage_set_critical(box->storage, "%s",
 
173
                                        eacces_error_get_creating("file_dotlock_create", path));
 
174
                        } else {
 
175
                                mail_storage_set_critical(box->storage,
 
176
                                        "file_dotlock_create(%s) failed: %m",
 
177
                                        path);
 
178
                        }
163
179
                        return -1;
164
180
                }
165
181
                /* the control dir doesn't exist. create it unless the whole
166
182
                   mailbox was just deleted. */
167
 
                if (maildir_set_deleted(uidlist->mbox))
 
183
                if (!maildir_set_deleted(uidlist->mbox))
168
184
                        return -1;
169
185
        }
170
186
 
171
187
        uidlist->lock_count++;
 
188
        uidlist->locked_refresh = FALSE;
172
189
 
173
190
        if (refresh) {
174
191
                /* make sure we have the latest changes before
183
200
 
184
201
int maildir_uidlist_lock(struct maildir_uidlist *uidlist)
185
202
{
186
 
        return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE);
 
203
        return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE, FALSE);
187
204
}
188
205
 
189
206
int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
190
207
{
191
 
        return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE);
 
208
        return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE, FALSE);
192
209
}
193
210
 
194
211
int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist)
210
227
        if (--uidlist->lock_count > 0)
211
228
                return;
212
229
 
 
230
        uidlist->locked_refresh = FALSE;
213
231
        (void)file_dotlock_delete(&uidlist->dotlock);
214
232
}
215
233
 
 
234
static bool dotlock_callback(unsigned int secs_left, bool stale, void *context)
 
235
{
 
236
        struct index_mailbox *ibox = context;
 
237
 
 
238
        index_storage_lock_notify(ibox, stale ?
 
239
                                  MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE :
 
240
                                  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
 
241
                                  secs_left);
 
242
        return TRUE;
 
243
}
 
244
 
216
245
struct maildir_uidlist *
217
246
maildir_uidlist_init_readonly(struct index_mailbox *ibox)
218
247
{
228
257
        uidlist->ibox = ibox;
229
258
        uidlist->path = i_strconcat(control_dir, "/"MAILDIR_UIDLIST_NAME, NULL);
230
259
        i_array_init(&uidlist->records, 128);
231
 
        uidlist->files = hash_create(default_pool, default_pool, 4096,
232
 
                                     maildir_filename_base_hash,
233
 
                                     maildir_filename_base_cmp);
 
260
        uidlist->files = hash_table_create(default_pool, default_pool, 4096,
 
261
                                           maildir_filename_base_hash,
 
262
                                           maildir_filename_base_cmp);
234
263
        uidlist->next_uid = 1;
235
264
        uidlist->hdr_extensions = str_new(default_pool, 128);
236
265
 
244
273
                MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT + 2;
245
274
        uidlist->dotlock_settings.stale_timeout =
246
275
                MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT;
 
276
        uidlist->dotlock_settings.callback = dotlock_callback;
 
277
        uidlist->dotlock_settings.context = ibox;
247
278
 
248
279
        return uidlist;
249
280
}
260
291
 
261
292
static void maildir_uidlist_close(struct maildir_uidlist *uidlist)
262
293
{
 
294
        struct mail_storage *storage = uidlist->ibox->box.storage;
 
295
 
263
296
        if (uidlist->fd != -1) {
264
 
                if (close(uidlist->fd) < 0)
265
 
                        i_error("close(%s) failed: %m", uidlist->path);
 
297
                if (close(uidlist->fd) < 0) {
 
298
                        mail_storage_set_critical(storage,
 
299
                                "close(%s) failed: %m", uidlist->path);
 
300
                }
266
301
                uidlist->fd = -1;
267
302
                uidlist->fd_ino = 0;
268
303
        }
270
305
        uidlist->read_line_count = 0;
271
306
}
272
307
 
 
308
static void maildir_uidlist_reset(struct maildir_uidlist *uidlist)
 
309
{
 
310
        maildir_uidlist_close(uidlist);
 
311
        uidlist->last_seen_uid = 0;
 
312
        uidlist->initial_hdr_read = FALSE;
 
313
 
 
314
        hash_table_clear(uidlist->files, FALSE);
 
315
        array_clear(&uidlist->records);
 
316
}
 
317
 
273
318
void maildir_uidlist_deinit(struct maildir_uidlist **_uidlist)
274
319
{
275
320
        struct maildir_uidlist *uidlist = *_uidlist;
280
325
        maildir_uidlist_update(uidlist);
281
326
        maildir_uidlist_close(uidlist);
282
327
 
283
 
        hash_destroy(&uidlist->files);
 
328
        hash_table_destroy(&uidlist->files);
284
329
        if (uidlist->record_pool != NULL)
285
330
                pool_unref(&uidlist->record_pool);
286
331
 
399
444
static bool maildir_uidlist_next(struct maildir_uidlist *uidlist,
400
445
                                 const char *line)
401
446
{
402
 
        struct maildir_uidlist_rec *rec, *old_rec;
 
447
        struct maildir_uidlist_rec *rec, *old_rec, *const *recs;
 
448
        unsigned int count;
403
449
        uint32_t uid;
404
450
 
405
451
        uid = 0;
420
466
                                              uid, uidlist->prev_read_uid);
421
467
                return FALSE;
422
468
        }
 
469
        if (uid >= (uint32_t)-1) {
 
470
                maildir_uidlist_set_corrupted(uidlist,
 
471
                                              "UID too high (%u)", uid);
 
472
                return FALSE;
 
473
        }
423
474
        uidlist->prev_read_uid = uid;
424
475
 
425
476
        if (uid <= uidlist->last_seen_uid) {
463
514
                return FALSE;
464
515
        }
465
516
 
466
 
        old_rec = hash_lookup(uidlist->files, line);
467
 
        if (old_rec != NULL) {
 
517
        old_rec = hash_table_lookup(uidlist->files, line);
 
518
        if (old_rec == NULL) {
 
519
                /* no conflicts */
 
520
        } else if (old_rec->uid == uid) {
 
521
                /* most likely this is a record we saved ourself, but couldn't
 
522
                   update last_seen_uid because uidlist wasn't refreshed while
 
523
                   it was locked.
 
524
 
 
525
                   another possibility is a duplicate file record. currently
 
526
                   it would be a bug, but not that big of a deal. also perhaps
 
527
                   in future such duplicate lines could be used to update
 
528
                   extended fields. so just let it through anyway.
 
529
 
 
530
                   we'll waste a bit of memory here by allocating the record
 
531
                   twice, but that's not really a problem.  */
 
532
                rec->filename = old_rec->filename;
 
533
                hash_table_insert(uidlist->files, rec->filename, rec);
 
534
                uidlist->unsorted = TRUE;
 
535
                return TRUE;
 
536
        } else {
468
537
                /* This can happen if expunged file is moved back and the file
469
538
                   was appended to uidlist. */
470
539
                i_warning("%s: Duplicate file entry at line %u: "
479
548
                uidlist->recreate = TRUE;
480
549
        }
481
550
 
 
551
        recs = array_get(&uidlist->records, &count);
 
552
        if (count > 0 && recs[count-1]->uid > uid) {
 
553
                /* we most likely have some records in the array that we saved
 
554
                   ourself without refreshing uidlist */
 
555
                uidlist->unsorted = TRUE;
 
556
        }
 
557
 
482
558
        rec->filename = p_strdup(uidlist->record_pool, line);
483
 
        hash_insert(uidlist->files, rec->filename, rec);
 
559
        hash_table_insert(uidlist->files, rec->filename, rec);
484
560
        array_append(&uidlist->records, &rec, 1);
485
561
        return TRUE;
486
562
}
488
564
static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist,
489
565
                                       struct istream *input)
490
566
{
491
 
        unsigned int uid_validity, next_uid;
 
567
        unsigned int uid_validity = 0, next_uid = 0;
492
568
        string_t *ext_hdr;
493
569
        const char *line;
494
570
        char key;
516
592
                                "Corrupted header (version 1)");
517
593
                        return 0;
518
594
                }
519
 
                if (uid_validity == uidlist->uid_validity &&
520
 
                    next_uid < uidlist->next_uid) {
521
 
                        maildir_uidlist_set_corrupted(uidlist,
522
 
                                "next_uid was lowered (v1, %u -> %u)",
523
 
                                uidlist->next_uid, next_uid);
524
 
                        return 0;
525
 
                }
526
595
                break;
527
596
        case UIDLIST_VERSION:
528
597
                ext_hdr = uidlist->hdr_extensions;
565
634
                return 0;
566
635
        }
567
636
 
 
637
        if (uid_validity == uidlist->uid_validity &&
 
638
            next_uid < uidlist->hdr_next_uid) {
 
639
                maildir_uidlist_set_corrupted(uidlist,
 
640
                        "next_uid header was lowered (%u -> %u)",
 
641
                        uidlist->hdr_next_uid, next_uid);
 
642
                return 0;
 
643
        }
 
644
 
568
645
        uidlist->uid_validity = uid_validity;
569
646
        uidlist->next_uid = next_uid;
 
647
        uidlist->hdr_next_uid = next_uid;
570
648
        return 1;
571
649
}
572
650
 
 
651
static void maildir_uidlist_records_sort_by_uid(struct maildir_uidlist *uidlist)
 
652
{
 
653
        struct maildir_uidlist_rec **recs;
 
654
        unsigned int count;
 
655
 
 
656
        recs = array_get_modifiable(&uidlist->records, &count);
 
657
        qsort(recs, count, sizeof(*recs), maildir_uid_cmp);
 
658
 
 
659
        uidlist->unsorted = FALSE;
 
660
}
 
661
 
573
662
static int
574
663
maildir_uidlist_update_read(struct maildir_uidlist *uidlist,
575
664
                            bool *retry_r, bool try_retry)
576
665
{
577
666
        struct mail_storage *storage = uidlist->ibox->box.storage;
578
667
        const char *line;
579
 
        unsigned int orig_next_uid;
 
668
        uint32_t orig_next_uid, orig_uid_validity;
580
669
        struct istream *input;
581
670
        struct stat st;
582
671
        uoff_t last_read_offset;
635
724
        input = i_stream_create_fd(fd, 4096, FALSE);
636
725
        i_stream_seek(input, last_read_offset);
637
726
 
 
727
        orig_uid_validity = uidlist->uid_validity;
638
728
        orig_next_uid = uidlist->next_uid;
639
729
        ret = input->v_offset != 0 ? 1 :
640
730
                maildir_uidlist_read_header(uidlist, input);
662
752
                if (input->stream_errno != 0)
663
753
                        ret = -1;
664
754
 
 
755
                if (uidlist->unsorted) {
 
756
                        uidlist->recreate = TRUE;
 
757
                        maildir_uidlist_records_sort_by_uid(uidlist);
 
758
                }
665
759
                if (uidlist->next_uid <= uidlist->prev_read_uid)
666
760
                        uidlist->next_uid = uidlist->prev_read_uid + 1;
667
 
                if (uidlist->next_uid < orig_next_uid) {
 
761
                if (ret > 0 && uidlist->uid_validity != orig_uid_validity &&
 
762
                    orig_uid_validity != 0) {
 
763
                        uidlist->recreate = TRUE;
 
764
                } else if (ret > 0 && uidlist->next_uid < orig_next_uid) {
668
765
                        mail_storage_set_critical(storage,
669
 
                                "%s: next_uid was lowered (%u -> %u)",
 
766
                                "%s: next_uid was lowered (%u -> %u, hdr=%u)",
670
767
                                uidlist->path, orig_next_uid,
671
 
                                uidlist->next_uid);
 
768
                                uidlist->next_uid, uidlist->hdr_next_uid);
672
769
                        uidlist->recreate = TRUE;
673
770
                        uidlist->next_uid = orig_next_uid;
674
771
                }
694
791
                        mail_storage_set_critical(storage,
695
792
                                "read(%s) failed: %m", uidlist->path);
696
793
                }
 
794
                uidlist->last_read_offset = 0;
697
795
        }
698
796
 
699
797
        i_stream_destroy(&input);
700
798
        if (ret <= 0) {
701
 
                if (close(fd) < 0)
702
 
                        i_error("close(%s) failed: %m", uidlist->path);
 
799
                if (close(fd) < 0) {
 
800
                        mail_storage_set_critical(storage,
 
801
                                "close(%s) failed: %m", uidlist->path);
 
802
                }
703
803
        }
704
804
        return ret;
705
805
}
734
834
        *recreated_r = FALSE;
735
835
 
736
836
        if ((ret = maildir_uidlist_stat(uidlist, &st)) <= 0)
737
 
                return ret;
 
837
                return ret < 0 ? -1 : 1;
738
838
 
739
839
        if (st.st_ino != uidlist->fd_ino ||
740
840
            !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) {
775
875
 
776
876
        if (uidlist->fd != -1) {
777
877
                ret = maildir_uidlist_has_changed(uidlist, &recreated);
778
 
                if (ret <= 0)
779
 
                        return ret;
 
878
                if (ret <= 0) {
 
879
                        if (UIDLIST_IS_LOCKED(uidlist))
 
880
                                uidlist->locked_refresh = TRUE;
 
881
                        return ret < 0 ? -1 : 1;
 
882
                }
780
883
 
781
884
                if (recreated)
782
885
                        maildir_uidlist_close(uidlist);
793
896
        if (ret >= 0) {
794
897
                uidlist->initial_read = TRUE;
795
898
                uidlist->initial_hdr_read = TRUE;
 
899
                if (UIDLIST_IS_LOCKED(uidlist))
 
900
                        uidlist->locked_refresh = TRUE;
796
901
        }
797
902
        return ret;
798
903
}
800
905
int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist)
801
906
{
802
907
        const struct maildir_index_header *mhdr = &uidlist->mbox->maildir_hdr;
 
908
        struct mail_index *index = uidlist->mbox->ibox.index;
 
909
        struct mail_index_view *view;
803
910
        const struct mail_index_header *hdr;
804
911
        struct stat st;
805
912
        int ret;
806
913
 
 
914
        i_assert(UIDLIST_IS_LOCKED(uidlist));
 
915
 
807
916
        if (uidlist->fd != -1)
808
917
                return maildir_uidlist_refresh(uidlist);
809
918
 
810
919
        if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0)
811
920
                return ret;
812
921
 
813
 
        if ((uoff_t)st.st_size == mhdr->uidlist_size &&
 
922
        if (ret > 0 && st.st_size == mhdr->uidlist_size &&
814
923
            st.st_mtime == (time_t)mhdr->uidlist_mtime &&
815
 
            ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), mhdr->uidlist_mtime_nsecs)) {
816
 
                /* index is up-to-date */
817
 
                hdr = mail_index_get_header(uidlist->mbox->ibox.view);
 
924
            ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), mhdr->uidlist_mtime_nsecs) &&
 
925
            (!mail_index_is_in_memory(index) || st.st_mtime < ioloop_time-1)) {
 
926
                /* index is up-to-date. look up the uidvalidity and next-uid
 
927
                   from it. we'll need to create a new view temporarily to
 
928
                   make sure we get the latest values. */
 
929
                view = mail_index_view_open(index);
 
930
                hdr = mail_index_get_header(view);
818
931
                uidlist->uid_validity = hdr->uid_validity;
819
932
                uidlist->next_uid = hdr->next_uid;
820
933
                uidlist->initial_hdr_read = TRUE;
 
934
                mail_index_view_close(&view);
821
935
                return 1;
822
936
        } else {
823
937
                return maildir_uidlist_refresh(uidlist);
824
938
        }
825
939
}
826
940
 
827
 
static struct maildir_uidlist_rec *
 
941
static int
828
942
maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid,
829
 
                           unsigned int *idx_r)
 
943
                           unsigned int *idx_r,
 
944
                           struct maildir_uidlist_rec **rec_r)
830
945
{
831
946
        struct maildir_uidlist_rec *const *recs;
832
947
        unsigned int idx, left_idx, right_idx;
834
949
        if (!uidlist->initial_read) {
835
950
                /* first time we need to read uidlist */
836
951
                if (maildir_uidlist_refresh(uidlist) < 0)
837
 
                        return NULL;
 
952
                        return -1;
838
953
        }
839
954
 
840
955
        idx = left_idx = 0;
848
963
                        right_idx = idx;
849
964
                else {
850
965
                        *idx_r = idx;
851
 
                        return recs[idx];
 
966
                        *rec_r = recs[idx];
 
967
                        return 1;
852
968
                }
853
969
        }
854
970
 
855
971
        if (idx > 0) idx--;
856
972
        *idx_r = idx;
857
 
        return NULL;
 
973
        return 0;
858
974
}
859
975
 
860
 
const char *
861
 
maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
862
 
                       enum maildir_uidlist_rec_flag *flags_r)
 
976
int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
 
977
                           enum maildir_uidlist_rec_flag *flags_r,
 
978
                           const char **fname_r)
863
979
{
864
 
        const char *fname;
 
980
        int ret;
865
981
 
866
 
        fname = maildir_uidlist_lookup_nosync(uidlist, uid, flags_r);
867
 
        if (fname == NULL) {
 
982
        ret = maildir_uidlist_lookup_nosync(uidlist, uid, flags_r, fname_r);
 
983
        if (ret <= 0) {
 
984
                if (ret < 0)
 
985
                        return -1;
868
986
                if (uidlist->fd != -1 || uidlist->mbox == NULL) {
869
987
                        /* refresh uidlist and check again in case it was added
870
988
                           after the last mailbox sync */
871
989
                        if (maildir_uidlist_refresh(uidlist) < 0)
872
 
                                return NULL;
 
990
                                return -1;
873
991
                } else {
874
992
                        /* the uidlist doesn't exist. */
875
993
                        if (maildir_storage_sync_force(uidlist->mbox, uid) < 0)
876
 
                                return NULL;
 
994
                                return -1;
877
995
                }
878
996
 
879
997
                /* try again */
880
 
                fname = maildir_uidlist_lookup_nosync(uidlist, uid, flags_r);
 
998
                ret = maildir_uidlist_lookup_nosync(uidlist, uid,
 
999
                                                    flags_r, fname_r);
881
1000
        }
882
1001
 
883
 
        return fname;
 
1002
        return ret;
884
1003
}
885
1004
 
886
 
const char *
887
 
maildir_uidlist_lookup_nosync(struct maildir_uidlist *uidlist, uint32_t uid,
888
 
                              enum maildir_uidlist_rec_flag *flags_r)
 
1005
int maildir_uidlist_lookup_nosync(struct maildir_uidlist *uidlist, uint32_t uid,
 
1006
                                  enum maildir_uidlist_rec_flag *flags_r,
 
1007
                                  const char **fname_r)
889
1008
{
890
 
        const struct maildir_uidlist_rec *rec;
 
1009
        struct maildir_uidlist_rec *rec;
891
1010
        unsigned int idx;
 
1011
        int ret;
892
1012
 
893
 
        rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);
894
 
        if (rec == NULL)
895
 
                return NULL;
 
1013
        if ((ret = maildir_uidlist_lookup_rec(uidlist, uid, &idx, &rec)) <= 0)
 
1014
                return ret;
896
1015
 
897
1016
        *flags_r = rec->flags;
898
 
        return rec->filename;
 
1017
        *fname_r = rec->filename;
 
1018
        return 1;
899
1019
}
900
1020
 
901
1021
const char *
902
1022
maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid,
903
1023
                           enum maildir_uidlist_rec_ext_key key)
904
1024
{
905
 
        const struct maildir_uidlist_rec *rec;
 
1025
        struct maildir_uidlist_rec *rec;
906
1026
        unsigned int idx;
907
1027
        const unsigned char *p;
908
 
        const char *value;
 
1028
        int ret;
909
1029
 
910
 
        rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);
911
 
        if (rec == NULL || rec->extensions == NULL)
 
1030
        ret = maildir_uidlist_lookup_rec(uidlist, uid, &idx, &rec);
 
1031
        if (ret <= 0 || rec->extensions == NULL)
912
1032
                return NULL;
913
1033
 
914
 
        p = rec->extensions; value = NULL;
 
1034
        p = rec->extensions;
915
1035
        while (*p != '\0') {
916
1036
                /* <key><value>\0 */
917
1037
                if (*p == (char)key)
943
1063
void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist,
944
1064
                                  uint32_t next_uid, bool force)
945
1065
{
946
 
        if (uidlist->next_uid < next_uid || force)
 
1066
        if (uidlist->next_uid < next_uid || force) {
 
1067
                i_assert(next_uid != 0);
947
1068
                uidlist->next_uid = next_uid;
 
1069
        }
948
1070
}
949
1071
 
950
1072
static void
957
1079
        const unsigned char *p;
958
1080
        buffer_t *buf;
959
1081
        unsigned int len;
960
 
 
961
 
        rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);
962
 
        if (rec == NULL) {
 
1082
        int ret;
 
1083
 
 
1084
        ret = maildir_uidlist_lookup_rec(uidlist, uid, &idx, &rec);
 
1085
        if (ret <= 0) {
 
1086
                if (ret < 0)
 
1087
                        return;
 
1088
 
963
1089
                /* maybe it's a new message */
964
1090
                if (maildir_uidlist_refresh(uidlist) < 0)
965
1091
                        return;
966
 
                rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);
967
 
                if (rec == NULL) {
 
1092
                if (maildir_uidlist_lookup_rec(uidlist, uid, &idx, &rec) <= 0) {
968
1093
                        /* message is already expunged, ignore */
969
1094
                        return;
970
1095
                }
992
1117
        rec->extensions = p_malloc(uidlist->record_pool, buf->used);
993
1118
        memcpy(rec->extensions, buf->data, buf->used);
994
1119
 
995
 
        uidlist->recreate = TRUE;
 
1120
        if (rec->uid != (uint32_t)-1) {
 
1121
                /* message already exists in uidlist, need to recreate it */
 
1122
                uidlist->recreate = TRUE;
 
1123
        }
996
1124
}
997
1125
 
998
1126
void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid,
1113
1241
                }
1114
1242
                /* the control dir doesn't exist. create it unless the whole
1115
1243
                   mailbox was just deleted. */
1116
 
                if (maildir_set_deleted(uidlist->mbox))
 
1244
                if (!maildir_set_deleted(uidlist->mbox))
1117
1245
                        return -1;
1118
1246
        }
1119
1247
 
1120
 
        if (box->file_create_gid != (gid_t)-1) {
1121
 
                if (fchown(fd, (uid_t)-1, box->file_create_gid) < 0) {
 
1248
        if (box->file_create_gid != (gid_t)-1 &&
 
1249
            fchown(fd, (uid_t)-1, box->file_create_gid) < 0) {
 
1250
                if (errno == EPERM) {
 
1251
                        mail_storage_set_critical(box->storage, "%s",
 
1252
                                eperm_error_get_chgrp("fchown", temp_path,
 
1253
                                                box->file_create_gid,
 
1254
                                                box->file_create_gid_origin));
 
1255
                } else {
1122
1256
                        mail_storage_set_critical(box->storage,
1123
1257
                                "fchown(%s) failed: %m", temp_path);
1124
1258
                }
1183
1317
{
1184
1318
        unsigned int min_rewrite_count;
1185
1319
 
1186
 
        if (!ctx->uidlist->initial_read)
 
1320
        if (!ctx->uidlist->locked_refresh)
1187
1321
                return FALSE;
 
1322
        if (ctx->uidlist->recreate)
 
1323
                return TRUE;
1188
1324
 
1189
1325
        min_rewrite_count =
1190
1326
                (ctx->uidlist->read_records_count + ctx->new_files_count) *
1196
1332
{
1197
1333
        struct maildir_uidlist *uidlist = ctx->uidlist;
1198
1334
 
1199
 
        if (uidlist->recreate ||
1200
 
            ctx->finish_change_counter != uidlist->change_counter)
 
1335
        if (!uidlist->locked_refresh)
 
1336
                return FALSE;
 
1337
 
 
1338
        if (ctx->finish_change_counter != uidlist->change_counter)
1201
1339
                return TRUE;
1202
 
 
1203
 
        if (!uidlist->initial_read)
1204
 
                return FALSE;
1205
 
 
1206
1340
        if (uidlist->fd == -1 || uidlist->version != UIDLIST_VERSION)
1207
1341
                return TRUE;
1208
1342
        return maildir_uidlist_want_compress(ctx);
1261
1395
        if ((uoff_t)st.st_size != file_size) {
1262
1396
                i_warning("%s: file size changed unexpectedly after write",
1263
1397
                          uidlist->path);
1264
 
        } else {
 
1398
        } else if (uidlist->locked_refresh) {
1265
1399
                uidlist->fd_size = st.st_size;
1266
1400
                uidlist->last_read_offset = st.st_size;
1267
1401
                maildir_uidlist_update_hdr(uidlist, &st);
1285
1419
        }
1286
1420
}
1287
1421
 
1288
 
int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
1289
 
                              enum maildir_uidlist_sync_flags sync_flags,
1290
 
                              struct maildir_uidlist_sync_ctx **sync_ctx_r)
 
1422
static int maildir_uidlist_sync_lock(struct maildir_uidlist *uidlist,
 
1423
                                     enum maildir_uidlist_sync_flags sync_flags,
 
1424
                                     bool *locked_r)
1291
1425
{
1292
 
        struct maildir_uidlist_sync_ctx *ctx;
1293
 
        bool nonblock, refresh, locked;
 
1426
        bool nonblock, refresh;
1294
1427
        int ret;
1295
1428
 
 
1429
        *locked_r = FALSE;
 
1430
 
 
1431
        if ((sync_flags & MAILDIR_UIDLIST_SYNC_NOLOCK) != 0) {
 
1432
                if (maildir_uidlist_refresh(uidlist) < 0)
 
1433
                        return -1;
 
1434
                return 1;
 
1435
        }
 
1436
 
1296
1437
        nonblock = (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0;
1297
1438
        refresh = (sync_flags & MAILDIR_UIDLIST_SYNC_NOREFRESH) == 0;
1298
1439
 
1299
 
        ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh);
 
1440
        ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh, refresh);
1300
1441
        if (ret <= 0) {
1301
1442
                if (ret < 0 || !nonblock)
1302
1443
                        return ret;
1307
1448
                /* forcing the sync anyway */
1308
1449
                if (maildir_uidlist_refresh(uidlist) < 0)
1309
1450
                        return -1;
1310
 
                locked = FALSE;
1311
1451
        } else {
1312
 
                locked = TRUE;
 
1452
                *locked_r = TRUE;
1313
1453
        }
 
1454
        return 1;
 
1455
}
 
1456
 
 
1457
void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist)
 
1458
{
 
1459
        maildir_uidlist_mark_all(uidlist, TRUE);
 
1460
}
 
1461
 
 
1462
int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
 
1463
                              enum maildir_uidlist_sync_flags sync_flags,
 
1464
                              struct maildir_uidlist_sync_ctx **sync_ctx_r)
 
1465
{
 
1466
        struct maildir_uidlist_sync_ctx *ctx;
 
1467
        bool locked;
 
1468
        int ret;
 
1469
 
 
1470
        ret = maildir_uidlist_sync_lock(uidlist, sync_flags, &locked);
 
1471
        if (ret <= 0)
 
1472
                return ret;
1314
1473
 
1315
1474
        *sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
1316
1475
        ctx->uidlist = uidlist;
1328
1487
                }
1329
1488
                return 1;
1330
1489
        }
 
1490
        i_assert(uidlist->locked_refresh);
1331
1491
 
1332
1492
        ctx->record_pool = pool_alloconly_create(MEMPOOL_GROWING
1333
1493
                                                 "maildir_uidlist_sync", 16384);
1334
 
        ctx->files = hash_create(default_pool, ctx->record_pool, 4096,
1335
 
                                 maildir_filename_base_hash,
1336
 
                                 maildir_filename_base_cmp);
 
1494
        ctx->files = hash_table_create(default_pool, ctx->record_pool, 4096,
 
1495
                                       maildir_filename_base_hash,
 
1496
                                       maildir_filename_base_cmp);
1337
1497
 
1338
1498
        i_array_init(&ctx->records, array_count(&uidlist->records));
1339
1499
        return 1;
1348
1508
        struct maildir_uidlist_rec *rec;
1349
1509
 
1350
1510
        /* we'll update uidlist directly */
1351
 
        rec = hash_lookup(uidlist->files, filename);
 
1511
        rec = hash_table_lookup(uidlist->files, filename);
1352
1512
        if (rec == NULL) {
1353
1513
                /* doesn't exist in uidlist */
1354
1514
                if (!ctx->locked) {
1376
1536
 
1377
1537
        rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
1378
1538
        rec->filename = p_strdup(uidlist->record_pool, filename);
1379
 
        hash_insert(uidlist->files, rec->filename, rec);
 
1539
        hash_table_insert(uidlist->files, rec->filename, rec);
1380
1540
 
1381
1541
        ctx->finished = FALSE;
1382
1542
}
1424
1584
                return 1;
1425
1585
        }
1426
1586
 
1427
 
        rec = hash_lookup(ctx->files, filename);
 
1587
        rec = hash_table_lookup(ctx->files, filename);
1428
1588
        if (rec != NULL) {
1429
1589
                if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
1430
1590
                                   MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) {
1438
1598
                rec->flags &= ~(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
1439
1599
                                MAILDIR_UIDLIST_REC_FLAG_MOVED);
1440
1600
        } else {
1441
 
                old_rec = hash_lookup(uidlist->files, filename);
 
1601
                old_rec = hash_table_lookup(uidlist->files, filename);
1442
1602
                i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist));
1443
1603
 
1444
1604
                rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1);
1460
1620
 
1461
1621
        rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
1462
1622
        rec->filename = p_strdup(ctx->record_pool, filename);
1463
 
        hash_insert(ctx->files, rec->filename, rec);
 
1623
        hash_table_insert(ctx->files, rec->filename, rec);
1464
1624
        return 1;
1465
1625
}
1466
1626
 
1471
1631
        unsigned int idx;
1472
1632
 
1473
1633
        i_assert(ctx->partial);
 
1634
        i_assert(ctx->uidlist->locked_refresh);
1474
1635
 
1475
 
        rec = hash_lookup(ctx->uidlist->files, filename);
 
1636
        rec = hash_table_lookup(ctx->uidlist->files, filename);
1476
1637
        i_assert(rec != NULL);
1477
1638
        i_assert(rec->uid != (uint32_t)-1);
1478
1639
 
1479
 
        hash_remove(ctx->uidlist->files, filename);
 
1640
        hash_table_remove(ctx->uidlist->files, filename);
1480
1641
        idx = maildir_uidlist_records_array_delete(ctx->uidlist, rec);
1481
1642
 
1482
1643
        if (ctx->first_unwritten_pos != (unsigned int)-1) {
1498
1659
{
1499
1660
        struct maildir_uidlist_rec *rec;
1500
1661
 
1501
 
        rec = hash_lookup(ctx->files, filename);
 
1662
        rec = hash_table_lookup(ctx->files, filename);
1502
1663
        return rec == NULL ? NULL : rec->filename;
1503
1664
}
1504
1665
 
1507
1668
{
1508
1669
        struct maildir_uidlist_rec *rec;
1509
1670
 
1510
 
        rec = hash_lookup(uidlist->files, filename);
 
1671
        rec = hash_table_lookup(uidlist->files, filename);
1511
1672
        if (rec == NULL)
1512
1673
                return FALSE;
1513
1674
 
1521
1682
{
1522
1683
        struct maildir_uidlist_rec *rec;
1523
1684
 
1524
 
        rec = hash_lookup(uidlist->files, filename);
 
1685
        rec = hash_table_lookup(uidlist->files, filename);
1525
1686
        return rec == NULL ? NULL : rec->filename;
1526
1687
}
1527
1688
 
1550
1711
 
1551
1712
        for (dest = ctx->first_nouid_pos; dest < count; dest++) {
1552
1713
                i_assert(recs[dest]->uid == (uint32_t)-1);
 
1714
                i_assert(ctx->uidlist->next_uid < (uint32_t)-1);
1553
1715
                recs[dest]->uid = ctx->uidlist->next_uid++;
1554
1716
                recs[dest]->flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
1555
1717
        }
1556
1718
 
 
1719
        if (ctx->uidlist->locked_refresh)
 
1720
                ctx->uidlist->last_seen_uid = ctx->uidlist->next_uid-1;
 
1721
 
1557
1722
        ctx->new_files_count = 0;
1558
1723
        ctx->first_nouid_pos = (unsigned int)-1;
1559
 
 
1560
 
        ctx->uidlist->last_seen_uid = ctx->uidlist->next_uid-1;
1561
1724
        ctx->uidlist->change_counter++;
1562
1725
        ctx->finish_change_counter = ctx->uidlist->change_counter;
1563
1726
}
1576
1739
        uidlist->records = ctx->records;
1577
1740
        ctx->records.arr.buffer = NULL;
1578
1741
 
1579
 
        hash_destroy(&uidlist->files);
 
1742
        hash_table_destroy(&uidlist->files);
1580
1743
        uidlist->files = ctx->files;
1581
1744
        ctx->files = NULL;
1582
1745
 
1599
1762
                if (!ctx->failed)
1600
1763
                        maildir_uidlist_swap(ctx);
1601
1764
        } else {
1602
 
                if (ctx->new_files_count != 0)
 
1765
                if (ctx->new_files_count != 0 && !ctx->failed) {
 
1766
                        i_assert(ctx->changed);
 
1767
                        i_assert(ctx->locked);
1603
1768
                        maildir_uidlist_assign_uids(ctx);
 
1769
                }
1604
1770
        }
1605
1771
 
1606
1772
        ctx->finished = TRUE;
1607
 
        ctx->uidlist->initial_sync = TRUE;
1608
1773
 
1609
1774
        i_assert(ctx->locked || !ctx->changed);
1610
 
        if ((ctx->changed || ctx->uidlist->recreate ||
1611
 
             maildir_uidlist_want_compress(ctx)) &&
 
1775
        if ((ctx->changed || maildir_uidlist_want_compress(ctx)) &&
1612
1776
            !ctx->failed && ctx->locked) T_BEGIN {
1613
 
                if (maildir_uidlist_sync_update(ctx) < 0)
 
1777
                if (maildir_uidlist_sync_update(ctx) < 0) {
 
1778
                        /* we couldn't write everything we wanted. make sure
 
1779
                           we don't continue using those UIDs */
 
1780
                        maildir_uidlist_reset(ctx->uidlist);
1614
1781
                        ctx->failed = TRUE;
 
1782
                }
1615
1783
        } T_END;
1616
1784
}
1617
1785
 
1618
 
int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx)
 
1786
int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx,
 
1787
                                bool success)
1619
1788
{
1620
1789
        struct maildir_uidlist_sync_ctx *ctx = *_ctx;
1621
 
        int ret = ctx->failed ? -1 : 0;
 
1790
        int ret;
1622
1791
 
1623
1792
        *_ctx = NULL;
1624
1793
 
 
1794
        if (!success)
 
1795
                ctx->failed = TRUE;
 
1796
        ret = ctx->failed ? -1 : 0;
 
1797
 
1625
1798
        if (!ctx->finished)
1626
1799
                maildir_uidlist_sync_finish(ctx);
1627
1800
        if (ctx->partial)
1630
1803
                maildir_uidlist_unlock(ctx->uidlist);
1631
1804
 
1632
1805
        if (ctx->files != NULL)
1633
 
                hash_destroy(&ctx->files);
 
1806
                hash_table_destroy(&ctx->files);
1634
1807
        if (ctx->record_pool != NULL)
1635
1808
                pool_unref(&ctx->record_pool);
1636
1809
        if (array_is_created(&ctx->records))
1645
1818
{
1646
1819
        struct maildir_uidlist_rec *rec;
1647
1820
 
1648
 
        rec = hash_lookup(uidlist->files, filename);
 
1821
        rec = hash_table_lookup(uidlist->files, filename);
1649
1822
        i_assert(rec != NULL);
1650
1823
 
1651
1824
        rec->flags |= flags;