9
9
#include "mail-cache.h"
10
#include "dbox-sync-rebuild.h"
10
#include "index-rebuild.h"
11
11
#include "mail-namespace.h"
12
12
#include "mailbox-list-private.h"
13
13
#include "mdbox-storage.h"
50
50
struct mdbox_map_mail_index_header orig_map_hdr;
51
struct hash_table *guid_hash;
52
ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *);
51
HASH_TABLE(uint8_t *, struct mdbox_rebuild_msg *) guid_hash;
52
ARRAY(struct mdbox_rebuild_msg *) msgs;
53
53
ARRAY_TYPE(seq_range) seen_file_ids;
55
55
uint32_t rebuild_count;
58
58
struct mailbox_list *default_list;
60
60
struct rebuild_msg_mailbox prev_msg;
62
unsigned int have_pop3_uidls:1;
63
unsigned int have_pop3_orders:1;
63
66
static struct mdbox_storage_rebuild_context *
72
75
ctx->storage = storage;
73
76
ctx->atomic = atomic;
74
77
ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
75
ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0,
76
guid_128_hash, guid_128_cmp);
78
hash_table_create(&ctx->guid_hash, ctx->pool, 0,
79
guid_128_hash, guid_128_cmp);
77
80
i_array_init(&ctx->msgs, 512);
78
81
i_array_init(&ctx->seen_file_ids, 128);
98
static int mdbox_rebuild_msg_offset_cmp(const void *p1, const void *p2)
102
mdbox_rebuild_msg_offset_cmp(struct mdbox_rebuild_msg *const *m1,
103
struct mdbox_rebuild_msg *const *m2)
100
const struct mdbox_rebuild_msg *const *m1 = p1, *const *m2 = p2;
102
105
if ((*m1)->file_id < (*m2)->file_id)
104
107
if ((*m1)->file_id > (*m2)->file_id)
132
static void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx,
133
struct dbox_file *file)
135
if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL)
136
ctx->have_pop3_uidls = TRUE;
137
if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL)
138
ctx->have_pop3_orders = TRUE;
129
141
static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
130
142
struct dbox_file *file, uint32_t file_id)
132
144
const char *guid;
133
146
struct mdbox_rebuild_msg *rec, *old_rec;
134
147
uoff_t offset, prev_offset;
135
148
bool last, first, fixed = FALSE;
186
200
i_assert(!guid_128_is_empty(rec->guid_128));
187
201
array_append(&ctx->msgs, &rec, 1);
189
old_rec = hash_table_lookup(ctx->guid_hash, rec->guid_128);
203
guid_p = rec->guid_128;
204
old_rec = hash_table_lookup(ctx->guid_hash, guid_p);
190
205
if (old_rec == NULL)
191
hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
206
hash_table_insert(ctx->guid_hash, guid_p, rec);
192
207
else if (rec->mail_size == old_rec->mail_size) {
193
208
/* two mails' GUID and size are the same, which quite
194
209
likely means that their contents are the same as
277
292
if (rebuild_rename_file(ctx, dir, &fname, &file_id) < 0)
280
seq_range_array_add(&ctx->seen_file_ids, 0, file_id);
295
seq_range_array_add(&ctx->seen_file_ids, file_id);
282
297
file = mdbox_file_init(ctx->storage, file_id);
283
298
if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted)
321
336
struct mdbox_map *map = ctx->storage->map;
322
337
const struct mail_index_header *hdr;
323
struct mdbox_rebuild_msg *const *msgs, **pos;
338
struct mdbox_rebuild_msg **pos;
324
339
struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
325
340
struct dbox_mail_lookup_rec rec;
329
343
array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp);
330
344
/* msgs now contains a list of all messages that exists in m.* files,
331
345
sorted by file_id,offset */
333
msgs = array_get_modifiable(&ctx->msgs, &count);
334
347
hdr = mail_index_get_header(ctx->atomic->sync_view);
335
348
for (seq = 1; seq <= hdr->messages_count; seq++) {
336
349
if (mdbox_map_view_lookup_rec(map, ctx->atomic->sync_view,
342
355
search_msg.file_id = rec.rec.file_id;
343
356
search_msg.offset = rec.rec.offset;
344
357
search_msg.rec_size = rec.rec.size;
345
pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs),
346
mdbox_rebuild_msg_offset_cmp);
358
pos = array_bsearch(&ctx->msgs, &search_msgp,
359
mdbox_rebuild_msg_offset_cmp);
347
360
if (pos == NULL || (*pos)->map_uid != 0) {
348
361
/* map record points to nonexistent or
349
362
a duplicate message. */
394
407
rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx,
395
struct dbox_sync_rebuild_context *rebuild_ctx,
408
struct index_rebuild_context *rebuild_ctx,
396
409
struct mdbox_mailbox *mbox,
397
410
struct mail_index_view *view,
398
411
struct mail_index_transaction *trans)
401
414
const struct mail_index_header *hdr;
402
415
struct mdbox_rebuild_msg *rec;
403
416
const void *data;
417
const uint8_t *guid_p;
405
418
uint32_t old_seq, new_seq, uid, map_uid;
407
420
/* Rebuild the mailbox's index. Note that index is reset at this point,
410
423
hdr = mail_index_get_header(view);
411
424
for (old_seq = 1; old_seq <= hdr->messages_count; old_seq++) {
412
425
mail_index_lookup_ext(view, old_seq, mbox->ext_id,
414
427
if (data == NULL) {
415
428
memset(&new_dbox_rec, 0, sizeof(new_dbox_rec));
422
435
mail_index_lookup_ext(view, old_seq, mbox->guid_ext_id,
425
439
/* see if we can find this message based on
426
440
1) GUID, 2) map_uid */
427
rec = data == NULL ? NULL :
428
hash_table_lookup(ctx->guid_hash, data);
441
rec = guid_p == NULL ? NULL :
442
hash_table_lookup(ctx->guid_hash, guid_p);
429
443
if (rec == NULL) {
430
444
/* multi-dbox message that wasn't found with GUID.
431
445
either it's lost or GUID has been corrupted. we can
452
466
mail_index_lookup_uid(view, old_seq, &uid);
453
467
mail_index_append(trans, uid, &new_seq);
454
dbox_sync_rebuild_index_metadata(rebuild_ctx,
468
index_rebuild_index_metadata(rebuild_ctx,
457
471
new_dbox_rec.map_uid = map_uid;
458
472
mail_index_update_ext(trans, new_seq, mbox->ext_id,
467
481
mdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id,
468
struct mdbox_index_header *hdr_r)
482
struct mdbox_index_header *hdr_r, bool *need_resize_r)
470
484
const void *data;
471
485
size_t data_size;
473
487
mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size);
474
488
memset(hdr_r, 0, sizeof(*hdr_r));
475
489
memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
490
*need_resize_r = data_size < sizeof(*hdr_r);
478
static void mdbox_header_update(struct dbox_sync_rebuild_context *rebuild_ctx,
493
static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx,
494
struct index_rebuild_context *rebuild_ctx,
479
495
struct mdbox_mailbox *mbox)
481
497
struct mdbox_index_header hdr, backup_hdr;
498
bool need_resize, need_resize_backup;
483
mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id, &hdr);
484
if (rebuild_ctx->backup_view == NULL)
500
mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id,
502
if (rebuild_ctx->backup_view == NULL) {
485
503
memset(&backup_hdr, 0, sizeof(backup_hdr));
487
506
mdbox_rebuild_get_header(rebuild_ctx->backup_view,
488
mbox->hdr_ext_id, &backup_hdr);
507
mbox->hdr_ext_id, &backup_hdr,
508
&need_resize_backup);
491
511
/* make sure we have valid mailbox guid */
501
521
/* update map's uid-validity */
502
522
hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map);
524
if (ctx->have_pop3_uidls)
525
hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
526
if (ctx->have_pop3_orders)
527
hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
504
529
/* and write changes */
531
mail_index_ext_resize_hdr(rebuild_ctx->trans, mbox->hdr_ext_id,
505
534
mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0,
506
535
&hdr, sizeof(hdr));
515
544
struct mail_index_sync_ctx *sync_ctx;
516
545
struct mail_index_view *view;
517
546
struct mail_index_transaction *trans;
518
struct dbox_sync_rebuild_context *rebuild_ctx;
547
struct index_rebuild_context *rebuild_ctx;
519
548
enum mail_error error;
522
551
box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY |
523
552
MAILBOX_FLAG_IGNORE_ACLS);
524
i_assert(box->storage == &ctx->storage->storage.storage);
553
if (box->storage != &ctx->storage->storage.storage) {
554
/* the namespace has multiple storages. */
525
558
if (mailbox_open(box) < 0) {
526
559
error = mailbox_get_last_mail_error(box);
527
560
i_error("Couldn't open mailbox '%s': %s",
538
571
MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
540
573
i_assert(ret != 0);
541
mail_storage_set_index_error(box);
574
mailbox_set_index_error(box);
542
575
mailbox_free(&box);
546
rebuild_ctx = dbox_sync_index_rebuild_init(&mbox->box, view, trans);
547
mdbox_header_update(rebuild_ctx, mbox);
579
rebuild_ctx = index_index_rebuild_init(&mbox->box, view, trans);
580
mdbox_header_update(ctx, rebuild_ctx, mbox);
548
581
rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
549
dbox_sync_index_rebuild_deinit(&rebuild_ctx);
582
index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next);
551
584
if (mail_index_sync_commit(&sync_ctx) < 0) {
552
mail_storage_set_index_error(box);
585
mailbox_set_index_error(box);
576
609
if ((info->flags & (MAILBOX_NONEXISTENT |
577
610
MAILBOX_NOSELECT)) == 0) {
579
ret = rebuild_mailbox(ctx, ns, info->name);
612
ret = rebuild_mailbox(ctx, ns, info->vname);
642
675
mailbox = mailbox_list_get_vname(ctx->default_list, mailbox);
643
676
mailbox = t_strdup(mailbox);
678
rebuild_scan_metadata(ctx, file);
646
680
dbox_file_unref(&file);
647
681
if (ret <= 0 || deleted) {
782
815
mail_index_lookup_ext(ctx->atomic->sync_view, seq,
783
816
ctx->storage->map->ref_ext_id,
786
819
if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
787
820
mail_index_update_ext(ctx->atomic->sync_trans, seq,
918
951
struct mdbox_storage_rebuild_context *ctx;
921
if (dbox_sync_rebuild_verify_alt_storage(storage->map->root_list) < 0) {
954
if (dbox_verify_alt_storage(storage->map->root_list) < 0) {
922
955
mail_storage_set_critical(&storage->storage.storage,
923
956
"mdbox rebuild: Alt storage %s not mounted, aborting",
924
957
storage->alt_storage_dir);
added
removed
src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
