1
/* Copyright (c) 2008-2009 Dovecot authors, see the included COPYING file */
5
#include "bsearch-insert-pos.h"
9
#include "file-dotlock.h"
11
#include "safe-mkstemp.h"
13
#include "mail-index-private.h"
14
#include "mail-index-strmap.h"
18
struct mail_index_strmap {
19
struct mail_index *index;
22
struct istream *input;
24
struct file_lock *file_lock;
25
struct dotlock *dotlock;
26
struct dotlock_settings dotlock_settings;
29
struct mail_index_strmap_view {
30
struct mail_index_strmap *strmap;
31
struct mail_index_view *view;
33
ARRAY_TYPE(mail_index_strmap_rec) recs;
34
ARRAY_DEFINE(recs_crc32, uint32_t);
35
struct hash2_table *hash;
37
mail_index_strmap_key_cmp_t *key_compare;
38
mail_index_strmap_rec_cmp_t *rec_compare;
39
mail_index_strmap_remap_t *remap_cb;
42
uoff_t last_read_block_offset;
43
uint32_t last_read_uid;
44
uint32_t last_added_uid;
45
uint32_t total_ref_count;
47
uint32_t last_ref_index;
48
uint32_t next_str_idx;
49
uint32_t lost_expunged_uid;
51
unsigned int desynced:1;
54
struct mail_index_strmap_read_context {
55
struct mail_index_strmap_view *view;
57
struct istream *input;
59
uint32_t highest_str_idx;
60
uint32_t uid_lookup_idx;
61
uint32_t lost_expunged_uid;
63
const unsigned char *data, *end, *str_idx_base;
64
struct mail_index_strmap_rec rec;
65
uint32_t next_ref_index;
66
unsigned int rec_size;
68
unsigned int too_large_uids:1;
71
struct mail_index_strmap_view_sync {
72
struct mail_index_strmap_view *view;
75
struct mail_index_strmap_hash_key {
80
/* number of bytes required to store one string idx */
81
#define STRMAP_FILE_STRIDX_SIZE (sizeof(uint32_t)*2)
83
/* renumber the string indexes when highest string idx becomes larger than
84
<number of indexes>*STRMAP_FILE_MAX_STRIDX_MULTIPLIER */
85
#define STRMAP_FILE_MAX_STRIDX_MULTIPLIER 2
87
#define STRIDX_MUST_RENUMBER(highest_idx, n_unique_indexes) \
88
(highest_idx > n_unique_indexes * STRMAP_FILE_MAX_STRIDX_MULTIPLIER)
90
#define MAIL_INDEX_STRMAP_TIMEOUT_SECS 10
92
const struct dotlock_settings default_dotlock_settings = {
93
MEMBER(temp_prefix) NULL,
94
MEMBER(lock_suffix) NULL,
96
MEMBER(timeout) MAIL_INDEX_STRMAP_TIMEOUT_SECS,
97
MEMBER(stale_timeout) 30
100
struct mail_index_strmap *
101
mail_index_strmap_init(struct mail_index *index, const char *suffix)
103
struct mail_index_strmap *strmap;
105
strmap = i_new(struct mail_index_strmap, 1);
106
strmap->index = index;
107
strmap->path = i_strconcat(index->filepath, suffix, NULL);
110
strmap->dotlock_settings = default_dotlock_settings;
111
strmap->dotlock_settings.use_excl_lock = index->use_excl_dotlocks;
112
strmap->dotlock_settings.nfs_flush = index->nfs_flush;
117
mail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,
121
mail_index_strmap_set_syscall_error(struct mail_index_strmap *strmap,
122
const char *function)
124
i_assert(function != NULL);
126
if (ENOSPACE(errno)) {
127
strmap->index->nodiskspace = TRUE;
131
mail_index_set_error(strmap->index,
132
"%s failed with strmap index file %s: %m",
133
function, strmap->path);
136
static void mail_index_strmap_close(struct mail_index_strmap *strmap)
138
if (strmap->file_lock != NULL)
139
file_lock_free(&strmap->file_lock);
140
else if (strmap->dotlock != NULL)
141
file_dotlock_delete(&strmap->dotlock);
143
if (strmap->fd != -1) {
144
if (close(strmap->fd) < 0)
145
mail_index_strmap_set_syscall_error(strmap, "close()");
148
if (strmap->input != NULL)
149
i_stream_unref(&strmap->input);
152
void mail_index_strmap_deinit(struct mail_index_strmap **_strmap)
154
struct mail_index_strmap *strmap = *_strmap;
157
mail_index_strmap_close(strmap);
158
i_free(strmap->path);
162
static unsigned int mail_index_strmap_hash_key(const void *_key)
164
const struct mail_index_strmap_hash_key *key = _key;
170
mail_index_strmap_hash_cmp(const void *_key, const void *_value, void *context)
172
const struct mail_index_strmap_hash_key *key = _key;
173
const struct mail_index_strmap_rec *rec = _value;
174
struct mail_index_strmap_view *view = context;
176
return view->key_compare(key->str, rec, view->cb_context);
179
struct mail_index_strmap_view *
180
mail_index_strmap_view_open(struct mail_index_strmap *strmap,
181
struct mail_index_view *idx_view,
182
mail_index_strmap_key_cmp_t *key_compare_cb,
183
mail_index_strmap_rec_cmp_t *rec_compare_cb,
184
mail_index_strmap_remap_t *remap_cb,
186
const ARRAY_TYPE(mail_index_strmap_rec) **recs_r,
187
const struct hash2_table **hash_r)
189
struct mail_index_strmap_view *view;
191
view = i_new(struct mail_index_strmap_view, 1);
192
view->strmap = strmap;
193
view->view = idx_view;
194
view->key_compare = key_compare_cb;
195
view->rec_compare = rec_compare_cb;
196
view->remap_cb = remap_cb;
197
view->cb_context = context;
198
view->next_str_idx = 1;
200
i_array_init(&view->recs, 64);
201
i_array_init(&view->recs_crc32, 64);
202
view->hash = hash2_create(0, sizeof(struct mail_index_strmap_rec),
203
mail_index_strmap_hash_key,
204
mail_index_strmap_hash_cmp, view);
205
*recs_r = &view->recs;
206
*hash_r = view->hash;
210
void mail_index_strmap_view_close(struct mail_index_strmap_view **_view)
212
struct mail_index_strmap_view *view = *_view;
215
array_free(&view->recs);
216
array_free(&view->recs_crc32);
217
hash2_destroy(&view->hash);
221
uint32_t mail_index_strmap_view_get_highest_idx(struct mail_index_strmap_view *view)
223
return view->next_str_idx-1;
226
static void mail_index_strmap_view_reset(struct mail_index_strmap_view *view)
228
view->remap_cb(NULL, 0, 0, view->cb_context);
229
array_clear(&view->recs);
230
array_clear(&view->recs_crc32);
231
hash2_clear(view->hash);
233
view->last_added_uid = 0;
234
view->lost_expunged_uid = 0;
235
view->desynced = FALSE;
239
mail_index_strmap_view_set_corrupted(struct mail_index_strmap_view *view)
241
mail_index_set_error(view->strmap->index,
242
"Corrupted strmap index file: %s",
244
(void)unlink(view->strmap->path);
245
mail_index_strmap_close(view->strmap);
246
mail_index_strmap_view_reset(view);
249
static int mail_index_strmap_open(struct mail_index_strmap_view *view)
251
struct mail_index_strmap *strmap = view->strmap;
252
const struct mail_index_header *idx_hdr;
253
struct mail_index_strmap_header hdr;
254
const unsigned char *data;
258
i_assert(strmap->fd == -1);
260
strmap->fd = open(strmap->path, O_RDWR);
261
if (strmap->fd == -1) {
264
mail_index_strmap_set_syscall_error(strmap, "open()");
267
strmap->input = i_stream_create_fd(strmap->fd, (size_t)-1, FALSE);
268
ret = i_stream_read_data(strmap->input, &data, &size, sizeof(hdr)-1);
271
mail_index_strmap_set_syscall_error(strmap, "read()");
272
mail_index_strmap_close(strmap);
275
mail_index_strmap_view_set_corrupted(view);
279
memcpy(&hdr, data, sizeof(hdr));
281
idx_hdr = mail_index_get_header(view->view);
282
if (hdr.version != MAIL_INDEX_STRMAP_VERSION ||
283
hdr.uid_validity != idx_hdr->uid_validity) {
284
/* need to rebuild. if we already had something in the strmap,
286
(void)unlink(strmap->path);
287
mail_index_strmap_close(strmap);
291
/* we'll read the entire file from the beginning */
292
view->last_added_uid = 0;
293
view->last_read_uid = 0;
294
view->total_ref_count = 0;
295
view->last_read_block_offset = sizeof(struct mail_index_strmap_header);
296
view->next_str_idx = 1;
298
mail_index_strmap_view_reset(view);
302
static bool mail_index_strmap_need_reopen(struct mail_index_strmap *strmap)
304
struct stat st1, st2;
306
/* FIXME: nfs flush */
307
if (fstat(strmap->fd, &st1) < 0) {
309
mail_index_strmap_set_syscall_error(strmap, "fstat()");
312
if (stat(strmap->path, &st2) < 0) {
313
mail_index_strmap_set_syscall_error(strmap, "stat()");
316
return st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev);
319
static int mail_index_strmap_refresh(struct mail_index_strmap_view *view)
323
if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index))
326
if (view->strmap->fd != -1) {
327
if (!mail_index_strmap_need_reopen(view->strmap)) {
328
if (view->lost_expunged_uid != 0) {
329
/* last read failed because view had a message
330
that didn't exist in the strmap (because it
331
was expunged by another session). if the
332
message still isn't expunged in this view,
333
just continue using the current strmap. */
334
if (mail_index_lookup_seq(view->view,
335
view->lost_expunged_uid, &seq))
337
} else if (view->desynced) {
338
/* our view isn't synced with the disk, we
339
can't read strmap without first resetting
342
i_stream_sync(view->strmap->input);
346
mail_index_strmap_close(view->strmap);
349
return mail_index_strmap_open(view);
353
mail_index_strmap_read_packed(struct mail_index_strmap_read_context *ctx,
356
const unsigned char *data;
357
const uint8_t *bytes, *p, *end;
361
ret = i_stream_read_data(ctx->input, &data, &size, sizeof(*num_r) - 1);
365
if (ctx->input->v_offset + size > ctx->end_offset)
366
size = ctx->end_offset - ctx->input->v_offset;
367
bytes = p = (const uint8_t *)data;
370
if (mail_index_unpack_num(&p, end, num_r) < 0)
372
i_stream_skip(ctx->input, p - bytes);
377
mail_index_strmap_uid_exists(struct mail_index_strmap_read_context *ctx,
380
const struct mail_index_record *rec;
382
if (ctx->uid_lookup_idx >= ctx->view->view->map->hdr.messages_count) {
383
if (uid >= ctx->view->view->map->hdr.next_uid) {
384
/* thread index has larger UIDs than what we've seen
385
in our view. we'll have to read them again later
386
when we know about them */
387
ctx->too_large_uids = TRUE;
392
rec = MAIL_INDEX_MAP_IDX(ctx->view->view->map, ctx->uid_lookup_idx);
393
if (rec->uid == uid) {
394
ctx->uid_lookup_idx++;
396
} else if (rec->uid > uid) {
399
/* record that exists in index is missing from strmap.
400
see if it's because the strmap is corrupted or because
401
our current view is a bit stale and the message has already
403
(void)mail_index_refresh(ctx->view->view->index);
404
if (mail_index_is_expunged(ctx->view->view,
405
ctx->uid_lookup_idx + 1))
406
ctx->lost_expunged_uid = rec->uid;
412
mail_index_strmap_read_rec_first(struct mail_index_strmap_read_context *ctx,
416
uint32_t n, i, count, str_idx;
419
/* <uid> <n> <crc32>*count <str_idx>*count
421
n = 0 -> count=1 (only Message-ID:)
422
n = 1 -> count=2 (Message-ID: + In-Reply-To:)
423
n = 2+ -> count=n (Message-ID: + References:)
425
if (mail_index_strmap_read_packed(ctx, &n) <= 0)
427
count = n < 2 ? n + 1 : n;
428
ctx->view->total_ref_count += count;
430
ctx->rec_size = count * (sizeof(ctx->rec.str_idx) + sizeof(*crc32_r));
431
ret = mail_index_strmap_uid_exists(ctx, ctx->rec.uid);
434
if (i_stream_read_data(ctx->view->strmap->input, &ctx->data, &size,
435
ctx->rec_size - 1) <= 0)
437
ctx->str_idx_base = ctx->data + count * sizeof(uint32_t);
440
/* this message has already been expunged, ignore it.
441
update highest string indexes anyway. */
442
for (i = 0; i < count; i++) {
443
memcpy(&str_idx, ctx->str_idx_base, sizeof(str_idx));
444
if (ctx->highest_str_idx < str_idx)
445
ctx->highest_str_idx = str_idx;
446
ctx->str_idx_base += sizeof(str_idx);
448
i_stream_skip(ctx->view->strmap->input, ctx->rec_size);
452
/* everything exists. save it. FIXME: these ref_index values
453
are thread index specific, perhaps something more generic
454
should be used some day */
455
ctx->end = ctx->data + count * sizeof(*crc32_r);
457
ctx->next_ref_index = 0;
458
if (!mail_index_strmap_read_rec_next(ctx, crc32_r))
460
ctx->next_ref_index = n == 1 ? 1 : 2;
465
mail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,
468
if (ctx->data == ctx->end) {
469
i_stream_skip(ctx->view->strmap->input, ctx->rec_size);
473
/* FIXME: str_idx could be stored as packed relative values
474
(first relative to highest_idx, the rest relative to the
477
/* read the record contents */
478
memcpy(&ctx->rec.str_idx, ctx->str_idx_base, sizeof(ctx->rec.str_idx));
479
memcpy(crc32_r, ctx->data, sizeof(*crc32_r));
481
ctx->rec.ref_index = ctx->next_ref_index++;
483
if (ctx->highest_str_idx < ctx->rec.str_idx)
484
ctx->highest_str_idx = ctx->rec.str_idx;
486
/* get to the next record */
487
ctx->data += sizeof(*crc32_r);
488
ctx->str_idx_base += sizeof(ctx->rec.str_idx);
493
strmap_read_block_init(struct mail_index_strmap_view *view,
494
struct mail_index_strmap_read_context *ctx)
496
struct mail_index_strmap *strmap = view->strmap;
497
const unsigned char *data;
499
uint32_t block_size, seq1, seq2;
502
if (view->last_read_uid + 1 >= view->view->map->hdr.next_uid) {
503
/* come back later when we know about the new UIDs */
507
memset(ctx, 0, sizeof(*ctx));
508
ret = i_stream_read_data(strmap->input, &data, &size,
509
sizeof(block_size)-1);
511
if (strmap->input->stream_errno == 0) {
515
mail_index_strmap_set_syscall_error(strmap, "read()");
518
memcpy(&block_size, data, sizeof(block_size));
519
block_size = mail_index_offset_to_uint32(block_size) >> 2;
520
if (block_size == 0) {
521
/* the rest of the file is either not written, or the previous
522
write didn't finish */
525
i_stream_skip(strmap->input, sizeof(block_size));
528
ctx->input = strmap->input;
529
ctx->end_offset = strmap->input->v_offset + block_size;
530
if (ctx->end_offset < strmap->input->v_offset) {
531
/* block size too large */
532
mail_index_strmap_view_set_corrupted(view);
535
ctx->rec.uid = view->last_read_uid + 1;
537
/* FIXME: when reading multiple blocks we shouldn't have to calculate
539
if (!mail_index_lookup_seq_range(view->view, ctx->rec.uid, (uint32_t)-1,
541
seq1 = mail_index_view_get_messages_count(view->view) + 1;
542
ctx->uid_lookup_idx = seq1 - 1;
547
strmap_read_block_next(struct mail_index_strmap_read_context *ctx,
553
if (mail_index_strmap_read_rec_next(ctx, crc32_r))
558
if (ctx->input->v_offset == ctx->end_offset) {
559
/* this block is done */
562
if (mail_index_strmap_read_packed(ctx, &uid_diff) < 0)
565
ctx->rec.uid += uid_diff;
566
ret = mail_index_strmap_read_rec_first(ctx, crc32_r);
572
strmap_read_block_deinit(struct mail_index_strmap_read_context *ctx, int ret,
573
bool update_block_offset)
575
struct mail_index_strmap_view *view = ctx->view;
576
struct mail_index_strmap *strmap = view->strmap;
578
if (ctx->highest_str_idx > view->total_ref_count) {
579
/* if all string indexes are unique, highest_str_index equals
580
total_ref_count. otherwise it's always lower. */
581
mail_index_set_error(strmap->index,
582
"Corrupted strmap index file %s: "
583
"String indexes too high "
584
"(highest=%u max=%u)",
585
strmap->path, ctx->highest_str_idx,
586
view->total_ref_count);
587
mail_index_strmap_view_set_corrupted(view);
590
if (ctx->lost_expunged_uid != 0) {
591
/* our view contained a message that had since been expunged. */
593
view->lost_expunged_uid = ctx->lost_expunged_uid;
594
} else if (ret < 0) {
595
if (strmap->input->stream_errno != 0)
596
mail_index_strmap_set_syscall_error(strmap, "read()");
598
mail_index_strmap_view_set_corrupted(view);
600
} else if (update_block_offset && !ctx->too_large_uids) {
601
view->last_read_block_offset = strmap->input->v_offset;
602
view->last_read_uid = ctx->rec.uid;
604
if (view->next_str_idx <= ctx->highest_str_idx)
605
view->next_str_idx = ctx->highest_str_idx + 1;
610
strmap_view_sync_handle_conflict(struct mail_index_strmap_read_context *ctx,
611
const struct mail_index_strmap_rec *hash_rec,
612
struct hash2_iter *iter)
616
/* hopefully it's a message that has since been expunged */
617
if (!mail_index_lookup_seq(ctx->view->view, hash_rec->uid, &seq)) {
618
/* message is no longer in our view. remove it completely. */
619
hash2_remove_iter(ctx->view->hash, iter);
622
if (mail_index_is_expunged(ctx->view->view, seq)) {
623
/* it's quite likely a conflict. we may not be able to verify
624
it, so just assume it is. nothing breaks even if we guess
625
wrong, the performance just suffers a bit. */
629
/* 0 means "doesn't match", which is the only acceptable case */
630
return ctx->view->rec_compare(&ctx->rec, hash_rec,
631
ctx->view->cb_context) == 0;
635
strmap_view_sync_block_check_conflicts(struct mail_index_strmap_read_context *ctx,
638
struct mail_index_strmap_rec *hash_rec;
639
struct hash2_iter iter;
642
/* unique string - there are no conflicts */
646
/* check for conflicting string indexes. they may happen if
648
1) msgid exists only for a message X that has been expunged
649
2) another process doesn't see X, but sees msgid for another
650
message and writes it using a new string index
651
3) if we still see X, we now see the same msgid with two
654
if we detect such a conflict, we can't continue using the
655
strmap index until X has been expunged. */
656
memset(&iter, 0, sizeof(iter));
657
while ((hash_rec = hash2_iterate(ctx->view->hash,
658
crc32, &iter)) != NULL &&
659
hash_rec->str_idx != ctx->rec.str_idx) {
660
/* CRC32 matches, but string index doesn't */
661
if (!strmap_view_sync_handle_conflict(ctx, hash_rec, &iter)) {
662
ctx->lost_expunged_uid = hash_rec->uid;
670
mail_index_strmap_view_sync_block(struct mail_index_strmap_read_context *ctx)
672
struct mail_index_strmap_rec *hash_rec;
673
uint32_t crc32, prev_uid = 0;
676
while ((ret = strmap_read_block_next(ctx, &crc32)) > 0) {
677
if (ctx->rec.uid <= ctx->view->last_added_uid) {
678
if (ctx->rec.uid < ctx->view->last_added_uid ||
679
prev_uid != ctx->rec.uid) {
680
/* we've already added this */
684
prev_uid = ctx->rec.uid;
686
if (strmap_view_sync_block_check_conflicts(ctx, crc32) < 0) {
690
ctx->view->last_added_uid = ctx->rec.uid;
692
/* add the record to records array */
693
array_append(&ctx->view->recs, &ctx->rec, 1);
694
array_append(&ctx->view->recs_crc32, &crc32, 1);
696
/* add a separate copy of the record to hash */
697
hash_rec = hash2_insert_hash(ctx->view->hash, crc32);
698
memcpy(hash_rec, &ctx->rec, sizeof(*hash_rec));
700
return strmap_read_block_deinit(ctx, ret, TRUE);
703
struct mail_index_strmap_view_sync *
704
mail_index_strmap_view_sync_init(struct mail_index_strmap_view *view,
705
uint32_t *last_uid_r)
707
struct mail_index_strmap_view_sync *sync;
708
struct mail_index_strmap_read_context ctx;
711
sync = i_new(struct mail_index_strmap_view_sync, 1);
714
if (mail_index_strmap_refresh(view) < 0) {
715
/* reading the strmap failed - just ignore and do
716
this in-memory based on whatever we knew last */
717
} else if (view->strmap->input != NULL) {
718
i_stream_seek(view->strmap->input,
719
view->last_read_block_offset);
720
while ((ret = strmap_read_block_init(view, &ctx)) > 0) {
721
if (mail_index_strmap_view_sync_block(&ctx) < 0) {
725
if (ctx.too_large_uids)
730
/* something failed - we can still use the strmap as far
731
as we managed to read it, but our view is now out
733
view->desynced = TRUE;
735
i_assert(view->lost_expunged_uid == 0);
738
*last_uid_r = view->last_added_uid;
742
static inline uint32_t crc32_str_nonzero(const char *str)
744
uint32_t value = crc32_str(str);
745
return value == 0 ? 1 : value;
748
void mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync,
749
uint32_t uid, uint32_t ref_index,
752
struct mail_index_strmap_view *view = sync->view;
753
struct mail_index_strmap_rec *rec, *old_rec;
754
struct mail_index_strmap_hash_key hash_key;
757
i_assert(uid > view->last_added_uid ||
758
(uid == view->last_added_uid &&
759
ref_index > view->last_ref_index));
762
hash_key.crc32 = crc32_str_nonzero(key);
764
old_rec = hash2_lookup(view->hash, &hash_key);
765
if (old_rec != NULL) {
766
/* The string already exists, use the same unique idx */
767
str_idx = old_rec->str_idx;
769
/* Newly seen string, assign a new unique idx to it */
770
str_idx = view->next_str_idx++;
772
i_assert(str_idx != 0);
774
rec = hash2_insert(view->hash, &hash_key);
776
rec->ref_index = ref_index;
777
rec->str_idx = str_idx;
778
array_append(&view->recs, rec, 1);
779
array_append(&view->recs_crc32, &hash_key.crc32, 1);
781
view->last_added_uid = uid;
782
view->last_ref_index = ref_index;
785
void mail_index_strmap_view_sync_add_unique(struct mail_index_strmap_view_sync *sync,
786
uint32_t uid, uint32_t ref_index)
788
struct mail_index_strmap_view *view = sync->view;
789
struct mail_index_strmap_rec rec;
791
i_assert(uid > view->last_added_uid ||
792
(uid == view->last_added_uid &&
793
ref_index > view->last_ref_index));
795
memset(&rec, 0, sizeof(rec));
797
rec.ref_index = ref_index;
798
rec.str_idx = view->next_str_idx++;
799
array_append(&view->recs, &rec, 1);
800
(void)array_append_space(&view->recs_crc32);
802
view->last_added_uid = uid;
803
view->last_ref_index = ref_index;
807
mail_index_strmap_zero_terminate(struct mail_index_strmap_view *view)
809
/* zero-terminate the records array */
810
(void)array_append_space(&view->recs);
811
array_delete(&view->recs, array_count(&view->recs)-1, 1);
814
static void mail_index_strmap_view_renumber(struct mail_index_strmap_view *view)
816
struct mail_index_strmap_read_context ctx;
817
struct mail_index_strmap_rec *recs, *hash_rec;
818
uint32_t prev_uid, str_idx, *recs_crc32, *renumber_map;
819
unsigned int i, dest, count, count2;
822
memset(&ctx, 0, sizeof(ctx));
825
/* create a map of old -> new index and remove records of
827
renumber_map = i_new(uint32_t, view->next_str_idx);
828
str_idx = 0; prev_uid = 0;
829
recs = array_get_modifiable(&view->recs, &count);
830
recs_crc32 = array_get_modifiable(&view->recs_crc32, &count2);
831
i_assert(count == count2);
833
for (i = dest = 0; i < count; ) {
834
if (prev_uid != recs[i].uid) {
835
/* see if this record should be removed */
836
prev_uid = recs[i].uid;
837
ret = mail_index_strmap_uid_exists(&ctx, prev_uid);
840
/* message expunged */
843
} while (i < count && recs[i].uid == prev_uid);
848
i_assert(recs[i].str_idx < view->next_str_idx);
849
if (renumber_map[recs[i].str_idx] == 0)
850
renumber_map[recs[i].str_idx] = ++str_idx;
852
recs[dest] = recs[i];
853
recs_crc32[dest] = recs_crc32[i];
857
i_assert(renumber_map[0] == 0);
858
array_delete(&view->recs, dest, i-dest);
859
array_delete(&view->recs_crc32, dest, i-dest);
860
mail_index_strmap_zero_terminate(view);
862
/* notify caller of the renumbering */
863
i_assert(str_idx <= view->next_str_idx);
864
view->remap_cb(renumber_map, view->next_str_idx, str_idx + 1,
867
/* renumber the indexes in-place and recreate the hash */
868
recs = array_get_modifiable(&view->recs, &count);
869
hash2_clear(view->hash);
870
for (i = 0; i < count; i++) {
871
recs[i].str_idx = renumber_map[recs[i].str_idx];
872
hash_rec = hash2_insert_hash(view->hash, recs_crc32[i]);
873
memcpy(hash_rec, &recs[i], sizeof(*hash_rec));
876
/* update the new next_str_idx only after remapping */
877
view->next_str_idx = str_idx + 1;
878
i_free(renumber_map);
881
static int mail_index_strmap_write_block(struct mail_index_strmap_view *view,
882
struct ostream *output,
883
unsigned int i, uint32_t base_uid)
885
const struct mail_index_strmap_rec *recs;
886
const uint32_t *crc32;
887
unsigned int j, n, count, count2, uid_rec_count;
889
uint8_t *p, packed[MAIL_INDEX_PACK_MAX_SIZE*2];
890
uoff_t block_offset, end_offset;
892
/* skip over the block size for now, we don't know it yet */
893
block_offset = output->offset;
895
o_stream_send(output, &block_size, sizeof(block_size));
898
recs = array_get(&view->recs, &count);
899
crc32 = array_get(&view->recs_crc32, &count2);
900
i_assert(count == count2);
902
/* @UNSAFE: <uid diff> */
904
mail_index_pack_num(&p, recs[i].uid - base_uid);
905
base_uid = recs[i].uid;
907
/* find how many records belong to this UID */
909
for (j = i + 1; j < count; j++) {
910
if (recs[j].uid != base_uid)
914
view->total_ref_count += uid_rec_count;
916
/* <n> <crc32>*count <str_idx>*count -
917
FIXME: thread index specific code */
918
i_assert(recs[i].ref_index == 0);
919
if (uid_rec_count == 1) {
920
/* Only Message-ID: header */
922
} else if (recs[i+1].ref_index == 1) {
923
/* In-Reply-To: header */
925
i_assert(uid_rec_count == 2);
927
/* References: header */
929
i_assert(recs[i+1].ref_index == 2);
932
mail_index_pack_num(&p, n);
933
o_stream_send(output, packed, p-packed);
934
for (j = 0; j < uid_rec_count; j++)
935
o_stream_send(output, &crc32[i+j], sizeof(crc32[i+j]));
936
for (j = 0; j < uid_rec_count; j++) {
937
i_assert(j < 2 || recs[i+j].ref_index == j+1);
938
o_stream_send(output, &recs[i+j].str_idx,
939
sizeof(recs[i+j].str_idx));
944
/* we know the block size now - write it */
945
block_size = output->offset - (block_offset + sizeof(block_size));
946
block_size = mail_index_uint32_to_offset(block_size << 2);
947
i_assert(block_size != 0);
949
end_offset = output->offset;
950
o_stream_seek(output, block_offset);
951
o_stream_send(output, &block_size, sizeof(block_size));
952
o_stream_seek(output, end_offset);
954
if (output->last_failed_errno != 0)
957
i_assert(view->last_added_uid == recs[count-1].uid);
958
view->last_read_uid = recs[count-1].uid;
959
view->last_read_block_offset = output->offset;
965
mail_index_strmap_recreate_write(struct mail_index_strmap_view *view,
966
struct ostream *output)
968
const struct mail_index_header *idx_hdr;
969
struct mail_index_strmap_header hdr;
971
idx_hdr = mail_index_get_header(view->view);
974
memset(&hdr, 0, sizeof(hdr));
975
hdr.version = MAIL_INDEX_STRMAP_VERSION;
976
hdr.uid_validity = idx_hdr->uid_validity;
977
o_stream_send(output, &hdr, sizeof(hdr));
979
view->total_ref_count = 0;
980
(void)mail_index_strmap_write_block(view, output, 0, 1);
983
static int mail_index_strmap_recreate(struct mail_index_strmap_view *view)
985
struct mail_index_strmap *strmap = view->strmap;
987
struct ostream *output;
988
const char *temp_path;
991
if (array_count(&view->recs) == 0) {
992
/* everything expunged - just unlink the existing index */
993
if (unlink(strmap->path) < 0 && errno != ENOENT)
994
mail_index_strmap_set_syscall_error(strmap, "unlink()");
998
str = t_str_new(256);
999
str_append(str, strmap->path);
1000
fd = safe_mkstemp_hostpid_group(str, view->view->index->mode,
1001
view->view->index->gid,
1002
view->view->index->gid_origin);
1003
temp_path = str_c(str);
1006
mail_index_set_error(strmap->index,
1007
"safe_mkstemp_hostpid(%s) failed: %m",
1011
output = o_stream_create_fd(fd, 0, FALSE);
1012
o_stream_cork(output);
1013
mail_index_strmap_recreate_write(view, output);
1014
if (output->last_failed_errno != 0) {
1015
errno = output->last_failed_errno;
1016
mail_index_set_error(strmap->index,
1017
"write(%s) failed: %m", temp_path);
1020
o_stream_destroy(&output);
1021
if (close(fd) < 0) {
1022
mail_index_set_error(strmap->index,
1023
"close(%s) failed: %m", temp_path);
1025
} else if (ret == 0 && rename(temp_path, strmap->path) < 0) {
1026
mail_index_set_error(strmap->index,
1027
"rename(%s, %s) failed: %m",
1028
temp_path, strmap->path);
1032
(void)unlink(temp_path);
1036
static int mail_index_strmap_lock(struct mail_index_strmap *strmap)
1040
i_assert(strmap->fd != -1);
1042
if (strmap->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
1043
i_assert(strmap->file_lock == NULL);
1045
ret = file_wait_lock(strmap->fd, strmap->path, F_WRLCK,
1046
strmap->index->lock_method,
1047
MAIL_INDEX_STRMAP_TIMEOUT_SECS,
1048
&strmap->file_lock);
1050
mail_index_strmap_set_syscall_error(strmap,
1051
"file_wait_lock()");
1054
i_assert(strmap->dotlock == NULL);
1056
ret = file_dotlock_create(&strmap->dotlock_settings,
1057
strmap->path, 0, &strmap->dotlock);
1059
mail_index_strmap_set_syscall_error(strmap,
1060
"file_dotlock_create()");
1066
static void mail_index_strmap_unlock(struct mail_index_strmap *strmap)
1068
if (strmap->file_lock != NULL)
1069
file_unlock(&strmap->file_lock);
1070
else if (strmap->dotlock != NULL)
1071
file_dotlock_delete(&strmap->dotlock);
1074
static int strmap_rec_cmp(const void *key, const void *value)
1076
const uint32_t *uid = key;
1077
const struct mail_index_strmap_rec *rec = value;
1079
return *uid < rec->uid ? -1 :
1080
(*uid > rec->uid ? 1 : 0);
1084
mail_index_strmap_write_append(struct mail_index_strmap_view *view)
1086
struct mail_index_strmap_read_context ctx;
1087
const struct mail_index_strmap_rec *old_recs;
1088
unsigned int i, old_count;
1089
struct ostream *output;
1090
uint32_t crc32, next_uid;
1094
/* Check first if another process had written new records to the file.
1095
If there are any, hopefully they're the same as what we would be
1096
writing. There are two problematic cases when messages have been
1099
1) The file contains UIDs that we don't have. This means the string
1100
indexes won't be compatible anymore, so we'll have to renumber ours
1101
to match the ones in the strmap file.
1103
Currently we don't bother handling 1) case. If indexes don't match
1104
what we have, we just don't write anything.
1106
2) We have UIDs that don't exist in the file. We can't simply skip
1107
those records, because other records may have pointers to them using
1108
different string indexes than we have. Even if we renumbered those,
1109
future appends by other processes might cause the same problem (they
1110
see the string for the first time and assign it a new index, but we
1111
already have internally given it another index). So the only
1112
sensible choice is to write nothing and hope that the message goes
1114
old_recs = array_get(&view->recs, &old_count);
1115
next_uid = view->last_read_uid + 1;
1116
(void)bsearch_insert_pos(&next_uid, old_recs, old_count,
1117
sizeof(*old_recs), strmap_rec_cmp, &i);
1118
if (i < old_count) {
1119
while (i > 0 && old_recs[i-1].uid == old_recs[i].uid)
1123
i_stream_sync(view->strmap->input);
1124
i_stream_seek(view->strmap->input, view->last_read_block_offset);
1125
full_block = TRUE; ret = 0;
1126
while (i < old_count &&
1127
(ret = strmap_read_block_init(view, &ctx)) > 0) {
1128
while ((ret = strmap_read_block_next(&ctx, &crc32)) > 0) {
1129
if (ctx.rec.uid != old_recs[i].uid ||
1130
ctx.rec.str_idx != old_recs[i].str_idx) {
1132
if (ctx.rec.uid > old_recs[i].uid) {
1134
ctx.lost_expunged_uid = ctx.rec.uid;
1135
} else if (ctx.rec.uid < old_recs[i].uid) {
1137
ctx.lost_expunged_uid = old_recs[i].uid;
1139
/* string index mismatch,
1145
if (++i == old_count) {
1150
if (strmap_read_block_deinit(&ctx, ret, full_block) < 0) {
1157
if (i == old_count) {
1158
/* nothing new to write */
1161
i_assert(full_block);
1162
i_assert(old_recs[i].uid > view->last_read_uid);
1164
/* write the new records */
1165
output = o_stream_create_fd(view->strmap->fd, 0, FALSE);
1166
o_stream_seek(output, view->last_read_block_offset);
1167
o_stream_cork(output);
1168
if (mail_index_strmap_write_block(view, output, i,
1169
view->last_read_uid + 1) < 0) {
1170
errno = output->last_failed_errno;
1171
mail_index_strmap_set_syscall_error(view->strmap, "write()");
1174
o_stream_destroy(&output);
1178
static int mail_index_strmap_write(struct mail_index_strmap_view *view)
1182
/* FIXME: this renumbering doesn't work well when running for a long
1183
time since records aren't removed from hash often enough */
1184
if (STRIDX_MUST_RENUMBER(view->next_str_idx - 1,
1185
hash2_count(view->hash))) {
1186
mail_index_strmap_view_renumber(view);
1187
if (!MAIL_INDEX_IS_IN_MEMORY(view->strmap->index)) {
1188
if (mail_index_strmap_recreate(view) < 0) {
1189
view->desynced = TRUE;
1196
if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index) || view->desynced)
1199
if (view->strmap->fd == -1) {
1200
/* initial file creation */
1201
if (mail_index_strmap_recreate(view) < 0) {
1202
view->desynced = TRUE;
1208
/* append the new records to the strmap file */
1209
if (mail_index_strmap_lock(view->strmap) <= 0) {
1210
/* timeout / error */
1212
} else if (mail_index_strmap_need_reopen(view->strmap)) {
1213
/* the file was already recreated - leave the syncing as it is
1214
for now and let the next sync re-read the file. */
1217
ret = mail_index_strmap_write_append(view);
1219
mail_index_strmap_unlock(view->strmap);
1221
view->desynced = TRUE;
1225
void mail_index_strmap_view_sync_commit(struct mail_index_strmap_view_sync **_sync)
1227
struct mail_index_strmap_view_sync *sync = *_sync;
1228
struct mail_index_strmap_view *view = sync->view;
1233
(void)mail_index_strmap_write(view);
1234
mail_index_strmap_zero_terminate(view);
1236
/* zero-terminate the records array */
1237
(void)array_append_space(&view->recs);
1238
array_delete(&view->recs, array_count(&view->recs)-1, 1);
1241
void mail_index_strmap_view_sync_rollback(struct mail_index_strmap_view_sync **_sync)
1243
struct mail_index_strmap_view_sync *sync = *_sync;
1247
mail_index_strmap_view_reset(sync->view);
1248
mail_index_strmap_zero_terminate(sync->view);
added
removed
src/lib-index/mail-index-strmap.c
