1
/* Copyright (C) 2000-2006 MySQL AB
3
This program is free software; you can redistribute it and/or modify
4
it under the terms of the GNU General Public License as published by
5
the Free Software Foundation; version 2 of the License.
7
This program is distributed in the hope that it will be useful,
8
but WITHOUT ANY WARRANTY; without even the implied warranty of
9
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
GNU General Public License for more details.
12
You should have received a copy of the GNU General Public License
13
along with this program; if not, write to the Free Software
14
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
16
/* Describe, check and repair of MyISAM tables */
19
About checksum calculation.
21
There are two types of checksums. Table checksum and row checksum.
23
Row checksum is an additional byte at the end of dynamic length
24
records. It must be calculated if the table is configured for them.
25
Otherwise they must not be used. The variable
26
MYISAM_SHARE::calc_checksum determines if row checksums are used.
27
MI_INFO::checksum is used as temporary storage during row handling.
28
For parallel repair we must assure that only one thread can use this
29
variable. There is no problem on the write side as this is done by one
30
thread only. But when checking a record after read this could go
31
wrong. But since all threads read through a common read buffer, it is
32
sufficient if only one thread checks it.
34
Table checksum is an eight byte value in the header of the index file.
35
It can be calculated even if row checksums are not used. The variable
36
MI_CHECK::glob_crc is calculated over all records.
37
MI_SORT_PARAM::calc_checksum determines if this should be done. This
38
variable is not part of MI_CHECK because it must be set per thread for
39
parallel repair. The global glob_crc must be changed by one thread
40
only. And it is sufficient to calculate the checksum once only.
46
#include <my_getopt.h>
47
#ifdef HAVE_SYS_VADVISE_H
48
#include <sys/vadvise.h>
50
#ifdef HAVE_SYS_MMAN_H
56
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
57
#define my_raid_delete(A,B,C) my_delete(A,B)
60
/* Functions defined in this file */
62
static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
63
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
64
my_off_t page, uchar *buff, ha_rows *keys,
65
ha_checksum *key_checksum, uint level);
66
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
67
static ha_checksum calc_checksum(ha_rows count);
68
static int writekeys(MI_SORT_PARAM *sort_param);
69
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
70
my_off_t pagepos, File new_file);
71
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
72
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
73
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
74
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
75
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
76
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
77
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
79
static int sort_insert_key(MI_SORT_PARAM *sort_param,
80
reg1 SORT_KEY_BLOCKS *key_block,
81
uchar *key, my_off_t prev_block);
82
static int sort_delete_record(MI_SORT_PARAM *sort_param);
83
/*static int flush_pending_blocks(MI_CHECK *param);*/
84
static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
86
static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
87
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
89
void myisamchk_init(MI_CHECK *param)
91
bzero((uchar*) param,sizeof(*param));
92
param->opt_follow_links=1;
93
param->keys_in_use= ~(ulonglong) 0;
94
param->search_after_block=HA_OFFSET_ERROR;
95
param->auto_increment_value= 0;
96
param->use_buffers=USE_BUFFER_INIT;
97
param->read_buffer_length=READ_BUFFER_INIT;
98
param->write_buffer_length=READ_BUFFER_INIT;
99
param->sort_buffer_length=SORT_BUFFER_INIT;
100
param->sort_key_blocks=BUFFERS_WHEN_SORTING;
101
param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
102
param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
103
param->start_check_pos=0;
104
param->max_record_length= LONGLONG_MAX;
105
param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
106
param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
108
param->need_print_msg_lock= 0;
112
/* Check the status flags for the table */
114
int chk_status(MI_CHECK *param, register MI_INFO *info)
116
MYISAM_SHARE *share=info->s;
118
if (mi_is_crashed_on_repair(info))
119
mi_check_print_warning(param,
120
"Table is marked as crashed and last repair failed");
121
else if (mi_is_crashed(info))
122
mi_check_print_warning(param,
123
"Table is marked as crashed");
124
if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
126
/* Don't count this as a real warning, as check can correct this ! */
127
uint save=param->warning_printed;
128
mi_check_print_warning(param,
129
share->state.open_count==1 ?
130
"%d client is using or hasn't closed the table properly" :
131
"%d clients are using or haven't closed the table properly",
132
share->state.open_count);
133
/* If this will be fixed by the check, forget the warning */
134
if (param->testflag & T_UPDATE_STATE)
135
param->warning_printed=save;
140
/* Check delete links */
142
int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
145
uint delete_link_length;
146
my_off_t empty,next_link,UNINIT_VAR(old_link);
147
char buff[22],buff2[22];
148
DBUG_ENTER("chk_del");
150
param->record_checksum=0;
151
delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
152
info->s->rec_reflength+1);
154
if (!(test_flag & T_SILENT))
155
puts("- check record delete-chain");
157
next_link=info->s->state.dellink;
158
if (info->state->del == 0)
160
if (test_flag & T_VERBOSE)
162
puts("No recordlinks");
167
if (test_flag & T_VERBOSE)
168
printf("Recordlinks: ");
170
for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
172
if (*killed_ptr(param))
174
if (test_flag & T_VERBOSE)
175
printf(" %9s",llstr(next_link,buff));
176
if (next_link >= info->state->data_file_length)
178
if (my_pread(info->dfile, (uchar*) buff,delete_link_length,
179
next_link,MYF(MY_NABP)))
181
if (test_flag & T_VERBOSE) puts("");
182
mi_check_print_error(param,"Can't read delete-link at filepos: %s",
183
llstr(next_link,buff));
188
if (test_flag & T_VERBOSE) puts("");
189
mi_check_print_error(param,"Record at pos: %s is not remove-marked",
190
llstr(next_link,buff));
193
if (info->s->options & HA_OPTION_PACK_RECORD)
195
my_off_t prev_link=mi_sizekorr(buff+12);
196
if (empty && prev_link != old_link)
198
if (test_flag & T_VERBOSE) puts("");
199
mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
203
next_link=mi_sizekorr(buff+4);
204
empty+=mi_uint3korr(buff+1);
208
param->record_checksum+=(ha_checksum) next_link;
209
next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
210
empty+=info->s->base.pack_reclength;
213
if (test_flag & T_VERBOSE)
215
if (empty != info->state->empty)
217
mi_check_print_warning(param,
218
"Found %s deleted space in delete link chain. Should be %s",
220
llstr(info->state->empty,buff));
222
if (next_link != HA_OFFSET_ERROR)
224
mi_check_print_error(param,
225
"Found more than the expected %s deleted rows in delete link chain",
226
llstr(info->state->del, buff));
231
mi_check_print_error(param,
232
"Found %s deleted rows in delete link chain. Should be %s",
233
llstr(info->state->del - i, buff2),
234
llstr(info->state->del, buff));
241
param->testflag|=T_RETRY_WITHOUT_QUICK;
242
if (test_flag & T_VERBOSE) puts("");
243
mi_check_print_error(param,"record delete-link-chain corrupted");
248
/* Check delete links in index file */
250
static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
253
uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
255
char llbuff[21], llbuff2[21];
257
DBUG_ENTER("check_k_link");
258
DBUG_PRINT("enter", ("block_size: %u", block_size));
260
if (param->testflag & T_VERBOSE)
261
printf("block_size %4u:", block_size); /* purecov: tested */
263
next_link=info->s->state.key_del[nr];
264
records= (ha_rows) (info->state->key_file_length / block_size);
265
while (next_link != HA_OFFSET_ERROR && records > 0)
267
if (*killed_ptr(param))
269
if (param->testflag & T_VERBOSE)
270
printf("%16s",llstr(next_link,llbuff));
272
/* Key blocks must lay within the key file length entirely. */
273
if (next_link + block_size > info->state->key_file_length)
275
/* purecov: begin tested */
276
mi_check_print_error(param, "Invalid key block position: %s "
277
"key block size: %u file_length: %s",
278
llstr(next_link, llbuff), block_size,
279
llstr(info->state->key_file_length, llbuff2));
284
/* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
285
if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
287
/* purecov: begin tested */
288
mi_check_print_error(param, "Mis-aligned key block: %s "
289
"minimum key block length: %u",
290
llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
296
Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
297
If the key cache block size is smaller than block_size, we can so
298
avoid unecessary eviction of cache block.
300
if (!(buff=key_cache_read(info->s->key_cache,
301
info->s->kfile, next_link, DFLT_INIT_HITS,
302
(uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
303
MI_MIN_KEY_BLOCK_LENGTH, 1)))
305
/* purecov: begin tested */
306
mi_check_print_error(param, "key cache read error for block: %s",
307
llstr(next_link,llbuff));
311
next_link=mi_sizekorr(buff);
313
param->key_file_blocks+=block_size;
315
if (param->testflag & T_VERBOSE)
317
if (next_link != HA_OFFSET_ERROR)
318
printf("%16s\n",llstr(next_link,llbuff));
322
DBUG_RETURN (next_link != HA_OFFSET_ERROR);
326
/* Check sizes of files */
328
int chk_size(MI_CHECK *param, register MI_INFO *info)
331
register my_off_t skr,size;
332
char buff[22],buff2[22];
333
DBUG_ENTER("chk_size");
335
if (!(param->testflag & T_SILENT)) puts("- check file-size");
337
/* The following is needed if called externally (not from myisamchk) */
338
flush_key_blocks(info->s->key_cache,
339
info->s->kfile, FLUSH_FORCE_WRITE);
341
size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
342
if ((skr=(my_off_t) info->state->key_file_length) != size)
344
/* Don't give error if file generated by myisampack */
345
if (skr > size && mi_is_any_key_active(info->s->state.key_map))
348
mi_check_print_error(param,
349
"Size of indexfile is: %-8s Should be: %s",
350
llstr(size,buff), llstr(skr,buff2));
353
mi_check_print_warning(param,
354
"Size of indexfile is: %-8s Should be: %s",
355
llstr(size,buff), llstr(skr,buff2));
357
if (!(param->testflag & T_VERY_SILENT) &&
358
! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
359
ulonglong2double(info->state->key_file_length) >
360
ulonglong2double(info->s->base.margin_key_file_length)*0.9)
361
mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
362
llstr(info->state->key_file_length,buff),
363
llstr(info->s->base.max_key_file_length-1,buff));
365
size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
366
skr=(my_off_t) info->state->data_file_length;
367
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
368
skr+= MEMMAP_EXTRA_MARGIN;
370
if (info->data_file_type == STATIC_RECORD &&
371
skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
372
skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
376
info->state->data_file_length=size; /* Skip other errors */
377
if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
380
mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
381
llstr(size,buff), llstr(skr,buff2));
382
param->testflag|=T_RETRY_WITHOUT_QUICK;
386
mi_check_print_warning(param,
387
"Size of datafile is: %-9s Should be: %s",
388
llstr(size,buff), llstr(skr,buff2));
391
if (!(param->testflag & T_VERY_SILENT) &&
392
!(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
393
ulonglong2double(info->state->data_file_length) >
394
(ulonglong2double(info->s->base.max_data_file_length)*0.9))
395
mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
396
llstr(info->state->data_file_length,buff),
397
llstr(info->s->base.max_data_file_length-1,buff2));
404
int chk_key(MI_CHECK *param, register MI_INFO *info)
406
uint key,found_keys=0,full_text_keys=0,result=0;
408
ha_checksum old_record_checksum,init_checksum;
409
my_off_t all_keydata,all_totaldata,key_totlength,length;
410
ulong *rec_per_key_part;
411
MYISAM_SHARE *share=info->s;
413
char buff[22],buff2[22];
414
DBUG_ENTER("chk_key");
416
if (!(param->testflag & T_SILENT))
417
puts("- check key delete-chain");
419
param->key_file_blocks=info->s->base.keystart;
420
for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
421
if (check_k_link(param,info,key))
423
if (param->testflag & T_VERBOSE) puts("");
424
mi_check_print_error(param,"key delete-link-chain corrupted");
428
if (!(param->testflag & T_SILENT)) puts("- check index reference");
430
all_keydata=all_totaldata=key_totlength=0;
431
old_record_checksum=0;
432
init_checksum=param->record_checksum;
433
if (!(share->options &
434
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
435
old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
436
share->base.pack_reclength;
437
rec_per_key_part= param->rec_per_key_part;
438
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
439
rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
441
param->key_crc[key]=0;
442
if (! mi_is_key_active(share->state.key_map, key))
444
/* Remember old statistics for key */
445
memcpy((char*) rec_per_key_part,
446
(char*) (share->state.rec_per_key_part +
447
(uint) (rec_per_key_part - param->rec_per_key_part)),
448
keyinfo->keysegs*sizeof(*rec_per_key_part));
453
param->record_checksum=init_checksum;
455
bzero((char*) ¶m->unique_count,sizeof(param->unique_count));
456
bzero((char*) ¶m->notnull_count,sizeof(param->notnull_count));
458
if ((!(param->testflag & T_SILENT)))
459
printf ("- check data record references index: %d\n",key+1);
460
if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
462
if (share->state.key_root[key] == HA_OFFSET_ERROR &&
463
(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
465
if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
466
DFLT_INIT_HITS,info->buff,0))
468
mi_check_print_error(param,"Can't read indexpage from filepos: %s",
469
llstr(share->state.key_root[key],buff));
470
if (!(param->testflag & T_INFO))
475
param->key_file_blocks+=keyinfo->block_length;
477
param->keydata=param->totaldata=0;
480
if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
481
&keys, param->key_crc+key,1))
483
if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
485
if (keys != info->state->records)
487
mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
488
llstr(info->state->records,buff2));
489
if (!(param->testflag & T_INFO))
494
if (found_keys - full_text_keys == 1 &&
496
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
497
(param->testflag & T_DONT_CHECK_CHECKSUM)))
498
old_record_checksum=param->record_checksum;
499
else if (old_record_checksum != param->record_checksum)
502
mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
505
mi_check_print_error(param,"Key 1 doesn't point at all records");
506
if (!(param->testflag & T_INFO))
512
if ((uint) share->base.auto_key -1 == key)
514
/* Check that auto_increment key is bigger than max key value */
515
ulonglong auto_increment;
517
_mi_read_key_record(info, 0L, info->rec_buff);
518
auto_increment= retrieve_auto_increment(info, info->rec_buff);
519
if (auto_increment > info->s->state.auto_increment)
521
mi_check_print_warning(param, "Auto-increment value: %s is smaller "
522
"than max used value: %s",
523
llstr(info->s->state.auto_increment,buff2),
524
llstr(auto_increment, buff));
526
if (param->testflag & T_AUTO_INC)
528
set_if_bigger(info->s->state.auto_increment,
530
set_if_bigger(info->s->state.auto_increment,
531
param->auto_increment_value);
534
/* Check that there isn't a row with auto_increment = 0 in the table */
535
mi_extra(info,HA_EXTRA_KEYREAD,0);
536
bzero(info->lastkey,keyinfo->seg->length);
537
if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
538
(key_part_map)1, HA_READ_KEY_EXACT))
540
/* Don't count this as a real warning, as myisamchk can't correct it */
541
uint save=param->warning_printed;
542
mi_check_print_warning(param, "Found row where the auto_increment "
543
"column has the value 0");
544
param->warning_printed=save;
546
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
549
length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
550
if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
551
printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
553
(int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
554
(int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
555
my_off_t2double(length)),
557
all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
560
if (param->testflag & T_STATISTICS)
561
update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
562
param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
563
param->notnull_count: NULL,
564
(ulonglong)info->state->records);
566
if (param->testflag & T_INFO)
568
if (all_totaldata != 0L && found_keys > 0)
569
printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
570
(int) (my_off_t2double(all_keydata)*100.0/
571
my_off_t2double(all_totaldata)),
572
(int) ((my_off_t2double(key_totlength) -
573
my_off_t2double(all_keydata))*100.0/
574
my_off_t2double(key_totlength)));
575
else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
578
if (param->key_file_blocks != info->state->key_file_length &&
579
param->keys_in_use != ~(ulonglong) 0)
580
mi_check_print_warning(param, "Some data are unreferenced in keyfile");
581
if (found_keys != full_text_keys)
582
param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
584
param->record_checksum=0;
589
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
590
my_off_t page, uchar *buff, ha_rows *keys,
591
ha_checksum *key_checksum, uint level)
593
char llbuff[22],llbuff2[22];
594
DBUG_ENTER("chk_index_down");
596
/* Key blocks must lay within the key file length entirely. */
597
if (page + keyinfo->block_length > info->state->key_file_length)
599
/* purecov: begin tested */
600
/* Give it a chance to fit in the real file size. */
601
my_off_t max_length= my_seek(info->s->kfile, 0L, MY_SEEK_END,
603
mi_check_print_error(param, "Invalid key block position: %s "
604
"key block size: %u file_length: %s",
605
llstr(page, llbuff), keyinfo->block_length,
606
llstr(info->state->key_file_length, llbuff2));
607
if (page + keyinfo->block_length > max_length)
609
/* Fix the remebered key file length. */
610
info->state->key_file_length= (max_length &
611
~ (my_off_t) (keyinfo->block_length - 1));
615
/* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
616
if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
618
/* purecov: begin tested */
619
mi_check_print_error(param, "Mis-aligned key block: %s "
620
"minimum key block length: %u",
621
llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
626
if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
628
mi_check_print_error(param,"Can't read key from filepos: %s",
632
param->key_file_blocks+=keyinfo->block_length;
633
if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
638
/* purecov: begin tested */
646
"Ignore NULLs" statistics collection method: process first index tuple.
649
mi_collect_stats_nonulls_first()
650
keyseg IN Array of key part descriptions
651
notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
652
tuples that don't contain NULLs)
653
key IN Key values tuple
656
Process the first index tuple - find out which prefix tuples don't
657
contain NULLs, and update the array of notnull counters accordingly.
661
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
665
first_null= (uint) (ha_find_null(keyseg, key) - keyseg);
667
All prefix tuples that don't include keypart_{first_null} are not-null
668
tuples (and all others aren't), increment counters for them.
670
for (kp= 0; kp < first_null; kp++)
676
"Ignore NULLs" statistics collection method: process next index tuple.
679
mi_collect_stats_nonulls_next()
680
keyseg IN Array of key part descriptions
681
notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
682
tuples that don't contain NULLs)
683
prev_key IN Previous key values tuple
684
last_key IN Next key values tuple
687
Process the next index tuple:
688
1. Find out which prefix tuples of last_key don't contain NULLs, and
689
update the array of notnull counters accordingly.
690
2. Find the first keypart number where the prev_key and last_key tuples
691
are different(A), or last_key has NULL value(B), and return it, so the
692
caller can count number of unique tuples for each key prefix. We don't
693
need (B) to be counted, and that is compensated back in
697
1 + number of first keypart where values differ or last_key tuple has NULL
701
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
702
uchar *prev_key, uchar *last_key)
705
uint first_null_seg, kp;
709
Find the first keypart where values are different or either of them is
710
NULL. We get results in diffs array:
711
diffs[0]= 1 + number of first different keypart
712
diffs[1]=offset: (last_key + diffs[1]) points to first value in
713
last_key that is NULL or different from corresponding
716
ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
717
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
718
seg= keyseg + diffs[0] - 1;
720
/* Find first NULL in last_key */
721
first_null_seg= (uint) (ha_find_null(seg, last_key + diffs[1]) - keyseg);
722
for (kp= 0; kp < first_null_seg; kp++)
726
Return 1+ number of first key part where values differ. Don't care if
727
these were NULLs and not .... We compensate for that in
734
/* Check if index is ok */
736
static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
737
my_off_t page, uchar *buff, ha_rows *keys,
738
ha_checksum *key_checksum, uint level)
741
uint used_length,comp_flag,nod_flag,key_length=0;
742
uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
743
my_off_t next_page,record;
746
DBUG_ENTER("chk_index");
747
DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
749
/* TODO: implement appropriate check for RTree keys */
750
if (keyinfo->flag & HA_SPATIAL)
753
if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
755
mi_check_print_error(param,"Not enough memory for keyblock");
759
if (keyinfo->flag & HA_NOSAME)
760
comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */
762
comp_flag=SEARCH_SAME; /* Keys in positionorder */
763
nod_flag=mi_test_if_nod(buff);
764
used_length=mi_getint(buff);
765
keypos=buff+2+nod_flag;
766
endpos=buff+used_length;
768
param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
770
if (level > param->max_level)
771
param->max_level=level;
773
if (used_length > keyinfo->block_length)
775
mi_check_print_error(param,"Wrong pageinfo at page: %s",
781
if (*killed_ptr(param))
783
memcpy((char*) info->lastkey,(char*) key,key_length);
784
info->lastkey_length=key_length;
787
next_page=_mi_kpos(nod_flag,keypos);
788
if (chk_index_down(param,info,keyinfo,next_page,
789
temp_buff,keys,key_checksum,level+1))
793
if (keypos >= endpos ||
794
(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
798
mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
802
(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
803
comp_flag, diff_pos)) >=0)
805
DBUG_DUMP("old",(uchar*) info->lastkey, info->lastkey_length);
806
DBUG_DUMP("new",(uchar*) key, key_length);
807
DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
809
if (comp_flag & SEARCH_FIND && flag == 0)
810
mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
812
mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
815
if (param->testflag & T_STATISTICS)
817
if (*keys != 1L) /* not first_key */
819
if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
820
ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
821
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
823
else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
825
diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
826
param->notnull_count,
829
param->unique_count[diff_pos[0]-1]++;
833
if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
834
mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
838
(*key_checksum)+= mi_byte_checksum((uchar*) key,
839
key_length- info->s->rec_reflength);
840
record= _mi_dpos(info,0,key+key_length);
841
if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
845
get_key_full_length_rdonly(off, key);
846
subkeys=ft_sintXkorr(key+off);
850
if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
851
temp_buff,&tmp_keys,key_checksum,1))
853
if (tmp_keys + subkeys)
855
mi_check_print_error(param,
856
"Number of words in the 2nd level tree "
857
"does not match the number in the header. "
858
"Parent word in on the page %s, offset %u",
859
llstr(page,llbuff), (uint) (old_keypos-buff));
867
if (record >= info->state->data_file_length)
870
char llbuff2[22], llbuff3[22];
872
mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
873
DBUG_PRINT("test",("page: %s record: %s filelength: %s",
874
llstr(page,llbuff),llstr(record,llbuff2),
875
llstr(info->state->data_file_length,llbuff3)));
876
DBUG_DUMP("key",(uchar*) key,key_length);
877
DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
880
param->record_checksum+=(ha_checksum) record;
882
if (keypos != endpos)
884
mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
885
llstr(page,llbuff), used_length, (keypos - buff));
888
my_afree((uchar*) temp_buff);
891
my_afree((uchar*) temp_buff);
896
/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
898
static ha_checksum calc_checksum(ha_rows count)
901
DBUG_ENTER("calc_checksum");
915
DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
916
DBUG_RETURN((ha_checksum) sum);
917
} /* calc_checksum */
920
/* Calc length of key in normal isam */
922
static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
926
DBUG_ENTER("isam_key_length");
928
length= info->s->rec_reflength;
929
for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
930
length+= keyseg->length;
932
DBUG_PRINT("exit",("length: %d",length));
937
/* Check that record-link is ok */
939
int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
941
int error,got_error,flag;
942
uint key,UNINIT_VAR(left_length),b_type,field;
943
ha_rows records,del_blocks;
944
my_off_t used,empty,pos,splits,UNINIT_VAR(start_recpos),
945
del_length,link_used,start_block;
946
uchar *record= 0, *UNINIT_VAR(to);
947
char llbuff[22],llbuff2[22],llbuff3[22];
948
ha_checksum intern_record_checksum;
949
ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
950
my_bool static_row_size;
952
MI_BLOCK_INFO block_info;
953
DBUG_ENTER("chk_data_link");
955
if (!(param->testflag & T_SILENT))
958
puts("- check records and index references");
960
puts("- check record links");
963
if (!mi_alloc_rec_buff(info, -1, &record))
965
mi_check_print_error(param,"Not enough memory for record");
968
records=del_blocks=0;
969
used=link_used=splits=del_length=0;
970
intern_record_checksum=param->glob_crc=0;
972
empty=info->s->pack.header_length;
974
/* Check how to calculate checksum of rows */
976
if (info->s->data_file_type == COMPRESSED_RECORD)
978
for (field=0 ; field < info->s->base.fields ; field++)
980
if (info->s->rec[field].base_type == FIELD_BLOB ||
981
info->s->rec[field].base_type == FIELD_VARCHAR)
989
pos=my_b_tell(¶m->read_cache);
990
bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
991
while (pos < info->state->data_file_length)
993
if (*killed_ptr(param))
995
switch (info->s->data_file_type) {
997
if (my_b_read(¶m->read_cache,(uchar*) record,
998
info->s->base.pack_reclength))
1001
pos+=info->s->base.pack_reclength;
1003
if (*record == '\0')
1006
del_length+=info->s->base.pack_reclength;
1007
continue; /* Record removed */
1009
param->glob_crc+= mi_static_checksum(info,record);
1010
used+=info->s->base.pack_reclength;
1012
case DYNAMIC_RECORD:
1013
flag=block_info.second_read=0;
1014
block_info.next_filepos=pos;
1017
if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header,
1018
(start_block=block_info.next_filepos),
1019
sizeof(block_info.header),
1020
(flag ? 0 : READING_NEXT) | READING_HEADER))
1022
if (start_block & (MI_DYN_ALIGN_SIZE-1))
1024
mi_check_print_error(param,"Wrong aligned block at %s",
1025
llstr(start_block,llbuff));
1028
b_type=_mi_get_block_info(&block_info,-1,start_block);
1029
if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
1032
if (b_type & BLOCK_SYNC_ERROR)
1036
mi_check_print_error(param,"Unexpected byte: %d at link: %s",
1037
(int) block_info.header[0],
1038
llstr(start_block,llbuff));
1041
pos=block_info.filepos+block_info.block_len;
1044
if (b_type & BLOCK_DELETED)
1046
if (block_info.block_len < info->s->base.min_block_length)
1048
mi_check_print_error(param,
1049
"Deleted block with impossible length %lu at %s",
1050
block_info.block_len,llstr(pos,llbuff));
1053
if ((block_info.next_filepos != HA_OFFSET_ERROR &&
1054
block_info.next_filepos >= info->state->data_file_length) ||
1055
(block_info.prev_filepos != HA_OFFSET_ERROR &&
1056
block_info.prev_filepos >= info->state->data_file_length))
1058
mi_check_print_error(param,"Delete link points outside datafile at %s",
1063
del_length+=block_info.block_len;
1064
pos=block_info.filepos+block_info.block_len;
1068
mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1069
block_info.header[0],block_info.header[1],
1070
block_info.header[2],
1071
llstr(start_block,llbuff));
1074
if (info->state->data_file_length < block_info.filepos+
1075
block_info.block_len)
1077
mi_check_print_error(param,
1078
"Recordlink that points outside datafile at %s",
1084
if (!flag++) /* First block */
1087
pos=block_info.filepos+block_info.block_len;
1088
if (block_info.rec_len > (uint) info->s->base.max_pack_length)
1090
mi_check_print_error(param,"Found too long record (%lu) at %s",
1091
(ulong) block_info.rec_len,
1092
llstr(start_recpos,llbuff));
1096
if (info->s->base.blobs)
1098
if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1101
mi_check_print_error(param,
1102
"Not enough memory (%lu) for blob at %s",
1103
(ulong) block_info.rec_len,
1104
llstr(start_recpos,llbuff));
1111
left_length=block_info.rec_len;
1113
if (left_length < block_info.data_len)
1115
mi_check_print_error(param,"Found too long record (%lu) at %s",
1116
(ulong) block_info.data_len,
1117
llstr(start_recpos,llbuff));
1121
if (_mi_read_cache(¶m->read_cache,(uchar*) to,block_info.filepos,
1122
(uint) block_info.data_len,
1123
flag == 1 ? READING_NEXT : 0))
1125
to+=block_info.data_len;
1126
link_used+= block_info.filepos-start_block;
1127
used+= block_info.filepos - start_block + block_info.data_len;
1128
empty+=block_info.block_len-block_info.data_len;
1129
left_length-=block_info.data_len;
1132
if (b_type & BLOCK_LAST)
1134
mi_check_print_error(param,
1135
"Wrong record length %s of %s at %s",
1136
llstr(block_info.rec_len-left_length,llbuff),
1137
llstr(block_info.rec_len, llbuff2),
1138
llstr(start_recpos,llbuff3));
1142
if (info->state->data_file_length < block_info.next_filepos)
1144
mi_check_print_error(param,
1145
"Found next-recordlink that points outside datafile at %s",
1146
llstr(block_info.filepos,llbuff));
1151
} while (left_length);
1154
if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1157
mi_check_print_error(param,"Found wrong record at %s",
1158
llstr(start_recpos,llbuff));
1163
info->checksum=mi_checksum(info,record);
1164
if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1166
if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1167
test(info->s->calc_checksum)))
1169
mi_check_print_error(param,"Found wrong packed record at %s",
1170
llstr(start_recpos,llbuff));
1175
param->glob_crc+= info->checksum;
1179
pos=block_info.filepos+block_info.block_len;
1181
case COMPRESSED_RECORD:
1182
if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header, pos,
1183
info->s->pack.ref_length, READING_NEXT))
1187
VOID(_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
1188
&info->rec_buff, -1, start_recpos));
1189
pos=block_info.filepos+block_info.rec_len;
1190
if (block_info.rec_len < (uint) info->s->min_pack_length ||
1191
block_info.rec_len > (uint) info->s->max_pack_length)
1193
mi_check_print_error(param,
1194
"Found block with wrong recordlength: %d at %s",
1195
block_info.rec_len, llstr(start_recpos,llbuff));
1199
if (_mi_read_cache(¶m->read_cache,(uchar*) info->rec_buff,
1200
block_info.filepos, block_info.rec_len, READING_NEXT))
1202
if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
1203
info->rec_buff, block_info.rec_len))
1205
mi_check_print_error(param,"Found wrong record at %s",
1206
llstr(start_recpos,llbuff));
1209
if (static_row_size)
1210
param->glob_crc+= mi_static_checksum(info,record);
1212
param->glob_crc+= mi_checksum(info,record);
1213
link_used+= (block_info.filepos - start_recpos);
1214
used+= (pos-start_recpos);
1217
assert(0); /* Impossible */
1221
intern_record_checksum+=(ha_checksum) start_recpos;
1223
if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1225
printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
1228
/* Check if keys match the record */
1230
for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1233
if (mi_is_key_active(info->s->state.key_map, key))
1235
if(!(keyinfo->flag & HA_FULLTEXT))
1237
uint key_length=_mi_make_key(info,key,info->lastkey,record,
1241
/* We don't need to lock the key tree here as we don't allow
1242
concurrent threads when running myisamchk
1245
#ifdef HAVE_RTREE_KEYS
1246
(keyinfo->flag & HA_SPATIAL) ?
1247
rtree_find_first(info, key, info->lastkey, key_length,
1248
MBR_EQUAL | MBR_DATA) :
1250
_mi_search(info,keyinfo,info->lastkey,key_length,
1251
SEARCH_SAME, info->s->state.key_root[key]);
1254
mi_check_print_error(param,"Record at: %10s "
1255
"Can't find key for index: %2d",
1256
llstr(start_recpos,llbuff),key+1);
1257
if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1262
key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
1271
if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1274
next:; /* Next record */
1276
if (param->testflag & T_WRITE_LOOP)
1278
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
1280
if (records != info->state->records)
1282
mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
1283
llstr(records,llbuff), llstr(info->state->records,llbuff2));
1286
else if (param->record_checksum &&
1287
param->record_checksum != intern_record_checksum)
1289
mi_check_print_error(param,
1290
"Keypointers and record positions doesn't match");
1293
else if (param->glob_crc != info->state->checksum &&
1295
(HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
1297
mi_check_print_warning(param,
1298
"Record checksum is not the same as checksum stored in the index file\n");
1303
for (key=0 ; key < info->s->base.keys; key++)
1305
if (key_checksum[key] != param->key_crc[key] &&
1306
!(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
1308
mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1315
if (del_length != info->state->empty)
1317
mi_check_print_warning(param,
1318
"Found %s deleted space. Should be %s",
1319
llstr(del_length,llbuff2),
1320
llstr(info->state->empty,llbuff));
1322
if (used+empty+del_length != info->state->data_file_length)
1324
mi_check_print_warning(param,
1325
"Found %s record-data and %s unused data and %s deleted-data",
1326
llstr(used,llbuff),llstr(empty,llbuff2),
1327
llstr(del_length,llbuff3));
1328
mi_check_print_warning(param,
1329
"Total %s, Should be: %s",
1330
llstr((used+empty+del_length),llbuff),
1331
llstr(info->state->data_file_length,llbuff2));
1333
if (del_blocks != info->state->del)
1335
mi_check_print_warning(param,
1336
"Found %10s deleted blocks Should be: %s",
1337
llstr(del_blocks,llbuff),
1338
llstr(info->state->del,llbuff2));
1340
if (splits != info->s->state.split)
1342
mi_check_print_warning(param,
1343
"Found %10s key parts. Should be: %s",
1344
llstr(splits,llbuff),
1345
llstr(info->s->state.split,llbuff2));
1347
if (param->testflag & T_INFO)
1349
if (param->warning_printed || param->error_printed)
1351
if (used != 0 && ! param->error_printed)
1353
printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1354
llstr(records,llbuff), (long)((used-link_used)/records),
1355
(info->s->base.blobs ? 0.0 :
1356
(ulonglong2double((ulonglong) info->s->base.reclength*records)-
1357
my_off_t2double(used))/
1358
ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
1359
printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1360
(ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
1361
(!records ? 100 : (int) (ulonglong2double(del_length+empty)/
1362
my_off_t2double(used)*100.0)),
1363
ulonglong2double(splits - del_blocks) / records);
1365
printf("Record blocks:%12s Delete blocks:%10s\n",
1366
llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1367
printf("Record data: %12s Deleted data: %10s\n",
1368
llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1369
printf("Lost space: %12s Linkdata: %10s\n",
1370
llstr(empty,llbuff),llstr(link_used,llbuff2));
1372
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1373
DBUG_RETURN (error);
1375
mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
1377
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1378
param->testflag|=T_RETRY_WITHOUT_QUICK;
1380
} /* chk_data_link */
1384
@brief Drop all indexes
1386
@param[in] param check parameters
1387
@param[in] info MI_INFO handle
1388
@param[in] force if to force drop all indexes
1395
Once allocated, index blocks remain part of the key file forever.
1396
When indexes are disabled, no block is freed. When enabling indexes,
1397
no block is freed either. The new indexes are create from new
1400
Before recreating formerly disabled indexes, the unused blocks
1401
must be freed. There are two options to do this:
1402
- Follow the tree of disabled indexes, add all blocks to the
1403
deleted blocks chain. Would require a lot of random I/O.
1404
- Drop all blocks by clearing all index root pointers and all
1405
delete chain pointers and resetting key_file_length to the end
1406
of the index file header. This requires to recreate all indexes,
1407
even those that may still be intact.
1408
The second method is probably faster in most cases.
1410
When disabling indexes, MySQL disables either all indexes or all
1411
non-unique indexes. When MySQL [re-]enables disabled indexes
1412
(T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
1413
index file, or there are no non-unique indexes. In the latter case,
1414
mi_repair*() would not be called as there would be no disabled
1417
If there would be more unique indexes than disabled (non-unique)
1418
indexes, we could do the first method. But this is not implemented
1419
yet. By now we drop and recreate all indexes when repair is called.
1421
However, there is an exception. Sometimes MySQL disables non-unique
1422
indexes when the table is empty (e.g. when copying a table in
1423
mysql_alter_table()). When enabling the non-unique indexes, they
1424
are still empty. So there is no index block that can be lost. This
1425
optimization is implemented in this function.
1427
Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
1428
recreate all enabled indexes unconditonally. We do not change the
1429
key_map. Otherwise we invert the key map temporarily (outside of
1430
this function) and recreate the then "seemingly" enabled indexes.
1431
When we cannot use the optimization, and drop all indexes, we
1432
pretend that all indexes were disabled. By the inversion, we will
1433
then recrate all indexes.
1436
static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force)
1438
MYISAM_SHARE *share= info->s;
1439
MI_STATE_INFO *state= &share->state;
1442
DBUG_ENTER("mi_drop_all_indexes");
1445
If any of the disabled indexes has a key block assigned, we must
1446
drop and recreate all indexes to avoid losing index blocks.
1448
If we want to recreate disabled indexes only _and_ all of these
1449
indexes are empty, we don't need to recreate the existing indexes.
1451
if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1453
DBUG_PRINT("repair", ("creating missing indexes"));
1454
for (i= 0; i < share->base.keys; i++)
1456
DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d",
1457
i, (long) state->key_root[i],
1458
mi_is_key_active(state->key_map, i)));
1459
if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1460
!mi_is_key_active(state->key_map, i))
1463
This index has at least one key block and it is disabled.
1464
We would lose its block(s) if would just recreate it.
1465
So we need to drop and recreate all indexes.
1467
DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
1471
if (i >= share->base.keys)
1474
All of the disabled indexes are empty. We can just recreate them.
1475
Flush dirty blocks of this index file from key cache and remove
1476
all blocks of this index file from key cache.
1478
DBUG_PRINT("repair", ("all disabled are empty: create missing"));
1479
error= flush_key_blocks(share->key_cache, share->kfile,
1484
We do now drop all indexes and declare them disabled. With the
1485
T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1486
disabled indexes and enable them.
1488
mi_clear_all_keys_active(state->key_map);
1489
DBUG_PRINT("repair", ("declared all indexes disabled"));
1492
/* Remove all key blocks of this index file from key cache. */
1493
if ((error= flush_key_blocks(share->key_cache, share->kfile,
1494
FLUSH_IGNORE_CHANGED)))
1495
goto end; /* purecov: inspected */
1497
/* Clear index root block pointers. */
1498
for (i= 0; i < share->base.keys; i++)
1499
state->key_root[i]= HA_OFFSET_ERROR;
1501
/* Clear the delete chains. */
1502
for (i= 0; i < state->header.max_block_size_index; i++)
1503
state->key_del[i]= HA_OFFSET_ERROR;
1505
/* Reset index file length to end of index file header. */
1506
info->state->key_file_length= share->base.keystart;
1508
DBUG_PRINT("repair", ("dropped all indexes"));
1509
/* error= 0; set by last (error= flush_key_bocks()). */
1516
/* Recover old table by reading each record and writing all keys */
1517
/* Save new datafile-name in temp_filename */
1519
int mi_repair(MI_CHECK *param, register MI_INFO *info,
1520
char * name, int rep_quick)
1522
int error,got_error;
1523
ha_rows start_records,new_header_length;
1526
MYISAM_SHARE *share=info->s;
1527
char llbuff[22],llbuff2[22];
1528
SORT_INFO sort_info;
1529
MI_SORT_PARAM sort_param;
1530
DBUG_ENTER("mi_repair");
1532
bzero((char *)&sort_info, sizeof(sort_info));
1533
bzero((char *)&sort_param, sizeof(sort_param));
1534
start_records=info->state->records;
1535
new_header_length=(param->testflag & T_UNPACK) ? 0L :
1536
share->pack.header_length;
1539
sort_param.sort_info=&sort_info;
1541
if (!(param->testflag & T_SILENT))
1543
printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1544
printf("Data records: %s\n", llstr(info->state->records,llbuff));
1546
param->testflag|=T_REP; /* for easy checking */
1548
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
1549
param->testflag|=T_CALC_CHECKSUM;
1551
DBUG_ASSERT(param->use_buffers < SIZE_T_MAX);
1553
if (!param->using_global_keycache)
1554
VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
1555
(size_t) param->use_buffers, 0, 0));
1557
if (init_io_cache(¶m->read_cache,info->dfile,
1558
(uint) param->read_buffer_length,
1559
READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1561
bzero(&info->rec_cache,sizeof(info->rec_cache));
1565
if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
1566
WRITE_CACHE, new_header_length, 1,
1567
MYF(MY_WME | MY_WAIT_IF_FULL)))
1569
info->opt_flag|=WRITE_CACHE_USED;
1570
if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
1571
!mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
1573
mi_check_print_error(param, "Not enough memory for extra record");
1579
/* Get real path for data file */
1580
if ((new_file=my_raid_create(fn_format(param->temp_filename,
1581
share->data_file_name, "",
1583
0,param->tmpfile_createflag,
1584
share->base.raid_type,
1585
share->base.raid_chunks,
1586
share->base.raid_chunksize,
1589
mi_check_print_error(param,"Can't create new tempfile: '%s'",
1590
param->temp_filename);
1593
if (new_header_length &&
1594
filecopy(param,new_file,info->dfile,0L,new_header_length,
1597
info->s->state.dellink= HA_OFFSET_ERROR;
1598
info->rec_cache.file=new_file;
1599
if (param->testflag & T_UNPACK)
1601
share->options&= ~HA_OPTION_COMPRESS_RECORD;
1602
mi_int2store(share->state.header.options,share->options);
1605
sort_info.info=info;
1606
sort_info.param = param;
1607
sort_param.read_cache=param->read_cache;
1608
sort_param.pos=sort_param.max_pos=share->pack.header_length;
1609
sort_param.filepos=new_header_length;
1610
param->read_cache.end_of_file=sort_info.filelength=
1611
my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
1613
sort_param.fix_datafile= (my_bool) (! rep_quick);
1614
sort_param.master=1;
1615
sort_info.max_records= ~(ha_rows) 0;
1617
set_data_file_type(&sort_info, share);
1618
del=info->state->del;
1619
info->state->records=info->state->del=share->state.split=0;
1620
info->state->empty=0;
1622
if (param->testflag & T_CALC_CHECKSUM)
1623
sort_param.calc_checksum= 1;
1625
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1627
/* This function always recreates all enabled indexes. */
1628
if (param->testflag & T_CREATE_MISSING_KEYS)
1629
mi_set_all_keys_active(share->state.key_map, share->base.keys);
1630
mi_drop_all_indexes(param, info, TRUE);
1632
lock_memory(param); /* Everything is alloced */
1634
/* Re-create all keys, which are set in key_map. */
1635
while (!(error=sort_get_next_record(&sort_param)))
1637
if (writekeys(&sort_param))
1639
if (my_errno != HA_ERR_FOUND_DUPP_KEY)
1641
DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
1642
mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1644
llstr(sort_param.start_recpos,llbuff),
1645
llstr(info->dupp_key_pos,llbuff2));
1646
if (param->testflag & T_VERBOSE)
1648
VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
1649
sort_param.record,0L));
1650
_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
1654
if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1656
param->testflag|=T_RETRY_WITHOUT_QUICK;
1657
param->error_printed=1;
1662
if (sort_write_record(&sort_param))
1665
if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
1666
flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
1669
if (param->testflag & T_WRITE_LOOP)
1671
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
1673
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
1675
mi_check_print_warning(param,
1676
"Can't change size of indexfile, error: %d",
1681
if (rep_quick && del+sort_info.dupp != info->state->del)
1683
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1684
mi_check_print_error(param,"Run recovery again without -q");
1686
param->retry_repair=1;
1687
param->testflag|=T_RETRY_WITHOUT_QUICK;
1690
if (param->testflag & T_SAFE_REPAIR)
1692
/* Don't repair if we loosed more than one row */
1693
if (info->state->records+1 < start_records)
1695
info->state->records=start_records;
1703
my_close(info->dfile,MYF(0));
1704
info->dfile=new_file;
1705
info->state->data_file_length=sort_param.filepos;
1706
share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
1710
info->state->data_file_length=sort_param.max_pos;
1712
if (param->testflag & T_CALC_CHECKSUM)
1713
info->state->checksum=param->glob_crc;
1715
if (!(param->testflag & T_SILENT))
1717
if (start_records != info->state->records)
1718
printf("Data records: %s\n", llstr(info->state->records,llbuff));
1720
mi_check_print_warning(param,
1721
"%s records have been removed",
1722
llstr(sort_info.dupp,llbuff));
1726
/* If invoked by external program that uses thr_lock */
1727
if (&share->state.state != info->state)
1728
memcpy( &share->state.state, info->state, sizeof(*info->state));
1733
/* Replace the actual file with the temporary file */
1736
my_close(new_file,MYF(0));
1737
info->dfile=new_file= -1;
1738
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1739
DATA_TMP_EXT, share->base.raid_chunks,
1740
(param->testflag & T_BACKUP_DATA ?
1741
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
1742
mi_open_datafile(info,share,name,-1))
1748
if (! param->error_printed)
1749
mi_check_print_error(param,"%d for record at pos %s",my_errno,
1750
llstr(sort_param.start_recpos,llbuff));
1753
VOID(my_close(new_file,MYF(0)));
1754
VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
1756
info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1758
mi_mark_crashed_on_repair(info);
1760
my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
1761
MYF(MY_ALLOW_ZERO_PTR));
1762
my_free(mi_get_rec_buff_ptr(info, sort_param.record),
1763
MYF(MY_ALLOW_ZERO_PTR));
1764
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
1765
VOID(end_io_cache(¶m->read_cache));
1766
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1767
VOID(end_io_cache(&info->rec_cache));
1768
got_error|=flush_blocks(param, share->key_cache, share->kfile);
1769
if (!got_error && param->testflag & T_UNPACK)
1771
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
1772
share->pack.header_length=0;
1773
share->data_file_type=sort_info.new_data_file_type;
1775
share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1776
STATE_NOT_ANALYZED);
1777
DBUG_RETURN(got_error);
1781
/* Uppate keyfile when doing repair */
1783
static int writekeys(MI_SORT_PARAM *sort_param)
1787
MI_INFO *info= sort_param->sort_info->info;
1788
uchar *buff= sort_param->record;
1789
my_off_t filepos= sort_param->filepos;
1790
DBUG_ENTER("writekeys");
1792
key=info->lastkey+info->s->base.max_key_length;
1793
for (i=0 ; i < info->s->base.keys ; i++)
1795
if (mi_is_key_active(info->s->state.key_map, i))
1797
if (info->s->keyinfo[i].flag & HA_FULLTEXT )
1799
if (_mi_ft_add(info, i, key, buff, filepos))
1803
else if (info->s->keyinfo[i].flag & HA_SPATIAL)
1805
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1806
if (rtree_insert(info, i, key, key_length))
1809
#endif /*HAVE_SPATIAL*/
1812
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1813
if (_mi_ck_write(info,i,key,key_length))
1821
if (my_errno == HA_ERR_FOUND_DUPP_KEY)
1823
info->errkey=(int) i; /* This key was found */
1826
if (mi_is_key_active(info->s->state.key_map, i))
1828
if (info->s->keyinfo[i].flag & HA_FULLTEXT)
1830
if (_mi_ft_del(info,i, key,buff,filepos))
1835
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1836
if (_mi_ck_delete(info,i,key,key_length))
1842
/* Remove checksum that was added to glob_crc in sort_get_next_record */
1843
if (sort_param->calc_checksum)
1844
sort_param->sort_info->param->glob_crc-= info->checksum;
1845
DBUG_PRINT("error",("errno: %d",my_errno));
1850
/* Change all key-pointers that points to a records */
1852
int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
1853
my_off_t newpos, uint prot_key)
1858
DBUG_ENTER("movepoint");
1860
key=info->lastkey+info->s->base.max_key_length;
1861
for (i=0 ; i < info->s->base.keys; i++)
1863
if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1865
key_length=_mi_make_key(info,i,key,record,oldpos);
1866
if (info->s->keyinfo[i].flag & HA_NOSAME)
1867
{ /* Change pointer direct */
1870
keyinfo=info->s->keyinfo+i;
1871
if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1872
(uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1873
info->s->state.key_root[i]))
1875
nod_flag=mi_test_if_nod(info->buff);
1876
_mi_dpointer(info,info->int_keypos-nod_flag-
1877
info->s->rec_reflength,newpos);
1878
if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1879
DFLT_INIT_HITS,info->buff))
1883
{ /* Change old key to new */
1884
if (_mi_ck_delete(info,i,key,key_length))
1886
key_length=_mi_make_key(info,i,key,record,newpos);
1887
if (_mi_ck_write(info,i,key,key_length))
1896
/* Tell system that we want all memory for our cache */
1898
void lock_memory(MI_CHECK *param __attribute__((unused)))
1900
#ifdef SUN_OS /* Key-cacheing thrases on sun 4.1 */
1901
if (param->opt_lock_memory)
1903
int success = mlockall(MCL_CURRENT); /* or plock(DATLOCK); */
1904
if (geteuid() == 0 && success != 0)
1905
mi_check_print_warning(param,
1906
"Failed to lock memory. errno %d",my_errno);
1912
/* Flush all changed blocks to disk */
1914
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
1916
if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
1918
mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
1921
if (!param->using_global_keycache)
1922
end_key_cache(key_cache,1);
1924
} /* flush_blocks */
1927
/* Sort index for more efficent reads */
1929
int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
1932
reg1 MI_KEYDEF *keyinfo;
1934
my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1935
uint r_locks,w_locks;
1937
MYISAM_SHARE *share=info->s;
1938
MI_STATE_INFO old_state;
1939
DBUG_ENTER("mi_sort_index");
1941
/* cannot sort index files with R-tree indexes */
1942
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1944
if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
1947
if (!(param->testflag & T_SILENT))
1948
printf("- Sorting index for MyISAM-table '%s'\n",name);
1950
/* Get real path for index file */
1951
fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1952
if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
1953
"", INDEX_TMP_EXT,2+4),
1954
0,param->tmpfile_createflag,MYF(0))) <= 0)
1956
mi_check_print_error(param,"Can't create new tempfile: '%s'",
1957
param->temp_filename);
1960
if (filecopy(param, new_file,share->kfile,0L,
1961
(ulong) share->base.keystart, "headerblock"))
1964
param->new_file_pos=share->base.keystart;
1965
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1968
if (! mi_is_key_active(info->s->state.key_map, key))
1971
if (share->state.key_root[key] != HA_OFFSET_ERROR)
1973
index_pos[key]=param->new_file_pos; /* Write first block here */
1974
if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1979
index_pos[key]= HA_OFFSET_ERROR; /* No blocks */
1982
/* Flush key cache for this file if we are calling this outside myisamchk */
1983
flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
1985
share->state.version=(ulong) time((time_t*) 0);
1986
old_state= share->state; /* save state if not stored */
1987
r_locks= share->r_locks;
1988
w_locks= share->w_locks;
1989
old_lock= info->lock_type;
1991
/* Put same locks as old file */
1992
share->r_locks= share->w_locks= share->tot_locks= 0;
1993
(void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1994
VOID(my_close(share->kfile,MYF(MY_WME)));
1996
VOID(my_close(new_file,MYF(MY_WME)));
1997
if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1999
mi_open_keyfile(share))
2001
info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */
2002
_mi_readinfo(info,F_WRLCK,0); /* Will lock the table */
2003
info->lock_type= old_lock;
2004
share->r_locks= r_locks;
2005
share->w_locks= w_locks;
2006
share->tot_locks= r_locks+w_locks;
2007
share->state= old_state; /* Restore old state */
2009
info->state->key_file_length=param->new_file_pos;
2010
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2011
for (key=0 ; key < info->s->base.keys ; key++)
2012
info->s->state.key_root[key]=index_pos[key];
2013
for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
2014
info->s->state.key_del[key]= HA_OFFSET_ERROR;
2016
info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
2020
VOID(my_close(new_file,MYF(MY_WME)));
2022
VOID(my_delete(param->temp_filename,MYF(MY_WME)));
2024
} /* mi_sort_index */
2027
/* Sort records recursive using one index */
2029
static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
2030
my_off_t pagepos, File new_file)
2032
uint length,nod_flag,used_length, key_length;
2033
uchar *buff,*keypos,*endpos;
2034
uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
2035
my_off_t new_page_pos,next_page;
2037
DBUG_ENTER("sort_one_index");
2039
/* cannot walk over R-tree indices */
2040
DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
2041
new_page_pos=param->new_file_pos;
2042
param->new_file_pos+=keyinfo->block_length;
2044
if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
2046
mi_check_print_error(param,"Not enough memory for key block");
2049
if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
2051
mi_check_print_error(param,"Can't read key block from filepos: %s",
2052
llstr(pagepos,llbuff));
2055
if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
2057
used_length=mi_getint(buff);
2058
keypos=buff+2+nod_flag;
2059
endpos=buff+used_length;
2064
next_page=_mi_kpos(nod_flag,keypos);
2065
_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2066
if (sort_one_index(param,info,keyinfo,next_page,new_file))
2069
("From page: %ld, keyoffset: %lu used_length: %d",
2070
(ulong) pagepos, (ulong) (keypos - buff),
2071
(int) used_length));
2072
DBUG_DUMP("buff",(uchar*) buff,used_length);
2076
if (keypos >= endpos ||
2077
(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
2079
DBUG_ASSERT(keypos <= endpos);
2080
if (keyinfo->flag & HA_FULLTEXT)
2084
get_key_full_length_rdonly(off, key);
2085
subkeys=ft_sintXkorr(key+off);
2088
next_page= _mi_dpos(info,0,key+key_length);
2089
_mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
2090
param->new_file_pos); /* Save new pos */
2091
if (sort_one_index(param,info,&info->s->ft2_keyinfo,
2092
next_page,new_file))
2099
/* Fill block with zero and write it to the new index file */
2100
length=mi_getint(buff);
2101
bzero((uchar*) buff+length,keyinfo->block_length-length);
2102
if (my_pwrite(new_file,(uchar*) buff,(uint) keyinfo->block_length,
2103
new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
2105
mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
2108
my_afree((uchar*) buff);
2111
my_afree((uchar*) buff);
2113
} /* sort_one_index */
2117
Let temporary file replace old file.
2118
This assumes that the new file was created in the same
2119
directory as given by realpath(filename).
2120
This will ensure that any symlinks that are used will still work.
2121
Copy stats from old file to new file, deletes orignal and
2122
changes new file name to old file name
2125
int change_to_newfile(const char * filename, const char * old_ext,
2126
const char * new_ext,
2127
uint raid_chunks __attribute__((unused)),
2130
char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
2133
return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
2134
fn_format(new_filename,filename,"",new_ext,2+4),
2136
MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2138
/* Get real path to filename */
2139
(void) fn_format(old_filename,filename,"",old_ext,2+4+32);
2140
return my_redel(old_filename,
2141
fn_format(new_filename,old_filename,"",new_ext,2+4),
2142
MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2143
} /* change_to_newfile */
2146
/* Locks a whole file */
2147
/* Gives an error-message if file can't be locked */
2149
int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
2150
const char *filetype, const char *filename)
2152
if (my_lock(file,lock_type,start,F_TO_EOF,
2153
param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
2154
MYF(MY_SEEK_NOT_DONE | MY_DONT_WAIT)))
2156
mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
2157
param->error_printed=2; /* Don't give that data is crashed */
2164
/* Copy a block between two files */
2166
int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
2167
my_off_t length, const char *type)
2169
char tmp_buff[IO_SIZE],*buff;
2171
DBUG_ENTER("filecopy");
2173
buff_length=(ulong) min(param->write_buffer_length,length);
2174
if (!(buff=my_malloc(buff_length,MYF(0))))
2176
buff=tmp_buff; buff_length=IO_SIZE;
2179
VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
2180
while (length > buff_length)
2182
if (my_read(from,(uchar*) buff,buff_length,MYF(MY_NABP)) ||
2183
my_write(to,(uchar*) buff,buff_length,param->myf_rw))
2185
length-= buff_length;
2187
if (my_read(from,(uchar*) buff,(uint) length,MYF(MY_NABP)) ||
2188
my_write(to,(uchar*) buff,(uint) length,param->myf_rw))
2190
if (buff != tmp_buff)
2191
my_free(buff,MYF(0));
2194
if (buff != tmp_buff)
2195
my_free(buff,MYF(0));
2196
mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
2203
Repair table or given index using sorting
2207
param Repair parameters
2208
info MyISAM handler to repair
2209
name Name of table (for warnings)
2210
rep_quick set to <> 0 if we should not change data file
2217
int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
2218
const char * name, int rep_quick)
2223
ha_rows start_records;
2224
my_off_t new_header_length,del;
2226
MI_SORT_PARAM sort_param;
2227
MYISAM_SHARE *share=info->s;
2229
ulong *rec_per_key_part;
2231
SORT_INFO sort_info;
2232
ulonglong UNINIT_VAR(key_map);
2233
DBUG_ENTER("mi_repair_by_sort");
2235
start_records=info->state->records;
2238
new_header_length=(param->testflag & T_UNPACK) ? 0 :
2239
share->pack.header_length;
2240
if (!(param->testflag & T_SILENT))
2242
printf("- recovering (with sort) MyISAM-table '%s'\n",name);
2243
printf("Data records: %s\n", llstr(start_records,llbuff));
2245
param->testflag|=T_REP; /* for easy checking */
2247
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2248
param->testflag|=T_CALC_CHECKSUM;
2250
bzero((char*)&sort_info,sizeof(sort_info));
2251
bzero((char *)&sort_param, sizeof(sort_param));
2252
if (!(sort_info.key_block=
2253
alloc_key_blocks(param,
2254
(uint) param->sort_key_blocks,
2255
share->base.max_key_block_length))
2256
|| init_io_cache(¶m->read_cache,info->dfile,
2257
(uint) param->read_buffer_length,
2258
READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
2260
init_io_cache(&info->rec_cache,info->dfile,
2261
(uint) param->write_buffer_length,
2262
WRITE_CACHE,new_header_length,1,
2263
MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
2265
sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2266
info->opt_flag|=WRITE_CACHE_USED;
2267
info->rec_cache.file=info->dfile; /* for sort_delete_record */
2269
if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
2270
!mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
2272
mi_check_print_error(param, "Not enough memory for extra record");
2277
/* Get real path for data file */
2278
if ((new_file=my_raid_create(fn_format(param->temp_filename,
2279
share->data_file_name, "",
2281
0,param->tmpfile_createflag,
2282
share->base.raid_type,
2283
share->base.raid_chunks,
2284
share->base.raid_chunksize,
2287
mi_check_print_error(param,"Can't create new tempfile: '%s'",
2288
param->temp_filename);
2291
if (new_header_length &&
2292
filecopy(param, new_file,info->dfile,0L,new_header_length,
2295
if (param->testflag & T_UNPACK)
2297
share->options&= ~HA_OPTION_COMPRESS_RECORD;
2298
mi_int2store(share->state.header.options,share->options);
2300
share->state.dellink= HA_OFFSET_ERROR;
2301
info->rec_cache.file=new_file;
2304
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2306
/* Optionally drop indexes and optionally modify the key_map. */
2307
mi_drop_all_indexes(param, info, FALSE);
2308
key_map= share->state.key_map;
2309
if (param->testflag & T_CREATE_MISSING_KEYS)
2311
/* Invert the copied key_map to recreate all disabled indexes. */
2315
sort_info.info=info;
2316
sort_info.param = param;
2318
set_data_file_type(&sort_info, share);
2319
sort_param.filepos=new_header_length;
2322
param->read_cache.end_of_file=sort_info.filelength=
2323
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2325
sort_param.wordlist=NULL;
2326
init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2328
if (share->data_file_type == DYNAMIC_RECORD)
2329
length=max(share->base.min_pack_length+1,share->base.min_block_length);
2330
else if (share->data_file_type == COMPRESSED_RECORD)
2331
length=share->base.min_block_length;
2333
length=share->base.pack_reclength;
2334
sort_info.max_records=
2335
((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2336
(ha_rows) (sort_info.filelength/length+1));
2337
sort_param.key_cmp=sort_key_cmp;
2338
sort_param.lock_in_memory=lock_memory;
2339
sort_param.tmpdir=param->tmpdir;
2340
sort_param.sort_info=&sort_info;
2341
sort_param.fix_datafile= (my_bool) (! rep_quick);
2342
sort_param.master =1;
2344
del=info->state->del;
2346
if (param->testflag & T_CALC_CHECKSUM)
2347
sort_param.calc_checksum= 1;
2349
rec_per_key_part= param->rec_per_key_part;
2350
for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2351
rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2353
sort_param.read_cache=param->read_cache;
2354
sort_param.keyinfo=share->keyinfo+sort_param.key;
2355
sort_param.seg=sort_param.keyinfo->seg;
2357
Skip this index if it is marked disabled in the copied
2358
(and possibly inverted) key_map.
2360
if (! mi_is_key_active(key_map, sort_param.key))
2362
/* Remember old statistics for key */
2363
memcpy((char*) rec_per_key_part,
2364
(char*) (share->state.rec_per_key_part +
2365
(uint) (rec_per_key_part - param->rec_per_key_part)),
2366
sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2367
DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
2372
if ((!(param->testflag & T_SILENT)))
2373
printf ("- Fixing index %d\n",sort_param.key+1);
2374
sort_param.max_pos=sort_param.pos=share->pack.header_length;
2375
keyseg=sort_param.seg;
2376
bzero((char*) sort_param.unique,sizeof(sort_param.unique));
2377
sort_param.key_length=share->rec_reflength;
2378
for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2380
sort_param.key_length+=keyseg[i].length;
2381
if (keyseg[i].flag & HA_SPACE_PACK)
2382
sort_param.key_length+=get_pack_length(keyseg[i].length);
2383
if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2384
sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2385
if (keyseg[i].flag & HA_NULL_PART)
2386
sort_param.key_length++;
2388
info->state->records=info->state->del=share->state.split=0;
2389
info->state->empty=0;
2391
if (sort_param.keyinfo->flag & HA_FULLTEXT)
2393
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2394
sort_param.keyinfo->seg->charset->mbmaxlen;
2395
sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2397
fulltext indexes may have much more entries than the
2398
number of rows in the table. We estimate the number here.
2400
Note, built-in parser is always nr. 0 - see ftparser_call_initializer()
2402
if (sort_param.keyinfo->ftkey_nr == 0)
2405
for built-in parser the number of generated index entries
2406
cannot be larger than the size of the data file divided
2407
by the minimal word's length
2409
sort_info.max_records=
2410
(ha_rows) (sort_info.filelength/ft_min_word_len+1);
2415
for external plugin parser we cannot tell anything at all :(
2416
so, we'll use all the sort memory and start from ~10 buffpeks.
2417
(see _create_index_by_sort)
2419
sort_info.max_records=
2420
10*param->sort_buffer_length/sort_param.key_length;
2423
sort_param.key_read=sort_ft_key_read;
2424
sort_param.key_write=sort_ft_key_write;
2428
sort_param.key_read=sort_key_read;
2429
sort_param.key_write=sort_key_write;
2432
if (_create_index_by_sort(&sort_param,
2433
(my_bool) (!(param->testflag & T_VERBOSE)),
2434
(uint) param->sort_buffer_length))
2436
param->retry_repair=1;
2439
/* No need to calculate checksum again. */
2440
sort_param.calc_checksum= 0;
2441
free_root(&sort_param.wordroot, MYF(0));
2443
/* Set for next loop */
2444
sort_info.max_records= (ha_rows) info->state->records;
2446
if (param->testflag & T_STATISTICS)
2447
update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2448
param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2449
sort_param.notnull: NULL,
2450
(ulonglong) info->state->records);
2451
/* Enable this index in the permanent (not the copied) key_map. */
2452
mi_set_key_active(share->state.key_map, sort_param.key);
2453
DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
2455
if (sort_param.fix_datafile)
2457
param->read_cache.end_of_file=sort_param.filepos;
2458
if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2460
if (param->testflag & T_SAFE_REPAIR)
2462
/* Don't repair if we loosed more than one row */
2463
if (info->state->records+1 < start_records)
2465
info->state->records=start_records;
2469
share->state.state.data_file_length = info->state->data_file_length=
2471
/* Only whole records */
2472
share->state.version=(ulong) time((time_t*) 0);
2473
my_close(info->dfile,MYF(0));
2474
info->dfile=new_file;
2475
share->data_file_type=sort_info.new_data_file_type;
2476
share->pack.header_length=(ulong) new_header_length;
2477
sort_param.fix_datafile=0;
2480
info->state->data_file_length=sort_param.max_pos;
2482
param->read_cache.file=info->dfile; /* re-init read cache */
2483
reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length,
2487
if (param->testflag & T_WRITE_LOOP)
2489
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
2492
if (rep_quick && del+sort_info.dupp != info->state->del)
2494
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2495
mi_check_print_error(param,"Run recovery again without -q");
2497
param->retry_repair=1;
2498
param->testflag|=T_RETRY_WITHOUT_QUICK;
2502
if (rep_quick & T_FORCE_UNIQUENESS)
2504
my_off_t skr=info->state->data_file_length+
2505
(share->options & HA_OPTION_COMPRESS_RECORD ?
2506
MEMMAP_EXTRA_MARGIN : 0);
2508
if (share->data_file_type == STATIC_RECORD &&
2509
skr < share->base.reloc*share->base.min_pack_length)
2510
skr=share->base.reloc*share->base.min_pack_length;
2512
if (skr != sort_info.filelength && !info->s->base.raid_type)
2513
if (my_chsize(info->dfile,skr,0,MYF(0)))
2514
mi_check_print_warning(param,
2515
"Can't change size of datafile, error: %d",
2518
if (param->testflag & T_CALC_CHECKSUM)
2519
info->state->checksum=param->glob_crc;
2521
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
2522
mi_check_print_warning(param,
2523
"Can't change size of indexfile, error: %d",
2526
if (!(param->testflag & T_SILENT))
2528
if (start_records != info->state->records)
2529
printf("Data records: %s\n", llstr(info->state->records,llbuff));
2531
mi_check_print_warning(param,
2532
"%s records have been removed",
2533
llstr(sort_info.dupp,llbuff));
2537
if (&share->state.state != info->state)
2538
memcpy( &share->state.state, info->state, sizeof(*info->state));
2541
got_error|= flush_blocks(param, share->key_cache, share->kfile);
2542
VOID(end_io_cache(&info->rec_cache));
2545
/* Replace the actual file with the temporary file */
2548
my_close(new_file,MYF(0));
2549
info->dfile=new_file= -1;
2550
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2551
DATA_TMP_EXT, share->base.raid_chunks,
2552
(param->testflag & T_BACKUP_DATA ?
2553
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
2554
mi_open_datafile(info,share,name,-1))
2560
if (! param->error_printed)
2561
mi_check_print_error(param,"%d when fixing table",my_errno);
2564
VOID(my_close(new_file,MYF(0)));
2565
VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
2567
if (info->dfile == new_file) /* Retry with key cache */
2568
if (unlikely(mi_open_datafile(info, share, name, -1)))
2569
param->retry_repair= 0; /* Safety */
2571
mi_mark_crashed_on_repair(info);
2573
else if (key_map == share->state.key_map)
2574
share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2575
share->state.changed|=STATE_NOT_SORTED_PAGES;
2577
my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
2578
MYF(MY_ALLOW_ZERO_PTR));
2579
my_free(mi_get_rec_buff_ptr(info, sort_param.record),
2580
MYF(MY_ALLOW_ZERO_PTR));
2581
my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
2582
my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
2583
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
2584
VOID(end_io_cache(¶m->read_cache));
2585
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2586
if (!got_error && (param->testflag & T_UNPACK))
2588
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
2589
share->pack.header_length=0;
2591
DBUG_RETURN(got_error);
2595
Threaded repair of table using sorting
2598
mi_repair_parallel()
2599
param Repair parameters
2600
info MyISAM handler to repair
2601
name Name of table (for warnings)
2602
rep_quick set to <> 0 if we should not change data file
2605
Same as mi_repair_by_sort but do it multithreaded
2606
Each key is handled by a separate thread.
2607
TODO: make a number of threads a parameter
2609
In parallel repair we use one thread per index. There are two modes:
2613
Only the indexes are rebuilt. All threads share a read buffer.
2614
Every thread that needs fresh data in the buffer enters the shared
2615
cache lock. The last thread joining the lock reads the buffer from
2616
the data file and wakes all other threads.
2620
The data file is rebuilt and all indexes are rebuilt to point to
2621
the new record positions. One thread is the master thread. It
2622
reads from the old data file and writes to the new data file. It
2623
also creates one of the indexes. The other threads read from a
2624
buffer which is filled by the master. If they need fresh data,
2625
they enter the shared cache lock. If the masters write buffer is
2626
full, it flushes it to the new data file and enters the shared
2627
cache lock too. When all threads joined in the lock, the master
2628
copies its write buffer to the read buffer for the other threads
2636
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
2637
const char * name, int rep_quick)
2640
return mi_repair_by_sort(param, info, name, rep_quick);
2643
uint i,key, total_key_length, istep;
2645
ha_rows start_records;
2646
my_off_t new_header_length,del;
2648
MI_SORT_PARAM *sort_param=0;
2649
MYISAM_SHARE *share=info->s;
2650
ulong *rec_per_key_part;
2653
IO_CACHE new_data_cache; /* For non-quick repair. */
2654
IO_CACHE_SHARE io_share;
2655
SORT_INFO sort_info;
2656
ulonglong UNINIT_VAR(key_map);
2657
pthread_attr_t thr_attr;
2658
ulong max_pack_reclength;
2659
DBUG_ENTER("mi_repair_parallel");
2661
start_records=info->state->records;
2664
new_header_length=(param->testflag & T_UNPACK) ? 0 :
2665
share->pack.header_length;
2666
if (!(param->testflag & T_SILENT))
2668
printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
2669
printf("Data records: %s\n", llstr(start_records,llbuff));
2671
param->testflag|=T_REP; /* for easy checking */
2673
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2674
param->testflag|=T_CALC_CHECKSUM;
2677
Quick repair (not touching data file, rebuilding indexes):
2679
Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2682
Non-quick repair (rebuilding data file and indexes):
2686
Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2687
Write cache is (MI_INFO *info)->rec_cache using new_file.
2691
Read cache is new_data_cache synced to master rec_cache.
2693
The final assignment of the filedescriptor for rec_cache is done
2694
after the cache creation.
2696
Don't check file size on new_data_cache, as the resulting file size
2699
As rec_cache and new_data_cache are synced, write_buffer_length is
2700
used for the read cache 'new_data_cache'. Both start at the same
2701
position 'new_header_length'.
2704
DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
2705
bzero((char*)&sort_info,sizeof(sort_info));
2706
/* Initialize pthread structures before goto err. */
2707
pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
2708
pthread_cond_init(&sort_info.cond, 0);
2709
pthread_mutex_init(¶m->print_msg_mutex, MY_MUTEX_INIT_FAST);
2710
param->need_print_msg_lock= 1;
2712
if (!(sort_info.key_block=
2713
alloc_key_blocks(param, (uint) param->sort_key_blocks,
2714
share->base.max_key_block_length)) ||
2715
init_io_cache(¶m->read_cache, info->dfile,
2716
(uint) param->read_buffer_length,
2717
READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
2719
(init_io_cache(&info->rec_cache, info->dfile,
2720
(uint) param->write_buffer_length,
2721
WRITE_CACHE, new_header_length, 1,
2722
MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
2723
init_io_cache(&new_data_cache, -1,
2724
(uint) param->write_buffer_length,
2725
READ_CACHE, new_header_length, 1,
2726
MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
2728
sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2729
info->opt_flag|=WRITE_CACHE_USED;
2730
info->rec_cache.file=info->dfile; /* for sort_delete_record */
2734
/* Get real path for data file */
2735
if ((new_file=my_raid_create(fn_format(param->temp_filename,
2736
share->data_file_name, "",
2739
0,param->tmpfile_createflag,
2740
share->base.raid_type,
2741
share->base.raid_chunks,
2742
share->base.raid_chunksize,
2745
mi_check_print_error(param,"Can't create new tempfile: '%s'",
2746
param->temp_filename);
2749
if (new_header_length &&
2750
filecopy(param, new_file,info->dfile,0L,new_header_length,
2753
if (param->testflag & T_UNPACK)
2755
share->options&= ~HA_OPTION_COMPRESS_RECORD;
2756
mi_int2store(share->state.header.options,share->options);
2758
share->state.dellink= HA_OFFSET_ERROR;
2759
info->rec_cache.file=new_file;
2762
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2764
/* Optionally drop indexes and optionally modify the key_map. */
2765
mi_drop_all_indexes(param, info, FALSE);
2766
key_map= share->state.key_map;
2767
if (param->testflag & T_CREATE_MISSING_KEYS)
2769
/* Invert the copied key_map to recreate all disabled indexes. */
2773
sort_info.info=info;
2774
sort_info.param = param;
2776
set_data_file_type(&sort_info, share);
2779
param->read_cache.end_of_file=sort_info.filelength=
2780
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2782
if (share->data_file_type == DYNAMIC_RECORD)
2783
rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
2784
else if (share->data_file_type == COMPRESSED_RECORD)
2785
rec_length=share->base.min_block_length;
2787
rec_length=share->base.pack_reclength;
2789
+1 below is required hack for parallel repair mode.
2790
The info->state->records value, that is compared later
2791
to sort_info.max_records and cannot exceed it, is
2792
increased in sort_key_write. In mi_repair_by_sort, sort_key_write
2793
is called after sort_key_read, where the comparison is performed,
2794
but in parallel mode master thread can call sort_key_write
2795
before some other repair thread calls sort_key_read.
2796
Furthermore I'm not even sure +1 would be enough.
2797
May be sort_info.max_records shold be always set to max value in
2800
sort_info.max_records=
2801
((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
2802
(ha_rows) (sort_info.filelength/rec_length+1));
2804
del=info->state->del;
2806
/* for compressed tables */
2807
max_pack_reclength= share->base.pack_reclength;
2808
if (share->options & HA_OPTION_COMPRESS_RECORD)
2809
set_if_bigger(max_pack_reclength, share->max_pack_length);
2810
if (!(sort_param=(MI_SORT_PARAM *)
2811
my_malloc((uint) share->base.keys *
2812
(sizeof(MI_SORT_PARAM) + max_pack_reclength),
2815
mi_check_print_error(param,"Not enough memory for key!");
2819
rec_per_key_part= param->rec_per_key_part;
2820
info->state->records=info->state->del=share->state.split=0;
2821
info->state->empty=0;
2823
for (i=key=0, istep=1 ; key < share->base.keys ;
2824
rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
2826
sort_param[i].key=key;
2827
sort_param[i].keyinfo=share->keyinfo+key;
2828
sort_param[i].seg=sort_param[i].keyinfo->seg;
2830
Skip this index if it is marked disabled in the copied
2831
(and possibly inverted) key_map.
2833
if (! mi_is_key_active(key_map, key))
2835
/* Remember old statistics for key */
2836
memcpy((char*) rec_per_key_part,
2837
(char*) (share->state.rec_per_key_part+
2838
(uint) (rec_per_key_part - param->rec_per_key_part)),
2839
sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
2844
if ((!(param->testflag & T_SILENT)))
2845
printf ("- Fixing index %d\n",key+1);
2846
if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2848
sort_param[i].key_read=sort_ft_key_read;
2849
sort_param[i].key_write=sort_ft_key_write;
2853
sort_param[i].key_read=sort_key_read;
2854
sort_param[i].key_write=sort_key_write;
2856
sort_param[i].key_cmp=sort_key_cmp;
2857
sort_param[i].lock_in_memory=lock_memory;
2858
sort_param[i].tmpdir=param->tmpdir;
2859
sort_param[i].sort_info=&sort_info;
2860
sort_param[i].master=0;
2861
sort_param[i].fix_datafile=0;
2862
sort_param[i].calc_checksum= 0;
2864
sort_param[i].filepos=new_header_length;
2865
sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;
2867
sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2868
(max_pack_reclength * i));
2869
if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
2871
mi_check_print_error(param,"Not enough memory!");
2875
sort_param[i].key_length=share->rec_reflength;
2876
for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
2879
sort_param[i].key_length+=keyseg->length;
2880
if (keyseg->flag & HA_SPACE_PACK)
2881
sort_param[i].key_length+=get_pack_length(keyseg->length);
2882
if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2883
sort_param[i].key_length+=2 + test(keyseg->length >= 127);
2884
if (keyseg->flag & HA_NULL_PART)
2885
sort_param[i].key_length++;
2887
total_key_length+=sort_param[i].key_length;
2889
if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2891
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2892
sort_param[i].keyinfo->seg->charset->mbmaxlen;
2893
sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2894
init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2897
sort_info.total_keys=i;
2898
sort_param[0].master= 1;
2899
sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2900
sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
2902
if (!ftparser_alloc_param(info))
2905
sort_info.got_error=0;
2906
pthread_mutex_lock(&sort_info.mutex);
2909
Initialize the I/O cache share for use with the read caches and, in
2910
case of non-quick repair, the write cache. When all threads join on
2911
the cache lock, the writer copies the write cache contents to the
2917
init_io_cache_share(¶m->read_cache, &io_share, NULL, i);
2919
init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
2922
io_share.total_threads= 0; /* share not used */
2924
(void) pthread_attr_init(&thr_attr);
2925
(void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);
2927
for (i=0 ; i < sort_info.total_keys ; i++)
2930
Copy the properly initialized IO_CACHE structure so that every
2931
thread has its own copy. In quick mode param->read_cache is shared
2932
for use by all threads. In non-quick mode all threads but the
2933
first copy the shared new_data_cache, which is synchronized to the
2934
write cache of the first thread. The first thread copies
2935
param->read_cache, which is not shared.
2937
sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
2939
DBUG_PRINT("io_cache_share", ("thread: %u read_cache: 0x%lx",
2940
i, (long) &sort_param[i].read_cache));
2943
two approaches: the same amount of memory for each thread
2944
or the memory for the same number of keys for each thread...
2945
In the second one all the threads will fill their sort_buffers
2946
(and call write_keys) at the same time, putting more stress on i/o.
2948
sort_param[i].sortbuff_size=
2949
#ifndef USING_SECOND_APPROACH
2950
param->sort_buffer_length/sort_info.total_keys;
2952
param->sort_buffer_length*sort_param[i].key_length/total_key_length;
2954
if (pthread_create(&sort_param[i].thr, &thr_attr,
2956
(void *) (sort_param+i)))
2958
mi_check_print_error(param,"Cannot start a repair thread");
2959
/* Cleanup: Detach from the share. Avoid others to be blocked. */
2960
if (io_share.total_threads)
2961
remove_io_thread(&sort_param[i].read_cache);
2962
DBUG_PRINT("error", ("Cannot start a repair thread"));
2963
sort_info.got_error=1;
2966
sort_info.threads_running++;
2968
(void) pthread_attr_destroy(&thr_attr);
2970
/* waiting for all threads to finish */
2971
while (sort_info.threads_running)
2972
pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
2973
pthread_mutex_unlock(&sort_info.mutex);
2975
if ((got_error= thr_write_keys(sort_param)))
2977
param->retry_repair=1;
2980
got_error=1; /* Assume the following may go wrong */
2982
if (sort_param[0].fix_datafile)
2985
Append some nuls to the end of a memory mapped file. Destroy the
2986
write cache. The master thread did already detach from the share
2987
by remove_io_thread() in sort.c:thr_find_all_keys().
2989
if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2991
if (param->testflag & T_SAFE_REPAIR)
2993
/* Don't repair if we loosed more than one row */
2994
if (info->state->records+1 < start_records)
2996
info->state->records=start_records;
3000
share->state.state.data_file_length= info->state->data_file_length=
3001
sort_param->filepos;
3002
/* Only whole records */
3003
share->state.version=(ulong) time((time_t*) 0);
3006
Exchange the data file descriptor of the table, so that we use the
3007
new file from now on.
3009
my_close(info->dfile,MYF(0));
3010
info->dfile=new_file;
3012
share->data_file_type=sort_info.new_data_file_type;
3013
share->pack.header_length=(ulong) new_header_length;
3016
info->state->data_file_length=sort_param->max_pos;
3018
if (rep_quick && del+sort_info.dupp != info->state->del)
3020
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
3021
mi_check_print_error(param,"Run recovery again without -q");
3022
param->retry_repair=1;
3023
param->testflag|=T_RETRY_WITHOUT_QUICK;
3027
if (rep_quick & T_FORCE_UNIQUENESS)
3029
my_off_t skr=info->state->data_file_length+
3030
(share->options & HA_OPTION_COMPRESS_RECORD ?
3031
MEMMAP_EXTRA_MARGIN : 0);
3033
if (share->data_file_type == STATIC_RECORD &&
3034
skr < share->base.reloc*share->base.min_pack_length)
3035
skr=share->base.reloc*share->base.min_pack_length;
3037
if (skr != sort_info.filelength && !info->s->base.raid_type)
3038
if (my_chsize(info->dfile,skr,0,MYF(0)))
3039
mi_check_print_warning(param,
3040
"Can't change size of datafile, error: %d",
3043
if (param->testflag & T_CALC_CHECKSUM)
3044
info->state->checksum=param->glob_crc;
3046
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
3047
mi_check_print_warning(param,
3048
"Can't change size of indexfile, error: %d", my_errno);
3050
if (!(param->testflag & T_SILENT))
3052
if (start_records != info->state->records)
3053
printf("Data records: %s\n", llstr(info->state->records,llbuff));
3055
mi_check_print_warning(param,
3056
"%s records have been removed",
3057
llstr(sort_info.dupp,llbuff));
3061
if (&share->state.state != info->state)
3062
memcpy(&share->state.state, info->state, sizeof(*info->state));
3065
got_error|= flush_blocks(param, share->key_cache, share->kfile);
3067
Destroy the write cache. The master thread did already detach from
3068
the share by remove_io_thread() or it was not yet started (if the
3069
error happend before creating the thread).
3071
VOID(end_io_cache(&info->rec_cache));
3073
Destroy the new data cache in case of non-quick repair. All slave
3074
threads did either detach from the share by remove_io_thread()
3075
already or they were not yet started (if the error happend before
3076
creating the threads).
3079
VOID(end_io_cache(&new_data_cache));
3082
/* Replace the actual file with the temporary file */
3085
my_close(new_file,MYF(0));
3086
info->dfile=new_file= -1;
3087
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
3088
DATA_TMP_EXT, share->base.raid_chunks,
3089
(param->testflag & T_BACKUP_DATA ?
3090
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
3091
mi_open_datafile(info,share,name,-1))
3097
if (! param->error_printed)
3098
mi_check_print_error(param,"%d when fixing table",my_errno);
3101
VOID(my_close(new_file,MYF(0)));
3102
VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
3104
if (info->dfile == new_file) /* Retry with key cache */
3105
if (unlikely(mi_open_datafile(info, share, name, -1)))
3106
param->retry_repair= 0; /* Safety */
3108
mi_mark_crashed_on_repair(info);
3110
else if (key_map == share->state.key_map)
3111
share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
3112
share->state.changed|=STATE_NOT_SORTED_PAGES;
3114
pthread_cond_destroy (&sort_info.cond);
3115
pthread_mutex_destroy(&sort_info.mutex);
3116
pthread_mutex_destroy(¶m->print_msg_mutex);
3117
param->need_print_msg_lock= 0;
3119
my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
3120
my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
3121
my_free((uchar*) sort_param,MYF(MY_ALLOW_ZERO_PTR));
3122
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
3123
VOID(end_io_cache(¶m->read_cache));
3124
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
3125
if (!got_error && (param->testflag & T_UNPACK))
3127
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
3128
share->pack.header_length=0;
3130
DBUG_RETURN(got_error);
3134
/* Read next record and return next key */
3136
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
3139
SORT_INFO *sort_info=sort_param->sort_info;
3140
MI_INFO *info=sort_info->info;
3141
DBUG_ENTER("sort_key_read");
3143
if ((error=sort_get_next_record(sort_param)))
3145
if (info->state->records == sort_info->max_records)
3147
mi_check_print_error(sort_info->param,
3148
"Key %d - Found too many records; Can't continue",
3152
sort_param->real_key_length=
3153
(info->s->rec_reflength+
3154
_mi_make_key(info, sort_param->key, (uchar*) key,
3155
sort_param->record, sort_param->filepos));
3157
bzero(key+sort_param->real_key_length,
3158
(sort_param->key_length-sort_param->real_key_length));
3160
DBUG_RETURN(sort_write_record(sort_param));
3161
} /* sort_key_read */
3163
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3166
SORT_INFO *sort_info=sort_param->sort_info;
3167
MI_INFO *info=sort_info->info;
3169
DBUG_ENTER("sort_ft_key_read");
3171
if (!sort_param->wordlist)
3175
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3176
if ((error=sort_get_next_record(sort_param)))
3178
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
3179
&sort_param->wordroot)))
3183
error=sort_write_record(sort_param);
3185
sort_param->wordptr=sort_param->wordlist=wptr;
3190
wptr=(FT_WORD*)(sort_param->wordptr);
3193
sort_param->real_key_length=(info->s->rec_reflength+
3194
_ft_make_key(info, sort_param->key,
3195
key, wptr++, sort_param->filepos));
3197
if (sort_param->key_length > sort_param->real_key_length)
3198
bzero(key+sort_param->real_key_length,
3199
(sort_param->key_length-sort_param->real_key_length));
3203
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3204
sort_param->wordlist=0;
3205
error=sort_write_record(sort_param);
3208
sort_param->wordptr=(void*)wptr;
3211
} /* sort_ft_key_read */
3215
Read next record from file using parameters in sort_info.
3218
sort_get_next_record()
3219
sort_param Information about and for the sort process
3223
Dynamic Records With Non-Quick Parallel Repair
3225
For non-quick parallel repair we use a synchronized read/write
3226
cache. This means that one thread is the master who fixes the data
3227
file by reading each record from the old data file and writing it
3228
to the new data file. By doing this the records in the new data
3229
file are written contiguously. Whenever the write buffer is full,
3230
it is copied to the read buffer. The slaves read from the read
3231
buffer, which is not associated with a file. Thus read_cache.file
3232
is -1. When using _mi_read_cache(), the slaves must always set
3233
flag to READING_NEXT so that the function never tries to read from
3234
file. This is safe because the records are contiguous. There is no
3235
need to read outside the cache. This condition is evaluated in the
3236
variable 'parallel_flag' for quick reference. read_cache.file must
3237
be >= 0 in every other case.
3245
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
3249
uint found_record,b_type,left_length;
3251
uchar *UNINIT_VAR(to);
3252
MI_BLOCK_INFO block_info;
3253
SORT_INFO *sort_info=sort_param->sort_info;
3254
MI_CHECK *param=sort_info->param;
3255
MI_INFO *info=sort_info->info;
3256
MYISAM_SHARE *share=info->s;
3257
char llbuff[22],llbuff2[22];
3258
DBUG_ENTER("sort_get_next_record");
3260
if (*killed_ptr(param))
3263
switch (share->data_file_type) {
3267
if (my_b_read(&sort_param->read_cache,sort_param->record,
3268
share->base.pack_reclength))
3270
if (sort_param->read_cache.error)
3271
param->out_flag |= O_DATA_LOST;
3272
param->retry_repair=1;
3273
param->testflag|=T_RETRY_WITHOUT_QUICK;
3276
sort_param->start_recpos=sort_param->pos;
3277
if (!sort_param->fix_datafile)
3279
sort_param->filepos=sort_param->pos;
3280
if (sort_param->master)
3281
share->state.split++;
3283
sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
3284
if (*sort_param->record)
3286
if (sort_param->calc_checksum)
3287
param->glob_crc+= (info->checksum=
3288
mi_static_checksum(info,sort_param->record));
3291
if (!sort_param->fix_datafile && sort_param->master)
3294
info->state->empty+=share->base.pack_reclength;
3297
case DYNAMIC_RECORD:
3299
pos=sort_param->pos;
3300
searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
3301
parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
3304
found_record=block_info.second_read= 0;
3308
pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
3309
param->testflag|=T_RETRY_WITHOUT_QUICK;
3310
sort_param->start_recpos=pos;
3314
if (pos > sort_param->max_pos)
3315
sort_param->max_pos=pos;
3316
if (pos & (MI_DYN_ALIGN_SIZE-1))
3318
if ((param->testflag & T_VERBOSE) || searching == 0)
3319
mi_check_print_info(param,"Wrong aligned block at %s",
3324
if (found_record && pos == param->search_after_block)
3325
mi_check_print_info(param,"Block: %s used by record at %s",
3326
llstr(param->search_after_block,llbuff),
3327
llstr(sort_param->start_recpos,llbuff2));
3328
if (_mi_read_cache(&sort_param->read_cache,
3329
(uchar*) block_info.header,pos,
3330
MI_BLOCK_INFO_HEADER_LENGTH,
3331
(! found_record ? READING_NEXT : 0) |
3332
parallel_flag | READING_HEADER))
3336
mi_check_print_info(param,
3337
"Can't read whole record at %s (errno: %d)",
3338
llstr(sort_param->start_recpos,llbuff),errno);
3343
if (searching && ! sort_param->fix_datafile)
3345
param->error_printed=1;
3346
param->retry_repair=1;
3347
param->testflag|=T_RETRY_WITHOUT_QUICK;
3348
DBUG_RETURN(1); /* Something wrong with data */
3350
b_type=_mi_get_block_info(&block_info,-1,pos);
3351
if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
3352
((b_type & BLOCK_FIRST) &&
3353
(block_info.rec_len < (uint) share->base.min_pack_length ||
3354
block_info.rec_len > (uint) share->base.max_pack_length)))
3357
if (param->testflag & T_VERBOSE || searching == 0)
3358
mi_check_print_info(param,
3359
"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
3360
block_info.header[0],block_info.header[1],
3361
block_info.header[2],llstr(pos,llbuff));
3364
block_info.second_read=0;
3366
/* Search after block in read header string */
3367
for (i=MI_DYN_ALIGN_SIZE ;
3368
i < MI_BLOCK_INFO_HEADER_LENGTH ;
3369
i+= MI_DYN_ALIGN_SIZE)
3370
if (block_info.header[i] >= 1 &&
3371
block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
3374
sort_param->start_recpos=pos;
3377
if (b_type & BLOCK_DELETED)
3380
if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3381
share->base.min_block_length)
3384
mi_check_print_info(param,
3385
"Deleted block with impossible length %u at %s",
3386
block_info.block_len,llstr(pos,llbuff));
3391
if ((block_info.next_filepos != HA_OFFSET_ERROR &&
3392
block_info.next_filepos >=
3393
info->state->data_file_length) ||
3394
(block_info.prev_filepos != HA_OFFSET_ERROR &&
3395
block_info.prev_filepos >= info->state->data_file_length))
3398
mi_check_print_info(param,
3399
"Delete link points outside datafile at %s",
3409
pos+= MI_DYN_ALIGN_SIZE;
3410
sort_param->start_recpos=pos;
3411
block_info.second_read=0;
3417
if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3418
share->base.min_block_length ||
3419
block_info.block_len > (uint) share->base.max_pack_length+
3423
mi_check_print_info(param,
3424
"Found block with impossible length %u at %s; Skipped",
3425
block_info.block_len+ (uint) (block_info.filepos-pos),
3430
pos+= MI_DYN_ALIGN_SIZE;
3431
sort_param->start_recpos=pos;
3432
block_info.second_read=0;
3436
if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
3438
if (!sort_param->fix_datafile && sort_param->master &&
3439
(b_type & BLOCK_DELETED))
3441
info->state->empty+=block_info.block_len;
3443
share->state.split++;
3449
pos+=MI_DYN_ALIGN_SIZE;
3450
sort_param->start_recpos=pos;
3453
pos=block_info.filepos+block_info.block_len;
3454
block_info.second_read=0;
3458
if (!sort_param->fix_datafile && sort_param->master)
3459
share->state.split++;
3460
if (! found_record++)
3462
sort_param->find_length=left_length=block_info.rec_len;
3463
sort_param->start_recpos=pos;
3464
if (!sort_param->fix_datafile)
3465
sort_param->filepos=sort_param->start_recpos;
3466
if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
3467
sort_param->pos=block_info.filepos+1;
3469
sort_param->pos=block_info.filepos+block_info.block_len;
3470
if (share->base.blobs)
3472
if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3473
&(sort_param->rec_buff))))
3475
if (param->max_record_length >= block_info.rec_len)
3477
mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
3478
llstr(sort_param->start_recpos,llbuff),
3479
(ulong) block_info.rec_len);
3484
mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
3485
llstr(sort_param->start_recpos,llbuff),
3486
(ulong) block_info.rec_len);
3492
to= sort_param->rec_buff;
3494
if (left_length < block_info.data_len || ! block_info.data_len)
3496
mi_check_print_info(param,
3497
"Found block with too small length at %s; Skipped",
3498
llstr(sort_param->start_recpos,llbuff));
3501
if (block_info.filepos + block_info.data_len >
3502
sort_param->read_cache.end_of_file)
3504
mi_check_print_info(param,
3505
"Found block that points outside data file at %s",
3506
llstr(sort_param->start_recpos,llbuff));
3510
Copy information that is already read. Avoid accessing data
3511
below the cache start. This could happen if the header
3512
streched over the end of the previous buffer contents.
3515
uint header_len= (uint) (block_info.filepos - pos);
3516
uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
3518
if (prefetch_len > block_info.data_len)
3519
prefetch_len= block_info.data_len;
3522
memcpy(to, block_info.header + header_len, prefetch_len);
3523
block_info.filepos+= prefetch_len;
3524
block_info.data_len-= prefetch_len;
3525
left_length-= prefetch_len;
3529
if (block_info.data_len &&
3530
_mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
3531
block_info.data_len,
3532
(found_record == 1 ? READING_NEXT : 0) |
3535
mi_check_print_info(param,
3536
"Read error for block at: %s (error: %d); Skipped",
3537
llstr(block_info.filepos,llbuff),my_errno);
3540
left_length-=block_info.data_len;
3541
to+=block_info.data_len;
3542
pos=block_info.next_filepos;
3543
if (pos == HA_OFFSET_ERROR && left_length)
3545
mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
3546
llstr(sort_param->start_recpos,llbuff));
3549
if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
3551
mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3553
llstr(sort_param->start_recpos,llbuff));
3556
} while (left_length);
3558
if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
3559
sort_param->find_length) != MY_FILE_ERROR)
3561
if (sort_param->read_cache.error < 0)
3563
if (sort_param->calc_checksum)
3564
info->checksum= mi_checksum(info, sort_param->record);
3565
if ((param->testflag & (T_EXTEND | T_REP)) || searching)
3567
if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3568
sort_param->find_length,
3569
(param->testflag & T_QUICK) &&
3570
sort_param->calc_checksum &&
3571
test(info->s->calc_checksum)))
3573
mi_check_print_info(param,"Found wrong packed record at %s",
3574
llstr(sort_param->start_recpos,llbuff));
3578
if (sort_param->calc_checksum)
3579
param->glob_crc+= info->checksum;
3583
mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
3585
llstr(sort_param->start_recpos,llbuff));
3587
pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
3590
case COMPRESSED_RECORD:
3591
for (searching=0 ;; searching=1, sort_param->pos++)
3593
if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
3595
share->pack.ref_length,READING_NEXT))
3597
if (searching && ! sort_param->fix_datafile)
3599
param->error_printed=1;
3600
param->retry_repair=1;
3601
param->testflag|=T_RETRY_WITHOUT_QUICK;
3602
DBUG_RETURN(1); /* Something wrong with data */
3604
sort_param->start_recpos=sort_param->pos;
3605
if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
3606
&sort_param->rec_buff, -1, sort_param->pos))
3608
if (!block_info.rec_len &&
3609
sort_param->pos + MEMMAP_EXTRA_MARGIN ==
3610
sort_param->read_cache.end_of_file)
3612
if (block_info.rec_len < (uint) share->min_pack_length ||
3613
block_info.rec_len > (uint) share->max_pack_length)
3616
mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3618
llstr(sort_param->pos,llbuff));
3621
if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3622
block_info.filepos, block_info.rec_len,
3626
mi_check_print_info(param,"Couldn't read whole record from %s",
3627
llstr(sort_param->pos,llbuff));
3630
if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
3631
sort_param->rec_buff, block_info.rec_len))
3634
mi_check_print_info(param,"Found wrong record at %s",
3635
llstr(sort_param->pos,llbuff));
3638
if (!sort_param->fix_datafile)
3640
sort_param->filepos=sort_param->pos;
3641
if (sort_param->master)
3642
share->state.split++;
3644
sort_param->max_pos=(sort_param->pos=block_info.filepos+
3645
block_info.rec_len);
3646
info->packed_length=block_info.rec_len;
3647
if (sort_param->calc_checksum)
3648
param->glob_crc+= (info->checksum=
3649
mi_checksum(info, sort_param->record));
3653
assert(0); /* Impossible */
3655
DBUG_RETURN(1); /* Impossible */
3660
Write record to new file.
3664
sort_param Sort parameters.
3667
This is only called by a master thread if parallel repair is used.
3674
int sort_write_record(MI_SORT_PARAM *sort_param)
3678
ulong block_length,reclength;
3680
uchar block_buff[8];
3681
SORT_INFO *sort_info=sort_param->sort_info;
3682
MI_CHECK *param=sort_info->param;
3683
MI_INFO *info=sort_info->info;
3684
MYISAM_SHARE *share=info->s;
3685
DBUG_ENTER("sort_write_record");
3687
if (sort_param->fix_datafile)
3689
switch (sort_info->new_data_file_type) {
3691
if (my_b_write(&info->rec_cache,sort_param->record,
3692
share->base.pack_reclength))
3694
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3697
sort_param->filepos+=share->base.pack_reclength;
3698
info->s->state.split++;
3699
/* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
3701
case DYNAMIC_RECORD:
3703
from=sort_param->rec_buff;
3706
/* must be sure that local buffer is big enough */
3707
reclength=info->s->base.pack_reclength+
3708
_my_calc_total_blob_length(info,sort_param->record)+
3709
ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
3710
MI_DYN_DELETE_BLOCK_HEADER;
3711
if (sort_info->buff_length < reclength)
3713
if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3714
MYF(MY_FREE_ON_ERROR |
3715
MY_ALLOW_ZERO_PTR))))
3717
sort_info->buff_length=reclength;
3719
from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
3721
/* We can use info->checksum here as only one thread calls this. */
3722
info->checksum=mi_checksum(info,sort_param->record);
3723
reclength=_mi_rec_pack(info,from,sort_param->record);
3725
/* sort_info->param->glob_crc+=info->checksum; */
3729
block_length=reclength+ 3 + test(reclength >= (65520-3));
3730
if (block_length < share->base.min_block_length)
3731
block_length=share->base.min_block_length;
3732
info->update|=HA_STATE_WRITE_AT_END;
3733
block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
3734
if (block_length > MI_MAX_BLOCK_LENGTH)
3735
block_length=MI_MAX_BLOCK_LENGTH;
3736
if (_mi_write_part_record(info,0L,block_length,
3737
sort_param->filepos+block_length,
3738
&from,&reclength,&flag))
3740
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3743
sort_param->filepos+=block_length;
3744
info->s->state.split++;
3745
} while (reclength);
3746
/* sort_info->param->glob_crc+=info->checksum; */
3748
case COMPRESSED_RECORD:
3749
reclength=info->packed_length;
3750
length= save_pack_length((uint) share->pack.version, block_buff,
3752
if (info->s->base.blobs)
3753
length+= save_pack_length((uint) share->pack.version,
3754
block_buff + length, info->blob_length);
3755
if (my_b_write(&info->rec_cache,block_buff,length) ||
3756
my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
3758
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3761
/* sort_info->param->glob_crc+=info->checksum; */
3762
sort_param->filepos+=reclength+length;
3763
info->s->state.split++;
3766
assert(0); /* Impossible */
3769
if (sort_param->master)
3771
info->state->records++;
3772
if ((param->testflag & T_WRITE_LOOP) &&
3773
(info->state->records % WRITE_COUNT) == 0)
3776
printf("%s\r", llstr(info->state->records,llbuff));
3777
VOID(fflush(stdout));
3781
} /* sort_write_record */
3784
/* Compare two keys from _create_index_by_sort */
3786
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
3790
return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3791
USE_WHOLE_KEY, SEARCH_SAME, not_used));
3792
} /* sort_key_cmp */
3795
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
3798
char llbuff[22],llbuff2[22];
3799
SORT_INFO *sort_info=sort_param->sort_info;
3800
MI_CHECK *param= sort_info->param;
3803
if (sort_info->key_block->inited)
3805
cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3806
(uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3808
if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
3809
ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3810
(uchar*) a, USE_WHOLE_KEY,
3811
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3812
else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3814
diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
3815
sort_param->notnull,
3816
sort_info->key_block->lastkey,
3819
sort_param->unique[diff_pos[0]-1]++;
3824
if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3825
mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
3828
if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
3831
sort_info->info->lastpos=get_record_for_key(sort_info->info,
3832
sort_param->keyinfo,
3834
mi_check_print_warning(param,
3835
"Duplicate key for record at %10s against record at %10s",
3836
llstr(sort_info->info->lastpos,llbuff),
3837
llstr(get_record_for_key(sort_info->info,
3838
sort_param->keyinfo,
3839
sort_info->key_block->
3842
param->testflag|=T_RETRY_WITHOUT_QUICK;
3843
if (sort_info->param->testflag & T_VERBOSE)
3844
_mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
3845
return (sort_delete_record(sort_param));
3850
mi_check_print_error(param,
3851
"Internal error: Keys are not in order from sort");
3855
return (sort_insert_key(sort_param,sort_info->key_block,
3856
(uchar*) a, HA_OFFSET_ERROR));
3857
} /* sort_key_write */
3859
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
3861
SORT_INFO *sort_info=sort_param->sort_info;
3862
SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3863
MYISAM_SHARE *share=sort_info->info->s;
3864
uint val_off, val_len;
3866
SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3869
val_len=share->ft2_keyinfo.keylength;
3870
get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3871
to=ft_buf->lastkey+val_off;
3875
/* flushing first-level tree */
3876
error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3878
for (from=to+val_len;
3879
!error && from < ft_buf->buf;
3882
memcpy(to, from, val_len);
3883
error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3888
/* flushing second-level tree keyblocks */
3889
error=flush_pending_blocks(sort_param);
3890
/* updating lastkey with second-level tree info */
3891
ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
3892
_mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
3893
share->state.key_root[sort_param->key]);
3894
/* restoring first level tree data in sort_info/sort_param */
3895
sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
3896
sort_param->keyinfo=share->keyinfo+sort_param->key;
3897
share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
3898
/* writing lastkey in first-level tree */
3899
return error ? error :
3900
sort_insert_key(sort_param,sort_info->key_block,
3901
ft_buf->lastkey,HA_OFFSET_ERROR);
3904
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
3906
uint a_len, val_off, val_len, error;
3908
SORT_INFO *sort_info=sort_param->sort_info;
3909
SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3910
SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3912
val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
3913
get_key_full_length_rdonly(a_len, (uchar *)a);
3918
use two-level tree only if key_reflength fits in rec_reflength place
3919
and row format is NOT static - for _mi_dpointer not to garble offsets
3921
if ((sort_info->info->s->base.key_reflength <=
3922
sort_info->info->s->base.rec_reflength) &&
3923
(sort_info->info->s->options &
3924
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
3925
ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
3926
sizeof(SORT_FT_BUF), MYF(MY_WME));
3930
sort_param->key_write=sort_key_write;
3931
return sort_key_write(sort_param, a);
3933
sort_info->ft_buf=ft_buf;
3934
goto word_init_ft_buf; /* no need to duplicate the code */
3936
get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3938
if (ha_compare_text(sort_param->seg->charset,
3939
((uchar *)a)+1,a_len-1,
3940
ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3942
if (!ft_buf->buf) /* store in second-level tree */
3945
return sort_insert_key(sort_param,key_block,
3946
((uchar *)a)+a_len, HA_OFFSET_ERROR);
3949
/* storing the key in the buffer. */
3950
memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3951
ft_buf->buf+=val_len;
3952
if (ft_buf->buf < ft_buf->end)
3955
/* converting to two-level tree */
3956
p=ft_buf->lastkey+val_off;
3958
while (key_block->inited)
3960
sort_info->key_block=key_block;
3961
sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
3962
ft_buf->count=(uint) (ft_buf->buf - p)/val_len;
3964
/* flushing buffer to second-level tree */
3965
for (error=0; !error && p < ft_buf->buf; p+= val_len)
3966
error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
3971
/* flushing buffer */
3972
if ((error=sort_ft_buf_flush(sort_param)))
3977
memcpy(ft_buf->lastkey, a, a_len);
3978
ft_buf->buf=ft_buf->lastkey+a_len;
3980
32 is just a safety margin here
3981
(at least max(val_len, sizeof(nod_flag)) should be there).
3982
May be better performance could be achieved if we'd put
3983
(sort_info->keyinfo->block_length-32)/XXX
3985
TODO: benchmark the best value for XXX.
3987
ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
3989
} /* sort_ft_key_write */
3992
/* get pointer to record from a key */
3994
static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
3997
return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
3998
} /* get_record_for_key */
4001
/* Insert a key in sort-key-blocks */
4003
static int sort_insert_key(MI_SORT_PARAM *sort_param,
4004
register SORT_KEY_BLOCKS *key_block, uchar *key,
4005
my_off_t prev_block)
4007
uint a_length,t_length,nod_flag;
4008
my_off_t filepos,key_file_length;
4009
uchar *anc_buff,*lastkey;
4010
MI_KEY_PARAM s_temp;
4012
MI_KEYDEF *keyinfo=sort_param->keyinfo;
4013
SORT_INFO *sort_info= sort_param->sort_info;
4014
MI_CHECK *param=sort_info->param;
4015
DBUG_ENTER("sort_insert_key");
4017
anc_buff=key_block->buff;
4018
info=sort_info->info;
4019
lastkey=key_block->lastkey;
4020
nod_flag= (key_block == sort_info->key_block ? 0 :
4021
info->s->base.key_reflength);
4023
if (!key_block->inited)
4025
key_block->inited=1;
4026
if (key_block == sort_info->key_block_end)
4028
mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
4031
a_length=2+nod_flag;
4032
key_block->end_pos=anc_buff+2;
4033
lastkey=0; /* No previous key in block */
4036
a_length=mi_getint(anc_buff);
4038
/* Save pointer to previous block */
4040
_mi_kpointer(info,key_block->end_pos,prev_block);
4042
t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
4043
(uchar*) 0,lastkey,lastkey,key,
4045
(*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
4047
mi_putint(anc_buff,a_length,nod_flag);
4048
key_block->end_pos+=t_length;
4049
if (a_length <= keyinfo->block_length)
4051
VOID(_mi_move_key(keyinfo,key_block->lastkey,key));
4052
key_block->last_length=a_length-t_length;
4056
/* Fill block with end-zero and write filled block */
4057
mi_putint(anc_buff,key_block->last_length,nod_flag);
4058
bzero((uchar*) anc_buff+key_block->last_length,
4059
keyinfo->block_length- key_block->last_length);
4060
key_file_length=info->state->key_file_length;
4061
if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4064
/* If we read the page from the key cache, we have to write it back to it */
4065
if (key_file_length == info->state->key_file_length)
4067
if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4070
else if (my_pwrite(info->s->kfile,(uchar*) anc_buff,
4071
(uint) keyinfo->block_length,filepos, param->myf_rw))
4073
DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
4075
/* Write separator-key to block in next level */
4076
if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
4079
/* clear old block and write new key in it */
4080
key_block->inited=0;
4081
DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
4082
} /* sort_insert_key */
4085
/* Delete record when we found a duplicated key */
4087
static int sort_delete_record(MI_SORT_PARAM *sort_param)
4092
SORT_INFO *sort_info=sort_param->sort_info;
4093
MI_CHECK *param=sort_info->param;
4094
MI_INFO *info=sort_info->info;
4095
DBUG_ENTER("sort_delete_record");
4097
if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
4099
mi_check_print_error(param,
4100
"Quick-recover aborted; Run recovery without switch -q or with switch -qq");
4103
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4105
mi_check_print_error(param,
4106
"Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
4110
old_file=info->dfile;
4111
info->dfile=info->rec_cache.file;
4112
if (sort_info->current_key)
4114
key=info->lastkey+info->s->base.max_key_length;
4115
if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
4116
error != HA_ERR_RECORD_DELETED)
4118
mi_check_print_error(param,"Can't read record to be removed");
4119
info->dfile=old_file;
4123
for (i=0 ; i < sort_info->current_key ; i++)
4125
uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
4126
if (_mi_ck_delete(info,i,key,key_length))
4128
mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
4129
info->dfile=old_file;
4133
if (sort_param->calc_checksum)
4134
param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
4136
error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
4137
info->dfile=old_file; /* restore actual value */
4138
info->state->records--;
4140
} /* sort_delete_record */
4142
/* Fix all pending blocks and flush everything to disk */
4144
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
4146
uint nod_flag,length;
4147
my_off_t filepos,key_file_length;
4148
SORT_KEY_BLOCKS *key_block;
4149
SORT_INFO *sort_info= sort_param->sort_info;
4150
myf myf_rw=sort_info->param->myf_rw;
4151
MI_INFO *info=sort_info->info;
4152
MI_KEYDEF *keyinfo=sort_param->keyinfo;
4153
DBUG_ENTER("flush_pending_blocks");
4155
filepos= HA_OFFSET_ERROR; /* if empty file */
4157
for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
4159
key_block->inited=0;
4160
length=mi_getint(key_block->buff);
4162
_mi_kpointer(info,key_block->end_pos,filepos);
4163
key_file_length=info->state->key_file_length;
4164
bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4165
if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4168
/* If we read the page from the key cache, we have to write it back */
4169
if (key_file_length == info->state->key_file_length)
4171
if (_mi_write_keypage(info, keyinfo, filepos,
4172
DFLT_INIT_HITS, key_block->buff))
4175
else if (my_pwrite(info->s->kfile,(uchar*) key_block->buff,
4176
(uint) keyinfo->block_length,filepos, myf_rw))
4178
DBUG_DUMP("buff",(uchar*) key_block->buff,length);
4181
info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
4183
} /* flush_pending_blocks */
4185
/* alloc space and pointers for key_blocks */
4187
static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
4191
SORT_KEY_BLOCKS *block;
4192
DBUG_ENTER("alloc_key_blocks");
4194
if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
4195
buffer_length+IO_SIZE)*blocks,
4198
mi_check_print_error(param,"Not enough memory for sort-key-blocks");
4201
for (i=0 ; i < blocks ; i++)
4204
block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
4207
} /* alloc_key_blocks */
4210
/* Check if file is almost full */
4212
int test_if_almost_full(MI_INFO *info)
4214
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4216
return my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)) / 10 * 9 >
4217
(my_off_t) info->s->base.max_key_file_length ||
4218
my_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
4219
(my_off_t) info->s->base.max_data_file_length;
4222
/* Recreate table with bigger more alloced record-data */
4224
int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
4229
MI_KEYDEF *keyinfo,*key,*key_end;
4230
HA_KEYSEG *keysegs,*keyseg;
4231
MI_COLUMNDEF *recdef,*rec,*end;
4232
MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
4233
MI_STATUS_INFO status_info;
4234
uint unpack,key_parts;
4235
ha_rows max_records;
4236
ulonglong file_length,tmp_length;
4237
MI_CREATE_INFO create_info;
4238
DBUG_ENTER("recreate_table");
4240
error=1; /* Default error */
4242
status_info= (*org_info)->state[0];
4243
info.state= &status_info;
4244
share= *(*org_info)->s;
4245
unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4246
(param->testflag & T_UNPACK);
4247
if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4249
memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
4250
(size_t) (sizeof(MI_KEYDEF)*share.base.keys));
4252
key_parts= share.base.all_key_parts;
4253
if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
4254
(key_parts+share.base.keys))))
4256
my_afree((uchar*) keyinfo);
4259
if (!(recdef=(MI_COLUMNDEF*)
4260
my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
4262
my_afree((uchar*) keyinfo);
4263
my_afree((uchar*) keysegs);
4266
if (!(uniquedef=(MI_UNIQUEDEF*)
4267
my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
4269
my_afree((uchar*) recdef);
4270
my_afree((uchar*) keyinfo);
4271
my_afree((uchar*) keysegs);
4275
/* Copy the column definitions */
4276
memcpy((uchar*) recdef,(uchar*) share.rec,
4277
(size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
4278
for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
4280
if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
4281
rec->type != FIELD_BLOB &&
4282
rec->type != FIELD_VARCHAR &&
4283
rec->type != FIELD_CHECK)
4284
rec->type=(int) FIELD_NORMAL;
4287
/* Change the new key to point at the saved key segments */
4288
memcpy((uchar*) keysegs,(uchar*) share.keyparts,
4289
(size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
4290
share.state.header.uniques)));
4292
for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
4295
for (; keyseg->type ; keyseg++)
4297
if (param->language)
4298
keyseg->language=param->language; /* change language */
4300
keyseg++; /* Skip end pointer */
4303
/* Copy the unique definitions and change them to point at the new key
4305
memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
4306
(size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
4307
for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
4308
u_ptr != u_end ; u_ptr++)
4311
keyseg+=u_ptr->keysegs+1;
4313
if (share.options & HA_OPTION_COMPRESS_RECORD)
4314
share.base.records=max_records=info.state->records;
4315
else if (share.base.min_pack_length)
4316
max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
4317
(ulong) share.base.min_pack_length);
4320
unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4321
(param->testflag & T_UNPACK);
4322
share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;
4324
file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
4325
tmp_length= file_length+file_length/10;
4326
set_if_bigger(file_length,param->max_data_file_length);
4327
set_if_bigger(file_length,tmp_length);
4328
set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);
4330
VOID(mi_close(*org_info));
4331
bzero((char*) &create_info,sizeof(create_info));
4332
create_info.max_rows=max(max_records,share.base.records);
4333
create_info.reloc_rows=share.base.reloc;
4334
create_info.old_options=(share.options |
4335
(unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));
4337
create_info.data_file_length=file_length;
4338
create_info.auto_increment=share.state.auto_increment;
4339
create_info.language = (param->language ? param->language :
4340
share.state.header.language);
4341
create_info.key_file_length= status_info.key_file_length;
4343
Allow for creating an auto_increment key. This has an effect only if
4344
an auto_increment key exists in the original table.
4346
create_info.with_auto_increment= TRUE;
4347
/* We don't have to handle symlinks here because we are using
4348
HA_DONT_TOUCH_DATA */
4349
if (mi_create(filename,
4350
share.base.keys - share.state.header.uniques,
4351
keyinfo, share.base.fields, recdef,
4352
share.state.header.uniques, uniquedef,
4354
HA_DONT_TOUCH_DATA))
4356
mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
4359
*org_info=mi_open(filename,O_RDWR,
4360
(param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
4361
(param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
4362
HA_OPEN_ABORT_IF_LOCKED);
4365
mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
4369
/* We are modifing */
4370
(*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
4371
VOID(_mi_readinfo(*org_info,F_WRLCK,0));
4372
(*org_info)->state->records=info.state->records;
4373
if (share.state.create_time)
4374
(*org_info)->s->state.create_time=share.state.create_time;
4375
(*org_info)->s->state.unique=(*org_info)->this_unique=
4377
(*org_info)->state->checksum=info.state->checksum;
4378
(*org_info)->state->del=info.state->del;
4379
(*org_info)->s->state.dellink=share.state.dellink;
4380
(*org_info)->state->empty=info.state->empty;
4381
(*org_info)->state->data_file_length=info.state->data_file_length;
4382
if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
4387
my_afree((uchar*) uniquedef);
4388
my_afree((uchar*) keyinfo);
4389
my_afree((uchar*) recdef);
4390
my_afree((uchar*) keysegs);
4395
/* write suffix to data file if neaded */
4397
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
4399
MI_INFO *info=sort_info->info;
4401
if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
4403
uchar buff[MEMMAP_EXTRA_MARGIN];
4404
bzero(buff,sizeof(buff));
4405
if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
4407
mi_check_print_error(sort_info->param,
4408
"%d when writing to datafile",my_errno);
4411
sort_info->param->read_cache.end_of_file+=sizeof(buff);
4416
/* Update state and myisamchk_time of indexfile */
4418
int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
4420
MYISAM_SHARE *share=info->s;
4422
if (update & UPDATE_OPEN_COUNT)
4424
share->state.open_count=0;
4425
share->global_changed=0;
4427
if (update & UPDATE_STAT)
4429
uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
4430
share->state.rec_per_key_rows=info->state->records;
4431
share->state.changed&= ~STATE_NOT_ANALYZED;
4432
if (info->state->records)
4434
for (i=0; i<key_parts; i++)
4436
if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
4437
share->state.changed|= STATE_NOT_ANALYZED;
4441
if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
4443
if (update & UPDATE_TIME)
4445
share->state.check_time= (long) time((time_t*) 0);
4446
if (!share->state.create_time)
4447
share->state.create_time=share->state.check_time;
4450
When tables are locked we haven't synched the share state and the
4451
real state for a while so we better do it here before synching
4452
the share state to disk. Only when table is write locked is it
4453
necessary to perform this synch.
4455
if (info->lock_type == F_WRLCK)
4456
share->state.state= *info->state;
4457
if (mi_state_info_write(share->kfile,&share->state,1+2))
4461
{ /* Force update of status */
4463
uint r_locks=share->r_locks,w_locks=share->w_locks;
4464
share->r_locks= share->w_locks= share->tot_locks= 0;
4465
error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4466
share->r_locks=r_locks;
4467
share->w_locks=w_locks;
4468
share->tot_locks=r_locks+w_locks;
4473
mi_check_print_error(param,"%d when updating keyfile",my_errno);
4478
Update auto increment value for a table
4479
When setting the 'repair_only' flag we only want to change the
4480
old auto_increment value if its wrong (smaller than some given key).
4481
The reason is that we shouldn't change the auto_increment value
4482
for a table without good reason when only doing a repair; If the
4483
user have inserted and deleted rows, the auto_increment value
4484
may be bigger than the biggest current row and this is ok.
4486
If repair_only is not set, we will update the flag to the value in
4487
param->auto_increment is bigger than the biggest key.
4490
void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
4491
my_bool repair_only)
4494
DBUG_ENTER("update_auto_increment_key");
4496
if (!info->s->base.auto_key ||
4497
! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
4499
if (!(param->testflag & T_VERY_SILENT))
4500
mi_check_print_info(param,
4501
"Table: %s doesn't have an auto increment key\n",
4502
param->isam_file_name);
4505
if (!(param->testflag & T_SILENT) &&
4506
!(param->testflag & T_REP))
4507
printf("Updating MyISAM file: %s\n", param->isam_file_name);
4509
We have to use an allocated buffer instead of info->rec_buff as
4510
_mi_put_key_in_record() may use info->rec_buff
4512
if (!mi_alloc_rec_buff(info, -1, &record))
4514
mi_check_print_error(param,"Not enough memory for extra record");
4518
mi_extra(info,HA_EXTRA_KEYREAD,0);
4519
if (mi_rlast(info, record, info->s->base.auto_key-1))
4521
if (my_errno != HA_ERR_END_OF_FILE)
4523
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4524
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4525
mi_check_print_error(param,"%d when reading last record",my_errno);
4529
info->s->state.auto_increment=param->auto_increment_value;
4533
ulonglong auto_increment= retrieve_auto_increment(info, record);
4534
set_if_bigger(info->s->state.auto_increment,auto_increment);
4536
set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
4538
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4539
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4540
update_state_info(param, info, UPDATE_AUTO_INC);
4546
Update statistics for each part of an index
4550
keyinfo IN Index information (only key->keysegs used)
4551
rec_per_key_part OUT Store statistics here
4552
unique IN Array of (#distinct tuples)
4553
notnull_tuples IN Array of (#tuples), or NULL
4554
records Number of records in the table
4557
This function is called produce index statistics values from unique and
4558
notnull_tuples arrays after these arrays were produced with sequential
4559
index scan (the scan is done in two places: chk_index() and
4562
This function handles all 3 index statistics collection methods.
4565
unique[0]= (#different values of {keypart1}) - 1
4566
unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
4569
For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
4570
notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
4571
notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
4572
keypart{i} are not NULL)
4574
For all other statistics collection methods notnull_tuples==NULL.
4577
rec_per_key_part[k] =
4578
= E(#records in the table such that keypart_1=c_1 AND ... AND
4579
keypart_k=c_k for arbitrary constants c_1 ... c_k)
4581
= {assuming that values have uniform distribution and index contains all
4582
tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
4585
= #tuples-in-the-index / #distinct-tuples-in-the-index.
4587
The #tuples-in-the-index and #distinct-tuples-in-the-index have different
4588
meaning depending on which statistics collection method is used:
4590
MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
4591
NULLS_EQUAL NULL == NULL all tuples in table
4592
NULLS_NOT_EQUAL NULL != NULL all tuples in table
4593
IGNORE_NULLS n/a tuples that don't have NULLs
4596
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4597
ulonglong *unique, ulonglong *notnull,
4600
ulonglong count=0,tmp, unique_tuples;
4601
ulonglong tuples= records;
4603
for (parts=0 ; parts < keyinfo->keysegs ; parts++)
4605
count+=unique[parts];
4606
unique_tuples= count + 1;
4609
tuples= notnull[parts];
4611
#(unique_tuples not counting tuples with NULLs) =
4612
#(unique_tuples counting tuples with NULLs as different) -
4613
#(tuples with NULLs)
4615
unique_tuples -= (records - notnull[parts]);
4618
if (unique_tuples == 0)
4620
else if (count == 0)
4621
tmp= tuples; /* 1 unique tuple */
4623
tmp= (tuples + unique_tuples/2) / unique_tuples;
4626
for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
4627
let's ensure it is not
4629
set_if_bigger(tmp,1);
4630
if (tmp >= (ulonglong) ~(ulong) 0)
4631
tmp=(ulonglong) ~(ulong) 0;
4633
*rec_per_key_part=(ulong) tmp;
4639
static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
4642
const uchar *end=buf+length;
4643
for (crc=0; buf != end; buf++)
4644
crc=((crc << 1) + *((uchar*) buf)) +
4645
test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
4649
static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
4651
uint key_maxlength=key->maxlength;
4652
if (key->flag & HA_FULLTEXT)
4654
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
4655
key->seg->charset->mbmaxlen;
4656
key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
4658
return (key->flag & HA_SPATIAL) ||
4659
(key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4660
((ulonglong) rows * key_maxlength >
4661
(ulonglong) myisam_max_temp_length));
4665
Deactivate all not unique index that can be recreated fast
4666
These include packed keys on which sorting will use more temporary
4667
space than the max allowed file length or for which the unpacked keys
4668
will take much more space than packed keys.
4669
Note that 'rows' may be zero for the case when we don't know how many
4670
rows we will put into the file.
4673
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4675
MYISAM_SHARE *share=info->s;
4676
MI_KEYDEF *key=share->keyinfo;
4679
DBUG_ASSERT(info->state->records == 0 &&
4680
(!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
4681
for (i=0 ; i < share->base.keys ; i++,key++)
4683
if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
4684
! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4686
mi_clear_key_active(share->state.key_map, i);
4687
info->update|= HA_STATE_CHANGED;
4694
Return TRUE if we can use repair by sorting
4695
One can set the force argument to force to use sorting
4696
even if the temporary file would be quite big!
4699
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
4700
ulonglong key_map, my_bool force)
4702
MYISAM_SHARE *share=info->s;
4703
MI_KEYDEF *key=share->keyinfo;
4707
mi_repair_by_sort only works if we have at least one key. If we don't
4708
have any keys, we should use the normal repair.
4710
if (! mi_is_any_key_active(key_map))
4711
return FALSE; /* Can't use sort */
4712
for (i=0 ; i < share->base.keys ; i++,key++)
4714
if (!force && mi_too_big_key_for_sort(key,rows))
4722
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4724
if ((sort_info->new_data_file_type=share->data_file_type) ==
4725
COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4729
if (share->options & HA_OPTION_PACK_RECORD)
4730
sort_info->new_data_file_type = DYNAMIC_RECORD;
4732
sort_info->new_data_file_type = STATIC_RECORD;
4734
/* Set delete_function for sort_delete_record() */
4735
memcpy((char*) &tmp, share, sizeof(*share));
4736
tmp.options= ~HA_OPTION_COMPRESS_RECORD;
4737
mi_setup_functions(&tmp);
4738
share->delete_record=tmp.delete_record;
added
removed
storage/myisam/mi_check.c
