« back to all changes in this revision
Viewing changes to patches/innodb_expand_import.patch
- browse files at revision 407
- compare with another revision
- download diff
- download tarball
- view history from revision 407
- Committer: Stewart Smith
- Date: 2011-11-24 08:14:40 UTC
- Revision ID: stewart@flamingspork.com-20111124081440-jffloqgkbgytzgl5
remove now unneeded patches as we're part of a happy bzr tree
- files removed:
- apply_patches
- patches
added
removed
patches/innodb_expand_import.patch
1
# name : innodb_expand_import.patch
2
# introduced : 11 or before
3
# maintainer : Yasufumi
4
#
5
#!!! notice !!!
6
# Any small change to this file in the main branch
7
# should be done or reviewed by the maintainer!
8
--- a/storage/innodb_plugin/btr/btr0btr.c
9
+++ b/storage/innodb_plugin/btr/btr0btr.c
10
@@ -837,7 +837,7 @@
11
/**************************************************************//**
12
Creates a new index page (not the root, and also not
13
used in page reorganization). @see btr_page_empty(). */
14
-static
15
+UNIV_INTERN
16
void
17
btr_page_create(
18
/*============*/
19
@@ -1704,7 +1704,7 @@
20
#ifndef UNIV_HOTBACKUP
21
/*************************************************************//**
22
Empties an index page. @see btr_page_create(). */
23
-static
24
+UNIV_INTERN
25
void
26
btr_page_empty(
27
/*===========*/
28
@@ -2266,7 +2266,7 @@
29
/**************************************************************//**
30
Attaches the halves of an index page on the appropriate level in an
31
index tree. */
32
-static
33
+UNIV_INTERN
34
void
35
btr_attach_half_pages(
36
/*==================*/
37
--- a/storage/innodb_plugin/fil/fil0fil.c
38
+++ b/storage/innodb_plugin/fil/fil0fil.c
39
@@ -40,6 +40,14 @@
40
#include "dict0dict.h"
41
#include "page0page.h"
42
#include "page0zip.h"
43
+#include "trx0trx.h"
44
+#include "trx0sys.h"
45
+#include "pars0pars.h"
46
+#include "row0mysql.h"
47
+#include "row0row.h"
48
+#include "que0que.h"
49
+#include "btr0btr.h"
50
+#include "btr0sea.h"
51
#ifndef UNIV_HOTBACKUP
52
# include "buf0lru.h"
53
# include "ibuf0ibuf.h"
54
@@ -2984,6 +2992,84 @@
55
}
56
57
/********************************************************************//**
58
+Checks if a page is corrupt. (for offline page)
59
+*/
60
+static
61
+ibool
62
+fil_page_buf_page_is_corrupted_offline(
63
+/*===================================*/
64
+ const byte* page, /*!< in: a database page */
65
+ ulint zip_size) /*!< in: size of compressed page;
66
+ 0 for uncompressed pages */
67
+{
68
+ ulint checksum_field;
69
+ ulint old_checksum_field;
70
+
71
+ if (!zip_size
72
+ && memcmp(page + FIL_PAGE_LSN + 4,
73
+ page + UNIV_PAGE_SIZE
74
+ - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) {
75
+ return(TRUE);
76
+ }
77
+
78
+ checksum_field = mach_read_from_4(page
79
+ + FIL_PAGE_SPACE_OR_CHKSUM);
80
+
81
+ if (zip_size) {
82
+ return(checksum_field != BUF_NO_CHECKSUM_MAGIC
83
+ && checksum_field
84
+ != page_zip_calc_checksum(page, zip_size));
85
+ }
86
+
87
+ old_checksum_field = mach_read_from_4(
88
+ page + UNIV_PAGE_SIZE
89
+ - FIL_PAGE_END_LSN_OLD_CHKSUM);
90
+
91
+ if (old_checksum_field != mach_read_from_4(page
92
+ + FIL_PAGE_LSN)
93
+ && old_checksum_field != BUF_NO_CHECKSUM_MAGIC
94
+ && old_checksum_field
95
+ != buf_calc_page_old_checksum(page)) {
96
+ return(TRUE);
97
+ }
98
+
99
+ if (checksum_field != 0
100
+ && checksum_field != BUF_NO_CHECKSUM_MAGIC
101
+ && checksum_field
102
+ != buf_calc_page_new_checksum(page)) {
103
+ return(TRUE);
104
+ }
105
+
106
+ return(FALSE);
107
+}
108
+
109
+/********************************************************************//**
110
+*/
111
+static
112
+void
113
+fil_page_buf_page_store_checksum(
114
+/*=============================*/
115
+ byte* page,
116
+ ulint zip_size)
117
+{
118
+ if (!zip_size) {
119
+ mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM,
120
+ srv_use_checksums
121
+ ? buf_calc_page_new_checksum(page)
122
+ : BUF_NO_CHECKSUM_MAGIC);
123
+ mach_write_to_4(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
124
+ srv_use_checksums
125
+ ? buf_calc_page_old_checksum(page)
126
+ : BUF_NO_CHECKSUM_MAGIC);
127
+ } else {
128
+ mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM,
129
+ srv_use_checksums
130
+ ? page_zip_calc_checksum(page, zip_size)
131
+ : BUF_NO_CHECKSUM_MAGIC);
132
+ }
133
+}
134
+
135
+/********************************************************************//**
136
Tries to open a single-table tablespace and optionally checks the space id is
137
right in it. If does not succeed, prints an error message to the .err log. This
138
function is used to open a tablespace when we start up mysqld, and also in
139
@@ -3029,7 +3115,7 @@
140
ut_a(!(flags & (~0UL << DICT_TF_BITS)));
141
142
file = os_file_create_simple_no_error_handling(
143
- filepath, OS_FILE_OPEN, OS_FILE_READ_ONLY, &success);
144
+ filepath, OS_FILE_OPEN, OS_FILE_READ_WRITE, &success);
145
if (!success) {
146
/* The following call prints an error message */
147
os_file_get_last_error(TRUE);
148
@@ -3076,6 +3162,445 @@
149
space_id = fsp_header_get_space_id(page);
150
space_flags = fsp_header_get_flags(page);
151
152
+ if (srv_expand_import) {
153
+
154
+ ibool file_is_corrupt = FALSE;
155
+ byte* buf3;
156
+ byte* descr_page;
157
+ ibool descr_is_corrupt = FALSE;
158
+ dulint old_id[31];
159
+ dulint new_id[31];
160
+ ulint root_page[31];
161
+ ulint n_index;
162
+ os_file_t info_file = -1;
163
+ char* info_file_path;
164
+ ulint i;
165
+ int len;
166
+ ib_uint64_t current_lsn;
167
+ ulint size_low, size_high, size, free_limit;
168
+ ib_int64_t size_bytes, free_limit_bytes;
169
+ dict_table_t* table;
170
+ dict_index_t* index;
171
+ fil_system_t* system;
172
+ fil_node_t* node = NULL;
173
+ fil_space_t* space;
174
+ ulint zip_size;
175
+
176
+ buf3 = ut_malloc(2 * UNIV_PAGE_SIZE);
177
+ descr_page = ut_align(buf3, UNIV_PAGE_SIZE);
178
+
179
+ current_lsn = log_get_lsn();
180
+
181
+ /* check the header page's consistency */
182
+ if (buf_page_is_corrupted(page,
183
+ dict_table_flags_to_zip_size(space_flags))) {
184
+ fprintf(stderr, "InnoDB: page 0 of %s seems corrupt.\n", filepath);
185
+ file_is_corrupt = TRUE;
186
+ descr_is_corrupt = TRUE;
187
+ }
188
+
189
+ /* store as first descr page */
190
+ memcpy(descr_page, page, UNIV_PAGE_SIZE);
191
+
192
+ zip_size = dict_table_flags_to_zip_size(flags);
193
+ ut_a(zip_size == dict_table_flags_to_zip_size(space_flags));
194
+
195
+ /* get free limit (page number) of the table space */
196
+/* these should be same to the definition in fsp0fsp.c */
197
+#define FSP_HEADER_OFFSET FIL_PAGE_DATA
198
+#define FSP_FREE_LIMIT 12
199
+ free_limit = mach_read_from_4(FSP_HEADER_OFFSET + FSP_FREE_LIMIT + page);
200
+ free_limit_bytes = (ib_int64_t)free_limit * (ib_int64_t)(zip_size ? zip_size : UNIV_PAGE_SIZE);
201
+
202
+ /* overwrite fsp header */
203
+ fsp_header_init_fields(page, id, flags);
204
+ mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, id);
205
+ space_id = id;
206
+ space_flags = flags;
207
+ if (mach_read_ull(page + FIL_PAGE_FILE_FLUSH_LSN) > current_lsn)
208
+ mach_write_ull(page + FIL_PAGE_FILE_FLUSH_LSN, current_lsn);
209
+
210
+ fil_page_buf_page_store_checksum(page, zip_size);
211
+
212
+ success = os_file_write(filepath, file, page, 0, 0, UNIV_PAGE_SIZE);
213
+
214
+ /* get file size */
215
+ os_file_get_size(file, &size_low, &size_high);
216
+ size_bytes = (((ib_int64_t)size_high) << 32)
217
+ + (ib_int64_t)size_low;
218
+
219
+ if (size_bytes < free_limit_bytes) {
220
+ free_limit_bytes = size_bytes;
221
+ if (size_bytes >= (ib_int64_t) (FSP_EXTENT_SIZE * (zip_size ? zip_size : UNIV_PAGE_SIZE))) {
222
+ fprintf(stderr, "InnoDB: free limit of %s is larger than its real size.\n", filepath);
223
+ file_is_corrupt = TRUE;
224
+ }
225
+ }
226
+
227
+ /* get cruster index information */
228
+ table = dict_table_get_low(name);
229
+ index = dict_table_get_first_index(table);
230
+ ut_a(index->page==3);
231
+
232
+ /* read metadata from .exp file */
233
+ n_index = 0;
234
+ memset(old_id, 0, sizeof(old_id));
235
+ memset(new_id, 0, sizeof(new_id));
236
+ memset(root_page, 0, sizeof(root_page));
237
+
238
+ info_file_path = fil_make_ibd_name(name, FALSE);
239
+ len = strlen(info_file_path);
240
+ info_file_path[len - 3] = 'e';
241
+ info_file_path[len - 2] = 'x';
242
+ info_file_path[len - 1] = 'p';
243
+
244
+ info_file = os_file_create_simple_no_error_handling(
245
+ info_file_path, OS_FILE_OPEN, OS_FILE_READ_ONLY, &success);
246
+ if (!success) {
247
+ fprintf(stderr, "InnoDB: cannot open %s\n", info_file_path);
248
+ file_is_corrupt = TRUE;
249
+ goto skip_info;
250
+ }
251
+ success = os_file_read(info_file, page, 0, 0, UNIV_PAGE_SIZE);
252
+ if (!success) {
253
+ fprintf(stderr, "InnoDB: cannot read %s\n", info_file_path);
254
+ file_is_corrupt = TRUE;
255
+ goto skip_info;
256
+ }
257
+ if (mach_read_from_4(page) != 0x78706f72UL
258
+ || mach_read_from_4(page + 4) != 0x74696e66UL) {
259
+ fprintf(stderr, "InnoDB: %s seems not to be a correct .exp file\n", info_file_path);
260
+ file_is_corrupt = TRUE;
261
+ goto skip_info;
262
+ }
263
+
264
+ fprintf(stderr, "InnoDB: import: extended import of %s is started.\n", name);
265
+
266
+ n_index = mach_read_from_4(page + 8);
267
+ fprintf(stderr, "InnoDB: import: %lu indexes are detected.\n", (ulong)n_index);
268
+ for (i = 0; i < n_index; i++) {
269
+ new_id[i] =
270
+ dict_table_get_index_on_name(table,
271
+ (char*)(page + (i + 1) * 512 + 12))->id;
272
+ old_id[i] = mach_read_from_8(page + (i + 1) * 512);
273
+ root_page[i] = mach_read_from_4(page + (i + 1) * 512 + 8);
274
+ }
275
+
276
+skip_info:
277
+ if (info_file != -1)
278
+ os_file_close(info_file);
279
+
280
+ /*
281
+ if (size_bytes >= 1024 * 1024) {
282
+ size_bytes = ut_2pow_round(size_bytes, 1024 * 1024);
283
+ }
284
+ */
285
+
286
+ if (zip_size) {
287
+ fprintf(stderr, "InnoDB: Warning: importing compressed table is still EXPERIMENTAL, currently.\n");
288
+ }
289
+
290
+ {
291
+ mem_heap_t* heap = NULL;
292
+ ulint offsets_[REC_OFFS_NORMAL_SIZE];
293
+ ulint* offsets = offsets_;
294
+ ib_int64_t offset;
295
+
296
+ size = (ulint) (size_bytes / (zip_size ? zip_size : UNIV_PAGE_SIZE));
297
+ /* over write space id of all pages */
298
+ rec_offs_init(offsets_);
299
+
300
+ fprintf(stderr, "InnoDB: Progress in %%:");
301
+
302
+ for (offset = 0; offset < free_limit_bytes;
303
+ offset += zip_size ? zip_size : UNIV_PAGE_SIZE) {
304
+ ibool page_is_corrupt;
305
+
306
+ success = os_file_read(file, page,
307
+ (ulint)(offset & 0xFFFFFFFFUL),
308
+ (ulint)(offset >> 32),
309
+ zip_size ? zip_size : UNIV_PAGE_SIZE);
310
+
311
+ page_is_corrupt = FALSE;
312
+
313
+ /* check consistency */
314
+ if (fil_page_buf_page_is_corrupted_offline(page, zip_size)) {
315
+ page_is_corrupt = TRUE;
316
+ }
317
+
318
+ if (mach_read_from_4(page + FIL_PAGE_OFFSET)
319
+ != offset / (zip_size ? zip_size : UNIV_PAGE_SIZE)) {
320
+
321
+ page_is_corrupt = TRUE;
322
+ }
323
+
324
+ /* if it is free page, inconsistency is acceptable */
325
+ if (!offset) {
326
+ /* header page*/
327
+ /* it should be overwritten already */
328
+ ut_a(!page_is_corrupt);
329
+
330
+ } else if (!((offset / (zip_size ? zip_size : UNIV_PAGE_SIZE))
331
+ % (zip_size ? zip_size : UNIV_PAGE_SIZE))) {
332
+ /* descr page (not header) */
333
+ if (page_is_corrupt) {
334
+ file_is_corrupt = TRUE;
335
+ descr_is_corrupt = TRUE;
336
+ } else {
337
+
338
+ descr_is_corrupt = FALSE;
339
+ }
340
+
341
+ /* store as descr page */
342
+ memcpy(descr_page, page, (zip_size ? zip_size : UNIV_PAGE_SIZE));
343
+
344
+ } else if (descr_is_corrupt) {
345
+ /* unknown state of the page */
346
+ if (page_is_corrupt) {
347
+ file_is_corrupt = TRUE;
348
+ }
349
+
350
+ } else {
351
+ /* check free page or not */
352
+ /* These definitions should be same to fsp0fsp.c */
353
+#define FSP_HEADER_SIZE (32 + 5 * FLST_BASE_NODE_SIZE)
354
+
355
+#define XDES_BITMAP (FLST_NODE_SIZE + 12)
356
+#define XDES_BITS_PER_PAGE 2
357
+#define XDES_FREE_BIT 0
358
+#define XDES_SIZE \
359
+ (XDES_BITMAP + UT_BITS_IN_BYTES(FSP_EXTENT_SIZE * XDES_BITS_PER_PAGE))
360
+#define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE)
361
+
362
+ /*descr = descr_page + XDES_ARR_OFFSET + XDES_SIZE * xdes_calc_descriptor_index(zip_size, offset)*/
363
+ /*xdes_get_bit(descr, XDES_FREE_BIT, page % FSP_EXTENT_SIZE, mtr)*/
364
+ byte* descr;
365
+ ulint index;
366
+ ulint byte_index;
367
+ ulint bit_index;
368
+
369
+ descr = descr_page + XDES_ARR_OFFSET
370
+ + XDES_SIZE * (ut_2pow_remainder(
371
+ (offset / (zip_size ? zip_size : UNIV_PAGE_SIZE)),
372
+ (zip_size ? zip_size : UNIV_PAGE_SIZE)) / FSP_EXTENT_SIZE);
373
+
374
+ index = XDES_FREE_BIT
375
+ + XDES_BITS_PER_PAGE * ((offset / (zip_size ? zip_size : UNIV_PAGE_SIZE)) % FSP_EXTENT_SIZE);
376
+ byte_index = index / 8;
377
+ bit_index = index % 8;
378
+
379
+ if (ut_bit_get_nth(mach_read_from_1(descr + XDES_BITMAP + byte_index), bit_index)) {
380
+ /* free page */
381
+ if (page_is_corrupt) {
382
+ goto skip_write;
383
+ }
384
+ } else {
385
+ /* not free */
386
+ if (page_is_corrupt) {
387
+ file_is_corrupt = TRUE;
388
+ }
389
+ }
390
+ }
391
+
392
+ if (page_is_corrupt) {
393
+ fprintf(stderr, " [errp:%lld]", offset / (zip_size ? zip_size : UNIV_PAGE_SIZE));
394
+
395
+ /* cannot treat corrupt page */
396
+ goto skip_write;
397
+ }
398
+
399
+ if (mach_read_from_4(page + FIL_PAGE_OFFSET) || !offset) {
400
+ mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, id);
401
+
402
+ for (i = 0; i < n_index; i++) {
403
+ if (offset / (zip_size ? zip_size : UNIV_PAGE_SIZE) == root_page[i]) {
404
+ if (fil_page_get_type(page) != FIL_PAGE_INDEX) {
405
+ file_is_corrupt = TRUE;
406
+ fprintf(stderr, " [etyp:%lld]",
407
+ offset / (zip_size ? zip_size : UNIV_PAGE_SIZE));
408
+ goto skip_write;
409
+ }
410
+ /* this is index root page */
411
+ mach_write_to_4(page + FIL_PAGE_DATA + PAGE_BTR_SEG_LEAF
412
+ + FSEG_HDR_SPACE, id);
413
+ mach_write_to_4(page + FIL_PAGE_DATA + PAGE_BTR_SEG_TOP
414
+ + FSEG_HDR_SPACE, id);
415
+ break;
416
+ }
417
+ }
418
+
419
+ if (fil_page_get_type(page) == FIL_PAGE_INDEX) {
420
+ dulint tmp = mach_read_from_8(page + (PAGE_HEADER + PAGE_INDEX_ID));
421
+
422
+ for (i = 0; i < n_index; i++) {
423
+ if (ut_dulint_cmp(old_id[i], tmp) == 0) {
424
+ mach_write_to_8(page + (PAGE_HEADER + PAGE_INDEX_ID), new_id[i]);
425
+ break;
426
+ }
427
+ }
428
+
429
+ if (!zip_size && mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL) == 0
430
+ && ut_dulint_cmp(old_id[0], tmp) == 0) {
431
+ /* leaf page of cluster index, reset trx_id of records */
432
+ rec_t* rec;
433
+ rec_t* supremum;
434
+ ulint n_recs;
435
+
436
+ supremum = page_get_supremum_rec(page);
437
+ rec = page_rec_get_next(page_get_infimum_rec(page));
438
+ n_recs = page_get_n_recs(page);
439
+
440
+ while (rec && rec != supremum && n_recs > 0) {
441
+ ulint n_fields;
442
+ ulint i;
443
+ ulint offset = index->trx_id_offset;
444
+ offsets = rec_get_offsets(rec, index, offsets,
445
+ ULINT_UNDEFINED, &heap);
446
+ n_fields = rec_offs_n_fields(offsets);
447
+ if (!offset) {
448
+ offset = row_get_trx_id_offset(index, offsets);
449
+ }
450
+ trx_write_trx_id(rec + offset, ut_dulint_create(0, 1));
451
+
452
+ for (i = 0; i < n_fields; i++) {
453
+ if (rec_offs_nth_extern(offsets, i)) {
454
+ ulint local_len;
455
+ byte* data;
456
+
457
+ data = rec_get_nth_field(rec, offsets, i, &local_len);
458
+
459
+ local_len -= BTR_EXTERN_FIELD_REF_SIZE;
460
+
461
+ mach_write_to_4(data + local_len + BTR_EXTERN_SPACE_ID, id);
462
+ }
463
+ }
464
+
465
+ rec = page_rec_get_next(rec);
466
+ n_recs--;
467
+ }
468
+ } else if (mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL) == 0
469
+ && ut_dulint_cmp(old_id[0], tmp) != 0) {
470
+ mach_write_to_8(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), ut_dulint_create(0, 1));
471
+ }
472
+ }
473
+
474
+ if (mach_read_ull(page + FIL_PAGE_LSN) > current_lsn) {
475
+ mach_write_ull(page + FIL_PAGE_LSN, current_lsn);
476
+ if (!zip_size) {
477
+ mach_write_ull(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
478
+ current_lsn);
479
+ }
480
+ }
481
+
482
+ fil_page_buf_page_store_checksum(page, zip_size);
483
+
484
+ success = os_file_write(filepath, file, page,
485
+ (ulint)(offset & 0xFFFFFFFFUL),
486
+ (ulint)(offset >> 32),
487
+ zip_size ? zip_size : UNIV_PAGE_SIZE);
488
+ }
489
+
490
+skip_write:
491
+ if (free_limit_bytes
492
+ && ((ib_int64_t)((offset + (zip_size ? zip_size : UNIV_PAGE_SIZE)) * 100) / free_limit_bytes)
493
+ != ((offset * 100) / free_limit_bytes)) {
494
+ fprintf(stderr, " %lu",
495
+ (ulong)((ib_int64_t)((offset + (zip_size ? zip_size : UNIV_PAGE_SIZE)) * 100) / free_limit_bytes));
496
+ }
497
+ }
498
+
499
+ fprintf(stderr, " done.\n");
500
+
501
+ /* update SYS_INDEXES set root page */
502
+ index = dict_table_get_first_index(table);
503
+ while (index) {
504
+ for (i = 0; i < n_index; i++) {
505
+ if (ut_dulint_cmp(new_id[i], index->id) == 0) {
506
+ break;
507
+ }
508
+ }
509
+
510
+ if (i != n_index
511
+ && root_page[i] != index->page) {
512
+ /* must update */
513
+ ulint error;
514
+ trx_t* trx;
515
+ pars_info_t* info = NULL;
516
+
517
+ trx = trx_allocate_for_mysql();
518
+ trx->op_info = "extended import";
519
+
520
+ info = pars_info_create();
521
+
522
+ pars_info_add_dulint_literal(info, "indexid", new_id[i]);
523
+ pars_info_add_int4_literal(info, "new_page", (lint) root_page[i]);
524
+
525
+ error = que_eval_sql(info,
526
+ "PROCEDURE UPDATE_INDEX_PAGE () IS\n"
527
+ "BEGIN\n"
528
+ "UPDATE SYS_INDEXES"
529
+ " SET PAGE_NO = :new_page"
530
+ " WHERE ID = :indexid;\n"
531
+ "COMMIT WORK;\n"
532
+ "END;\n",
533
+ FALSE, trx);
534
+
535
+ if (error != DB_SUCCESS) {
536
+ fprintf(stderr, "InnoDB: failed to update SYS_INDEXES\n");
537
+ }
538
+
539
+ trx_commit_for_mysql(trx);
540
+
541
+ trx_free_for_mysql(trx);
542
+
543
+ index->page = root_page[i];
544
+ }
545
+
546
+ index = dict_table_get_next_index(index);
547
+ }
548
+ if (UNIV_LIKELY_NULL(heap)) {
549
+ mem_heap_free(heap);
550
+ }
551
+ }
552
+ /* .exp file should be removed */
553
+ success = os_file_delete(info_file_path);
554
+ if (!success) {
555
+ success = os_file_delete_if_exists(info_file_path);
556
+ }
557
+ mem_free(info_file_path);
558
+
559
+ system = fil_system;
560
+ mutex_enter(&(system->mutex));
561
+ space = fil_space_get_by_id(id);
562
+ if (space)
563
+ node = UT_LIST_GET_FIRST(space->chain);
564
+ if (node && node->size < size) {
565
+ space->size += (size - node->size);
566
+ node->size = size;
567
+ }
568
+ mutex_exit(&(system->mutex));
569
+
570
+ ut_free(buf3);
571
+
572
+ if (file_is_corrupt) {
573
+ ut_print_timestamp(stderr);
574
+ fputs(" InnoDB: Error: file ",
575
+ stderr);
576
+ ut_print_filename(stderr, filepath);
577
+ fprintf(stderr, " seems to be corrupt.\n"
578
+ "InnoDB: anyway, all not corrupt pages were tried to be converted to salvage.\n"
579
+ "InnoDB: ##### CAUTION #####\n"
580
+ "InnoDB: ## The .ibd must cause to crash InnoDB, though re-import would seem to be succeeded.\n"
581
+ "InnoDB: ## If you don't have knowledge about salvaging data from .ibd, you should not use the file.\n"
582
+ "InnoDB: ###################\n");
583
+ success = FALSE;
584
+
585
+ ut_free(buf2);
586
+
587
+ goto func_exit;
588
+ }
589
+ }
590
+
591
ut_free(buf2);
592
593
if (UNIV_UNLIKELY(space_id != id
594
@@ -3117,6 +3642,269 @@
595
os_file_close(file);
596
mem_free(filepath);
597
598
+ if (srv_expand_import && dict_table_flags_to_zip_size(flags)) {
599
+ ulint page_no;
600
+ ulint zip_size;
601
+ ulint height;
602
+ rec_t* node_ptr;
603
+ dict_table_t* table;
604
+ dict_index_t* index;
605
+ buf_block_t* block;
606
+ page_t* page;
607
+ page_zip_des_t* page_zip;
608
+ mtr_t mtr;
609
+
610
+ mem_heap_t* heap = NULL;
611
+ ulint offsets_[REC_OFFS_NORMAL_SIZE];
612
+ ulint* offsets = offsets_;
613
+
614
+ rec_offs_init(offsets_);
615
+
616
+ zip_size = dict_table_flags_to_zip_size(flags);
617
+
618
+ table = dict_table_get_low(name);
619
+ index = dict_table_get_first_index(table);
620
+ page_no = dict_index_get_page(index);
621
+ ut_a(page_no == 3);
622
+
623
+ fprintf(stderr, "InnoDB: It is compressed .ibd file. need to convert additionaly on buffer pool.\n");
624
+
625
+ /* down to leaf */
626
+ mtr_start(&mtr);
627
+ mtr_set_log_mode(&mtr, MTR_LOG_NONE);
628
+
629
+ height = ULINT_UNDEFINED;
630
+
631
+ for (;;) {
632
+ block = buf_page_get(space_id, zip_size, page_no,
633
+ RW_NO_LATCH, &mtr);
634
+ page = buf_block_get_frame(block);
635
+
636
+ block->check_index_page_at_flush = TRUE;
637
+
638
+ if (height == ULINT_UNDEFINED) {
639
+ height = btr_page_get_level(page, &mtr);
640
+ }
641
+
642
+ if (height == 0) {
643
+ break;
644
+ }
645
+
646
+ node_ptr = page_rec_get_next(page_get_infimum_rec(page));
647
+
648
+ height--;
649
+
650
+ offsets = rec_get_offsets(node_ptr, index, offsets, ULINT_UNDEFINED, &heap);
651
+ page_no = btr_node_ptr_get_child_page_no(node_ptr, offsets);
652
+ }
653
+
654
+ mtr_commit(&mtr);
655
+
656
+ fprintf(stderr, "InnoDB: pages needs split are ...");
657
+
658
+ /* scan reaf pages */
659
+ while (page_no != FIL_NULL) {
660
+ rec_t* rec;
661
+ rec_t* supremum;
662
+ ulint n_recs;
663
+
664
+ mtr_start(&mtr);
665
+
666
+ block = buf_page_get(space_id, zip_size, page_no,
667
+ RW_X_LATCH, &mtr);
668
+ page = buf_block_get_frame(block);
669
+ page_zip = buf_block_get_page_zip(block);
670
+
671
+ if (!page_zip) {
672
+ /*something wrong*/
673
+ fprintf(stderr, "InnoDB: Something wrong with reading page %lu.\n", page_no);
674
+convert_err_exit:
675
+ mtr_commit(&mtr);
676
+ mutex_enter(&fil_system->mutex);
677
+ fil_space_free(space_id, FALSE);
678
+ mutex_exit(&fil_system->mutex);
679
+ success = FALSE;
680
+ goto convert_exit;
681
+ }
682
+
683
+ supremum = page_get_supremum_rec(page);
684
+ rec = page_rec_get_next(page_get_infimum_rec(page));
685
+ n_recs = page_get_n_recs(page);
686
+
687
+ /* illegal operation as InnoDB online system. so not logged */
688
+ while (rec && rec != supremum && n_recs > 0) {
689
+ ulint n_fields;
690
+ ulint i;
691
+ ulint offset = index->trx_id_offset;
692
+
693
+ offsets = rec_get_offsets(rec, index, offsets,
694
+ ULINT_UNDEFINED, &heap);
695
+ n_fields = rec_offs_n_fields(offsets);
696
+ if (!offset) {
697
+ offset = row_get_trx_id_offset(index, offsets);
698
+ }
699
+ trx_write_trx_id(rec + offset, ut_dulint_create(0, 1));
700
+
701
+ for (i = 0; i < n_fields; i++) {
702
+ if (rec_offs_nth_extern(offsets, i)) {
703
+ ulint local_len;
704
+ byte* data;
705
+
706
+ data = rec_get_nth_field(rec, offsets, i, &local_len);
707
+
708
+ local_len -= BTR_EXTERN_FIELD_REF_SIZE;
709
+
710
+ mach_write_to_4(data + local_len + BTR_EXTERN_SPACE_ID, id);
711
+ }
712
+ }
713
+
714
+ rec = page_rec_get_next(rec);
715
+ n_recs--;
716
+ }
717
+
718
+ /* dummy logged update for along with modified page path */
719
+ if (ut_dulint_cmp(index->id, btr_page_get_index_id(page)) != 0) {
720
+ /* this should be adjusted already */
721
+ fprintf(stderr, "InnoDB: The page %lu seems to be converted wrong.\n", page_no);
722
+ goto convert_err_exit;
723
+ }
724
+ btr_page_set_index_id(page, page_zip, index->id, &mtr);
725
+
726
+ /* confirm whether fits to the page size or not */
727
+ if (!page_zip_compress(page_zip, page, index, &mtr)
728
+ && !btr_page_reorganize(block, index, &mtr)) {
729
+ buf_block_t* new_block;
730
+ page_t* new_page;
731
+ page_zip_des_t* new_page_zip;
732
+ rec_t* split_rec;
733
+ ulint n_uniq;
734
+
735
+ /* split page is needed */
736
+ fprintf(stderr, " %lu", page_no);
737
+
738
+ mtr_x_lock(dict_index_get_lock(index), &mtr);
739
+
740
+ n_uniq = dict_index_get_n_unique_in_tree(index);
741
+
742
+ if(page_get_n_recs(page) < 2) {
743
+ /* no way to make smaller */
744
+ fprintf(stderr, "InnoDB: The page %lu cannot be store to the page size.\n", page_no);
745
+ goto convert_err_exit;
746
+ }
747
+
748
+ if (UNIV_UNLIKELY(page_no == dict_index_get_page(index))) {
749
+ ulint new_page_no;
750
+ dtuple_t* node_ptr;
751
+ ulint level;
752
+ rec_t* node_ptr_rec;
753
+ page_cur_t page_cursor;
754
+
755
+ /* it is root page, need to raise before split */
756
+
757
+ level = btr_page_get_level(page, &mtr);
758
+
759
+ new_block = btr_page_alloc(index, 0, FSP_NO_DIR, level, &mtr);
760
+ new_page = buf_block_get_frame(new_block);
761
+ new_page_zip = buf_block_get_page_zip(new_block);
762
+ btr_page_create(new_block, new_page_zip, index, level, &mtr);
763
+
764
+ btr_page_set_next(new_page, new_page_zip, FIL_NULL, &mtr);
765
+ btr_page_set_prev(new_page, new_page_zip, FIL_NULL, &mtr);
766
+
767
+ page_zip_copy_recs(new_page_zip, new_page,
768
+ page_zip, page, index, &mtr);
769
+ btr_search_move_or_delete_hash_entries(new_block, block, index);
770
+
771
+ rec = page_rec_get_next(page_get_infimum_rec(new_page));
772
+ new_page_no = buf_block_get_page_no(new_block);
773
+
774
+ node_ptr = dict_index_build_node_ptr(index, rec, new_page_no, heap,
775
+ level);
776
+ dtuple_set_info_bits(node_ptr,
777
+ dtuple_get_info_bits(node_ptr)
778
+ | REC_INFO_MIN_REC_FLAG);
779
+ btr_page_empty(block, page_zip, index, level + 1, &mtr);
780
+
781
+ btr_page_set_next(page, page_zip, FIL_NULL, &mtr);
782
+ btr_page_set_prev(page, page_zip, FIL_NULL, &mtr);
783
+
784
+ page_cur_set_before_first(block, &page_cursor);
785
+
786
+ node_ptr_rec = page_cur_tuple_insert(&page_cursor, node_ptr,
787
+ index, 0, &mtr);
788
+ ut_a(node_ptr_rec);
789
+
790
+ if (!btr_page_reorganize(block, index, &mtr)) {
791
+ fprintf(stderr, "InnoDB: failed to store the page %lu.\n", page_no);
792
+ goto convert_err_exit;
793
+ }
794
+
795
+ /* move to the raised page */
796
+ page_no = new_page_no;
797
+ block = new_block;
798
+ page = new_page;
799
+ page_zip = new_page_zip;
800
+
801
+ fprintf(stderr, "(raise_to:%lu)", page_no);
802
+ }
803
+
804
+ split_rec = page_get_middle_rec(page);
805
+
806
+ new_block = btr_page_alloc(index, page_no + 1, FSP_UP,
807
+ btr_page_get_level(page, &mtr), &mtr);
808
+ new_page = buf_block_get_frame(new_block);
809
+ new_page_zip = buf_block_get_page_zip(new_block);
810
+ btr_page_create(new_block, new_page_zip, index,
811
+ btr_page_get_level(page, &mtr), &mtr);
812
+
813
+ offsets = rec_get_offsets(split_rec, index, offsets, n_uniq, &heap);
814
+
815
+ btr_attach_half_pages(index, block,
816
+ split_rec, new_block, FSP_UP, &mtr);
817
+
818
+ page_zip_copy_recs(new_page_zip, new_page,
819
+ page_zip, page, index, &mtr);
820
+ page_delete_rec_list_start(split_rec - page + new_page,
821
+ new_block, index, &mtr);
822
+ btr_search_move_or_delete_hash_entries(new_block, block, index);
823
+ page_delete_rec_list_end(split_rec, block, index,
824
+ ULINT_UNDEFINED, ULINT_UNDEFINED, &mtr);
825
+
826
+ fprintf(stderr, "(new:%lu)", buf_block_get_page_no(new_block));
827
+
828
+ /* Are they needed? */
829
+ if (!btr_page_reorganize(block, index, &mtr)) {
830
+ fprintf(stderr, "InnoDB: failed to store the page %lu.\n", page_no);
831
+ goto convert_err_exit;
832
+ }
833
+ if (!btr_page_reorganize(new_block, index, &mtr)) {
834
+ fprintf(stderr, "InnoDB: failed to store the page %lu.\n", buf_block_get_page_no(new_block));
835
+ goto convert_err_exit;
836
+ }
837
+ }
838
+
839
+ page_no = btr_page_get_next(page, &mtr);
840
+
841
+ mtr_commit(&mtr);
842
+
843
+ if (heap) {
844
+ mem_heap_empty(heap);
845
+ }
846
+ }
847
+
848
+ fprintf(stderr, "...done.\nInnoDB: waiting the flush batch of the additional conversion.\n");
849
+
850
+ /* should wait for the not-logged changes are all flushed */
851
+ buf_flush_batch(BUF_FLUSH_LIST, ULINT_MAX, mtr.end_lsn + 1);
852
+ buf_flush_wait_batch_end(BUF_FLUSH_LIST);
853
+
854
+ fprintf(stderr, "InnoDB: done.\n");
855
+convert_exit:
856
+ if (UNIV_LIKELY_NULL(heap)) {
857
+ mem_heap_free(heap);
858
+ }
859
+ }
860
+
861
return(success);
862
}
863
#endif /* !UNIV_HOTBACKUP */
864
--- a/storage/innodb_plugin/handler/ha_innodb.cc
865
+++ b/storage/innodb_plugin/handler/ha_innodb.cc
866
@@ -7102,6 +7102,14 @@
867
err = row_discard_tablespace_for_mysql(dict_table->name, trx);
868
} else {
869
err = row_import_tablespace_for_mysql(dict_table->name, trx);
870
+
871
+ /* in expanded import mode re-initialize auto_increment again */
872
+ if ((err == DB_SUCCESS) && srv_expand_import &&
873
+ (table->found_next_number_field != NULL)) {
874
+ dict_table_autoinc_lock(dict_table);
875
+ innobase_initialize_autoinc();
876
+ dict_table_autoinc_unlock(dict_table);
877
+ }
878
}
879
880
err = convert_error_code_to_mysql(err, dict_table->flags, NULL);
881
@@ -11378,6 +11386,11 @@
882
"Enable/Disable unsafe group commit when support_xa=OFF and use with binlog or other XA storage engine.",
883
NULL, NULL, 0, 0, 1, 0);
884
885
+static MYSQL_SYSVAR_ULONG(expand_import, srv_expand_import,
886
+ PLUGIN_VAR_RQCMDARG,
887
+ "Enable/Disable converting automatically *.ibd files when import tablespace.",
888
+ NULL, NULL, 0, 0, 1, 0);
889
+
890
static MYSQL_SYSVAR_ULONG(extra_rsegments, srv_extra_rsegments,
891
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
892
"Number of extra user rollback segments when create new database.",
893
@@ -11455,6 +11468,7 @@
894
MYSQL_SYSVAR(adaptive_checkpoint),
895
MYSQL_SYSVAR(flush_log_at_trx_commit_session),
896
MYSQL_SYSVAR(enable_unsafe_group_commit),
897
+ MYSQL_SYSVAR(expand_import),
898
MYSQL_SYSVAR(extra_rsegments),
899
MYSQL_SYSVAR(dict_size_limit),
900
MYSQL_SYSVAR(use_sys_malloc),
901
--- a/storage/innodb_plugin/handler/innodb_patch_info.h
902
+++ b/storage/innodb_plugin/handler/innodb_patch_info.h
903
@@ -32,6 +32,7 @@
904
{"innodb_extra_rseg","allow to create extra rollback segments","When create new db, the new parameter allows to create more rollback segments","http://www.percona.com/docs/wiki/percona-xtradb"},
905
{"innodb_overwrite_relay_log_info","overwrite relay-log.info when slave recovery","Building as plugin, it is not used.","http://www.percona.com/docs/wiki/percona-xtradb:innodb_overwrite_relay_log_info"},
906
{"innodb_thread_concurrency_timer_based","use InnoDB timer based concurrency throttling (backport from MySQL 5.4.0)","",""},
907
+{"innodb_expand_import","convert .ibd file automatically when import tablespace","the files are generated by xtrabackup export mode.","http://www.percona.com/docs/wiki/percona-xtradb"},
908
{"innodb_dict_size_limit","Limit dictionary cache size","Variable innodb_dict_size_limit in bytes","http://www.percona.com/docs/wiki/percona-xtradb"},
909
{"innodb_split_buf_pool_mutex","More fix of buffer_pool mutex","Spliting buf_pool_mutex and optimizing based on innodb_opt_lru_count","http://www.percona.com/docs/wiki/percona-xtradb"},
910
{NULL, NULL, NULL, NULL}
911
--- a/storage/innodb_plugin/include/btr0btr.h
912
+++ b/storage/innodb_plugin/include/btr0btr.h
913
@@ -208,6 +208,17 @@
914
@return the uncompressed page frame */
915
# define btr_page_get(space,zip_size,page_no,mode,mtr) \
916
buf_block_get_frame(btr_block_get(space,zip_size,page_no,mode,mtr))
917
+/**************************************************************//**
918
+Sets the index id field of a page. */
919
+UNIV_INLINE
920
+void
921
+btr_page_set_index_id(
922
+/*==================*/
923
+ page_t* page, /*!< in: page to be created */
924
+ page_zip_des_t* page_zip,/*!< in: compressed page whose uncompressed
925
+ part will be updated, or NULL */
926
+ dulint id, /*!< in: index id */
927
+ mtr_t* mtr); /*!< in: mtr */
928
#endif /* !UNIV_HOTBACKUP */
929
/**************************************************************//**
930
Gets the index id field of a page.
931
@@ -245,6 +256,17 @@
932
const page_t* page, /*!< in: index page */
933
mtr_t* mtr); /*!< in: mini-transaction handle */
934
/********************************************************//**
935
+Sets the next index page field. */
936
+UNIV_INLINE
937
+void
938
+btr_page_set_next(
939
+/*==============*/
940
+ page_t* page, /*!< in: index page */
941
+ page_zip_des_t* page_zip,/*!< in: compressed page whose uncompressed
942
+ part will be updated, or NULL */
943
+ ulint next, /*!< in: next page number */
944
+ mtr_t* mtr); /*!< in: mini-transaction handle */
945
+/********************************************************//**
946
Gets the previous index page number.
947
@return prev page number */
948
UNIV_INLINE
949
@@ -253,6 +275,17 @@
950
/*==============*/
951
const page_t* page, /*!< in: index page */
952
mtr_t* mtr); /*!< in: mini-transaction handle */
953
+/********************************************************//**
954
+Sets the previous index page field. */
955
+UNIV_INLINE
956
+void
957
+btr_page_set_prev(
958
+/*==============*/
959
+ page_t* page, /*!< in: index page */
960
+ page_zip_des_t* page_zip,/*!< in: compressed page whose uncompressed
961
+ part will be updated, or NULL */
962
+ ulint prev, /*!< in: previous page number */
963
+ mtr_t* mtr); /*!< in: mini-transaction handle */
964
/*************************************************************//**
965
Gets pointer to the previous user record in the tree. It is assumed
966
that the caller has appropriate latches on the page and its neighbor.
967
@@ -298,6 +331,18 @@
968
/*===========================*/
969
const rec_t* rec, /*!< in: node pointer record */
970
const ulint* offsets);/*!< in: array returned by rec_get_offsets() */
971
+/**************************************************************//**
972
+Creates a new index page (not the root, and also not
973
+used in page reorganization). @see btr_page_empty(). */
974
+UNIV_INTERN
975
+void
976
+btr_page_create(
977
+/*============*/
978
+ buf_block_t* block, /*!< in/out: page to be created */
979
+ page_zip_des_t* page_zip,/*!< in/out: compressed page, or NULL */
980
+ dict_index_t* index, /*!< in: index */
981
+ ulint level, /*!< in: the B-tree level of the page */
982
+ mtr_t* mtr); /*!< in: mtr */
983
/************************************************************//**
984
Creates the root node for a new index tree.
985
@return page number of the created root, FIL_NULL if did not succeed */
986
@@ -368,6 +413,17 @@
987
dict_index_t* index, /*!< in: record descriptor */
988
mtr_t* mtr); /*!< in: mtr */
989
/*************************************************************//**
990
+Empties an index page. @see btr_page_create(). */
991
+UNIV_INTERN
992
+void
993
+btr_page_empty(
994
+/*===========*/
995
+ buf_block_t* block, /*!< in: page to be emptied */
996
+ page_zip_des_t* page_zip,/*!< out: compressed page, or NULL */
997
+ dict_index_t* index, /*!< in: index of the page */
998
+ ulint level, /*!< in: the B-tree level of the page */
999
+ mtr_t* mtr); /*!< in: mtr */
1000
+/*************************************************************//**
1001
Decides if the page should be split at the convergence point of
1002
inserts converging to left.
1003
@return TRUE if split recommended */
1004
@@ -426,6 +482,20 @@
1005
# define btr_insert_on_non_leaf_level(i,l,t,m) \
1006
btr_insert_on_non_leaf_level_func(i,l,t,__FILE__,__LINE__,m)
1007
#endif /* !UNIV_HOTBACKUP */
1008
+/**************************************************************//**
1009
+Attaches the halves of an index page on the appropriate level in an
1010
+index tree. */
1011
+UNIV_INTERN
1012
+void
1013
+btr_attach_half_pages(
1014
+/*==================*/
1015
+ dict_index_t* index, /*!< in: the index tree */
1016
+ buf_block_t* block, /*!< in/out: page to be split */
1017
+ const rec_t* split_rec, /*!< in: first record on upper
1018
+ half page */
1019
+ buf_block_t* new_block, /*!< in/out: the new half page */
1020
+ ulint direction, /*!< in: FSP_UP or FSP_DOWN */
1021
+ mtr_t* mtr); /*!< in: mtr */
1022
/****************************************************************//**
1023
Sets a record as the predefined minimum record. */
1024
UNIV_INTERN
1025
--- a/storage/innodb_plugin/include/srv0srv.h
1026
+++ b/storage/innodb_plugin/include/srv0srv.h
1027
@@ -219,6 +219,8 @@
1028
extern ulint srv_read_ahead;
1029
extern ulint srv_adaptive_checkpoint;
1030
1031
+extern ulint srv_expand_import;
1032
+
1033
extern ulint srv_extra_rsegments;
1034
extern ulint srv_dict_size_limit;
1035
/*-------------------------------------------*/
1036
--- a/storage/innodb_plugin/row/row0mysql.c
1037
+++ b/storage/innodb_plugin/row/row0mysql.c
1038
@@ -2520,6 +2520,11 @@
1039
1040
current_lsn = log_get_lsn();
1041
1042
+ /* Enlarge the fatal lock wait timeout during import. */
1043
+ mutex_enter(&kernel_mutex);
1044
+ srv_fatal_semaphore_wait_threshold += 7200; /* 2 hours */
1045
+ mutex_exit(&kernel_mutex);
1046
+
1047
/* It is possible, though very improbable, that the lsn's in the
1048
tablespace to be imported have risen above the current system lsn, if
1049
a lengthy purge, ibuf merge, or rollback was performed on a backup
1050
@@ -2631,6 +2636,11 @@
1051
1052
trx->op_info = "";
1053
1054
+ /* Restore the fatal semaphore wait timeout */
1055
+ mutex_enter(&kernel_mutex);
1056
+ srv_fatal_semaphore_wait_threshold -= 7200; /* 2 hours */
1057
+ mutex_exit(&kernel_mutex);
1058
+
1059
return((int) err);
1060
}
1061
1062
--- a/storage/innodb_plugin/srv/srv0srv.c
1063
+++ b/storage/innodb_plugin/srv/srv0srv.c
1064
@@ -392,6 +392,8 @@
1065
UNIV_INTERN ulint srv_read_ahead = 3; /* 1: random 2: linear 3: Both */
1066
UNIV_INTERN ulint srv_adaptive_checkpoint = 0; /* 0: none 1: reflex 2: estimate */
1067
1068
+UNIV_INTERN ulint srv_expand_import = 0; /* 0:disable 1:enable */
1069
+
1070
UNIV_INTERN ulint srv_extra_rsegments = 0; /* extra rseg for users */
1071
UNIV_INTERN ulint srv_dict_size_limit = 0;
1072
/*-------------------------------------------*/
Loggerhead is a web-based interface for Breezy
Version: 2.0.1