1
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
5
COPYING CONDITIONS NOTICE:
7
This program is free software; you can redistribute it and/or modify
8
it under the terms of version 2 of the GNU General Public License as
9
published by the Free Software Foundation, and provided that the
10
following conditions are met:
12
* Redistributions of source code must retain this COPYING
13
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
14
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
15
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
18
* Redistributions in binary form must reproduce this COPYING
19
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
20
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
21
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
22
GRANT (below) in the documentation and/or other materials
23
provided with the distribution.
25
You should have received a copy of the GNU General Public License
26
along with this program; if not, write to the Free Software
27
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32
TokuDB, Tokutek Fractal Tree Indexing Library.
33
Copyright (C) 2007-2013 Tokutek, Inc.
37
This program is distributed in the hope that it will be useful, but
38
WITHOUT ANY WARRANTY; without even the implied warranty of
39
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
40
General Public License for more details.
42
UNIVERSITY PATENT NOTICE:
44
The technology is licensed by the Massachusetts Institute of
45
Technology, Rutgers State University of New Jersey, and the Research
46
Foundation of State University of New York at Stony Brook under
47
United States of America Serial No. 11/760379 and to the patents
48
and/or patent applications resulting from it.
50
PATENT MARKING NOTICE:
52
This software is covered by US Patent No. 8,185,551.
53
This software is covered by US Patent No. 8,489,638.
57
"THIS IMPLEMENTATION" means the copyrightable works distributed by
58
Tokutek as part of the Fractal Tree project.
60
"PATENT CLAIMS" means the claims of patents that are owned or
61
licensable by Tokutek, both currently or in the future; and that in
62
the absence of this license would be infringed by THIS
63
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
65
"PATENT CHALLENGE" shall mean a challenge to the validity,
66
patentability, enforceability and/or non-infringement of any of the
67
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
69
Tokutek hereby grants to you, for the term and geographical scope of
70
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
71
irrevocable (except as stated in this section) patent license to
72
make, have made, use, offer to sell, sell, import, transfer, and
73
otherwise run, modify, and propagate the contents of THIS
74
IMPLEMENTATION, where such license applies only to the PATENT
75
CLAIMS. This grant does not include claims that would be infringed
76
only as a consequence of further modifications of THIS
77
IMPLEMENTATION. If you or your agent or licensee institute or order
78
or agree to the institution of patent litigation against any entity
79
(including a cross-claim or counterclaim in a lawsuit) alleging that
80
THIS IMPLEMENTATION constitutes direct or contributory patent
81
infringement, or inducement of patent infringement, then any rights
82
granted to you under this License shall terminate as of the date
83
such litigation is filed. If you or your agent or exclusive
84
licensee institute or order or agree to the institution of a PATENT
85
CHALLENGE, then Tokutek may terminate any rights granted to you
89
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
90
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
92
/* This file defines the logformat in an executable fashion.
93
* This code is used to generate
94
* The code that writes into the log.
95
* The code that reads the log and prints it to stdout (the log_print utility)
96
* The code that reads the log for recovery.
97
* The struct definitions.
98
* The Latex documentation.
106
#include <sys/stat.h>
107
#include <sys/types.h>
109
#include <toku_portability.h>
110
#include <toku_assert.h>
113
typedef struct field {
116
const char *format; // optional format string
119
#define NULLFIELD {0,0,0}
122
enum log_begin_action {
125
ASSERT_BEGIN_WAS_LOGGED,
126
LOG_BEGIN_ACTION_NA = IGNORE_LOG_BEGIN
131
unsigned int command_and_flags;
132
struct field *fields;
133
enum log_begin_action log_begin_action;
136
// In the fields, don't mention the command, the LSN, the CRC or the trailing LEN.
138
const struct logtype rollbacks[] = {
139
//TODO: #2037 Add dname
140
{"fdelete", 'U', FA{{"FILENUM", "filenum", 0},
141
NULLFIELD}, LOG_BEGIN_ACTION_NA},
142
//TODO: #2037 Add dname
143
{"fcreate", 'F', FA{{"FILENUM", "filenum", 0},
144
{"BYTESTRING", "iname", 0},
145
NULLFIELD}, LOG_BEGIN_ACTION_NA},
146
// cmdinsert is used to insert a key-value pair into a DB. For rollback we don't need the data.
147
{"cmdinsert", 'i', FA{
148
{"FILENUM", "filenum", 0},
149
{"BYTESTRING", "key", 0},
150
NULLFIELD}, LOG_BEGIN_ACTION_NA},
151
{"cmddelete", 'd', FA{
152
{"FILENUM", "filenum", 0},
153
{"BYTESTRING", "key", 0},
154
NULLFIELD}, LOG_BEGIN_ACTION_NA},
155
{"rollinclude", 'r', FA{{"TXNID_PAIR", "xid", 0},
156
{"uint64_t", "num_nodes", 0},
157
{"BLOCKNUM", "spilled_head", 0},
158
{"BLOCKNUM", "spilled_tail", 0},
159
NULLFIELD}, LOG_BEGIN_ACTION_NA},
160
{"load", 'l', FA{{"FILENUM", "old_filenum", 0},
161
{"BYTESTRING", "new_iname", 0},
162
NULLFIELD}, LOG_BEGIN_ACTION_NA},
164
{"hot_index", 'h', FA{{"FILENUMS", "hot_index_filenums", 0},
165
NULLFIELD}, LOG_BEGIN_ACTION_NA},
166
{"dictionary_redirect", 'R', FA{{"FILENUM", "old_filenum", 0},
167
{"FILENUM", "new_filenum", 0},
168
NULLFIELD}, LOG_BEGIN_ACTION_NA},
169
{"cmdupdate", 'u', FA{{"FILENUM", "filenum", 0},
170
{"BYTESTRING", "key", 0},
171
NULLFIELD}, LOG_BEGIN_ACTION_NA},
172
{"cmdupdatebroadcast", 'B', FA{{"FILENUM", "filenum", 0},
173
{"bool", "is_resetting_op", 0},
174
NULLFIELD}, LOG_BEGIN_ACTION_NA},
175
{"change_fdescriptor", 'D', FA{{"FILENUM", "filenum", 0},
176
{"BYTESTRING", "old_descriptor", 0},
177
NULLFIELD}, LOG_BEGIN_ACTION_NA},
178
{0,0,FA{NULLFIELD}, LOG_BEGIN_ACTION_NA}
181
const struct logtype logtypes[] = {
182
// Records produced by checkpoints
183
#if 0 // no longer used, but reserve the type
184
{"local_txn_checkpoint", 'c', FA{{"TXNID", "xid", 0}, NULLFIELD}},
186
{"begin_checkpoint", 'x', FA{{"uint64_t", "timestamp", 0}, {"TXNID", "last_xid", 0}, NULLFIELD}, IGNORE_LOG_BEGIN},
187
{"end_checkpoint", 'X', FA{{"LSN", "lsn_begin_checkpoint", 0},
188
{"uint64_t", "timestamp", 0},
189
{"uint32_t", "num_fassociate_entries", 0}, // how many files were checkpointed
190
{"uint32_t", "num_xstillopen_entries", 0}, // how many txns were checkpointed
191
NULLFIELD}, IGNORE_LOG_BEGIN},
192
//TODO: #2037 Add dname
193
{"fassociate", 'f', FA{{"FILENUM", "filenum", 0},
194
{"uint32_t", "treeflags", 0},
195
{"BYTESTRING", "iname", 0}, // pathname of file
196
{"uint8_t", "unlink_on_close", 0},
197
NULLFIELD}, IGNORE_LOG_BEGIN},
198
//We do not use a TXNINFO struct since recovery log has
199
//FILENUMS and TOKUTXN has FTs (for open_fts)
200
{"xstillopen", 's', FA{{"TXNID_PAIR", "xid", 0},
201
{"TXNID_PAIR", "parentxid", 0},
202
{"uint64_t", "rollentry_raw_count", 0},
203
{"FILENUMS", "open_filenums", 0},
204
{"uint8_t", "force_fsync_on_commit", 0},
205
{"uint64_t", "num_rollback_nodes", 0},
206
{"uint64_t", "num_rollentries", 0},
207
{"BLOCKNUM", "spilled_rollback_head", 0},
208
{"BLOCKNUM", "spilled_rollback_tail", 0},
209
{"BLOCKNUM", "current_rollback", 0},
210
NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED}, // record all transactions
211
// prepared txns need a gid
212
{"xstillopenprepared", 'p', FA{{"TXNID_PAIR", "xid", 0},
213
{"XIDP", "xa_xid", 0}, // prepared transactions need a gid, and have no parentxid.
214
{"uint64_t", "rollentry_raw_count", 0},
215
{"FILENUMS", "open_filenums", 0},
216
{"uint8_t", "force_fsync_on_commit", 0},
217
{"uint64_t", "num_rollback_nodes", 0},
218
{"uint64_t", "num_rollentries", 0},
219
{"BLOCKNUM", "spilled_rollback_head", 0},
220
{"BLOCKNUM", "spilled_rollback_tail", 0},
221
{"BLOCKNUM", "current_rollback", 0},
222
NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED}, // record all transactions
223
// Records produced by transactions
224
{"xbegin", 'b', FA{{"TXNID_PAIR", "xid", 0},{"TXNID_PAIR", "parentxid", 0},NULLFIELD}, IGNORE_LOG_BEGIN},
225
{"xcommit",'C', FA{{"TXNID_PAIR", "xid", 0},NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
226
{"xprepare",'P', FA{{"TXNID_PAIR", "xid", 0}, {"XIDP", "xa_xid", 0}, NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
227
{"xabort", 'q', FA{{"TXNID_PAIR", "xid", 0},NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
228
//TODO: #2037 Add dname
229
{"fcreate", 'F', FA{{"TXNID_PAIR", "xid", 0},
230
{"FILENUM", "filenum", 0},
231
{"BYTESTRING", "iname", 0},
232
{"uint32_t", "mode", "0%o"},
233
{"uint32_t", "treeflags", 0},
234
{"uint32_t", "nodesize", 0},
235
{"uint32_t", "basementnodesize", 0},
236
{"uint32_t", "compression_method", 0},
237
NULLFIELD}, SHOULD_LOG_BEGIN},
238
//TODO: #2037 Add dname
239
{"fopen", 'O', FA{{"BYTESTRING", "iname", 0},
240
{"FILENUM", "filenum", 0},
241
{"uint32_t", "treeflags", 0},
242
NULLFIELD}, IGNORE_LOG_BEGIN},
243
//TODO: #2037 Add dname
244
{"fclose", 'e', FA{{"BYTESTRING", "iname", 0},
245
{"FILENUM", "filenum", 0},
246
NULLFIELD}, IGNORE_LOG_BEGIN},
247
//TODO: #2037 Add dname
248
{"fdelete", 'U', FA{{"TXNID_PAIR", "xid", 0},
249
{"FILENUM", "filenum", 0},
250
NULLFIELD}, SHOULD_LOG_BEGIN},
251
{"enq_insert", 'I', FA{{"FILENUM", "filenum", 0},
252
{"TXNID_PAIR", "xid", 0},
253
{"BYTESTRING", "key", 0},
254
{"BYTESTRING", "value", 0},
255
NULLFIELD}, SHOULD_LOG_BEGIN},
256
{"enq_insert_no_overwrite", 'i', FA{{"FILENUM", "filenum", 0},
257
{"TXNID_PAIR", "xid", 0},
258
{"BYTESTRING", "key", 0},
259
{"BYTESTRING", "value", 0},
260
NULLFIELD}, SHOULD_LOG_BEGIN},
261
{"enq_delete_any", 'E', FA{{"FILENUM", "filenum", 0},
262
{"TXNID_PAIR", "xid", 0},
263
{"BYTESTRING", "key", 0},
264
NULLFIELD}, SHOULD_LOG_BEGIN},
265
{"enq_insert_multiple", 'm', FA{{"FILENUM", "src_filenum", 0},
266
{"FILENUMS", "dest_filenums", 0},
267
{"TXNID_PAIR", "xid", 0},
268
{"BYTESTRING", "src_key", 0},
269
{"BYTESTRING", "src_val", 0},
270
NULLFIELD}, SHOULD_LOG_BEGIN},
271
{"enq_delete_multiple", 'M', FA{{"FILENUM", "src_filenum", 0},
272
{"FILENUMS", "dest_filenums", 0},
273
{"TXNID_PAIR", "xid", 0},
274
{"BYTESTRING", "src_key", 0},
275
{"BYTESTRING", "src_val", 0},
276
NULLFIELD}, SHOULD_LOG_BEGIN},
277
{"comment", 'T', FA{{"uint64_t", "timestamp", 0},
278
{"BYTESTRING", "comment", 0},
279
NULLFIELD}, IGNORE_LOG_BEGIN},
280
// Note: shutdown_up_to_19 log entry is NOT ALLOWED TO BE CHANGED.
281
// Do not change the letter ('Q'), do not add fields,
282
// do not remove fields.
283
// TODO: Kill this logentry entirely once we no longer support version 19.
284
{"shutdown_up_to_19", 'Q', FA{{"uint64_t", "timestamp", 0},
285
NULLFIELD}, IGNORE_LOG_BEGIN},
286
// Note: Shutdown log entry is NOT ALLOWED TO BE CHANGED.
287
// Do not change the letter ('0'), do not add fields,
288
// do not remove fields.
289
// You CAN leave this alone and add a new one, but then you have
290
// to deal with the upgrade mechanism again.
291
// This is how we detect clean shutdowns from OLDER VERSIONS.
292
// This log entry must always be readable for future versions.
293
// If you DO change it, you need to write a separate log upgrade mechanism.
294
{"shutdown", '0', FA{{"uint64_t", "timestamp", 0},
295
{"TXNID", "last_xid", 0},
296
NULLFIELD}, IGNORE_LOG_BEGIN},
297
{"load", 'l', FA{{"TXNID_PAIR", "xid", 0},
298
{"FILENUM", "old_filenum", 0},
299
{"BYTESTRING", "new_iname", 0},
300
NULLFIELD}, SHOULD_LOG_BEGIN},
302
{"hot_index", 'h', FA{{"TXNID_PAIR", "xid", 0},
303
{"FILENUMS", "hot_index_filenums", 0},
304
NULLFIELD}, SHOULD_LOG_BEGIN},
305
{"enq_update", 'u', FA{{"FILENUM", "filenum", 0},
306
{"TXNID_PAIR", "xid", 0},
307
{"BYTESTRING", "key", 0},
308
{"BYTESTRING", "extra", 0},
309
NULLFIELD}, SHOULD_LOG_BEGIN},
310
{"enq_updatebroadcast", 'B', FA{{"FILENUM", "filenum", 0},
311
{"TXNID_PAIR", "xid", 0},
312
{"BYTESTRING", "extra", 0},
313
{"bool", "is_resetting_op", 0},
314
NULLFIELD}, SHOULD_LOG_BEGIN},
315
{"change_fdescriptor", 'D', FA{{"FILENUM", "filenum", 0},
316
{"TXNID_PAIR", "xid", 0},
317
{"BYTESTRING", "old_descriptor", 0},
318
{"BYTESTRING", "new_descriptor", 0},
319
{"bool", "update_cmp_descriptor", 0},
320
NULLFIELD}, SHOULD_LOG_BEGIN},
321
{0,0,FA{NULLFIELD}, (enum log_begin_action) 0}
325
#define DO_STRUCTS(lt, array, body) do { \
326
const struct logtype *lt; \
327
for (lt=&array[0]; lt->name; lt++) { \
331
#define DO_ROLLBACKS(lt, body) DO_STRUCTS(lt, rollbacks, body)
333
#define DO_LOGTYPES(lt, body) DO_STRUCTS(lt, logtypes, body)
335
#define DO_LOGTYPES_AND_ROLLBACKS(lt, body) (DO_ROLLBACKS(lt,body), DO_LOGTYPES(lt, body))
337
#define DO_FIELDS(fld, lt, body) do { \
339
for (fld=lt->fields; fld->type; fld++) { \
344
static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
347
va_start(ap, format);
348
r=vfprintf(f1, format, ap); assert(r>=0);
350
va_start(ap, format);
351
r=vfprintf(f2, format, ap); assert(r>=0);
355
FILE *hf=0, *cf=0, *pf=0;
358
generate_enum_internal (const char *enum_name, const char *enum_prefix, const struct logtype *lts) {
361
memset(used_cmds, 0, 256);
362
fprintf(hf, "enum %s {", enum_name);
365
unsigned char cmd = (unsigned char)(lt->command_and_flags&0xff);
366
if (count!=0) fprintf(hf, ",");
369
fprintf(hf," %s_%-16s = '%c'", enum_prefix, lt->name, cmd);
370
if (used_cmds[cmd]!=0) { fprintf(stderr, "%s:%d: error: Command %d (%c) was used twice (second time for %s)\n", __FILE__, __LINE__, cmd, cmd, lt->name); abort(); }
373
fprintf(hf, "\n};\n\n");
378
generate_enum (void) {
379
generate_enum_internal("lt_cmd", "LT", logtypes);
380
generate_enum_internal("rt_cmd", "RT", rollbacks);
384
generate_log_struct (void) {
386
{ fprintf(hf, "struct logtype_%s {\n", lt->name);
387
fprintf(hf, " %-16s lsn;\n", "LSN");
388
DO_FIELDS(field_type, lt,
389
fprintf(hf, " %-16s %s;\n", field_type->type, field_type->name));
390
fprintf(hf, " %-16s crc;\n", "uint32_t");
391
fprintf(hf, " %-16s len;\n", "uint32_t");
393
//fprintf(hf, "void toku_recover_%s (LSN lsn", lt->name);
394
//DO_FIELDS(field_type, lt, fprintf(hf, ", %s %s", field_type->type, field_type->name));
395
//fprintf(hf, ");\n");
398
{ fprintf(hf, "struct rolltype_%s {\n", lt->name);
399
DO_FIELDS(field_type, lt,
400
fprintf(hf, " %-16s %s;\n", field_type->type, field_type->name));
402
fprintf(hf, "int toku_rollback_%s (", lt->name);
403
DO_FIELDS(field_type, lt, fprintf(hf, "%s %s,", field_type->type, field_type->name));
404
fprintf(hf, "TOKUTXN txn, LSN oplsn);\n");
405
fprintf(hf, "int toku_commit_%s (", lt->name);
406
DO_FIELDS(field_type, lt, fprintf(hf, "%s %s,", field_type->type, field_type->name));
407
fprintf(hf, "TOKUTXN txn, LSN oplsn);\n");
409
fprintf(hf, "struct log_entry {\n");
410
fprintf(hf, " enum lt_cmd cmd;\n");
411
fprintf(hf, " union {\n");
412
DO_LOGTYPES(lt, fprintf(hf," struct logtype_%s %s;\n", lt->name, lt->name));
413
fprintf(hf, " } u;\n");
416
fprintf(hf, "struct roll_entry {\n");
417
fprintf(hf, " enum rt_cmd cmd;\n");
418
fprintf(hf, " struct roll_entry *prev; /* for in-memory list of log entries. Threads from newest to oldest. */\n");
419
fprintf(hf, " union {\n");
420
DO_ROLLBACKS(lt, fprintf(hf," struct rolltype_%s %s;\n", lt->name, lt->name));
421
fprintf(hf, " } u;\n");
427
generate_dispatch (void) {
428
fprintf(hf, "#define rolltype_dispatch(s, funprefix) ({ switch((s)->cmd) {\\\n");
429
DO_ROLLBACKS(lt, fprintf(hf, " case RT_%s: funprefix ## %s (&(s)->u.%s); break;\\\n", lt->name, lt->name, lt->name));
430
fprintf(hf, " }})\n");
432
fprintf(hf, "#define logtype_dispatch_assign(s, funprefix, var, ...) do { switch((s)->cmd) {\\\n");
433
DO_LOGTYPES(lt, fprintf(hf, " case LT_%s: var = funprefix ## %s (&(s)->u.%s, __VA_ARGS__); break;\\\n", lt->name, lt->name, lt->name));
434
fprintf(hf, " }} while (0)\n");
436
fprintf(hf, "#define rolltype_dispatch_assign(s, funprefix, var, ...) do { \\\n");
437
fprintf(hf, " switch((s)->cmd) {\\\n");
439
fprintf(hf, " case RT_%s: var = funprefix ## %s (", lt->name, lt->name);
441
DO_FIELDS(field_type, lt, {
442
if (fieldcount>0) fprintf(hf, ",");
443
fprintf(hf, "(s)->u.%s.%s", lt->name, field_type->name);
446
fprintf(hf, ", __VA_ARGS__); break;\\\n");
448
fprintf(hf, " default: assert(0);} } while (0)\n");
450
fprintf(hf, "#define logtype_dispatch_args(s, funprefix, ...) do { switch((s)->cmd) {\\\n");
453
fprintf(hf, " case LT_%s: funprefix ## %s ((s)->u.%s.lsn", lt->name, lt->name, lt->name);
454
DO_FIELDS(field_type, lt, fprintf(hf, ",(s)->u.%s.%s", lt->name, field_type->name));
455
fprintf(hf, ", __VA_ARGS__); break;\\\n");
457
fprintf(hf, " }} while (0)\n");
461
generate_get_timestamp(void) {
462
fprintf(cf, "static uint64_t toku_get_timestamp(void) {\n");
463
fprintf(cf, " struct timeval tv; int r = gettimeofday(&tv, NULL);\n");
464
fprintf(cf, " assert(r==0);\n");
465
fprintf(cf, " return tv.tv_sec * 1000000ULL + tv.tv_usec;\n");
470
generate_log_writer (void) {
471
generate_get_timestamp();
473
//TODO(yoni): The overhead variables are NOT correct for BYTESTRING, FILENUMS (or any other variable length type)
474
// We should switch to something like using toku_logsizeof_*.
475
fprintf(hf, "static const size_t toku_log_%s_overhead = (+4+1+8", lt->name);
476
DO_FIELDS(field_type, lt, fprintf(hf, "+sizeof(%s)", field_type->type));
477
fprintf(hf, "+8);\n");
478
fprintf2(cf, hf, "void toku_log_%s (TOKULOGGER logger, LSN *lsnp, int do_fsync", lt->name);
479
switch (lt->log_begin_action) {
480
case SHOULD_LOG_BEGIN:
481
case ASSERT_BEGIN_WAS_LOGGED: {
482
fprintf2(cf, hf, ", TOKUTXN txn");
485
case IGNORE_LOG_BEGIN: break;
487
DO_FIELDS(field_type, lt, fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name));
489
fprintf(cf, ") {\n");
490
fprintf(cf, " if (logger == NULL) {\n");
491
fprintf(cf, " return;\n");
493
switch (lt->log_begin_action) {
494
case SHOULD_LOG_BEGIN: {
495
fprintf(cf, " //txn can be NULL during tests\n");
496
fprintf(cf, " //never null when not checkpoint.\n");
497
fprintf(cf, " if (txn && !txn->begin_was_logged) {\n");
498
fprintf(cf, " invariant(!txn_declared_read_only(txn));\n");
499
fprintf(cf, " toku_maybe_log_begin_txn_for_write_operation(txn);\n");
503
case ASSERT_BEGIN_WAS_LOGGED: {
504
fprintf(cf, " //txn can be NULL during tests\n");
505
fprintf(cf, " invariant(!txn || txn->begin_was_logged);\n");
506
fprintf(cf, " invariant(!txn || !txn_declared_read_only(txn));\n");
509
case IGNORE_LOG_BEGIN: break;
511
fprintf(cf, " if (!logger->write_log_files) {\n");
512
fprintf(cf, " ml_lock(&logger->input_lock);\n");
513
fprintf(cf, " logger->lsn.lsn++;\n");
514
fprintf(cf, " if (lsnp) *lsnp=logger->lsn;\n");
515
fprintf(cf, " ml_unlock(&logger->input_lock);\n");
516
fprintf(cf, " return;\n");
518
fprintf(cf, " const unsigned int buflen= (+4 // len at the beginning\n");
519
fprintf(cf, " +1 // log command\n");
520
fprintf(cf, " +8 // lsn\n");
521
DO_FIELDS(field_type, lt,
522
fprintf(cf, " +toku_logsizeof_%s(%s)\n", field_type->type, field_type->name));
523
fprintf(cf, " +8 // crc + len\n");
524
fprintf(cf, " );\n");
525
fprintf(cf, " struct wbuf wbuf;\n");
526
fprintf(cf, " ml_lock(&logger->input_lock);\n");
527
fprintf(cf, " toku_logger_make_space_in_inbuf(logger, buflen);\n");
528
fprintf(cf, " wbuf_nocrc_init(&wbuf, logger->inbuf.buf+logger->inbuf.n_in_buf, buflen);\n");
529
fprintf(cf, " wbuf_nocrc_int(&wbuf, buflen);\n");
530
fprintf(cf, " wbuf_nocrc_char(&wbuf, '%c');\n", (char)(0xff<->command_and_flags));
531
fprintf(cf, " logger->lsn.lsn++;\n");
532
fprintf(cf, " logger->inbuf.max_lsn_in_buf = logger->lsn;\n");
533
fprintf(cf, " wbuf_nocrc_LSN(&wbuf, logger->lsn);\n");
534
fprintf(cf, " if (lsnp) *lsnp=logger->lsn;\n");
535
DO_FIELDS(field_type, lt,
536
if (strcmp(field_type->name, "timestamp") == 0)
537
fprintf(cf, " if (timestamp == 0) timestamp = toku_get_timestamp();\n");
538
fprintf(cf, " wbuf_nocrc_%s(&wbuf, %s);\n", field_type->type, field_type->name));
539
fprintf(cf, " wbuf_nocrc_int(&wbuf, toku_x1764_memory(wbuf.buf, wbuf.ndone));\n");
540
fprintf(cf, " wbuf_nocrc_int(&wbuf, buflen);\n");
541
fprintf(cf, " assert(wbuf.ndone==buflen);\n");
542
fprintf(cf, " logger->inbuf.n_in_buf += buflen;\n");
543
fprintf(cf, " toku_logger_maybe_fsync(logger, logger->lsn, do_fsync, true);\n");
544
fprintf(cf, "}\n\n");
549
generate_log_reader (void) {
551
fprintf(cf, "static int toku_log_fread_%s (FILE *infile, uint32_t len1, struct logtype_%s *data, struct x1764 *checksum)", lt->name, lt->name);
553
fprintf(cf, " int r=0;\n");
554
fprintf(cf, " uint32_t actual_len=5; // 1 for the command, 4 for the first len.\n");
555
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", "LSN", "lsn");
556
DO_FIELDS(field_type, lt,
557
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", field_type->type, field_type->name));
558
fprintf(cf, " uint32_t checksum_in_file, len_in_file;\n");
559
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &checksum_in_file); actual_len+=4; if (r!=0) return r;\n");
560
fprintf(cf, " r=toku_fread_uint32_t_nocrclen(infile, &len_in_file); actual_len+=4; if (r!=0) return r;\n");
561
fprintf(cf, " if (checksum_in_file!=toku_x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
562
fprintf(cf, " return 0;\n");
563
fprintf(cf, "}\n\n");
565
fprintf2(cf, hf, "int toku_log_fread (FILE *infile, struct log_entry *le)");
568
fprintf(cf, " uint32_t len1; int r;\n");
569
fprintf(cf, " uint32_t ignorelen=0;\n");
570
fprintf(cf, " struct x1764 checksum;\n");
571
fprintf(cf, " toku_x1764_init(&checksum);\n");
572
fprintf(cf, " r = toku_fread_uint32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
573
fprintf(cf, " int cmd=fgetc(infile);\n");
574
fprintf(cf, " if (cmd==EOF) return EOF;\n");
575
fprintf(cf, " char cmdchar = (char)cmd;\n");
576
fprintf(cf, " toku_x1764_add(&checksum, &cmdchar, 1);\n");
577
fprintf(cf, " le->cmd=(enum lt_cmd)cmd;\n");
578
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
580
fprintf(cf, " case LT_%s:\n", lt->name);
581
fprintf(cf, " return toku_log_fread_%s (infile, len1, &le->u.%s, &checksum);\n", lt->name, lt->name);
583
fprintf(cf, " };\n");
584
fprintf(cf, " return DB_BADFORMAT;\n"); // Should read past the record using the len field.
585
fprintf(cf, "}\n\n");
586
//fprintf2(cf, hf, "// Return 0 if there is something to read, return -1 if nothing to read, abort if an error.\n");
587
fprintf2(cf, hf, "// Return 0 if there is something to read, -1 if nothing to read, >0 on error\n");
588
fprintf2(cf, hf, "int toku_log_fread_backward (FILE *infile, struct log_entry *le)");
591
fprintf(cf, " memset(le, 0, sizeof(*le));\n");
592
fprintf(cf, " long pos = ftell(infile);\n");
593
fprintf(cf, " if (pos<=12) return -1;\n");
594
fprintf(cf, " int r = fseek(infile, -4, SEEK_CUR); \n");// assert(r==0);\n");
595
fprintf(cf, " if (r!=0) return get_error_errno();\n");
596
fprintf(cf, " uint32_t len;\n");
597
fprintf(cf, " r = toku_fread_uint32_t_nocrclen(infile, &len); \n");// assert(r==0);\n");
598
fprintf(cf, " if (r!=0) return 1;\n");
599
fprintf(cf, " r = fseek(infile, -(int)len, SEEK_CUR) ; \n");// assert(r==0);\n");
600
fprintf(cf, " if (r!=0) return get_error_errno();\n");
601
fprintf(cf, " r = toku_log_fread(infile, le); \n");// assert(r==0);\n");
602
fprintf(cf, " if (r!=0) return 1;\n");
603
fprintf(cf, " long afterpos = ftell(infile);\n");
604
fprintf(cf, " if (afterpos != pos) return 1;\n");
605
fprintf(cf, " r = fseek(infile, -(int)len, SEEK_CUR); \n");// assert(r==0);\n");
606
fprintf(cf, " if (r!=0) return get_error_errno();\n");
607
fprintf(cf, " return 0;\n");
608
fprintf(cf, "}\n\n");
611
fprintf(cf, "static void toku_log_free_log_entry_%s_resources (struct logtype_%s *data", lt->name, lt->name);
612
if (!lt->fields->type) fprintf(cf, " __attribute__((__unused__))");
613
fprintf(cf, ") {\n");
614
DO_FIELDS(field_type, lt,
615
fprintf(cf, " toku_free_%s(data->%s);\n", field_type->type, field_type->name);
617
fprintf(cf, "}\n\n");
619
fprintf2(cf, hf, "void toku_log_free_log_entry_resources (struct log_entry *le)");
622
fprintf(cf, " switch ((enum lt_cmd)le->cmd) {\n");
624
fprintf(cf, " case LT_%s:\n", lt->name);
625
fprintf(cf, " return toku_log_free_log_entry_%s_resources (&(le->u.%s));\n", lt->name, lt->name);
627
fprintf(cf, " };\n");
628
fprintf(cf, " return;\n");
629
fprintf(cf, "}\n\n");
633
generate_logprint (void) {
634
unsigned maxnamelen=0;
635
fprintf2(pf, hf, "int toku_logprint_one_record(FILE *outf, FILE *f)");
638
fprintf(pf, " int cmd, r;\n");
639
fprintf(pf, " uint32_t len1, crc_in_file;\n");
640
fprintf(pf, " uint32_t ignorelen=0;\n");
641
fprintf(pf, " struct x1764 checksum;\n");
642
fprintf(pf, " toku_x1764_init(&checksum);\n");
643
fprintf(pf, " r=toku_fread_uint32_t(f, &len1, &checksum, &ignorelen);\n");
644
fprintf(pf, " if (r==EOF) return EOF;\n");
645
fprintf(pf, " cmd=fgetc(f);\n");
646
fprintf(pf, " if (cmd==EOF) return DB_BADFORMAT;\n");
647
fprintf(pf, " uint32_t len_in_file, len=1+4; // cmd + len1\n");
648
fprintf(pf, " char charcmd = (char)cmd;\n");
649
fprintf(pf, " toku_x1764_add(&checksum, &charcmd, 1);\n");
650
fprintf(pf, " switch ((enum lt_cmd)cmd) {\n");
651
DO_LOGTYPES(lt, { if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); });
653
unsigned char cmd = (unsigned char)(0xff<->command_and_flags);
654
fprintf(pf, " case LT_%s: \n", lt->name);
655
// We aren't using the log reader here because we want better diagnostics as soon as things go wrong.
656
fprintf(pf, " fprintf(outf, \"%%-%us \", \"%s\");\n", maxnamelen, lt->name);
657
if (isprint(cmd)) fprintf(pf," fprintf(outf, \" '%c':\");\n", cmd);
658
else fprintf(pf," fprintf(outf, \"0%03o:\");\n", cmd);
659
fprintf(pf, " r = toku_logprint_%-16s(outf, f, \"lsn\", &checksum, &len, 0); if (r!=0) return r;\n", "LSN");
660
DO_FIELDS(field_type, lt, {
661
fprintf(pf, " r = toku_logprint_%-16s(outf, f, \"%s\", &checksum, &len,", field_type->type, field_type->name);
662
if (field_type->format) fprintf(pf, "\"%s\"", field_type->format);
663
else fprintf(pf, "0");
664
fprintf(pf, "); if (r!=0) return r;\n");
667
fprintf(pf, " uint32_t actual_murmur = toku_x1764_finish(&checksum);\n");
668
fprintf(pf, " r = toku_fread_uint32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
669
fprintf(pf, " fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
670
fprintf(pf, " if (crc_in_file!=actual_murmur) fprintf(outf, \" checksum=%%08x\", actual_murmur);\n");
671
fprintf(pf, " r = toku_fread_uint32_t_nocrclen (f, &len_in_file); len+=4; if (r!=0) return r;\n");
672
fprintf(pf, " fprintf(outf, \" len=%%u\", len_in_file);\n");
673
fprintf(pf, " if (len_in_file!=len) fprintf(outf, \" actual_len=%%u\", len);\n");
674
fprintf(pf, " if (len_in_file!=len || crc_in_file!=actual_murmur) return DB_BADFORMAT;\n");
675
fprintf(pf, " };\n");
676
fprintf(pf, " fprintf(outf, \"\\n\");\n");
677
fprintf(pf, " return 0;\n\n");
680
fprintf(pf, " fprintf(outf, \"Unknown command %%d ('%%c')\", cmd, cmd);\n");
681
fprintf(pf, " return DB_BADFORMAT;\n");
682
fprintf(pf, "}\n\n");
686
generate_rollbacks (void) {
688
fprintf2(cf, hf, "void toku_logger_save_rollback_%s (TOKUTXN txn", lt->name);
689
DO_FIELDS(field_type, lt, {
690
if ( strcmp(field_type->type, "BYTESTRING") == 0 ) {
691
fprintf2(cf, hf, ", BYTESTRING *%s_ptr", field_type->name);
693
else if ( strcmp(field_type->type, "FILENUMS") == 0 ) {
694
fprintf2(cf, hf, ", FILENUMS *%s_ptr", field_type->name);
697
fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name);
702
fprintf(cf, ") {\n");
703
fprintf(cf, " toku_txn_lock(txn);\n");
704
fprintf(cf, " ROLLBACK_LOG_NODE log;\n");
705
fprintf(cf, " toku_get_and_pin_rollback_log_for_new_entry(txn, &log);\n");
706
// 'memdup' all BYTESTRINGS here
707
DO_FIELDS(field_type, lt, {
708
if ( strcmp(field_type->type, "BYTESTRING") == 0 ) {
709
fprintf(cf, " BYTESTRING %s = {\n"
710
" .len = %s_ptr->len,\n"
711
" .data = cast_to_typeof(%s.data) toku_memdup_in_rollback(log, %s_ptr->data, %s_ptr->len)\n"
713
field_type->name, field_type->name, field_type->name, field_type->name, field_type->name);
715
if ( strcmp(field_type->type, "FILENUMS") == 0 ) {
716
fprintf(cf, " FILENUMS %s = {\n"
717
" .num = %s_ptr->num,\n"
718
" .filenums = cast_to_typeof(%s.filenums) toku_memdup_in_rollback(log, %s_ptr->filenums, %s_ptr->num * (sizeof (FILENUM)))\n"
720
field_type->name, field_type->name, field_type->name, field_type->name, field_type->name);
725
fprintf(cf, " uint32_t rollback_fsize = toku_logger_rollback_fsize_%s(", lt->name);
726
DO_FIELDS(field_type, lt, fprintf(cf, "%s%s", (count++>0)?", ":"", field_type->name));
729
fprintf(cf, " struct roll_entry *v;\n");
730
fprintf(cf, " size_t mem_needed = sizeof(v->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
731
fprintf(cf, " CAST_FROM_VOIDP(v, toku_malloc_in_rollback(log, mem_needed));\n");
732
fprintf(cf, " assert(v);\n");
733
fprintf(cf, " v->cmd = (enum rt_cmd)%u;\n", lt->command_and_flags&0xff);
734
DO_FIELDS(field_type, lt, fprintf(cf, " v->u.%s.%s = %s;\n", lt->name, field_type->name, field_type->name));
735
fprintf(cf, " v->prev = log->newest_logentry;\n");
736
fprintf(cf, " if (log->oldest_logentry==NULL) log->oldest_logentry=v;\n");
737
fprintf(cf, " log->newest_logentry = v;\n");
738
fprintf(cf, " log->rollentry_resident_bytecount += rollback_fsize;\n");
739
fprintf(cf, " txn->roll_info.rollentry_raw_count += rollback_fsize;\n");
740
fprintf(cf, " txn->roll_info.num_rollentries++;\n");
741
fprintf(cf, " log->dirty = true;\n");
742
fprintf(cf, " // spill and unpin assert success internally\n");
743
fprintf(cf, " toku_maybe_spill_rollbacks(txn, log);\n");
744
fprintf(cf, " toku_rollback_log_unpin(txn, log);\n");
745
fprintf(cf, " toku_txn_unlock(txn);\n");
750
fprintf2(cf, hf, "void toku_logger_rollback_wbuf_nocrc_write_%s (struct wbuf *wbuf", lt->name);
751
DO_FIELDS(field_type, lt, fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name));
752
fprintf2(cf, hf, ")");
758
fprintf(cf, " uint32_t rollback_fsize = toku_logger_rollback_fsize_%s(", lt->name);
759
DO_FIELDS(field_type, lt, fprintf(cf, "%s%s", (count++>0)?", ":"", field_type->name));
761
fprintf(cf, " wbuf_nocrc_int(wbuf, rollback_fsize);\n");
763
fprintf(cf, " wbuf_nocrc_char(wbuf, '%c');\n", (char)(0xff<->command_and_flags));
764
DO_FIELDS(field_type, lt, fprintf(cf, " wbuf_nocrc_%s(wbuf, %s);\n", field_type->type, field_type->name));
767
fprintf2(cf, hf, "void toku_logger_rollback_wbuf_nocrc_write (struct wbuf *wbuf, struct roll_entry *r)");
769
fprintf(cf, " {\n switch (r->cmd) {\n");
771
fprintf(cf, " case RT_%s: toku_logger_rollback_wbuf_nocrc_write_%s(wbuf", lt->name, lt->name);
772
DO_FIELDS(field_type, lt, fprintf(cf, ", r->u.%s.%s", lt->name, field_type->name));
773
fprintf(cf, "); return;\n");
775
fprintf(cf, " }\n assert(0);\n");
778
fprintf2(cf, hf, "uint32_t toku_logger_rollback_fsize_%s (", lt->name);
780
DO_FIELDS(field_type, lt, fprintf2(cf, hf, "%s%s %s", (count++>0)?", ":"", field_type->type, field_type->name));
782
fprintf(cf, ") {\n");
783
fprintf(cf, " return 1 /* the cmd*/\n");
784
fprintf(cf, " + 4 /* the int at the end saying the size */");
785
DO_FIELDS(field_type, lt,
786
fprintf(cf, "\n + toku_logsizeof_%s(%s)", field_type->type, field_type->name));
787
fprintf(cf, ";\n}\n");
789
fprintf2(cf, hf, "uint32_t toku_logger_rollback_fsize(struct roll_entry *item)");
791
fprintf(cf, "{\n switch(item->cmd) {\n");
793
fprintf(cf, " case RT_%s: return toku_logger_rollback_fsize_%s(", lt->name, lt->name);
795
DO_FIELDS(field_type, lt, fprintf(cf, "%sitem->u.%s.%s", (count++>0)?", ":"", lt->name, field_type->name));
798
fprintf(cf, " }\n assert(0);\n return 0;\n");
801
fprintf2(cf, hf, "int toku_parse_rollback(unsigned char *buf, uint32_t n_bytes, struct roll_entry **itemp, MEMARENA ma)");
803
fprintf(cf, " {\n assert(n_bytes>0);\n struct roll_entry *item;\n enum rt_cmd cmd = (enum rt_cmd)(buf[0]);\n size_t mem_needed;\n");
804
fprintf(cf, " struct rbuf rc = {buf, n_bytes, 1};\n");
805
fprintf(cf, " switch(cmd) {\n");
807
fprintf(cf, " case RT_%s:\n", lt->name);
808
fprintf(cf, " mem_needed = sizeof(item->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
809
fprintf(cf, " CAST_FROM_VOIDP(item, toku_memarena_malloc(ma, mem_needed));\n");
810
fprintf(cf, " item->cmd = cmd;\n");
811
DO_FIELDS(field_type, lt, fprintf(cf, " rbuf_ma_%s(&rc, ma, &item->u.%s.%s);\n", field_type->type, lt->name, field_type->name));
812
fprintf(cf, " *itemp = item;\n");
813
fprintf(cf, " return 0;\n");
815
fprintf(cf, " }\n return EINVAL;\n}\n");
819
generate_log_entry_functions(void) {
820
fprintf(hf, "LSN toku_log_entry_get_lsn(struct log_entry *);\n");
821
fprintf(cf, "LSN toku_log_entry_get_lsn(struct log_entry *le) {\n");
822
fprintf(cf, " return le->u.begin_checkpoint.lsn;\n");
826
const char codefile[] = "log_code.cc";
827
const char printfile[] = "log_print.cc";
828
const char headerfile[] = "log_header.h";
829
int main (int argc, const char *const argv[]) {
830
assert(argc==2); // the single argument is the directory into which to put things
831
const char *dir = argv[1];
832
size_t codepathlen = sizeof(codefile) + strlen(dir) + 4;
833
size_t printpathlen = sizeof(printfile) + strlen(dir) + 4;
834
size_t headerpathlen = sizeof(headerfile) + strlen(dir) + 4;
835
char codepath[codepathlen];
836
char printpath[printpathlen];
837
char headerpath[headerpathlen];
838
{ int r = snprintf(codepath, codepathlen, "%s/%s", argv[1], codefile); assert(r<(int)codepathlen); }
839
{ int r = snprintf(printpath, printpathlen, "%s/%s", argv[1], printfile); assert(r<(int)printpathlen); }
840
{ int r = snprintf(headerpath, headerpathlen, "%s/%s", argv[1], headerfile); assert(r<(int)headerpathlen); }
841
chmod(codepath, S_IRUSR|S_IWUSR);
842
chmod(headerpath, S_IRUSR|S_IWUSR);
845
cf = fopen(codepath, "w");
846
if (cf==0) { int r = get_error_errno(); printf("fopen of %s failed because of errno=%d (%s)\n", codepath, r, strerror(r)); } // sometimes this is failing, so let's make a better diagnostic
848
hf = fopen(headerpath, "w"); assert(hf!=0);
849
pf = fopen(printpath, "w"); assert(pf!=0);
850
fprintf2(cf, hf, "/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */\n");
851
fprintf2(cf, hf, "// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:\n");
852
fprintf(hf, "#ifndef LOG_HEADER_H\n");
853
fprintf(hf, "#define LOG_HEADER_H\n");
854
fprintf2(cf, hf, "/* Do not edit this file. This code generated by logformat.c. Copyright (c) 2007-2013 Tokutek Inc. */\n");
855
fprintf2(cf, hf, "#ident \"Copyright (c) 2007-2013 Tokutek Inc. All rights reserved.\"\n");
856
fprintf2(cf, pf, "#include <stdint.h>\n");
857
fprintf2(cf, pf, "#include <sys/time.h>\n");
858
fprintf2(cf, pf, "#include <ft/fttypes.h>\n");
859
fprintf2(cf, pf, "#include <ft/log-internal.h>\n");
860
fprintf(hf, "#include <ft/ft-internal.h>\n");
861
fprintf(hf, "#include <util/memarena.h>\n");
863
generate_log_struct();
865
generate_log_writer();
866
generate_log_reader();
867
generate_rollbacks();
868
generate_log_entry_functions();
870
fprintf(hf, "#endif\n");
872
int r=fclose(hf); assert(r==0);
873
r=fclose(cf); assert(r==0);
874
r=fclose(pf); assert(r==0);
875
// Make it tougher to modify by mistake
876
chmod(codepath, S_IRUSR|S_IRGRP|S_IROTH);
877
chmod(headerpath, S_IRUSR|S_IRGRP|S_IROTH);
added
removed
storage/tokudb/ft-index/ft/logformat.cc
