1
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
4
COPYING CONDITIONS NOTICE:
6
This program is free software; you can redistribute it and/or modify
7
it under the terms of version 2 of the GNU General Public License as
8
published by the Free Software Foundation, and provided that the
9
following conditions are met:
11
* Redistributions of source code must retain this COPYING
12
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
13
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
14
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
17
* Redistributions in binary form must reproduce this COPYING
18
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
19
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
20
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
21
GRANT (below) in the documentation and/or other materials
22
provided with the distribution.
24
You should have received a copy of the GNU General Public License
25
along with this program; if not, write to the Free Software
26
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
31
TokuDB, Tokutek Fractal Tree Indexing Library.
32
Copyright (C) 2007-2013 Tokutek, Inc.
36
This program is distributed in the hope that it will be useful, but
37
WITHOUT ANY WARRANTY; without even the implied warranty of
38
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
39
General Public License for more details.
41
UNIVERSITY PATENT NOTICE:
43
The technology is licensed by the Massachusetts Institute of
44
Technology, Rutgers State University of New Jersey, and the Research
45
Foundation of State University of New York at Stony Brook under
46
United States of America Serial No. 11/760379 and to the patents
47
and/or patent applications resulting from it.
49
PATENT MARKING NOTICE:
51
This software is covered by US Patent No. 8,185,551.
52
This software is covered by US Patent No. 8,489,638.
56
"THIS IMPLEMENTATION" means the copyrightable works distributed by
57
Tokutek as part of the Fractal Tree project.
59
"PATENT CLAIMS" means the claims of patents that are owned or
60
licensable by Tokutek, both currently or in the future; and that in
61
the absence of this license would be infringed by THIS
62
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
64
"PATENT CHALLENGE" shall mean a challenge to the validity,
65
patentability, enforceability and/or non-infringement of any of the
66
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
68
Tokutek hereby grants to you, for the term and geographical scope of
69
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
70
irrevocable (except as stated in this section) patent license to
71
make, have made, use, offer to sell, sell, import, transfer, and
72
otherwise run, modify, and propagate the contents of THIS
73
IMPLEMENTATION, where such license applies only to the PATENT
74
CLAIMS. This grant does not include claims that would be infringed
75
only as a consequence of further modifications of THIS
76
IMPLEMENTATION. If you or your agent or licensee institute or order
77
or agree to the institution of patent litigation against any entity
78
(including a cross-claim or counterclaim in a lawsuit) alleging that
79
THIS IMPLEMENTATION constitutes direct or contributory patent
80
infringement, or inducement of patent infringement, then any rights
81
granted to you under this License shall terminate as of the date
82
such litigation is filed. If you or your agent or exclusive
83
licensee institute or order or agree to the institution of a PATENT
84
CHALLENGE, then Tokutek may terminate any rights granted to you
88
#ident "Copyright (c) 2010-2013 Tokutek Inc. All rights reserved."
92
#include "toku_pthread.h"
95
#include "toku_random.h"
101
uint32_t USE_COMPRESS=0;
103
bool do_check = false;
104
uint32_t num_rows = 1;
105
uint32_t which_db_to_fail = (uint32_t) -1;
106
uint32_t which_row_to_fail = (uint32_t) -1;
107
enum how_to_fail { FAIL_NONE, FAIL_KSIZE, FAIL_VSIZE } how_to_fail = FAIL_NONE;
109
static struct random_data random_data[NUM_DBS];
110
char random_buf[NUM_DBS][8];
112
static int put_multiple_generate(DB *dest_db,
113
DB *src_db __attribute__((__unused__)),
114
DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals,
115
const DBT *src_key, const DBT *src_val __attribute__((__unused__))) {
116
toku_dbt_array_resize(dest_keys, 1);
117
toku_dbt_array_resize(dest_vals, 1);
118
DBT *dest_key = &dest_keys->dbts[0];
119
DBT *dest_val = &dest_vals->dbts[0];
121
uint32_t which = *(uint32_t*)dest_db->app_private;
122
assert(src_key->size==4);
123
uint32_t rownum = *(uint32_t*)src_key->data;
125
uint32_t ksize, vsize;
126
const uint32_t kmax=32*1024, vmax=32*1024*1024;
127
if (which==which_db_to_fail && rownum==which_row_to_fail) {
128
switch (how_to_fail) {
129
case FAIL_NONE: ksize=kmax; vsize=vmax; goto gotsize;
130
case FAIL_KSIZE: ksize=kmax+1; vsize=vmax; goto gotsize;
131
case FAIL_VSIZE: ksize=kmax; vsize=vmax+1; goto gotsize;
138
assert(dest_key->flags==DB_DBT_REALLOC);
139
if (dest_key->ulen < ksize) {
140
dest_key->data = toku_xrealloc(dest_key->data, ksize);
141
dest_key->ulen = ksize;
143
assert(dest_val->flags==DB_DBT_REALLOC);
144
if (dest_val->ulen < vsize) {
145
dest_val->data = toku_xrealloc(dest_val->data, vsize);
146
dest_val->ulen = vsize;
148
assert(ksize>=sizeof(uint32_t));
149
for (uint32_t i=0; i<ksize; i++) ((char*)dest_key->data)[i] = myrandom_r(&random_data[which]);
150
for (uint32_t i=0; i<vsize; i++) ((char*)dest_val->data)[i] = myrandom_r(&random_data[which]);
151
*(uint32_t*)dest_key->data = rownum;
152
dest_key->size = ksize;
153
dest_val->size = vsize;
163
static void error_callback (DB *db __attribute__((__unused__)), int which_db, int err, DBT *key __attribute__((__unused__)), DBT *val __attribute__((__unused__)), void *extra) {
164
struct error_extra *e =(struct error_extra *)extra;
165
assert(which_db==(int)which_db_to_fail);
167
assert(e->error_count==0);
171
static void reset_random(void) {
174
for (int i = 0; i < NUM_DBS; i++) {
175
ZERO_STRUCT(random_data[i]);
176
ZERO_ARRAY(random_buf[i]);
177
r = myinitstate_r(i, random_buf[i], 8, &random_data[i]);
182
static void test_loader_maxsize(DB **dbs, DB **check_dbs)
187
uint32_t db_flags[NUM_DBS];
188
uint32_t dbt_flags[NUM_DBS];
189
for(int i=0;i<NUM_DBS;i++) {
190
db_flags[i] = DB_NOOVERWRITE;
193
uint32_t loader_flags = USE_COMPRESS; // set with -p option
195
// create and initialize loader
196
r = env->txn_begin(env, NULL, &txn, 0);
198
r = env->create_loader(env, txn, &loader, nullptr, NUM_DBS, dbs, db_flags, dbt_flags, loader_flags);
199
assert(which_db_to_fail != 0);
201
struct error_extra error_extra = {.bad_i=0,.error_count=0};
202
r = loader->set_error_callback(loader, error_callback, (void*)&error_extra);
204
r = loader->set_poll_function(loader, NULL, NULL);
208
// using loader->put, put values into DB
211
for(uint32_t i=0;i<num_rows;i++) {
214
dbt_init(&key, &k, sizeof(unsigned int));
215
dbt_init(&val, &v, sizeof(unsigned int));
216
r = loader->put(loader, &key, &val);
221
if (verbose) { printf("closing"); fflush(stdout); }
222
r = loader->close(loader);
223
if (verbose) { printf(" done\n"); }
224
switch(how_to_fail) {
225
case FAIL_NONE: assert(r==0); assert(error_extra.error_count==0); goto checked;
226
case FAIL_KSIZE: assert(r==EINVAL); assert(error_extra.error_count==1); goto checked;
227
case FAIL_VSIZE: assert(r==EINVAL); assert(error_extra.error_count==1); goto checked;
231
r = txn->commit(txn, 0);
234
if (do_check && how_to_fail==FAIL_NONE) {
235
r = env->txn_begin(env, NULL, &txn, 0);
240
uint32_t flags[NUM_DBS];
241
for (int i = 0; i < NUM_DBS; i++) {
242
dbt_init_realloc(&keys[i]);
243
dbt_init_realloc(&vals[i]);
247
for(uint32_t i=0;i<num_rows;i++) {
250
dbt_init(&key, &k, sizeof(unsigned int));
251
dbt_init(&val, &v, sizeof(unsigned int));
252
r = env_put_multiple_test_no_array(env, nullptr, txn, &key, &val, NUM_DBS, check_dbs, keys, vals, flags);
255
r = txn->commit(txn, 0);
257
r = env->txn_begin(env, NULL, &txn, 0);
260
for (int i = 0; i < NUM_DBS; i++) {
263
r = dbs[i]->cursor(dbs[i], txn, &loader_cursor, 0);
265
r = dbs[i]->cursor(check_dbs[i], txn, &check_cursor, 0);
271
dbt_init_realloc(&loader_key);
272
dbt_init_realloc(&loader_val);
273
dbt_init_realloc(&check_key);
274
dbt_init_realloc(&check_val);
275
for (uint32_t x = 0; x <= num_rows; x++) {
276
int r_loader = loader_cursor->c_get(loader_cursor, &loader_key, &loader_val, DB_NEXT);
277
int r_check = check_cursor->c_get(check_cursor, &check_key, &check_val, DB_NEXT);
278
assert(r_loader == r_check);
280
CKERR2(r_loader, DB_NOTFOUND);
281
CKERR2(r_check, DB_NOTFOUND);
286
assert(loader_key.size == check_key.size);
287
assert(loader_val.size == check_val.size);
288
assert(memcmp(loader_key.data, check_key.data, loader_key.size) == 0);
289
assert(memcmp(loader_val.data, check_val.data, loader_val.size) == 0);
291
toku_free(loader_key.data);
292
toku_free(loader_val.data);
293
toku_free(check_key.data);
294
toku_free(check_val.data);
295
loader_cursor->c_close(loader_cursor);
296
check_cursor->c_close(check_cursor);
299
for (int i = 0; i < NUM_DBS; i++) {
300
toku_free(keys[i].data);
301
toku_free(vals[i].data);
302
dbt_init_realloc(&keys[i]);
303
dbt_init_realloc(&vals[i]);
305
r = txn->commit(txn, 0);
312
char *free_me = NULL;
313
const char *env_dir = TOKU_TEST_FILENAME; // the default env_dir
315
static void create_and_open_dbs(DB **dbs, const char *suffix, int *idx) {
318
dbt_init(&desc, "foo", sizeof("foo"));
320
char name[MAX_NAME*2];
322
for(int i=0;i<NUM_DBS;i++) {
324
r = db_create(&dbs[i], env, 0); CKERR(r);
325
dbs[i]->app_private = &idx[i];
326
snprintf(name, sizeof(name), "db_%04x_%s", i, suffix);
327
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
328
IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
329
{ int chk_r = dbs[i]->change_descriptor(dbs[i], txn_desc, &desc, 0); CKERR(chk_r); }
335
uint_or_size_dbt_cmp (DB *db, const DBT *a, const DBT *b) {
336
assert(db && a && b);
337
if (a->size == sizeof(unsigned int) && b->size == sizeof(unsigned int)) {
338
return uint_dbt_cmp(db, a, b);
340
return a->size - b->size;
343
static void run_test(uint32_t nr, uint32_t wdb, uint32_t wrow, enum how_to_fail htf) {
344
num_rows = nr; which_db_to_fail = wdb; which_row_to_fail = wrow; how_to_fail = htf;
347
toku_os_recursive_delete(env_dir);
348
r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
350
r = db_env_create(&env, 0); CKERR(r);
351
r = env->set_default_bt_compare(env, uint_or_size_dbt_cmp); CKERR(r);
352
r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
354
int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOG | DB_CREATE | DB_PRIVATE;
355
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
356
env->set_errfile(env, stderr);
357
//Disable auto-checkpointing
358
r = env->checkpointing_set_period(env, 0); CKERR(r);
360
DB **XMALLOC_N(NUM_DBS, dbs);
361
DB **XMALLOC_N(NUM_DBS, check_dbs);
364
create_and_open_dbs(dbs, "loader", &idx[0]);
365
if (do_check && how_to_fail==FAIL_NONE) {
366
create_and_open_dbs(check_dbs, "check", &idx[0]);
369
if (verbose) printf("running test_loader()\n");
370
// -------------------------- //
371
test_loader_maxsize(dbs, check_dbs);
372
// -------------------------- //
373
if (verbose) printf("done test_loader()\n");
375
for(int i=0;i<NUM_DBS;i++) {
376
dbs[i]->close(dbs[i], 0); CKERR(r);
378
if (do_check && how_to_fail==FAIL_NONE) {
379
check_dbs[i]->close(check_dbs[i], 0); CKERR(r);
383
r = env->close(env, 0); CKERR(r);
385
toku_free(check_dbs);
388
// ------------ infrastructure ----------
389
static void do_args(int argc, char * const argv[]);
391
int num_rows_set = false;
393
int test_main(int argc, char * const *argv) {
396
run_test(1, (uint32_t) -1, (uint32_t) -1, FAIL_NONE);
397
run_test(1, 1, 0, FAIL_NONE);
398
run_test(1, 1, 0, FAIL_KSIZE);
399
run_test(1, 1, 0, FAIL_VSIZE);
401
run_test(1000000, 1, 500000, FAIL_KSIZE);
402
run_test(1000000, 1, 500000, FAIL_VSIZE);
408
static void do_args(int argc, char * const argv[]) {
413
if (strcmp(argv[0], "-h")==0) {
416
fprintf(stderr, "Usage: %s [-h] [-v] [-q] [-p] [-f]\n", cmd);
417
fprintf(stderr, " where -e <env> uses <env> to construct the directory (so that different tests can run concurrently)\n");
418
fprintf(stderr, " -h help\n");
419
fprintf(stderr, " -v verbose\n");
420
fprintf(stderr, " -q quiet\n");
421
fprintf(stderr, " -z compress intermediates\n");
422
fprintf(stderr, " -c compare with regular dbs\n");
423
fprintf(stderr, " -f fast (suitable for vgrind)\n");
425
} else if (strcmp(argv[0], "-c")==0) {
427
} else if (strcmp(argv[0], "-v")==0) {
429
} else if (strcmp(argv[0],"-q")==0) {
431
if (verbose<0) verbose=0;
432
} else if (strcmp(argv[0], "-z")==0) {
433
USE_COMPRESS = LOADER_COMPRESS_INTERMEDIATES;
434
} else if (strcmp(argv[0], "-f")==0) {
437
fprintf(stderr, "Unknown arg: %s\n", argv[0]);