1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 #ident "$Id$"
3 /*======
4 This file is part of PerconaFT.
5
6
7 Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
8
9 PerconaFT is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License, version 2,
11 as published by the Free Software Foundation.
12
13 PerconaFT is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
20
21 ----------------------------------------
22
23 PerconaFT is free software: you can redistribute it and/or modify
24 it under the terms of the GNU Affero General Public License, version 3,
25 as published by the Free Software Foundation.
26
27 PerconaFT is distributed in the hope that it will be useful,
28 but WITHOUT ANY WARRANTY; without even the implied warranty of
29 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
30 GNU Affero General Public License for more details.
31
32 You should have received a copy of the GNU Affero General Public License
33 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
34 ======= */
35
36 #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
37
38 // Need to use malloc for the malloc instrumentation tests
39 #ifndef TOKU_ALLOW_DEPRECATED
40 #define TOKU_ALLOW_DEPRECATED
41 #endif
42
43 #include "test.h"
44 #include "toku_pthread.h"
45 #include <db.h>
46 #include <sys/stat.h>
47 #include "ydb-internal.h"
48 #include <memory.h>
49 #include <dlfcn.h>
50
51 DB_ENV *env;
52 enum {MAX_NAME=128};
53 enum {MAX_DBS=1024};
54 int NUM_DBS=1;
55 int NUM_ROWS=1000000;
56 int CHECK_RESULTS=1;
57 int DISALLOW_PUTS=0;
58 int COMPRESS=0;
59 enum { old_default_cachesize=1024 }; // MB
60 int CACHESIZE=old_default_cachesize;
61 int ALLOW_DUPS=0;
62 enum {MAGIC=311};
63 char *datadir = NULL;
64 bool check_est = true; // do check the estimates by default
65 bool footprint_print = false; // print memory footprint info
66 bool upgrade_test = false;
67
68 // Code for showing memory footprint information.
69 pthread_mutex_t my_lock = PTHREAD_MUTEX_INITIALIZER;
70 size_t hiwater;
71 size_t water;
72 size_t hiwater_start;
73 static long long mcount = 0, fcount=0;
74
75
my_free(void * p)76 static void my_free(void*p) {
77 if (p) {
78 water-=toku_malloc_usable_size(p);
79 }
80 free(p);
81 }
82
my_malloc(size_t size)83 static void *my_malloc(size_t size) {
84 void *r = malloc(size);
85 if (r) {
86 water += toku_malloc_usable_size(r);
87 if (water>hiwater) hiwater=water;
88 }
89 return r;
90 }
91
my_realloc(void * p,size_t size)92 static void *my_realloc(void *p, size_t size) {
93 size_t old_usable = p ? toku_malloc_usable_size(p) : 0;
94 void *r = realloc(p, size);
95 if (r) {
96 water -= old_usable;
97 water += toku_malloc_usable_size(r);
98 }
99 return r;
100 }
101
102 //
103 // Functions to create unique key/value pairs, row generators, checkers, ... for each of NUM_DBS
104 //
105
106 // a is the bit-wise permute table. For DB[i], permute bits as described in a[i] using 'twiddle32'
107 // inv is the inverse bit-wise permute of a[]. To get the original value from a twiddled value, twiddle32 (again) with inv[]
108 int a[MAX_DBS][32];
109 int inv[MAX_DBS][32];
110
111
112 static const char *loader_temp_prefix = "tokuld"; // #2536
113
114 // return number of temp files
115 static int
count_temp(char * dirname)116 count_temp(char * dirname) {
117 int n = 0;
118
119 DIR * dir = opendir(dirname);
120
121 struct dirent *ent;
122 while ((ent=readdir(dir))) {
123 if ((ent->d_type==DT_REG || ent->d_type==DT_UNKNOWN) && strncmp(ent->d_name, loader_temp_prefix, 6)==0) {
124 n++;
125 if (verbose) {
126 printf("Temp files (%d)\n", n);
127 printf(" %s/%s\n", dirname, ent->d_name);
128 }
129 }
130 }
131 closedir(dir);
132 return n;
133 }
134
135 // rotate right and left functions
rotr32(const unsigned int x,const unsigned int num)136 static inline unsigned int rotr32(const unsigned int x, const unsigned int num) {
137 if (num == 0) {
138 return x;
139 } else {
140 const unsigned int n = num % 32;
141 return (x >> n) | ( x << (32 - n));
142 }
143 }
rotl32(const unsigned int x,const unsigned int num)144 static inline unsigned int rotl32(const unsigned int x, const unsigned int num) {
145 if (num == 0) {
146 return x;
147 } else {
148 const unsigned int n = num % 32;
149 return (x << n) | ( x >> (32 - n));
150 }
151 }
152
generate_permute_tables(void)153 static void generate_permute_tables(void) {
154 int i, j, tmp;
155 for(int db=0;db<MAX_DBS;db++) {
156 for(i=0;i<32;i++) {
157 a[db][i] = i;
158 }
159 for(i=0;i<32;i++) {
160 j = random() % (i + 1);
161 tmp = a[db][j];
162 a[db][j] = a[db][i];
163 a[db][i] = tmp;
164 }
165 // if(db < NUM_DBS){ printf("a[%d] = ", db); for(i=0;i<32;i++) { printf("%2d ", a[db][i]); } printf("\n");}
166 for(i=0;i<32;i++) {
167 inv[db][a[db][i]] = i;
168 }
169 }
170 }
171
172 // permute bits of x based on permute table bitmap
twiddle32(unsigned int x,int db)173 static unsigned int twiddle32(unsigned int x, int db)
174 {
175 unsigned int b = 0;
176 for(int i=0;i<32;i++) {
177 b |= (( x >> i ) & 1) << a[db][i];
178 }
179 return b;
180 }
181
182 // permute bits of x based on inverse permute table bitmap
inv_twiddle32(unsigned int x,int db)183 static unsigned int inv_twiddle32(unsigned int x, int db)
184 {
185 unsigned int b = 0;
186 for(int i=0;i<32;i++) {
187 b |= (( x >> i ) & 1) << inv[db][i];
188 }
189 return b;
190 }
191
192 // generate val from key, index
generate_val(int key,int i)193 static unsigned int generate_val(int key, int i) {
194 return rotl32((key + MAGIC), i);
195 }
pkey_for_val(int key,int i)196 static unsigned int pkey_for_val(int key, int i) {
197 return rotr32(key, i) - MAGIC;
198 }
199
200 // There is no handlerton in this test, so this function is a local replacement
201 // for the handlerton's generate_row_for_put().
put_multiple_generate(DB * dest_db,DB * src_db,DBT_ARRAY * dest_keys,DBT_ARRAY * dest_vals,const DBT * src_key,const DBT * src_val)202 static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
203 toku_dbt_array_resize(dest_keys, 1);
204 toku_dbt_array_resize(dest_vals, 1);
205 DBT *dest_key = &dest_keys->dbts[0];
206 DBT *dest_val = &dest_vals->dbts[0];
207
208 (void) src_db;
209
210 uint32_t which = *(uint32_t*)dest_db->app_private;
211
212 if ( which == 0 ) {
213 if (dest_key->flags==DB_DBT_REALLOC) {
214 if (dest_key->data) toku_free(dest_key->data);
215 dest_key->flags = 0;
216 dest_key->ulen = 0;
217 }
218 if (dest_val->flags==DB_DBT_REALLOC) {
219 if (dest_val->data) toku_free(dest_val->data);
220 dest_val->flags = 0;
221 dest_val->ulen = 0;
222 }
223 dbt_init(dest_key, src_key->data, src_key->size);
224 dbt_init(dest_val, src_val->data, src_val->size);
225 }
226 else {
227 assert(dest_key->flags==DB_DBT_REALLOC);
228 if (dest_key->ulen < sizeof(unsigned int)) {
229 dest_key->data = toku_xrealloc(dest_key->data, sizeof(unsigned int));
230 dest_key->ulen = sizeof(unsigned int);
231 }
232 assert(dest_val->flags==DB_DBT_REALLOC);
233 if (dest_val->ulen < sizeof(unsigned int)) {
234 dest_val->data = toku_xrealloc(dest_val->data, sizeof(unsigned int));
235 dest_val->ulen = sizeof(unsigned int);
236 }
237 unsigned int *new_key = (unsigned int *)dest_key->data;
238 unsigned int *new_val = (unsigned int *)dest_val->data;
239
240 *new_key = twiddle32(*(unsigned int*)src_key->data, which);
241 *new_val = generate_val(*(unsigned int*)src_key->data, which);
242
243 dest_key->size = sizeof(unsigned int);
244 dest_val->size = sizeof(unsigned int);
245 //data is already set above
246 }
247
248 // printf("dest_key.data = %d\n", *(int*)dest_key->data);
249 // printf("dest_val.data = %d\n", *(int*)dest_val->data);
250
251 return 0;
252 }
253
254
uint_cmp(const void * ap,const void * bp)255 static int uint_cmp(const void *ap, const void *bp) {
256 unsigned int an = *(unsigned int *)ap;
257 unsigned int bn = *(unsigned int *)bp;
258 if (an < bn)
259 return -1;
260 if (an > bn)
261 return +1;
262 return 0;
263 }
264
check_results(DB ** dbs)265 static void check_results(DB **dbs) {
266 for(int j=0;j<NUM_DBS;j++) {
267 unsigned int prev_k = 0;
268
269 DBT key, val;
270 unsigned int k=0, v=0;
271 dbt_init(&key, &k, sizeof(unsigned int));
272 dbt_init(&val, &v, sizeof(unsigned int));
273
274 int r;
275
276 DB_TXN *txn;
277 r = env->txn_begin(env, NULL, &txn, 0);
278 CKERR(r);
279
280 DBC *cursor;
281 r = dbs[j]->cursor(dbs[j], txn, &cursor, 0);
282 CKERR(r);
283
284 // generate the expected keys
285 unsigned int *expected_key = (unsigned int *) toku_malloc(NUM_ROWS * sizeof (unsigned int));
286 for (int i = 0; i < NUM_ROWS; i++) {
287 expected_key[i] = j == 0 ? (unsigned int)(i+1) : twiddle32(i+1, j);
288 }
289 // sort the keys
290 qsort(expected_key, NUM_ROWS, sizeof (unsigned int), uint_cmp);
291
292 for (int i = 0; i < NUM_ROWS+1; i++) {
293 r = cursor->c_get(cursor, &key, &val, DB_NEXT);
294 if (DISALLOW_PUTS) {
295 CKERR2(r, DB_NOTFOUND);
296 break;
297 }
298 if (r == DB_NOTFOUND) {
299 assert(i == NUM_ROWS); // check that there are exactly NUM_ROWS in the dictionary
300 break;
301 }
302 CKERR(r);
303
304 k = *(unsigned int*)key.data;
305
306 unsigned int pkey_for_db_key = (j == 0) ? k : inv_twiddle32(k, j);
307 v = *(unsigned int*)val.data;
308 // test that we have the expected keys and values
309 assert((unsigned int)pkey_for_db_key == (unsigned int)pkey_for_val(v, j));
310 // printf(" DB[%d] key = %10u, val = %10u, pkey_for_db_key = %10u, pkey_for_val=%10d\n", j, v, k, pkey_for_db_key, pkey_for_val(v, j));
311
312 // check the expected keys
313 assert(k == expected_key[i]);
314
315 // check prev_key < key
316 if (i > 0)
317 assert(prev_k < k);
318
319 // update prev = current
320 prev_k = k;
321 }
322
323 toku_free(expected_key);
324
325 if ( verbose ) {printf("."); fflush(stdout);}
326 r = cursor->c_close(cursor);
327 CKERR(r);
328
329 r = txn->commit(txn, 0);
330 CKERR(r);
331 }
332 if ( verbose ) printf("\nCheck OK\n");
333 }
334
delete_all(DB ** dbs)335 static void delete_all(DB **dbs) {
336 for(int j=0;j<NUM_DBS;j++) {
337
338 int r;
339
340 DB_TXN *txn;
341 r = env->txn_begin(env, NULL, &txn, 0);
342 CKERR(r);
343
344 // generate the expected keys
345 unsigned int *expected_key = (unsigned int *) toku_malloc(NUM_ROWS * sizeof (unsigned int));
346 for (int i = 0; i < NUM_ROWS; i++)
347 expected_key[i] = j == 0 ? (unsigned int)(i+1) : twiddle32(i+1, j);
348 // sort the keys
349 qsort(expected_key, NUM_ROWS, sizeof (unsigned int), uint_cmp);
350
351 // delete all of the keys
352 for (int i = 0; i < NUM_ROWS; i++) {
353 DBT key;
354 dbt_init(&key, &expected_key[i], sizeof expected_key[i]);
355 r = dbs[j]->del(dbs[j], txn, &key, DB_DELETE_ANY);
356 assert(r == 0);
357 }
358
359 // verify empty
360 DBC *cursor;
361 r = dbs[j]->cursor(dbs[j], txn, &cursor, 0);
362 CKERR(r);
363
364 DBT key, val;
365 unsigned int k=0, v=0;
366 dbt_init(&key, &k, sizeof(unsigned int));
367 dbt_init(&val, &v, sizeof(unsigned int));
368
369 r = cursor->c_get(cursor, &key, &val, DB_NEXT);
370 assert(r == DB_NOTFOUND);
371
372 toku_free(expected_key);
373
374 if ( verbose ) {printf("."); fflush(stdout);}
375 r = cursor->c_close(cursor);
376 CKERR(r);
377
378 r = txn->commit(txn, 0);
379 CKERR(r);
380 }
381 if ( verbose ) printf("\nCheck OK\n");
382 }
383
384 static void *expect_poll_void = &expect_poll_void;
385 static uint64_t poll_count=0;
386 static uint64_t bomb_after_poll_count=UINT64_MAX;
387
388 static struct progress_info {
389 double time;
390 double progress;
391 } *progress_infos=NULL;
392 static int progress_infos_count=0;
393 static int progress_infos_limit=0;
394
395 // timing
396 static bool did_start=false;
397 static struct timeval start;
398
poll_function(void * extra,float progress)399 static int poll_function (void *extra, float progress) {
400 if (verbose>=2) {
401 assert(did_start);
402 struct timeval now;
403 gettimeofday(&now, 0);
404 double elapsed = now.tv_sec - start.tv_sec + 1e-6*(now.tv_usec - start.tv_usec);
405 printf("Progress: %6.6fs %5.1f%%\n", elapsed, progress*100);
406 if (progress_infos_count>=progress_infos_limit) {
407 progress_infos_limit = 2*progress_infos_limit + 1;
408 XREALLOC_N(progress_infos_limit, progress_infos);
409 }
410 progress_infos[progress_infos_count++] = (struct progress_info){elapsed, progress};
411 }
412 assert(extra==expect_poll_void);
413 assert(0.0<=progress && progress<=1.0);
414 poll_count++; // Calls to poll_function() are protected by a lock, so we don't have to do this atomically.
415 if (poll_count>bomb_after_poll_count)
416 return TOKUDB_CANCELED;
417 else
418 return 0;
419 }
420
421 static struct timeval starttime;
elapsed_time(void)422 static double elapsed_time (void) {
423 struct timeval now;
424 gettimeofday(&now, NULL);
425 return now.tv_sec - starttime.tv_sec + 1e-6*(now.tv_usec - starttime.tv_usec);
426 }
427
test_loader(DB ** dbs)428 static void test_loader(DB **dbs)
429 {
430 gettimeofday(&starttime, NULL);
431 int r;
432 DB_TXN *txn;
433 DB_LOADER *loader;
434 uint32_t db_flags[MAX_DBS];
435 uint32_t dbt_flags[MAX_DBS];
436 uint32_t flags = DB_NOOVERWRITE;
437 if ( (DISALLOW_PUTS != 0) && (ALLOW_DUPS == 1) ) flags = 0;
438 for(int i=0;i<MAX_DBS;i++) {
439 db_flags[i] = flags;
440 dbt_flags[i] = 0;
441 }
442
443 uint32_t loader_flags = DISALLOW_PUTS | COMPRESS; // set with -p option
444
445 // create and initialize loader
446 r = env->txn_begin(env, NULL, &txn, 0);
447 CKERR(r);
448 hiwater_start = hiwater;
449 if (footprint_print) printf("%s:%d Hiwater=%ld water=%ld\n", __FILE__, __LINE__, hiwater, water);
450 r = env->create_loader(env, txn, &loader, dbs[0], NUM_DBS, dbs, db_flags, dbt_flags, loader_flags);
451 CKERR(r);
452 if (footprint_print) printf("%s:%d Hiwater=%ld water=%ld\n", __FILE__, __LINE__, hiwater, water);
453 r = loader->set_error_callback(loader, NULL, NULL);
454 CKERR(r);
455 r = loader->set_poll_function(loader, poll_function, expect_poll_void);
456 CKERR(r);
457
458 // using loader->put, put values into DB
459 DBT key, val;
460 unsigned int k, v;
461 for(int i=1;i<=NUM_ROWS;i++) {
462 k = i;
463 v = generate_val(i, 0);
464 dbt_init(&key, &k, sizeof(unsigned int));
465 dbt_init(&val, &v, sizeof(unsigned int));
466 r = loader->put(loader, &key, &val);
467 if (DISALLOW_PUTS) {
468 CKERR2(r, EINVAL);
469 } else {
470 CKERR(r);
471 }
472 if ( verbose) { if((i%10000) == 0){printf("."); fflush(stdout);} }
473 }
474 if ( verbose ) {printf("\n"); fflush(stdout);}
475
476 poll_count=0;
477
478 int n = count_temp(env->i->real_data_dir);
479 if (verbose) printf("Num temp files = %d\n", n);
480
481 did_start = true;
482 gettimeofday(&start, 0);
483
484 // close the loader
485 if ( verbose ) printf("%9.6fs closing\n", elapsed_time());
486 if (footprint_print) printf("%s:%d Hiwater=%ld water=%ld\n", __FILE__, __LINE__, hiwater, water);
487 r = loader->close(loader);
488 if (footprint_print) printf("%s:%d Hiwater=%ld water=%ld (extra hiwater=%ldM)\n", __FILE__, __LINE__, hiwater, water, (hiwater-hiwater_start)/(1024*1024));
489 if ( verbose ) printf("%9.6fs done\n", elapsed_time());
490 CKERR2s(r,0,TOKUDB_CANCELED);
491
492 if (r==0) {
493 if ( DISALLOW_PUTS == 0 ) {
494 if (poll_count == 0) printf("%s:%d\n", __FILE__, __LINE__);
495 assert(poll_count>0);
496 }
497
498 r = txn->commit(txn, 0);
499 CKERR(r);
500
501 // verify the DBs
502 if ( CHECK_RESULTS ) {
503 check_results(dbs);
504 delete_all(dbs);
505 }
506
507 } else {
508 r = txn->abort(txn);
509 CKERR(r);
510 }
511 }
512
513 static const char *envdir = TOKU_TEST_FILENAME;
514 const char *tmp_subdir = "tmp.subdir";
515
516 #define OLDDATADIR "../../../../tokudb.data/"
517 const char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.emptydictionaries.cleanshutdown";
518
setup(void)519 static void setup(void) {
520 int r;
521 int len = 256;
522 char syscmd[len];
523 const char * src_db_dir;
524
525 src_db_dir = db_v4_dir;
526
527 r = snprintf(syscmd, len, "cp -r %s %s", src_db_dir, envdir);
528 assert(r<len);
529 r = system(syscmd);
530 CKERR(r);
531 }
532
run_test(void)533 static void run_test(void)
534 {
535 int r;
536
537 int cmdlen = strlen(envdir) + strlen(tmp_subdir) + 10;
538 char tmpdir[cmdlen];
539 r = snprintf(tmpdir, cmdlen, "%s/%s", envdir, tmp_subdir);
540 assert(r<cmdlen);
541
542 // first delete anything left from previous run of this test
543 {
544 int len = strlen(envdir) + 20;
545 char syscmd[len];
546 r = snprintf(syscmd, len, "rm -rf %s", envdir);
547 assert(r<len);
548 r = system(syscmd); CKERR(r);
549 }
550 if (upgrade_test) {
551 setup();
552 }
553 else {
554 r = toku_os_mkdir(envdir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
555 r = toku_os_mkdir(tmpdir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
556 }
557
558 r = db_env_create(&env, 0); CKERR(r);
559 r = env->set_tmp_dir(env, tmp_subdir); CKERR(r);
560
561 r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
562 if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE);
563 r = env->set_cachesize(env, CACHESIZE / 1024, (CACHESIZE % 1024)*1024*1024, 1); CKERR(r);
564 if (datadir) {
565 r = env->set_data_dir(env, datadir); CKERR(r);
566 }
567 r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
568 CKERR(r);
569 int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
570 r = env->open(env, envdir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
571 env->set_errfile(env, stderr);
572 r = env->checkpointing_set_period(env, 60); CKERR(r);
573
574 DBT desc;
575 dbt_init(&desc, "foo", sizeof("foo"));
576 char name[MAX_NAME*2];
577
578 DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS);
579 assert(dbs != NULL);
580 int idx[MAX_DBS];
581 for(int i=0;i<NUM_DBS;i++) {
582 idx[i] = i;
583 r = db_create(&dbs[i], env, 0); CKERR(r);
584 dbs[i]->app_private = &idx[i];
585 snprintf(name, sizeof(name), "db_%04x", i);
586 r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
587 IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
588 { int chk_r = dbs[i]->change_descriptor(dbs[i], txn_desc, &desc, 0); CKERR(chk_r); }
589 });
590 }
591
592 generate_permute_tables();
593
594 // -------------------------- //
595 test_loader(dbs);
596 // -------------------------- //
597
598 for(int i=0;i<NUM_DBS;i++) {
599 dbs[i]->close(dbs[i], 0); CKERR(r);
600 dbs[i] = NULL;
601 }
602 if (verbose >= 2)
603 print_engine_status(env);
604 r = env->close(env, 0); CKERR(r);
605 toku_free(dbs);
606 }
607
608
609 // ------------ infrastructure ----------
610 static void do_args(int argc, char * const argv[]);
611
test_main(int argc,char * const * argv)612 int test_main(int argc, char * const *argv) {
613 do_args(argc, argv);
614
615 run_test();
616
617 if (progress_infos) {
618 if (verbose>=2) {
619 double ratio=progress_infos[progress_infos_count-1].time/progress_infos[progress_infos_count-1].progress;
620 printf("Progress ratios:\n");
621 for (int i=0; i<progress_infos_count; i++) {
622 printf(" %5.3f\n", (progress_infos[i].time/progress_infos[i].progress)/ratio);
623 }
624 }
625 toku_free(progress_infos);
626 }
627 if (footprint_print) {
628 printf("%s:%d Hiwater=%ld water=%ld (extra hiwater=%ldM) mcount=%lld fcount=%lld\n", __FILE__, __LINE__, hiwater, water, (hiwater-hiwater_start)/(1024*1024), mcount, fcount);
629 typedef void (*malloc_stats_fun_t)(void);
630 malloc_stats_fun_t malloc_stats_f = (malloc_stats_fun_t) dlsym(RTLD_DEFAULT, "malloc_stats");
631 if (malloc_stats_f) {
632 malloc_stats_f();
633 }
634 }
635 return 0;
636 }
637
do_args(int argc,char * const argv[])638 static void do_args(int argc, char * const argv[]) {
639
640 // Must look for "-f" right away before we malloc anything.
641 for (int i=1; i<argc; i++) {
642
643 if (strcmp(argv[i], "-f")) {
644 db_env_set_func_malloc(my_malloc);
645 db_env_set_func_realloc(my_realloc);
646 db_env_set_func_free(my_free);
647 }
648 }
649
650 int resultcode;
651 char *cmd = argv[0];
652 argc--; argv++;
653
654 CACHESIZE = (toku_os_get_phys_memory_size() / (1024*1024))/2; //MB
655
656 while (argc>0) {
657 if (strcmp(argv[0], "-v")==0) {
658 verbose++;
659 } else if (strcmp(argv[0],"-q")==0) {
660 verbose--;
661 if (verbose<0) verbose=0;
662 } else if (strcmp(argv[0], "-h")==0) {
663 resultcode=0;
664 do_usage:
665 fprintf(stderr, "Usage: -h -c -d <num_dbs> -r <num_rows> [ -b <num_calls> ] [-m <megabytes>] [-M]\n%s\n", cmd);
666 fprintf(stderr, " where -d <num_dbs> is the number of dictionaries to build (primary & secondary). (Default=%d)\n", NUM_DBS);
667 fprintf(stderr, " -b <num_calls> causes the poll function to return nonzero after <num_calls>\n");
668 fprintf(stderr, " -e <env> uses <env> to construct the directory (so that different tests can run concurrently)\n");
669 fprintf(stderr, " -m <m> use m MB of memory for the cachetable (default is %d MB)\n", CACHESIZE);
670 fprintf(stderr, " -M use %d MB of memory for the cachetable\n", old_default_cachesize);
671 fprintf(stderr, " -s use size factor of 1 and count temporary files\n");
672 fprintf(stderr, " -f print memory footprint information at various points in the load\n");
673 exit(resultcode);
674 } else if (strcmp(argv[0], "-d")==0) {
675 argc--; argv++;
676 NUM_DBS = atoi(argv[0]);
677 if ( NUM_DBS > MAX_DBS ) {
678 fprintf(stderr, "max value for -d field is %d\n", MAX_DBS);
679 resultcode=1;
680 goto do_usage;
681 }
682 } else if (strcmp(argv[0], "-e")==0) {
683 argc--; argv++;
684 envdir = argv[0];
685 } else if (strcmp(argv[0], "-v")==0) {
686 verbose++;
687 } else if (strcmp(argv[0],"-q")==0) {
688 verbose--;
689 if (verbose<0) verbose=0;
690 } else if (strcmp(argv[0], "-f")==0) {
691 footprint_print = true;
692 } else if (strcmp(argv[0], "-r")==0) {
693 argc--; argv++;
694 NUM_ROWS = atoi(argv[0]);
695 } else if (strcmp(argv[0], "-c")==0) {
696 CHECK_RESULTS = 1;
697 } else if (strcmp(argv[0], "-p")==0) {
698 DISALLOW_PUTS = LOADER_DISALLOW_PUTS;
699 } else if (strcmp(argv[0], "-z")==0) {
700 COMPRESS = LOADER_COMPRESS_INTERMEDIATES;
701 } else if (strcmp(argv[0], "-m")==0) {
702 argc--; argv++;
703 CACHESIZE = atoi(argv[0]);
704 } else if (strcmp(argv[0], "-M")==0) {
705 CACHESIZE = old_default_cachesize;
706 } else if (strcmp(argv[0], "-y")==0) {
707 ALLOW_DUPS = 1;
708 } else if (strcmp(argv[0], "-s")==0) {
709 //printf("\nTesting loader with size_factor=1\n");
710 db_env_set_loader_size_factor(1);
711 } else if (strcmp(argv[0], "-b")==0) {
712 argc--; argv++;
713 char *end;
714 errno=0;
715 bomb_after_poll_count = strtoll(argv[0], &end, 10);
716 assert(errno==0);
717 assert(*end==0); // make sure we consumed the whole integer.
718 } else if (strcmp(argv[0], "--datadir") == 0 && argc > 1) {
719 argc--; argv++;
720 datadir = argv[0];
721 } else if (strcmp(argv[0], "--dont_check_est") == 0) {
722 check_est = false;
723 } else if (strcmp(argv[0], "-u")==0) {
724 upgrade_test = true;
725 } else {
726 fprintf(stderr, "Unknown arg: %s\n", argv[0]);
727 resultcode=1;
728 goto do_usage;
729 }
730 argc--;
731 argv++;
732 }
733 }
734