1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 // vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
3 #ident "$Id$"
4 /*======
5 This file is part of PerconaFT.
6
7
8 Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
9
10 PerconaFT is free software: you can redistribute it and/or modify
11 it under the terms of the GNU General Public License, version 2,
12 as published by the Free Software Foundation.
13
14 PerconaFT is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
21
22 ----------------------------------------
23
24 PerconaFT is free software: you can redistribute it and/or modify
25 it under the terms of the GNU Affero General Public License, version 3,
26 as published by the Free Software Foundation.
27
28 PerconaFT is distributed in the hope that it will be useful,
29 but WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 GNU Affero General Public License for more details.
32
33 You should have received a copy of the GNU Affero General Public License
34 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
35 ======= */
36
37 #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
38
39 /*********************
40 *
41 * Purpose is to preload a set of dictionaries using nested transactions,
42 * to be used to test version upgrade.
43 *
44 * Each row will be inserted using nested transactions MAXDEPTH deep.
45 * Each nested transaction will insert a value one greater than the parent transaction.
46 * For each row, a single transaction will be aborted, the rest will be committed.
47 * The transaction to be aborted will be the row number mod MAXDEPTH.
48 * So, for row 0, the outermost transaction will be aborted and the row will not appear in the database.
49 * For row 1, transaction 1 will be aborted, so the inserted value will be the original generated value.
50 * For each row, the inserted value will be:
51 * if row%MAXDEPTH == 0 no row
52 * else value = generated value + (row%MAXDEPTH -1)
53 *
54 *
55 * For each row
56 * generate k,v pair
57 * for txndepth = 0 to MAXDEPTH-1 {
58 * add txndepth to v
59 * begin txn
60 * insert
61 * if txndepth = row%MAXDEPTH abort
62 * else commit
63 * }
64 * }
65 *
66 */
67
68
69
70
71
72 #define kv_pair_funcs 1 // pull in kv_pair generators from test.h
73
74 #include "test.h"
75 #include "toku_pthread.h"
76 #include <db.h>
77 #include <sys/stat.h>
78 #include "ydb-internal.h"
79
80 #include "test_kv_gen.h"
81 /*
82 */
83
84 DB_ENV *env;
85 enum {MAX_NAME=128};
86 enum {ROWS_PER_TRANSACTION=10000};
87 uint NUM_DBS=1;
88 uint NUM_ROWS=100000;
89 int CHECK_RESULTS=0;
90 int optimize=0;
91 int littlenode = 0;
92 enum { old_default_cachesize=1024 }; // MB
93 int CACHESIZE=old_default_cachesize;
94 int ALLOW_DUPS=0;
95
96 // max depth of nested transactions for this test
97 //#define MAXDEPTH 128
98 #define MAXDEPTH 64
99
100 static void
101 nested_insert(DB ** dbs, uint depth, DB_TXN *parent_txn, uint k, uint generated_value);
102
103
104 static void
check_results_nested(DB ** dbs,const uint num_rows)105 check_results_nested(DB ** dbs, const uint num_rows) {
106 int num_dbs = 1; // maybe someday increase
107 for(int j=0;j<num_dbs;j++){
108 DBT key, val;
109 unsigned int k=0, v=0;
110 dbt_init(&key, &k, sizeof(unsigned int));
111 dbt_init(&val, &v, sizeof(unsigned int));
112 int r;
113
114 DB_TXN *txn;
115 r = env->txn_begin(env, NULL, &txn, 0);
116 CKERR(r);
117
118 DBC *cursor;
119 r = dbs[j]->cursor(dbs[j], txn, &cursor, 0);
120 CKERR(r);
121 for(uint i=0;i<num_rows;i++) {
122 if (i % MAXDEPTH) {
123 r = cursor->c_get(cursor, &key, &val, DB_NEXT);
124 CKERR(r);
125 uint observed_k = *(unsigned int*)key.data;
126 uint observed_v = *(unsigned int*)val.data;
127 uint expected_k = i;
128 uint generated_value = generate_val(i, 0);
129 uint expected_v = generated_value + (i%MAXDEPTH - 1);
130 if (verbose >= 3)
131 printf("expected key %d, observed key %d, expected val %d, observed val %d\n",
132 expected_k, observed_k, expected_v, observed_v);
133 // test that we have the expected keys and values
134 assert(observed_k == expected_k);
135 assert(observed_v == expected_v);
136 }
137 dbt_init(&key, NULL, sizeof(unsigned int));
138 dbt_init(&val, NULL, sizeof(unsigned int));
139 if ( verbose && (i%10000 == 0)) {printf("."); fflush(stdout);}
140 }
141 r = cursor->c_close(cursor);
142 CKERR(r);
143 r = txn->commit(txn, DB_TXN_NOSYNC);
144 CKERR(r);
145 }
146 if ( verbose ) {printf("ok");fflush(stdout);}
147 }
148
149
150
151
152
153
154 static struct timeval starttime;
UU()155 static double UU() elapsed_time (void) {
156 struct timeval now;
157 gettimeofday(&now, NULL);
158 return now.tv_sec - starttime.tv_sec + 1e-6*(now.tv_usec - starttime.tv_usec);
159 }
160
preload_dbs(DB ** dbs)161 static void preload_dbs(DB **dbs)
162 {
163 gettimeofday(&starttime, NULL);
164 uint row;
165
166 if ( verbose ) { printf("loading");fflush(stdout); }
167
168 for(row = 0; row <= NUM_ROWS; row++) {
169 uint generated_value = generate_val(row, 0);
170 nested_insert(dbs, 0, NULL, row, generated_value);
171 }
172
173 if (optimize) {
174 if (verbose) { printf("\noptimizing");fflush(stdout);}
175 do_hot_optimize_on_dbs(env, dbs, 1);
176 }
177
178 if ( CHECK_RESULTS) {
179 if ( verbose ) {printf("\nchecking");fflush(stdout);}
180 check_results_nested(&dbs[0], NUM_ROWS);
181 }
182 if ( verbose) {printf("\ndone\n");fflush(stdout);}
183 }
184
185 static void
nested_insert(DB ** dbs,uint depth,DB_TXN * parent_txn,uint k,uint generated_value)186 nested_insert(DB ** dbs, uint depth, DB_TXN *parent_txn, uint k, uint generated_value) {
187 if (depth < MAXDEPTH) {
188 DBT key, val;
189 dbt_init_realloc(&key);
190 dbt_init_realloc(&val);
191 uint v = generated_value + depth;
192 DB_TXN * txn;
193 int r = env->txn_begin(env, parent_txn, &txn, 0);
194 CKERR(r);
195 dbt_init(&key, &k, sizeof(unsigned int));
196 dbt_init(&val, &v, sizeof(unsigned int));
197 int db = 0; // maybe later replace with loop
198 r = dbs[db]->put(dbs[db], txn, &key, &val, 0);
199 CKERR(r);
200 if (key.flags == 0) { dbt_init_realloc(&key); }
201 if (val.flags == 0) { dbt_init_realloc(&val); }
202 nested_insert(dbs, depth+1, txn, k, generated_value);
203 if (depth == (k % MAXDEPTH)) {
204 r = txn->abort(txn);
205 CKERR(r);
206 if (verbose>=3)
207 printf("abort k = %d, v= %d, depth = %d\n", k, v, depth);
208 }
209 else {
210 r = txn->commit(txn, DB_TXN_NOSYNC);
211 CKERR(r);
212 if (verbose>=3)
213 printf("commit k = %d, v= %d, depth = %d\n", k, v, depth);
214 }
215 if ( verbose && (k%10000 == 0)) {printf(".");fflush(stdout);}
216
217 if ( key.flags ) { toku_free(key.data); key.data = NULL; }
218 if ( val.flags ) { toku_free(val.data); key.data = NULL; }
219 }
220 }
221
222
223 char *free_me = NULL;
224 const char *env_dir = TOKU_TEST_FILENAME; // the default env_dir.
225
run_test(void)226 static void run_test(void)
227 {
228 int r;
229 {
230 int len = strlen(env_dir) + 20;
231 char syscmd[len];
232 r = snprintf(syscmd, len, "rm -rf %s", env_dir);
233 assert(r<len);
234 r = system(syscmd); CKERR(r);
235 }
236 r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
237
238 r = db_env_create(&env, 0); CKERR(r);
239 r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
240 // r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r);
241 // if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE);
242 // r = env->set_cachesize(env, CACHESIZE / 1024, (CACHESIZE % 1024)*1024*1024, 1); CKERR(r);
243 // CKERR(r);
244 int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
245 r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
246 env->set_errfile(env, stderr);
247 r = env->checkpointing_set_period(env, 0); CKERR(r);
248
249 DBT desc;
250 dbt_init(&desc, "foo", sizeof("foo"));
251 char name[MAX_NAME*2];
252
253 DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS);
254 assert(dbs != NULL);
255 int idx[MAX_DBS];
256 for(uint i=0;i<NUM_DBS;i++) {
257 idx[i] = i;
258 r = db_create(&dbs[i], env, 0); CKERR(r);
259 if (littlenode) {
260 r=dbs[i]->set_pagesize(dbs[i], 4096);
261 CKERR(0);
262 }
263 dbs[i]->app_private = &idx[i];
264 snprintf(name, sizeof(name), "db_%04x", i);
265 r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
266 IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
267 { int chk_r = dbs[i]->change_descriptor(dbs[i], txn_desc, &desc, 0); CKERR(chk_r); }
268 });
269 }
270
271 generate_permute_tables();
272
273 // -------------------------- //
274 preload_dbs(dbs);
275 // -------------------------- //
276
277 for(uint i=0;i<NUM_DBS;i++) {
278 r = dbs[i]->close(dbs[i], 0); CKERR(r);
279 dbs[i] = NULL;
280 }
281
282 if (verbose >= 2)
283 print_engine_status(env);
284 r = env->close(env, 0); CKERR(r);
285 toku_free(dbs);
286
287 }
288
289 // ------------ infrastructure ----------
290 static void do_args(int argc, char * const argv[]);
291
test_main(int argc,char * const argv[])292 int test_main(int argc, char * const argv[]) {
293 do_args(argc, argv);
294 run_test();
295 if (free_me) toku_free(free_me);
296 return 0;
297 }
298
do_args(int argc,char * const argv[])299 static void do_args(int argc, char * const argv[]) {
300 int resultcode;
301 char *cmd = argv[0];
302 argc--; argv++;
303
304 while (argc>0) {
305 if (strcmp(argv[0], "-v")==0) {
306 verbose++;
307 } else if (strcmp(argv[0],"-q")==0) {
308 verbose--;
309 if (verbose<0) verbose=0;
310 } else if (strcmp(argv[0], "-h")==0) {
311 resultcode=0;
312 do_usage:
313 fprintf(stderr, "Usage: -h -c -n -d <num_dbs> -r <num_rows> %s\n", cmd);
314 exit(resultcode);
315 } else if (strcmp(argv[0], "-d")==0) {
316 argc--; argv++;
317 NUM_DBS = atoi(argv[0]);
318 if ( NUM_DBS > MAX_DBS ) {
319 fprintf(stderr, "max value for -d field is %d\n", MAX_DBS);
320 resultcode=1;
321 goto do_usage;
322 }
323 } else if (strcmp(argv[0], "-r")==0) {
324 argc--; argv++;
325 NUM_ROWS = atoi(argv[0]);
326 } else if (strcmp(argv[0], "-c")==0) {
327 CHECK_RESULTS = 1;
328 } else if (strcmp(argv[0], "-n")==0) {
329 littlenode = 1;
330 } else if (strcmp(argv[0], "-o")==0) {
331 optimize = 1;
332 } else {
333 fprintf(stderr, "Unknown arg: %s\n", argv[0]);
334 resultcode=1;
335 goto do_usage;
336 }
337 argc--;
338 argv++;
339 }
340 }
341