1
2 #include "lsmtest.h"
3 #include <sqlite3.h>
4
test_failed()5 void test_failed(){
6 assert( 0 );
7 return;
8 }
9
10 #define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__)
testSetErrorFunc(int rc,int * pRc,const char * zFile,int iLine)11 static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
12 if( rc ){
13 *pRc = rc;
14 fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
15 test_failed();
16 }
17 }
18
lsm_memcmp(u8 * a,u8 * b,int c)19 static int lsm_memcmp(u8 *a, u8 *b, int c){
20 int i;
21 for(i=0; i<c; i++){
22 if( a[i]!=b[i] ) return a[i] - b[i];
23 }
24 return 0;
25 }
26
27 /*
28 ** A test utility function.
29 */
testFetch(TestDb * pDb,void * pKey,int nKey,void * pVal,int nVal,int * pRc)30 void testFetch(
31 TestDb *pDb, /* Database handle */
32 void *pKey, int nKey, /* Key to query database for */
33 void *pVal, int nVal, /* Expected value */
34 int *pRc /* IN/OUT: Error code */
35 ){
36 if( *pRc==0 ){
37 void *pDbVal;
38 int nDbVal;
39 int rc;
40
41 static int nCall = 0; nCall++;
42
43 rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal);
44 testSetError(rc);
45 if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){
46 testSetError(1);
47 }
48 }
49 }
50
testWrite(TestDb * pDb,void * pKey,int nKey,void * pVal,int nVal,int * pRc)51 void testWrite(
52 TestDb *pDb, /* Database handle */
53 void *pKey, int nKey, /* Key to query database for */
54 void *pVal, int nVal, /* Value to write */
55 int *pRc /* IN/OUT: Error code */
56 ){
57 if( *pRc==0 ){
58 int rc;
59 static int nCall = 0;
60 nCall++;
61 rc = tdb_write(pDb, pKey, nKey, pVal, nVal);
62 testSetError(rc);
63 }
64 }
testDelete(TestDb * pDb,void * pKey,int nKey,int * pRc)65 void testDelete(
66 TestDb *pDb, /* Database handle */
67 void *pKey, int nKey, /* Key to query database for */
68 int *pRc /* IN/OUT: Error code */
69 ){
70 if( *pRc==0 ){
71 int rc;
72 *pRc = rc = tdb_delete(pDb, pKey, nKey);
73 testSetError(rc);
74 }
75 }
testDeleteRange(TestDb * pDb,void * pKey1,int nKey1,void * pKey2,int nKey2,int * pRc)76 void testDeleteRange(
77 TestDb *pDb, /* Database handle */
78 void *pKey1, int nKey1,
79 void *pKey2, int nKey2,
80 int *pRc /* IN/OUT: Error code */
81 ){
82 if( *pRc==0 ){
83 int rc;
84 *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2);
85 testSetError(rc);
86 }
87 }
88
testBegin(TestDb * pDb,int iTrans,int * pRc)89 void testBegin(TestDb *pDb, int iTrans, int *pRc){
90 if( *pRc==0 ){
91 int rc;
92 rc = tdb_begin(pDb, iTrans);
93 testSetError(rc);
94 }
95 }
testCommit(TestDb * pDb,int iTrans,int * pRc)96 void testCommit(TestDb *pDb, int iTrans, int *pRc){
97 if( *pRc==0 ){
98 int rc;
99 rc = tdb_commit(pDb, iTrans);
100 testSetError(rc);
101 }
102 }
103 #if 0 /* unused */
104 static void testRollback(TestDb *pDb, int iTrans, int *pRc){
105 if( *pRc==0 ){
106 int rc;
107 rc = tdb_rollback(pDb, iTrans);
108 testSetError(rc);
109 }
110 }
111 #endif
112
testWriteStr(TestDb * pDb,const char * zKey,const char * zVal,int * pRc)113 void testWriteStr(
114 TestDb *pDb, /* Database handle */
115 const char *zKey, /* Key to query database for */
116 const char *zVal, /* Value to write */
117 int *pRc /* IN/OUT: Error code */
118 ){
119 int nVal = (zVal ? strlen(zVal) : 0);
120 testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
121 }
122
123 #if 0 /* unused */
124 static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){
125 testDelete(pDb, (void *)zKey, strlen(zKey), pRc);
126 }
127 #endif
testFetchStr(TestDb * pDb,const char * zKey,const char * zVal,int * pRc)128 void testFetchStr(
129 TestDb *pDb, /* Database handle */
130 const char *zKey, /* Key to query database for */
131 const char *zVal, /* Value to write */
132 int *pRc /* IN/OUT: Error code */
133 ){
134 int nVal = (zVal ? strlen(zVal) : 0);
135 testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
136 }
137
testFetchCompare(TestDb * pControl,TestDb * pDb,void * pKey,int nKey,int * pRc)138 void testFetchCompare(
139 TestDb *pControl,
140 TestDb *pDb,
141 void *pKey, int nKey,
142 int *pRc
143 ){
144 int rc;
145 void *pDbVal1;
146 void *pDbVal2;
147 int nDbVal1;
148 int nDbVal2;
149
150 static int nCall = 0;
151 nCall++;
152
153 rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1);
154 testSetError(rc);
155
156 rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2);
157 testSetError(rc);
158
159 if( *pRc==0
160 && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
161 ){
162 testSetError(1);
163 }
164 }
165
166 typedef struct ScanResult ScanResult;
167 struct ScanResult {
168 TestDb *pDb;
169
170 int nRow;
171 u32 cksum1;
172 u32 cksum2;
173 void *pKey1; int nKey1;
174 void *pKey2; int nKey2;
175
176 int bReverse;
177 int nPrevKey;
178 u8 aPrevKey[256];
179 };
180
keyCompare(void * pKey1,int nKey1,void * pKey2,int nKey2)181 static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){
182 int res;
183 res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2));
184 if( res==0 ){
185 res = nKey1 - nKey2;
186 }
187 return res;
188 }
189
190 int test_scan_debug = 0;
191
scanCompareCb(void * pCtx,void * pKey,int nKey,void * pVal,int nVal)192 static void scanCompareCb(
193 void *pCtx,
194 void *pKey, int nKey,
195 void *pVal, int nVal
196 ){
197 ScanResult *p = (ScanResult *)pCtx;
198 u8 *aKey = (u8 *)pKey;
199 u8 *aVal = (u8 *)pVal;
200 int i;
201
202 if( test_scan_debug ){
203 printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey);
204 fflush(stdout);
205 }
206 #if 0
207 if( test_scan_debug ) printf("%.20s\n", (char *)pVal);
208 #endif
209
210 #if 0
211 /* Check tdb_fetch() matches */
212 int rc = 0;
213 testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc);
214 assert( rc==0 );
215 #endif
216
217 /* Update the checksum data */
218 p->nRow++;
219 for(i=0; i<nKey; i++){
220 p->cksum1 += ((int)aKey[i] << (i&0x0F));
221 p->cksum2 += p->cksum1;
222 }
223 for(i=0; i<nVal; i++){
224 p->cksum1 += ((int)aVal[i] << (i&0x0F));
225 p->cksum2 += p->cksum1;
226 }
227
228 /* Check that the delivered row is not out of order. */
229 if( nKey<(int)sizeof(p->aPrevKey) ){
230 if( p->nPrevKey ){
231 int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey);
232 if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){
233 testPrintError("Returned key out of order at %s:%d\n",
234 __FILE__, __LINE__
235 );
236 }
237 }
238
239 p->nPrevKey = nKey;
240 memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey));
241 }
242
243 /* Check that the delivered row is within range. */
244 if( p->pKey1 && (
245 (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0)
246 || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey)
247 )){
248 testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__);
249 }
250 if( p->pKey2 && (
251 (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0)
252 || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey)
253 )){
254 testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__);
255 }
256
257 }
258
259 /*
260 ** Scan the contents of the two databases. Check that they match.
261 */
testScanCompare(TestDb * pDb1,TestDb * pDb2,int bReverse,void * pKey1,int nKey1,void * pKey2,int nKey2,int * pRc)262 void testScanCompare(
263 TestDb *pDb1, /* Control (trusted) database */
264 TestDb *pDb2, /* Database being tested */
265 int bReverse,
266 void *pKey1, int nKey1,
267 void *pKey2, int nKey2,
268 int *pRc
269 ){
270 static int nCall = 0; nCall++;
271 if( *pRc==0 ){
272 ScanResult res1;
273 ScanResult res2;
274 void *pRes1 = (void *)&res1;
275 void *pRes2 = (void *)&res2;
276
277 memset(&res1, 0, sizeof(ScanResult));
278 memset(&res2, 0, sizeof(ScanResult));
279
280 res1.pDb = pDb1;
281 res1.nKey1 = nKey1; res1.pKey1 = pKey1;
282 res1.nKey2 = nKey2; res1.pKey2 = pKey2;
283 res1.bReverse = bReverse;
284 res2.pDb = pDb2;
285 res2.nKey1 = nKey1; res2.pKey1 = pKey1;
286 res2.nKey2 = nKey2; res2.pKey2 = pKey2;
287 res2.bReverse = bReverse;
288
289 tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
290 if( test_scan_debug ) printf("\n\n\n");
291 tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
292 if( test_scan_debug ) printf("\n\n\n");
293
294 if( res1.nRow!=res2.nRow
295 || res1.cksum1!=res2.cksum1
296 || res1.cksum2!=res2.cksum2
297 ){
298 printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2);
299 printf("got: %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2);
300 testSetError(1);
301 *pRc = 1;
302 }
303 }
304 }
305
testClose(TestDb ** ppDb)306 void testClose(TestDb **ppDb){
307 tdb_close(*ppDb);
308 *ppDb = 0;
309 }
310
testOpen(const char * zSystem,int bClear,int * pRc)311 TestDb *testOpen(const char *zSystem, int bClear, int *pRc){
312 TestDb *pDb = 0;
313 if( *pRc==0 ){
314 int rc;
315 rc = tdb_open(zSystem, 0, bClear, &pDb);
316 if( rc!=0 ){
317 testSetError(rc);
318 *pRc = rc;
319 }
320 }
321 return pDb;
322 }
323
testReopen(TestDb ** ppDb,int * pRc)324 void testReopen(TestDb **ppDb, int *pRc){
325 if( *pRc==0 ){
326 const char *zLib;
327 zLib = tdb_library_name(*ppDb);
328 testClose(ppDb);
329 *pRc = tdb_open(zLib, 0, 0, ppDb);
330 }
331 }
332
333
334 #if 0 /* unused */
335 static void testSystemSelect(const char *zSys, int *piSel, int *pRc){
336 if( *pRc==0 ){
337 struct SysName { const char *zName; } *aName;
338 int nSys;
339 int i;
340
341 for(nSys=0; tdb_system_name(nSys); nSys++);
342 aName = malloc(sizeof(struct SysName) * (nSys+1));
343 for(i=0; i<=nSys; i++){
344 aName[i].zName = tdb_system_name(i);
345 }
346
347 *pRc = testArgSelect(aName, "db", zSys, piSel);
348 free(aName);
349 }
350 }
351 #endif
352
testMallocVPrintf(const char * zFormat,va_list ap)353 char *testMallocVPrintf(const char *zFormat, va_list ap){
354 int nByte;
355 va_list copy;
356 char *zRet;
357
358 __va_copy(copy, ap);
359 nByte = vsnprintf(0, 0, zFormat, copy);
360 va_end(copy);
361
362 assert( nByte>=0 );
363 zRet = (char *)testMalloc(nByte+1);
364 vsnprintf(zRet, nByte+1, zFormat, ap);
365 return zRet;
366 }
367
testMallocPrintf(const char * zFormat,...)368 char *testMallocPrintf(const char *zFormat, ...){
369 va_list ap;
370 char *zRet;
371
372 va_start(ap, zFormat);
373 zRet = testMallocVPrintf(zFormat, ap);
374 va_end(ap);
375
376 return zRet;
377 }
378
379
380 /*
381 ** A wrapper around malloc(3).
382 **
383 ** This function should be used for all allocations made by test procedures.
384 ** It has the following properties:
385 **
386 ** * Test code may assume that allocations may not fail.
387 ** * Returned memory is always zeroed.
388 **
389 ** Allocations made using testMalloc() should be freed using testFree().
390 */
testMalloc(int n)391 void *testMalloc(int n){
392 u8 *p = (u8*)malloc(n + 8);
393 memset(p, 0, n+8);
394 *(int*)p = n;
395 return (void*)&p[8];
396 }
397
testMallocCopy(void * pCopy,int nByte)398 void *testMallocCopy(void *pCopy, int nByte){
399 void *pRet = testMalloc(nByte);
400 memcpy(pRet, pCopy, nByte);
401 return pRet;
402 }
403
testRealloc(void * ptr,int n)404 void *testRealloc(void *ptr, int n){
405 if( ptr ){
406 u8 *p = (u8*)ptr - 8;
407 int nOrig = *(int*)p;
408 p = (u8*)realloc(p, n+8);
409 if( nOrig<n ){
410 memset(&p[8+nOrig], 0, n-nOrig);
411 }
412 *(int*)p = n;
413 return (void*)&p[8];
414 }
415 return testMalloc(n);
416 }
417
418 /*
419 ** Free an allocation made by an earlier call to testMalloc().
420 */
testFree(void * ptr)421 void testFree(void *ptr){
422 if( ptr ){
423 u8 *p = (u8*)ptr - 8;
424 memset(p, 0x55, *(int*)p + 8);
425 free(p);
426 }
427 }
428
429 /*
430 ** String zPattern contains a glob pattern. Return true if zStr matches
431 ** the pattern, or false if it does not.
432 */
testGlobMatch(const char * zPattern,const char * zStr)433 int testGlobMatch(const char *zPattern, const char *zStr){
434 int i = 0;
435 int j = 0;
436
437 while( zPattern[i] ){
438 char p = zPattern[i];
439
440 if( p=='*' || p=='%' ){
441 do {
442 if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1;
443 }while( zStr[j++] );
444 return 0;
445 }
446
447 if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){
448 /* Match failed. */
449 return 0;
450 }
451
452 j++;
453 i++;
454 }
455
456 return (zPattern[i]==0 && zStr[j]==0);
457 }
458
459 /*
460 ** End of test utilities
461 **************************************************************************/
462
do_test(int nArg,char ** azArg)463 int do_test(int nArg, char **azArg){
464 int j;
465 int rc;
466 int nFail = 0;
467 const char *zPattern = 0;
468
469 if( nArg>1 ){
470 testPrintError("Usage: test ?PATTERN?\n");
471 return 1;
472 }
473 if( nArg==1 ){
474 zPattern = azArg[0];
475 }
476
477 for(j=0; tdb_system_name(j); j++){
478 rc = 0;
479
480 test_data_1(tdb_system_name(j), zPattern, &rc);
481 test_data_2(tdb_system_name(j), zPattern, &rc);
482 test_data_3(tdb_system_name(j), zPattern, &rc);
483 test_data_4(tdb_system_name(j), zPattern, &rc);
484 test_rollback(tdb_system_name(j), zPattern, &rc);
485 test_mc(tdb_system_name(j), zPattern, &rc);
486 test_mt(tdb_system_name(j), zPattern, &rc);
487
488 if( rc ) nFail++;
489 }
490
491 rc = 0;
492 test_oom(zPattern, &rc);
493 if( rc ) nFail++;
494
495 rc = 0;
496 test_api(zPattern, &rc);
497 if( rc ) nFail++;
498
499 rc = 0;
500 do_crash_test(zPattern, &rc);
501 if( rc ) nFail++;
502
503 rc = 0;
504 do_writer_crash_test(zPattern, &rc);
505 if( rc ) nFail++;
506
507 return (nFail!=0);
508 }
509
configure_lsm_db(TestDb * pDb)510 static lsm_db *configure_lsm_db(TestDb *pDb){
511 lsm_db *pLsm;
512 pLsm = tdb_lsm(pDb);
513 if( pLsm ){
514 tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4");
515 }
516 return pLsm;
517 }
518
519 typedef struct WriteHookEvent WriteHookEvent;
520 struct WriteHookEvent {
521 i64 iOff;
522 int nData;
523 int nUs;
524 };
525 WriteHookEvent prev = {0, 0, 0};
526
flushPrev(FILE * pOut)527 static void flushPrev(FILE *pOut){
528 if( prev.nData ){
529 fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs);
530 prev.nData = 0;
531 }
532 }
533
534 #if 0 /* unused */
535 static void do_speed_write_hook2(
536 void *pCtx,
537 int bLog,
538 i64 iOff,
539 int nData,
540 int nUs
541 ){
542 FILE *pOut = (FILE *)pCtx;
543 if( bLog ) return;
544
545 if( prev.nData && nData && iOff==prev.iOff+prev.nData ){
546 prev.nData += nData;
547 prev.nUs += nUs;
548 }else{
549 flushPrev(pOut);
550 if( nData==0 ){
551 fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs);
552 }else{
553 prev.iOff = iOff;
554 prev.nData = nData;
555 prev.nUs = nUs;
556 }
557 }
558 }
559 #endif
560
561 #define ST_REPEAT 0
562 #define ST_WRITE 1
563 #define ST_PAUSE 2
564 #define ST_FETCH 3
565 #define ST_SCAN 4
566 #define ST_NSCAN 5
567 #define ST_KEYSIZE 6
568 #define ST_VALSIZE 7
569 #define ST_TRANS 8
570
571
print_speed_test_help()572 static void print_speed_test_help(){
573 printf(
574 "\n"
575 "Repeat the following $repeat times:\n"
576 " 1. Insert $write key-value pairs. One transaction for each write op.\n"
577 " 2. Pause for $pause ms.\n"
578 " 3. Perform $fetch queries on the database.\n"
579 "\n"
580 " Keys are $keysize bytes in size. Values are $valsize bytes in size\n"
581 " Both keys and values are pseudo-randomly generated\n"
582 "\n"
583 "Options are:\n"
584 " -repeat $repeat (default value 10)\n"
585 " -write $write (default value 10000)\n"
586 " -pause $pause (default value 0)\n"
587 " -fetch $fetch (default value 0)\n"
588 " -keysize $keysize (default value 12)\n"
589 " -valsize $valsize (default value 100)\n"
590 " -system $system (default value \"lsm\")\n"
591 " -trans $trans (default value 0)\n"
592 "\n"
593 );
594 }
595
do_speed_test2(int nArg,char ** azArg)596 int do_speed_test2(int nArg, char **azArg){
597 struct Option {
598 const char *zOpt;
599 int eVal;
600 int iDefault;
601 } aOpt[] = {
602 { "-repeat", ST_REPEAT, 10},
603 { "-write", ST_WRITE, 10000},
604 { "-pause", ST_PAUSE, 0},
605 { "-fetch", ST_FETCH, 0},
606 { "-scan", ST_SCAN, 0},
607 { "-nscan", ST_NSCAN, 0},
608 { "-keysize", ST_KEYSIZE, 12},
609 { "-valsize", ST_VALSIZE, 100},
610 { "-trans", ST_TRANS, 0},
611 { "-system", -1, 0},
612 { "help", -2, 0},
613 {0, 0, 0}
614 };
615 int i;
616 int aParam[9];
617 int rc = 0;
618 int bReadonly = 0;
619 int nContent = 0;
620
621 TestDb *pDb;
622 Datasource *pData;
623 DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 };
624 char *zSystem = "";
625 int bLsm = 1;
626 FILE *pLog = 0;
627
628 #ifdef NDEBUG
629 /* If NDEBUG is defined, disable the dynamic memory related checks in
630 ** lsmtest_mem.c. They slow things down. */
631 testMallocUninstall(tdb_lsm_env());
632 #endif
633
634 /* Initialize aParam[] with default values. */
635 for(i=0; i<ArraySize(aOpt); i++){
636 if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault;
637 }
638
639 /* Process the command line switches. */
640 for(i=0; i<nArg; i+=2){
641 int iSel;
642 rc = testArgSelect(aOpt, "switch", azArg[i], &iSel);
643 if( rc ){
644 return rc;
645 }
646 if( aOpt[iSel].eVal==-2 ){
647 print_speed_test_help();
648 return 0;
649 }
650 if( i+1==nArg ){
651 testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
652 return 1;
653 }
654 if( aOpt[iSel].eVal>=0 ){
655 aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]);
656 }else{
657 zSystem = azArg[i+1];
658 bLsm = 0;
659 #if 0
660 for(j=0; zSystem[j]; j++){
661 if( zSystem[j]=='=' ) bLsm = 1;
662 }
663 #endif
664 }
665 }
666
667 printf("#");
668 for(i=0; i<ArraySize(aOpt); i++){
669 if( aOpt[i].zOpt ){
670 if( aOpt[i].eVal>=0 ){
671 printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]);
672 }else if( aOpt[i].eVal==-1 ){
673 printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem);
674 }
675 }
676 }
677 printf("\n");
678
679 defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE];
680 defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE];
681 pData = testDatasourceNew(&defn);
682
683 if( aParam[ST_WRITE]==0 ){
684 bReadonly = 1;
685 }
686
687 if( bLsm ){
688 rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb);
689 }else{
690 pDb = testOpen(zSystem, !bReadonly, &rc);
691 }
692 if( rc!=0 ) return rc;
693 if( bReadonly ){
694 nContent = testCountDatabase(pDb);
695 }
696
697 #if 0
698 pLog = fopen("/tmp/speed.log", "w");
699 tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
700 #endif
701
702 for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){
703 int msWrite, msFetch;
704 int iFetch;
705 int nWrite = aParam[ST_WRITE];
706
707 if( bReadonly ){
708 msWrite = 0;
709 }else{
710 testTimeInit();
711
712 if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc);
713 testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc);
714 if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
715
716 msWrite = testTimeGet();
717 nContent += nWrite;
718 }
719
720 if( aParam[ST_PAUSE] ){
721 if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000);
722 if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000));
723 }
724
725 if( aParam[ST_FETCH] ){
726 testTimeInit();
727 if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc);
728 for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){
729 int iKey = testPrngValue(i*nWrite+iFetch) % nContent;
730 #ifndef NDEBUG
731 testDatasourceFetch(pDb, pData, iKey, &rc);
732 #else
733 void *pKey; int nKey; /* Database key to query for */
734 void *pVal; int nVal; /* Result of query */
735
736 testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0);
737 rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal);
738 if( rc==0 && nVal<0 ) rc = 1;
739 if( rc ) break;
740 #endif
741 }
742 if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
743 msFetch = testTimeGet();
744 }else{
745 msFetch = 0;
746 }
747
748 if( i==(aParam[ST_REPEAT]-1) ){
749 testTimeInit();
750 testClose(&pDb);
751 msWrite += testTimeGet();
752 }
753
754 printf("%d %d %d\n", i, msWrite, msFetch);
755 fflush(stdout);
756 }
757
758 testClose(&pDb);
759 testDatasourceFree(pData);
760
761 if( pLog ){
762 flushPrev(pLog);
763 fclose(pLog);
764 }
765 return rc;
766 }
767
do_speed_tests(int nArg,char ** azArg)768 int do_speed_tests(int nArg, char **azArg){
769
770 struct DbSystem {
771 const char *zLibrary;
772 const char *zColor;
773 } aSys[] = {
774 { "sqlite3", "black" },
775 { "leveldb", "blue" },
776 { "lsm", "red" },
777 { "lsm_mt2", "orange" },
778 { "lsm_mt3", "purple" },
779 { "kyotocabinet", "green" },
780 {0, 0}
781 };
782
783 int i;
784 int j;
785 int rc;
786 int nSleep = 0; /* ms of rest allowed between INSERT tests */
787 int nRow = 0; /* Number of rows to insert into database */
788 int nStep; /* Measure INSERT time after this many rows */
789 int nSelStep; /* Measure SELECT time after this many rows */
790 int nSelTest; /* Number of SELECTs to run for timing */
791 int doReadTest = 1;
792 int doWriteTest = 1;
793
794 int *aTime; /* INSERT timing data */
795 int *aWrite; /* Writes per nStep inserts */
796 int *aSelTime; /* SELECT timing data */
797 int isFirst = 1;
798 int bSleep = 0;
799
800 /* File to write gnuplot script to. */
801 const char *zOut = "lsmtest_speed.gnuplot";
802
803 u32 sys_mask = 0;
804
805 testMallocUninstall(tdb_lsm_env());
806
807 for(i=0; i<nArg; i++){
808 struct Opt {
809 const char *zOpt;
810 int isSwitch;
811 } aOpt[] = {
812 { "sqlite3" , 0},
813 { "leveldb" , 0},
814 { "lsm" , 0},
815 { "lsm_mt2" , 0},
816 { "lsm_mt3" , 0},
817 { "kyotocabinet" , 0},
818 { "-rows" , 1},
819 { "-sleep" , 2},
820 { "-testmode" , 3},
821 { "-out" , 4},
822 { 0, 0}
823 };
824 int iSel;
825
826 rc = testArgSelect(aOpt, "argument", azArg[i], &iSel);
827 if( rc ) return rc;
828
829 if( aOpt[iSel].isSwitch ){
830 i++;
831
832 if( i>=nArg ){
833 testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
834 return 1;
835 }
836 if( aOpt[iSel].isSwitch==1 ){
837 nRow = atoi(azArg[i]);
838 }
839 if( aOpt[iSel].isSwitch==2 ){
840 nSleep = atoi(azArg[i]);
841 }
842 if( aOpt[iSel].isSwitch==3 ){
843 struct Mode {
844 const char *zMode;
845 int doReadTest;
846 int doWriteTest;
847 } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}};
848 int iMode;
849 rc = testArgSelect(aMode, "option", azArg[i], &iMode);
850 if( rc ) return rc;
851 doReadTest = aMode[iMode].doReadTest;
852 doWriteTest = aMode[iMode].doWriteTest;
853 }
854 if( aOpt[iSel].isSwitch==4 ){
855 /* The "-out FILE" switch. This option is used to specify a file to
856 ** write the gnuplot script to. */
857 zOut = azArg[i];
858 }
859 }else{
860 /* A db name */
861 rc = testArgSelect(aOpt, "system", azArg[i], &iSel);
862 if( rc ) return rc;
863 sys_mask |= (1<<iSel);
864 }
865 }
866
867 if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
868 nRow = MAX(nRow, 100000);
869 nStep = nRow/100;
870 nSelStep = nRow/10;
871 nSelTest = (nSelStep > 100000) ? 100000 : nSelStep;
872
873 aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep);
874 aWrite = malloc(sizeof(int) * nRow/nStep);
875 aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep);
876
877 /* This loop collects the INSERT speed data. */
878 if( doWriteTest ){
879 printf("Writing output to file \"%s\".\n", zOut);
880
881 for(j=0; aSys[j].zLibrary; j++){
882 FILE *pLog = 0;
883 TestDb *pDb; /* Database being tested */
884 lsm_db *pLsm;
885 int iDot = 0;
886
887 if( ((1<<j)&sys_mask)==0 ) continue;
888 if( bSleep && nSleep ) sqlite3_sleep(nSleep);
889 bSleep = 1;
890
891 testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary);
892
893 rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
894 if( rc ) return rc;
895
896 pLsm = configure_lsm_db(pDb);
897 #if 0
898 pLog = fopen("/tmp/speed.log", "w");
899 tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
900 #endif
901
902 testTimeInit();
903 for(i=0; i<nRow; i+=nStep){
904 int iStep;
905 int nWrite1 = 0, nWrite2 = 0;
906 testCaseProgress(i, nRow, testCaseNDot(), &iDot);
907 if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1);
908 for(iStep=0; iStep<nStep; iStep++){
909 u32 aKey[4]; /* 16-byte key */
910 u32 aVal[25]; /* 100 byte value */
911 testPrngArray(i+iStep, aKey, ArraySize(aKey));
912 testPrngArray(i+iStep, aVal, ArraySize(aVal));
913 rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
914 }
915 aTime[(j*nRow+i)/nStep] = testTimeGet();
916 if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2);
917 aWrite[i/nStep] = nWrite2 - nWrite1;
918 }
919
920 tdb_close(pDb);
921 if( pLog ) fclose(pLog);
922 testCaseFinish(rc);
923 }
924 }
925
926 /* This loop collects the SELECT speed data. */
927 if( doReadTest ){
928 for(j=0; aSys[j].zLibrary; j++){
929 int iDot = 0;
930 TestDb *pDb; /* Database being tested */
931
932 if( ((1<<j)&sys_mask)==0 ) continue;
933 if( bSleep && nSleep ) sqlite3_sleep(nSleep);
934 bSleep = 1;
935
936 testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary);
937
938 if( doWriteTest ){
939 rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
940 if( rc ) return rc;
941 configure_lsm_db(pDb);
942
943 for(i=0; i<nRow; i+=nSelStep){
944 int iStep;
945 int iSel;
946 testCaseProgress(i, nRow, testCaseNDot(), &iDot);
947 for(iStep=0; iStep<nSelStep; iStep++){
948 u32 aKey[4]; /* 16-byte key */
949 u32 aVal[25]; /* 100 byte value */
950 testPrngArray(i+iStep, aKey, ArraySize(aKey));
951 testPrngArray(i+iStep, aVal, ArraySize(aVal));
952 rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
953 }
954
955 testTimeInit();
956 for(iSel=0; iSel<nSelTest; iSel++){
957 void *pDummy;
958 int nDummy;
959 u32 iKey;
960 u32 aKey[4]; /* 16-byte key */
961
962 iKey = testPrngValue(iSel) % (i+nSelStep);
963 testPrngArray(iKey, aKey, ArraySize(aKey));
964 rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
965 }
966 aSelTime[(j*nRow+i)/nSelStep] = testTimeGet();
967 tdb_fetch(pDb, 0, 0, 0, 0);
968 }
969 }else{
970 int t;
971 int iSel;
972
973 rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb);
974 configure_lsm_db(pDb);
975
976 testTimeInit();
977 for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){
978 void *pDummy;
979 int nDummy;
980 u32 iKey;
981 u32 aKey[4]; /* 16-byte key */
982 #ifndef NDEBUG
983 u32 aVal[25]; /* 100 byte value */
984 #endif
985
986 testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot);
987
988 iKey = testPrngValue(iSel) % nRow;
989 testPrngArray(iKey, aKey, ArraySize(aKey));
990 rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
991
992 #ifndef NDEBUG
993 testPrngArray(iKey, aVal, ArraySize(aVal));
994 assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 );
995 #endif
996 }
997 if( rc!=LSM_OK ) return rc;
998
999 t = testTimeGet();
1000 tdb_fetch(pDb, 0, 0, 0, 0);
1001
1002 printf("%s: %d selects/second\n",
1003 aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t)
1004 );
1005 }
1006
1007 tdb_close(pDb);
1008 testCaseFinish(rc);
1009 }
1010 }
1011
1012
1013 if( doWriteTest ){
1014 FILE *pOut = fopen(zOut, "w");
1015 if( !pOut ){
1016 printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno));
1017 return 1;
1018 }
1019
1020 fprintf(pOut, "set xlabel \"Rows Inserted\"\n");
1021 fprintf(pOut, "set ylabel \"Inserts per second\"\n");
1022 if( doReadTest ){
1023 fprintf(pOut, "set y2label \"Selects per second\"\n");
1024 }else if( sys_mask==(1<<2) ){
1025 fprintf(pOut, "set y2label \"Page writes per insert\"\n");
1026 }
1027 fprintf(pOut, "set yrange [0:*]\n");
1028 fprintf(pOut, "set y2range [0:*]\n");
1029 fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) );
1030 fprintf(pOut, "set ytics nomirror\n");
1031 fprintf(pOut, "set y2tics nomirror\n");
1032 fprintf(pOut, "set key box lw 0.01\n");
1033 fprintf(pOut, "plot ");
1034
1035 for(j=0; aSys[j].zLibrary; j++){
1036 if( (1<<j)&sys_mask ){
1037 const char *zLib = aSys[j].zLibrary;
1038 fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ",
1039 (isFirst?"":", "), zLib, aSys[j].zColor
1040 );
1041 if( doReadTest ){
1042 fprintf(pOut, ", \"-\" ti \"%s SELECT\" "
1043 "axis x1y2 with points lw 3 lc rgb \"%s\""
1044 , zLib, aSys[j].zColor
1045 );
1046 }
1047 isFirst = 0;
1048 }
1049 }
1050
1051 assert( strcmp(aSys[2].zLibrary, "lsm")==0 );
1052 if( sys_mask==(1<<2) && !doReadTest ){
1053 fprintf(pOut, ", \"-\" ti \"lsm pages written\" "
1054 "axis x1y2 with boxes lw 1 lc rgb \"grey\""
1055 );
1056 }
1057
1058 fprintf(pOut, "\n");
1059
1060 for(j=0; aSys[j].zLibrary; j++){
1061 if( ((1<<j)&sys_mask)==0 ) continue;
1062 fprintf(pOut, "# Rows Inserts per second\n");
1063 for(i=0; i<nRow; i+=nStep){
1064 int iTime = aTime[(j*nRow+i)/nStep];
1065 int ips = (int)((i+nStep)*1000.0 / (double)iTime);
1066 fprintf(pOut, "%d %d\n", i+nStep, ips);
1067 }
1068 fprintf(pOut, "end\n");
1069
1070 if( doReadTest ){
1071 fprintf(pOut, "# Rows Selects per second\n");
1072 for(i=0; i<nRow; i+=nSelStep){
1073 int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]);
1074 fprintf(pOut, "%d %d\n", i+nSelStep, sps);
1075 }
1076 fprintf(pOut, "end\n");
1077 }else if( sys_mask==(1<<2) ){
1078 for(i=0; i<(nRow/nStep); i++){
1079 fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep);
1080 }
1081 fprintf(pOut, "end\n");
1082 }
1083 }
1084
1085 fprintf(pOut, "pause -1\n");
1086 fclose(pOut);
1087 }
1088
1089 free(aTime);
1090 free(aSelTime);
1091 free(aWrite);
1092 testMallocInstall(tdb_lsm_env());
1093 return 0;
1094 }
1095
1096 /*
1097 ** Usage: lsmtest random ?N?
1098 **
1099 ** This command prints a sequence of zero or more numbers from the PRNG
1100 ** system to stdout. If the "N" argument is missing, values the first 10
1101 ** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N.
1102 **
1103 ** This was added to verify that the PRNG values do not change between
1104 ** runs of the lsmtest program.
1105 */
do_random_tests(int nArg,char ** azArg)1106 int do_random_tests(int nArg, char **azArg){
1107 int i;
1108 int nRand;
1109 if( nArg==0 ){
1110 nRand = 10;
1111 }else if( nArg==1 ){
1112 nRand = atoi(azArg[0]);
1113 }else{
1114 testPrintError("Usage: random ?N?\n");
1115 return -1;
1116 }
1117 for(i=0; i<nRand; i++){
1118 printf("0x%x\n", testPrngValue(i));
1119 }
1120 return 0;
1121 }
1122
testFormatSize(char * aBuf,int nBuf,i64 nByte)1123 static int testFormatSize(char *aBuf, int nBuf, i64 nByte){
1124 int res;
1125 if( nByte<(1<<10) ){
1126 res = snprintf(aBuf, nBuf, "%d byte", (int)nByte);
1127 }else if( nByte<(1<<20) ){
1128 res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10)));
1129 }else{
1130 res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20)));
1131 }
1132 return res;
1133 }
1134
testReadSize(char * z)1135 static i64 testReadSize(char *z){
1136 int n = strlen(z);
1137 char c = z[n-1];
1138 i64 nMul = 1;
1139
1140 switch( c ){
1141 case 'g': case 'G':
1142 nMul = (1<<30);
1143 break;
1144
1145 case 'm': case 'M':
1146 nMul = (1<<20);
1147 break;
1148
1149 case 'k': case 'K':
1150 nMul = (1<<10);
1151 break;
1152
1153 default:
1154 nMul = 1;
1155 }
1156
1157 return nMul * (i64)atoi(z);
1158 }
1159
1160 /*
1161 ** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE
1162 */
do_writer_test(int nArg,char ** azArg)1163 static int do_writer_test(int nArg, char **azArg){
1164 int nBlock;
1165 int nSize;
1166 int i;
1167 int fd;
1168 int ms;
1169 char aFilesize[32];
1170 char aBlockSize[32];
1171
1172 char *aPage;
1173 int *aOrder;
1174 int nSync;
1175
1176 i64 filesize;
1177 i64 blocksize;
1178 i64 syncsize;
1179 int nPage = 4096;
1180
1181 /* How long to sleep before running a trial (in ms). */
1182 #if 0
1183 const int nSleep = 10000;
1184 #endif
1185 const int nSleep = 0;
1186
1187 if( nArg!=3 ){
1188 testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE");
1189 return -1;
1190 }
1191
1192 filesize = testReadSize(azArg[0]);
1193 blocksize = testReadSize(azArg[1]);
1194 syncsize = testReadSize(azArg[2]);
1195
1196 nBlock = (int)(filesize / blocksize);
1197 nSize = (int)blocksize;
1198 nSync = (int)(syncsize / blocksize);
1199
1200 aPage = (char *)malloc(4096);
1201 aOrder = (int *)malloc(nBlock * sizeof(int));
1202 for(i=0; i<nBlock; i++) aOrder[i] = i;
1203 for(i=0; i<(nBlock*25); i++){
1204 int tmp;
1205 u32 a = testPrngValue(i);
1206 u32 b = testPrngValue(a);
1207 a = a % nBlock;
1208 b = b % nBlock;
1209 tmp = aOrder[a];
1210 aOrder[a] = aOrder[b];
1211 aOrder[b] = tmp;
1212 }
1213
1214 testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize);
1215 testFormatSize(aBlockSize, sizeof(aFilesize), nSize);
1216
1217 printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize);
1218 if( nSync==1 ){
1219 printf("Sync after each block.\n");
1220 }else{
1221 printf("Sync after each %d blocks.\n", nSync);
1222 }
1223
1224 printf("Preparing file... ");
1225 fflush(stdout);
1226 unlink("writer.out");
1227 fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664);
1228 if( fd<0 ){
1229 testPrintError("open(): %d - %s\n", errno, strerror(errno));
1230 return -1;
1231 }
1232 testTimeInit();
1233 for(i=0; i<nBlock; i++){
1234 int iPg;
1235 memset(aPage, i&0xFF, nPage);
1236 for(iPg=0; iPg<(nSize/nPage); iPg++){
1237 write(fd, aPage, nPage);
1238 }
1239 }
1240 fsync(fd);
1241 printf("ok (%d ms)\n", testTimeGet());
1242
1243 for(i=0; i<5; i++){
1244 int j;
1245
1246 sqlite3_sleep(nSleep);
1247 printf("Now writing sequentially... ");
1248 fflush(stdout);
1249
1250 lseek(fd, 0, SEEK_SET);
1251 testTimeInit();
1252 for(j=0; j<nBlock; j++){
1253 int iPg;
1254 if( ((j+1)%nSync)==0 ) fdatasync(fd);
1255 memset(aPage, j&0xFF, nPage);
1256 for(iPg=0; iPg<(nSize/nPage); iPg++){
1257 write(fd, aPage, nPage);
1258 }
1259 }
1260 fdatasync(fd);
1261 ms = testTimeGet();
1262 printf("%d ms\n", ms);
1263 sqlite3_sleep(nSleep);
1264 printf("Now in an arbitrary order... ");
1265
1266 fflush(stdout);
1267 testTimeInit();
1268 for(j=0; j<nBlock; j++){
1269 int iPg;
1270 if( ((j+1)%nSync)==0 ) fdatasync(fd);
1271 lseek(fd, aOrder[j]*nSize, SEEK_SET);
1272 memset(aPage, j&0xFF, nPage);
1273 for(iPg=0; iPg<(nSize/nPage); iPg++){
1274 write(fd, aPage, nPage);
1275 }
1276 }
1277 fdatasync(fd);
1278 ms = testTimeGet();
1279 printf("%d ms\n", ms);
1280 }
1281
1282 close(fd);
1283 free(aPage);
1284 free(aOrder);
1285
1286 return 0;
1287 }
1288
do_insert_work_hook(lsm_db * db,void * p)1289 static void do_insert_work_hook(lsm_db *db, void *p){
1290 char *z = 0;
1291 lsm_info(db, LSM_INFO_DB_STRUCTURE, &z);
1292 if( z ){
1293 printf("%s\n", z);
1294 fflush(stdout);
1295 lsm_free(lsm_get_env(db), z);
1296 }
1297
1298 unused_parameter(p);
1299 }
1300
1301 typedef struct InsertWriteHook InsertWriteHook;
1302 struct InsertWriteHook {
1303 FILE *pOut;
1304 int bLog;
1305 i64 iOff;
1306 int nData;
1307 };
1308
flushHook(InsertWriteHook * pHook)1309 static void flushHook(InsertWriteHook *pHook){
1310 if( pHook->nData ){
1311 fprintf(pHook->pOut, "write %s %d %d\n",
1312 (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData
1313 );
1314 pHook->nData = 0;
1315 fflush(pHook->pOut);
1316 }
1317 }
1318
do_insert_write_hook(void * pCtx,int bLog,i64 iOff,int nData,int nUs)1319 static void do_insert_write_hook(
1320 void *pCtx,
1321 int bLog,
1322 i64 iOff,
1323 int nData,
1324 int nUs
1325 ){
1326 InsertWriteHook *pHook = (InsertWriteHook *)pCtx;
1327 if( bLog ) return;
1328
1329 if( nData==0 ){
1330 flushHook(pHook);
1331 fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db"));
1332 }else if( pHook->nData
1333 && bLog==pHook->bLog
1334 && iOff==(pHook->iOff+pHook->nData)
1335 ){
1336 pHook->nData += nData;
1337 }else{
1338 flushHook(pHook);
1339 pHook->bLog = bLog;
1340 pHook->iOff = iOff;
1341 pHook->nData = nData;
1342 }
1343 }
1344
do_replay(int nArg,char ** azArg)1345 static int do_replay(int nArg, char **azArg){
1346 char aBuf[4096];
1347 FILE *pInput;
1348 FILE *pClose = 0;
1349 const char *zDb;
1350
1351 lsm_env *pEnv;
1352 lsm_file *pOut;
1353 int rc;
1354
1355 if( nArg!=2 ){
1356 testPrintError("Usage: replay WRITELOG FILE\n");
1357 return 1;
1358 }
1359
1360 if( strcmp(azArg[0], "-")==0 ){
1361 pInput = stdin;
1362 }else{
1363 pClose = pInput = fopen(azArg[0], "r");
1364 }
1365 zDb = azArg[1];
1366 pEnv = tdb_lsm_env();
1367 rc = pEnv->xOpen(pEnv, zDb, 0, &pOut);
1368 if( rc!=LSM_OK ) return rc;
1369
1370 while( feof(pInput)==0 ){
1371 char zLine[80];
1372 fgets(zLine, sizeof(zLine)-1, pInput);
1373 zLine[sizeof(zLine)-1] = '\0';
1374
1375 if( 0==memcmp("sync db", zLine, 7) ){
1376 rc = pEnv->xSync(pOut);
1377 if( rc!=0 ) break;
1378 }else{
1379 int iOff;
1380 int nData;
1381 int nMatch;
1382 nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData);
1383 if( nMatch==2 ){
1384 int i;
1385 for(i=0; i<nData; i+=sizeof(aBuf)){
1386 memset(aBuf, i&0xFF, sizeof(aBuf));
1387 rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf));
1388 if( rc!=0 ) break;
1389 }
1390 }
1391 }
1392 }
1393 if( pClose ) fclose(pClose);
1394 pEnv->xClose(pOut);
1395
1396 return rc;
1397 }
1398
do_insert(int nArg,char ** azArg)1399 static int do_insert(int nArg, char **azArg){
1400 const char *zDb = "lsm";
1401 TestDb *pDb = 0;
1402 int i;
1403 int rc;
1404 const int nRow = 1 * 1000 * 1000;
1405
1406 DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 };
1407 Datasource *pData = 0;
1408
1409 if( nArg>1 ){
1410 testPrintError("Usage: insert ?DATABASE?\n");
1411 return 1;
1412 }
1413 if( nArg==1 ){ zDb = azArg[0]; }
1414
1415 testMallocUninstall(tdb_lsm_env());
1416 for(i=0; zDb[i] && zDb[i]!='='; i++);
1417 if( zDb[i] ){
1418 rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb);
1419 }else{
1420 rc = tdb_open(zDb, 0, 1, &pDb);
1421 }
1422
1423 if( rc!=0 ){
1424 testPrintError("Error opening db \"%s\": %d\n", zDb, rc);
1425 }else{
1426 InsertWriteHook hook;
1427 memset(&hook, 0, sizeof(hook));
1428 hook.pOut = fopen("writelog.txt", "w");
1429
1430 pData = testDatasourceNew(&defn);
1431 tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0);
1432 tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook);
1433
1434 if( rc==0 ){
1435 for(i=0; i<nRow; i++){
1436 void *pKey; int nKey; /* Database key to insert */
1437 void *pVal; int nVal; /* Database value to insert */
1438 testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
1439 tdb_write(pDb, pKey, nKey, pVal, nVal);
1440 }
1441 }
1442
1443 testDatasourceFree(pData);
1444 tdb_close(pDb);
1445 flushHook(&hook);
1446 fclose(hook.pOut);
1447 }
1448 testMallocInstall(tdb_lsm_env());
1449
1450 return rc;
1451 }
1452
st_do_show(int a,char ** b)1453 static int st_do_show(int a, char **b) { return do_show(a, b); }
st_do_work(int a,char ** b)1454 static int st_do_work(int a, char **b) { return do_work(a, b); }
st_do_io(int a,char ** b)1455 static int st_do_io(int a, char **b) { return do_io(a, b); }
1456
1457 #ifdef __linux__
1458 #include <sys/time.h>
1459 #include <sys/resource.h>
1460
lsmtest_rusage_report(void)1461 static void lsmtest_rusage_report(void){
1462 struct rusage r;
1463 memset(&r, 0, sizeof(r));
1464
1465 getrusage(RUSAGE_SELF, &r);
1466 printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n",
1467 (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock
1468 );
1469 }
1470 #else
lsmtest_rusage_report(void)1471 static void lsmtest_rusage_report(void){
1472 /* no-op */
1473 }
1474 #endif
1475
main(int argc,char ** argv)1476 int main(int argc, char **argv){
1477 struct TestFunc {
1478 const char *zName;
1479 int bRusageReport;
1480 int (*xFunc)(int, char **);
1481 } aTest[] = {
1482 {"random", 1, do_random_tests},
1483 {"writespeed", 1, do_writer_test},
1484 {"io", 1, st_do_io},
1485
1486 {"insert", 1, do_insert},
1487 {"replay", 1, do_replay},
1488
1489 {"speed", 1, do_speed_tests},
1490 {"speed2", 1, do_speed_test2},
1491 {"show", 0, st_do_show},
1492 {"work", 1, st_do_work},
1493 {"test", 1, do_test},
1494
1495 {0, 0}
1496 };
1497 int rc; /* Return Code */
1498 int iFunc; /* Index into aTest[] */
1499
1500 int nLeakAlloc = 0; /* Allocations leaked by lsm */
1501 int nLeakByte = 0; /* Bytes leaked by lsm */
1502
1503 #ifdef LSM_DEBUG_MEM
1504 FILE *pReport = 0; /* lsm malloc() report file */
1505 const char *zReport = "malloc.txt generated";
1506 #else
1507 const char *zReport = "malloc.txt NOT generated";
1508 #endif
1509
1510 testMallocInstall(tdb_lsm_env());
1511
1512 if( argc<2 ){
1513 testPrintError("Usage: %s sub-command ?args...?\n", argv[0]);
1514 return -1;
1515 }
1516
1517 /* Initialize error reporting */
1518 testErrorInit(argc, argv);
1519
1520 /* Initialize PRNG system */
1521 testPrngInit();
1522
1523 rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc);
1524 if( rc==0 ){
1525 rc = aTest[iFunc].xFunc(argc-2, &argv[2]);
1526 }
1527
1528 #ifdef LSM_DEBUG_MEM
1529 pReport = fopen("malloc.txt", "w");
1530 testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport);
1531 fclose(pReport);
1532 #else
1533 testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0);
1534 #endif
1535
1536 if( nLeakAlloc ){
1537 testPrintError("Leaked %d bytes in %d allocations (%s)\n",
1538 nLeakByte, nLeakAlloc, zReport
1539 );
1540 if( rc==0 ) rc = -1;
1541 }
1542 testMallocUninstall(tdb_lsm_env());
1543
1544 if( aTest[iFunc].bRusageReport ){
1545 lsmtest_rusage_report();
1546 }
1547 return rc;
1548 }
1549