1 /*
2 ##############################################################################
3 # Copyright (c) 2000-2006 All rights reserved
4 # Alberto Reggiori <areggiori@webweaving.org>
5 # Dirk-Willem van Gulik <dirkx@webweaving.org>
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions
9 # are met:
10 #
11 # 1. Redistributions of source code must retain the above copyright
12 # notice, this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in
16 # the documentation and/or other materials provided with the
17 # distribution.
18 #
19 # 3. The end-user documentation included with the redistribution,
20 # if any, must include the following acknowledgment:
21 # "This product includes software developed by
22 # Alberto Reggiori <areggiori@webweaving.org> and
23 # Dirk-Willem van Gulik <dirkx@webweaving.org>."
24 # Alternately, this acknowledgment may appear in the software itself,
25 # if and wherever such third-party acknowledgments normally appear.
26 #
27 # 4. All advertising materials mentioning features or use of this software
28 # must display the following acknowledgement:
29 # This product includes software developed by the University of
30 # California, Berkeley and its contributors.
31 #
32 # 5. Neither the name of the University nor the names of its contributors
33 # may be used to endorse or promote products derived from this software
34 # without specific prior written permission.
35 #
36 # 6. Products derived from this software may not be called "RDFStore"
37 # nor may "RDFStore" appear in their names without prior written
38 # permission.
39 #
40 # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
41 # ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
43 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
44 # FOR ANY DIRECT, INDIRECT, INCIDENTAL,
45 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
49 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
51 # OF THE POSSIBILITY OF SUCH DAMAGE.
52 #
53 # ====================================================================
54 #
55 # This software consists of work developed by Alberto Reggiori and
56 # Dirk-Willem van Gulik. The RDF specific part is based based on public
57 # domain software written at the Stanford University Database Group by
58 # Sergey Melnik. For more information on the RDF API Draft work,
59 # please see <http://www-db.stanford.edu/~melnik/rdf/api.html>
60 # The DBMS TCP/IP server part is based on software originally written
61 # by Dirk-Willem van Gulik for Web Weaving Internet Engineering m/v Enschede,
62 # The Netherlands.
63 #
64 ##############################################################################
65 #
66 # $Id: backend_bdb_store.c,v 1.21 2006/06/19 10:10:21 areggiori Exp $
67 */
68
69 #include "dbms.h"
70 #include "dbms_compat.h"
71 #include "dbms_comms.h"
72
73 #include "rdfstore_flat_store.h"
74 #include "rdfstore_log.h"
75 #include "rdfstore.h"
76
77 #include "backend_store.h"
78
79 #include "backend_bdb_store.h"
80 #include "backend_bdb_store_private.h"
81
82 #include <sys/stat.h>
83
84 static char *mkpath(char *base, char *infile);
85
86 /*
87 * Some default call back functions.
88 */
89 static void
default_myfree(void * adr)90 default_myfree(void *adr)
91 {
92 RDFSTORE_FREE(adr);
93 }
94 static void *
default_mymalloc(size_t x)95 default_mymalloc(size_t x)
96 {
97 return RDFSTORE_MALLOC(x);
98 }
99 static void
default_myerror(char * err,int erx)100 default_myerror(char *err, int erx)
101 {
102 fprintf(stderr, "backend_bdb Error[%d]: %s\n", erx, err);
103 }
104
105 /* backend_bdb error management */
106 static char _backend_bdb_erm[256] = "\0";
107
108 /* human-readable error codes */
109 static char *backend_bdb_error[] = {
110 /* FLAT_STORE_E_UNDEF 2000 */
111 "Not defined",
112 /* FLAT_STORE_E_NONNUL 2001 */
113 "Undefined Error",
114 /* FLAT_STORE_E_NOMEM 2002 */
115 "Out of memory",
116 /* FLAT_STORE_E_NOPE 2003 */
117 "No such database",
118 /* FLAT_STORE_E_KEYEMPTY 2004 */
119 "Key/data deleted or never created",
120 /* FLAT_STORE_E_KEYEXIST 2005 */
121 "The key/data pair already exists",
122 /* FLAT_STORE_E_NOTFOUND 2006 */
123 "Key/data pair not found",
124 /* FLAT_STORE_E_OLD_VERSION 2007 */
125 "Out-of-date version",
126 /* FLAT_STORE_E_DBMS 2008 */
127 "DBMS error",
128 /* FLAT_STORE_E_CANNOTOPEN 2009 */
129 "Cannot open database",
130 /* FLAT_STORE_E_BUG 2010 */
131 "Conceptual error"
132 };
133
134 void
backend_bdb_set_error(void * eme,char * msg,rdfstore_flat_store_error_t erx)135 backend_bdb_set_error(void *eme, char *msg, rdfstore_flat_store_error_t erx)
136 {
137 backend_bdb_t *me = (backend_bdb_t *) eme;
138 bzero(me->err, sizeof(me->err));
139 if (erx == FLAT_STORE_E_DBMS) {
140 snprintf(me->err, sizeof(me->err), "DBMS Error %s: %s\n", msg,
141 errno == 0 ? "" : (strlen(strerror(errno)) <= sizeof(me->err)) ? strerror(errno) : ""); /* not enough... */
142 } else {
143 if ((erx > FLAT_STORE_E_UNDEF) && (erx <= FLAT_STORE_E_BUG)) {
144 strcpy(me->err, backend_bdb_error[erx - FLAT_STORE_E_UNDEF]);
145 } else {
146 if (strlen(strerror(erx)) <= sizeof(me->err))
147 strcpy(me->err, strerror(erx));
148 };
149 };
150 if (strlen(me->err) <= sizeof(_backend_bdb_erm))
151 strcpy(_backend_bdb_erm, me->err);
152
153 #ifdef VERBOSE
154 if (me->error)
155 (*(me->error)) (me->err, erx);
156 #endif
157 }
158
159
160 char *
backend_bdb_get_error(void * eme)161 backend_bdb_get_error(void *eme)
162 {
163 backend_bdb_t *me = (backend_bdb_t *) eme;
164 if (me == NULL)
165 return _backend_bdb_erm;
166 else
167 return me->err;
168 };
169
170 /* clone a key or value for older BDB */
171 DBT
backend_bdb_kvdup(void * eme,DBT data)172 backend_bdb_kvdup(void *eme, DBT data)
173 {
174 backend_bdb_t *me = (backend_bdb_t *) eme;
175 DBT dup;
176
177 memset(&dup, 0, sizeof(dup));
178
179 if (data.size == 0) {
180 dup.data = NULL;
181 return dup;
182 };
183
184 dup.size = data.size;
185
186 if ((dup.data = (char *) me->malloc(data.size + 1)) == NULL) {
187 perror("Out of memory");
188 exit(1);
189 };
190
191
192 memcpy(dup.data, data.data, data.size);
193 memcpy(dup.data + data.size, "\0", 1);
194
195 return dup;
196 };
197
198 void
backend_bdb_reset_debuginfo(void * eme)199 backend_bdb_reset_debuginfo(
200 void *eme
201 )
202 {
203 #ifdef RDFSTORE_FLAT_STORE_DEBUG
204 backend_bdb_t *me = (backend_bdb_t *) eme;
205 me->num_store = 0;
206 me->num_fetch = 0;
207 me->num_inc = 0;
208 me->num_dec = 0;
209 me->num_sync = 0;
210 me->num_next = 0;
211 me->num_from = 0;
212 me->num_first = 0;
213 me->num_delete = 0;
214 me->num_clear = 0;
215 me->num_exists = 0;
216 #endif
217 };
218
219 #ifdef BERKELEY_DB_1_OR_2
220 static int rdfstore_backend_bdb_compare_int(
221 const DBT *a,
222 const DBT *b );
223 #else
224 static int rdfstore_backend_bdb_compare_int(
225 DB *file,
226 const DBT *a,
227 const DBT *b );
228 #endif
229
230 #ifdef BERKELEY_DB_1_OR_2
231 static int rdfstore_backend_bdb_compare_double(
232 const DBT *a,
233 const DBT *b );
234 #else
235 static int rdfstore_backend_bdb_compare_double(
236 DB *file,
237 const DBT *a,
238 const DBT *b );
239 #endif
240
241 /*
242 * NOTE: all the functions return 0 on success and non zero value if error
243 * (see above and include/backend_bdb.h for known error codes)
244 */
245 rdfstore_flat_store_error_t
backend_bdb_open(int remote,int ro,void ** emme,char * dir,char * name,unsigned int local_hash_flags,char * host,int port,void * (* _my_malloc)(size_t size),void (* _my_free)(void *),void (* _my_report)(dbms_cause_t cause,int count),void (* _my_error)(char * err,int erx),int bt_compare_fcn_type)246 backend_bdb_open(
247 int remote,
248 int ro,
249 void **emme,
250 char *dir,
251 char *name,
252 unsigned int local_hash_flags,
253 char *host,
254 int port,
255 void *(*_my_malloc) (size_t size),
256 void (*_my_free) (void *),
257 void (*_my_report) (dbms_cause_t cause, int count),
258 void (*_my_error) (char *err, int erx),
259 int bt_compare_fcn_type
260 )
261 {
262 backend_bdb_t **mme = (backend_bdb_t **) emme;
263 backend_bdb_t *me;
264 char *buff;
265 struct stat s;
266 #if 0
267 HASHINFO priv = {
268 16 * 1024, /* bsize; hash bucked size */
269 8, /* ffactor, # keys/bucket */
270 3000, /* nelements, guestimate */
271 512 * 1024, /* cache size */
272 NULL, /* hash function */
273 0 /* use current host order */
274 };
275 #endif
276
277 #ifdef BERKELEY_DB_1_OR_2 /* Berkeley DB Version 1 or 2 */
278 #ifdef DB_VERSION_MAJOR
279 DB_INFO btreeinfo;
280 memset(&btreeinfo, 0, sizeof(btreeinfo));
281 btreeinfo.bt_compare = ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ? rdfstore_backend_bdb_compare_int : ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ? rdfstore_backend_bdb_compare_double : NULL ;
282 #else
283 BTREEINFO btreeinfo;
284 memset(&btreeinfo, 0, sizeof(btreeinfo));
285 btreeinfo.compare = ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ? rdfstore_backend_bdb_compare_int : ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ? rdfstore_backend_bdb_compare_double : NULL ;
286 #endif
287 #endif
288
289 *mme = NULL;
290
291 if (_my_error == NULL)
292 _my_error = default_myerror;
293
294 if (_my_malloc == NULL)
295 _my_malloc = default_mymalloc;
296
297 if (_my_free == NULL)
298 _my_free = default_myfree;
299
300 me = (backend_bdb_t *) _my_malloc(sizeof(backend_bdb_t));
301 if (me == NULL) {
302 perror("backend_bdb_open");
303 return FLAT_STORE_E_NOMEM;
304 };
305
306 me->error = _my_error;
307 me->malloc = _my_malloc;
308 me->free = _my_free;
309
310 me->bt_compare_fcn_type = bt_compare_fcn_type;
311
312 bzero(me->err, sizeof(me->err));
313
314 if (_my_report != NULL)
315 me->callback = _my_report;
316
317 backend_bdb_reset_debuginfo(me);
318
319 if (remote) {
320 backend_bdb_set_error(me, "BDB backend does not do remote storage", FLAT_STORE_E_DBMS);
321 perror("backend_bdb_open");
322 _my_free(me);
323 return FLAT_STORE_E_CANNOTOPEN;
324 };
325
326 /* use local Berkeley DB either in-memory or physical files on disk */
327 if ((dir) &&
328 (name)) {
329 if(ro==1) { /* do not even try to go further if open read-only */
330 if ( (stat(dir, &s) != 0) ||
331 ((s.st_mode & S_IFDIR) == 0) ) {
332 backend_bdb_set_error(me, "Could not open database", FLAT_STORE_E_NOPE);
333 perror("backend_bdb_open");
334 fprintf(stderr, "Could not open database'%s'\n", dir);
335 _my_free(me);
336 return FLAT_STORE_E_CANNOTOPEN;
337 };
338 };
339 /* make path */
340 if (!(buff = mkpath(dir, name))) {
341 backend_bdb_set_error(me, "Could not create or open database", FLAT_STORE_E_NOPE);
342 perror("backend_bdb_open");
343 fprintf(stderr, "Could not create or open database'%s'\n", dir);
344 _my_free(me);
345 return FLAT_STORE_E_CANNOTOPEN;
346 };
347 strcpy(me->filename, buff);
348 umask(0);
349 } else {
350 strcpy(me->filename, "\0");
351 buff = NULL;
352 };
353
354 /* something strange with BDB - it gives 'Bus error' if DB file not there and open in DB_RDONLY - why seg fault? must be DB_ENV stuff... */
355 if( (buff!=NULL) &&
356 (ro==1) ) {
357 if ( (stat(buff, &s) != 0) ||
358 ((s.st_mode & S_IFREG) == 0) ) {
359 backend_bdb_set_error(me, "Could not open database", FLAT_STORE_E_NOPE);
360 perror("backend_bdb_open");
361 fprintf(stderr, "Could not open database '%s'\n", dir);
362 _my_free(me);
363 return FLAT_STORE_E_CANNOTOPEN;
364 };
365 };
366
367 #ifdef BERKELEY_DB_1_OR_2 /* Berkeley DB Version 1 or 2 */
368
369 #ifdef DB_VERSION_MAJOR
370 if ( (db_open( buff,
371 DB_BTREE, ((ro==0 || buff==NULL) ? ( DB_CREATE ) : ( DB_RDONLY ) ),
372 0666, NULL, &btreeinfo, &me->bdb )) ||
373 #if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 6
374 ((me->bdb->cursor)(me->bdb, NULL, &me->cursor))
375 #else
376 ((me->bdb->cursor)(me->bdb, NULL, &me->cursor, 0))
377 #endif
378 ) {
379 #else
380
381 #if defined(DB_LIBRARY_COMPATIBILITY_API) && DB_VERSION_MAJOR > 2
382 if (!(me->bdb = (DB *)__db185_open( buff,
383 ((ro==0 || buff==NULL) ? (O_RDWR | O_CREAT) : ( O_RDONLY ) ),
384 0666, DB_BTREE, &btreeinfo ))) {
385 #else
386 /* for unpatched db-1.85 when use in-memory DB_BTREE due to mkstemp() call in hash/hash_page.c open_temp()
387 i.e. HASHVERSION==2 we use DB_BTREE instead in CGI/mod_perl environments to avoid problems with errors
388 like 'Permission denied' due the Web server running in a different user under a different directory */
389
390 #if DIRKX_DEBUG
391 BTREEINFO openinfo = {
392 0,
393 32 * 1024 * 1024,
394 0,
395 atoi(getenv("PSIZE")),
396 64 * 1024,
397 NULL, NULL, 0
398 };
399 #endif
400 if (!(me->bdb = (DB *)dbopen( buff,
401 ((ro==0 || buff==NULL) ? (O_RDWR | O_CREAT) : ( O_RDONLY ) ),
402 0666,
403 #if HASHVERSION == 2
404 ( ( (buff==NULL) && (getenv("GATEWAY_INTERFACE") != NULL) ) ? DB_BTREE : DB_BTREE ),
405 #else
406 DB_BTREE,
407 #endif
408 #if DIRKX_DEBUG
409 &openinfo))) {
410 #else
411 &btreeinfo ))) {
412 #endif
413
414 #endif /* DB_LIBRARY_COMPATIBILITY_API */
415
416 #endif
417
418 #else /* Berkeley DB Version > 2 */
419 if (db_create(&me->bdb, NULL,0)) {
420 rdfstore_flat_store_set_error((void*)me,"Could not create environment",FLAT_STORE_E_CANNOTOPEN);
421 perror("rdfstore_flat_store_open");
422 fprintf(stderr,"Could not open/create '%s':\n",buff);
423 _my_free(me);
424 return FLAT_STORE_E_CANNOTOPEN;
425 };
426
427 /* set the b-tree comparinson function to the one passed */
428 if( bt_compare_fcn_type != 0 ) {
429 me->bdb->set_bt_compare(me->bdb, ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ?
430 rdfstore_backend_bdb_compare_int : ( bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ?
431 rdfstore_backend_bdb_compare_double : NULL );
432 };
433
434 me->bdb->set_errfile(me->bdb,stderr);
435 me->bdb->set_errpfx(me->bdb,"BerkelyDB");
436 #if DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR < 3
437 me->bdb->set_malloc( me->bdb, me->malloc );
438 #elif DB_VERSION_MAJOR > 3 || DB_VERSION_MINOR >= 3
439 /* This interface appeared in 3.3 */
440 me->bdb->set_alloc( me->bdb, me->malloc, NULL, NULL ); /* could also pass me->free as 4th param but not sure how memoeyr is managed still */
441 #endif
442
443 #if DB_VERSION_MAJOR >= 4 && DB_VERSION_MINOR > 0 && DB_VERSION_PATCH >= 17
444 if ( (me->bdb->open( me->bdb,
445 NULL,
446 buff,
447 NULL,
448 DB_BTREE, ((ro==0 || buff==NULL) ? ( DB_CREATE ) : ( DB_RDONLY ) ),
449 0666 )) ||
450 #else
451 if ( (me->bdb->open( me->bdb,
452 buff,
453 NULL,
454 DB_BTREE, ((ro==0 || buff==NULL) ? ( DB_CREATE ) : ( DB_RDONLY ) ),
455 0666 )) ||
456 #endif
457 ((me->bdb->cursor)(me->bdb, NULL, &me->cursor, 0)) ) {
458 #endif /* Berkeley DB Version > 2 */
459
460 rdfstore_flat_store_set_error((void*)me,"Could not open/create database",FLAT_STORE_E_CANNOTOPEN);
461 perror("rdfstore_flat_store_open");
462 fprintf(stderr,"Could not open/create '%s':\n",buff);
463 _my_free(me);
464 return FLAT_STORE_E_CANNOTOPEN;
465 };
466
467 #ifndef BERKELEY_DB_1_OR_2 /* Berkeley DB Version > 2 */
468 /*
469 (void)me->bdb->set_h_ffactor(me->bdb, 1024);
470 (void)me->bdb->set_h_nelem(me->bdb, (u_int32_t)6000);
471 */
472 #endif
473
474 #ifdef RDFSTORE_FLAT_STORE_DEBUG
475 fprintf(stderr,"rdfstore_flat_store_open '%s'\n",me->filename);
476 #endif
477
478 *mme = me;
479 return 0;
480 }
481
482 rdfstore_flat_store_error_t
483 backend_bdb_close(
484 void *eme
485 )
486 {
487 backend_bdb_t *me = (backend_bdb_t *) eme;
488 void (*_my_free) (void *) = me->free;
489 int retval = 0;
490
491 #ifdef DB_VERSION_MAJOR
492 me->cursor->c_close(me->cursor);
493 (me->bdb->close) (me->bdb, 0);
494 #else
495 (me->bdb->close) (me->bdb);
496 #endif
497 _my_free(me);
498
499 #ifdef RDFSTORE_FLAT_STORE_DEBUG
500 fprintf(stderr, "backend_bdb_close '%s'\n", me->filename);
501 #endif
502
503 return retval;
504 };
505
506 rdfstore_flat_store_error_t
507 backend_bdb_fetch(
508 void * eme,
509 DBT key,
510 DBT * val
511 )
512 {
513 backend_bdb_t * me = (backend_bdb_t *) eme;
514 int retval = 0;
515
516 #ifdef RDFSTORE_FLAT_STORE_DEBUG
517 fprintf(stderr, "backend_bdb_fetch num=%d from '%s'\n", ++(me->num_fetch), me->filename);
518 #endif
519
520 #if DB_VERSION_MAJOR >= 2
521 memset(val, 0, sizeof(*val));
522 (*val).flags = DB_DBT_MALLOC;
523 retval = ((me->bdb)->get) (me->bdb, NULL, &key, val, 0);
524 #else
525 retval = ((me->bdb)->get) (me->bdb, &key, val, 0);
526 #endif
527
528 /* need to add proper client side BDB error management */
529 if (retval != 0) {
530 #if DB_VERSION_MAJOR >= 2
531 if ((*val).data && (*val).size)
532 me->free((*val).data);
533 #endif
534 memset(val, 0, sizeof(*val));
535 (*val).data = NULL;
536
537 #ifdef DB_VERSION_MAJOR
538 if (retval == DB_NOTFOUND)
539 #else
540 if (retval == 1)
541 #endif
542 {
543 backend_bdb_set_error(me, "Could not fetch key/value", FLAT_STORE_E_NOTFOUND);
544 return FLAT_STORE_E_NOTFOUND;
545 } else {
546 backend_bdb_set_error(me, "Could not fetch key/value", FLAT_STORE_E_NOTFOUND);
547 perror("backend_bdb_fetch");
548 fprintf(stderr, "Could not fetch '%s': %s\n", me->filename, (char *) key.data);
549 return FLAT_STORE_E_NOTFOUND;
550 };
551 } else {
552 #if DB_VERSION_MAJOR < 2
553 /*
554 * Berkeley DB 1.85 don't malloc the data for the caller
555 * application duplicate the returned value to ensure
556 * reentrancy
557 */
558 (*val) = backend_bdb_kvdup(me, *val);
559 #endif
560 return retval;
561 };
562 }
563
564 rdfstore_flat_store_error_t
565 backend_bdb_fetch_compressed(
566 void * eme,
567 void (*func_decode)(unsigned int,unsigned char*, unsigned int *, unsigned char *),
568 DBT key,
569 unsigned int * outsize_p, unsigned char * outchar
570 )
571 {
572 backend_bdb_t *me = (backend_bdb_t *) eme;
573 int retval = 0;
574 DBT val;
575 memset(&val,0,sizeof(val));
576
577 if ((retval = backend_bdb_fetch(eme,key,&val)))
578 return retval;
579
580 (*func_decode)(val.size,val.data,outsize_p,outchar);
581 (me->free)(val.data);
582
583 return retval;
584 }
585
586 rdfstore_flat_store_error_t
587 backend_bdb_store(
588 void *eme,
589 DBT key,
590 DBT val
591 )
592 {
593 backend_bdb_t *me = (backend_bdb_t *) eme;
594 int retval = 0;
595
596 #ifdef RDFSTORE_FLAT_STORE_DEBUG
597 fprintf(stderr, "backend_bdb_store num=%d in '%s'\n", ++(me->num_store), me->filename);
598 #endif
599
600 #ifdef DB_VERSION_MAJOR
601 retval = ((me->bdb)->put) (me->bdb, NULL, &key, &val, 0);
602 #else
603 retval = ((me->bdb)->put) (me->bdb, &key, &val, 0);
604 #endif
605 if (retval != 0) {
606 #ifdef DB_VERSION_MAJOR
607 if (retval == DB_KEYEXIST)
608 #else
609 if (retval == 1)
610 #endif
611 {
612 backend_bdb_set_error(me, "Could not store key/value", FLAT_STORE_E_KEYEXIST);
613 return FLAT_STORE_E_KEYEXIST;
614 };
615
616 backend_bdb_set_error(me, "Could not store key/value", FLAT_STORE_E_NONNUL);
617 fprintf(stderr, "Could not store '%s': %s(%d) = %s(%d) E=%d\n", me->filename,
618 (char *) key.data, (int)key.size,
619 (char *) val.data, (int)val.size,
620 retval);
621 return FLAT_STORE_E_NONNUL;
622 }
623 return 0;
624 }
625
626 rdfstore_flat_store_error_t
627 backend_bdb_store_compressed(
628 void * eme,
629 void (*func_encode)(unsigned int,unsigned char*, unsigned int *, unsigned char *),
630 DBT key,
631 unsigned int insize , unsigned char * inchar,
632 unsigned char * outbuff
633 )
634 {
635 unsigned int outsize;
636 DBT val;
637 memset(&val,0,sizeof(val));
638
639 (*func_encode)(insize, inchar, &outsize, outbuff);
640
641 val.data = outbuff;
642 val.size = outsize;
643
644 return backend_bdb_store(eme,key,val);
645 }
646
647 rdfstore_flat_store_error_t
648 backend_bdb_exists(
649 void *eme,
650 DBT key
651 )
652 {
653 backend_bdb_t *me = (backend_bdb_t *) eme;
654 int retval;
655 DBT val;
656 memset(&val,0,sizeof(val));
657
658 #ifdef RDFSTORE_FLAT_STORE_DEBUG
659 fprintf(stderr, "backend_bdb_exists num=%d from '%s'\n", ++(me->num_exists), me->filename);
660 #endif
661
662
663 memset(&val, 0, sizeof(val));
664
665 /*
666 * here we do not care about memory management due that we just want
667 * to know whether or not the given key exists
668 */
669
670 #if DB_VERSION_MAJOR >= 2
671 retval = ((me->bdb)->get) (me->bdb, NULL, &key, &val, 0);
672 #else
673 retval = ((me->bdb)->get) (me->bdb, &key, &val, 0);
674 #endif
675 return retval;
676 };
677
678 rdfstore_flat_store_error_t
679 backend_bdb_delete(
680 void *eme,
681 DBT key
682 )
683 {
684 backend_bdb_t *me = (backend_bdb_t *) eme;
685 int retval;
686
687 #ifdef RDFSTORE_FLAT_STORE_DEBUG
688 fprintf(stderr, "backend_bdb_delete num=%d from '%s'\n", ++(me->num_delete), me->filename);
689 #endif
690
691 #ifdef DB_VERSION_MAJOR
692 retval = ((me->bdb)->del) (me->bdb, NULL, &key, 0);
693 if( retval == DB_NOTFOUND )
694 return FLAT_STORE_E_NOTFOUND;
695 #else
696 retval = ((me->bdb)->del) (me->bdb, &key, 0);
697 if ( retval == 1 )
698 return FLAT_STORE_E_NOTFOUND;
699 #endif
700 return retval;
701 };
702
703 rdfstore_flat_store_error_t
704 backend_bdb_clear(
705 void *eme
706 )
707 {
708 backend_bdb_t *me = (backend_bdb_t *) eme;
709 char *buff;
710
711 #ifdef BERKELEY_DB_1_OR_2 /* Berkeley DB Version 1 or 2 */
712 #ifdef DB_VERSION_MAJOR
713 DB_INFO btreeinfo;
714 memset(&btreeinfo, 0, sizeof(btreeinfo));
715 btreeinfo.bt_compare = ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ? rdfstore_backend_bdb_compare_int : ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ? rdfstore_backend_bdb_compare_double : NULL ;
716 #else
717 BTREEINFO btreeinfo;
718 memset(&btreeinfo, 0, sizeof(btreeinfo));
719 btreeinfo.compare = ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ? rdfstore_backend_bdb_compare_int : ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ? rdfstore_backend_bdb_compare_double : NULL ;
720 #endif
721 #endif
722
723 #ifdef RDFSTORE_FLAT_STORE_DEBUG
724 me->num_store = 0;
725 me->num_fetch = 0;
726 me->num_inc = 0;
727 me->num_dec = 0;
728 me->num_sync = 0;
729 me->num_next = 0;
730 me->num_from = 0;
731 me->num_first = 0;
732 me->num_delete = 0;
733 me->num_exists = 0;
734 fprintf(stderr, "backend_bdb_clear num=%d in '%s'\n", ++(me->num_clear), me->filename);
735 #endif
736
737
738 /* close the database, remove the file, and repoen... ? */
739 /* close */
740 #ifdef DB_VERSION_MAJOR
741 me->cursor->c_close(me->cursor);
742 (me->bdb->close) (me->bdb, 0);
743 #else
744 (me->bdb->close) (me->bdb);
745 #endif
746
747 if (strcmp(me->filename, "\0") != 0) {
748 /* remove db file (not the directory!) */
749 if (unlink(me->filename)) {
750 perror("backend_bdb_clear");
751 fprintf(stderr, "Could not remove '%s' while clearing\n", me->filename);
752 return -1;
753 };
754 buff = me->filename;
755 umask(0);
756 } else {
757 buff = NULL;
758 };
759
760 /* re-open */
761
762 #ifdef BERKELEY_DB_1_OR_2 /* Berkeley DB Version 1 or 2 */
763
764 #ifdef DB_VERSION_MAJOR
765 if ((db_open(buff,
766 DB_BTREE, DB_CREATE,
767 0666, NULL, &btreeinfo, &me->bdb)) ||
768 #if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 6
769 ((me->bdb->cursor) (me->bdb, NULL, &me->cursor))
770 #else
771 ((me->bdb->cursor) (me->bdb, NULL, &me->cursor, 0))
772 #endif
773 ) {
774 #else
775
776 #if defined(DB_LIBRARY_COMPATIBILITY_API) && DB_VERSION_MAJOR > 2
777 if (!(me->bdb = (DB *) __db185_open(buff,
778 O_RDWR | O_CREAT,
779 0666, DB_BTREE, &btreeinfo))) {
780 #else
781 if (!(me->bdb = (DB *) dbopen(buff,
782 O_RDWR | O_CREAT,
783 0666, DB_BTREE, &btreeinfo))) {
784 #endif /* DB_LIBRARY_COMPATIBILITY_API */
785
786 #endif
787
788 #else /* Berkeley DB Version > 2 */
789 if (db_create(&me->bdb, NULL, 0)) {
790 backend_bdb_set_error(me, "Could not open/create database", FLAT_STORE_E_CANNOTOPEN);
791 perror("backend_bdb_open");
792 fprintf(stderr, "Could not open/create '%s':\n", buff);
793 return FLAT_STORE_E_CANNOTOPEN;
794 };
795
796 /* set the b-tree comparinson function to the one passed */
797 if( me->bt_compare_fcn_type != 0 ) {
798 me->bdb->set_bt_compare(me->bdb, ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_INT ) ?
799 rdfstore_backend_bdb_compare_int : ( me->bt_compare_fcn_type == FLAT_STORE_BT_COMP_DOUBLE ) ?
800 rdfstore_backend_bdb_compare_double : NULL );
801 };
802
803 me->bdb->set_errfile(me->bdb,stderr);
804 me->bdb->set_errpfx(me->bdb,"BerkelyDB");
805 #if DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR < 3
806 me->bdb->set_malloc(me->bdb, me->malloc);
807 #elif DB_VERSION_MAJOR > 3 || DB_VERSION_MINOR >= 3
808 /* This interface appeared in 3.3 */
809 me->bdb->set_alloc(me->bdb, me->malloc, NULL, NULL); /* could also pass
810 * me->free as 4th param
811 * but not sure how
812 * memoeyr is managed
813 * still */
814 #endif
815
816 #if DB_VERSION_MAJOR >= 4 && DB_VERSION_MINOR > 0 && DB_VERSION_PATCH >= 17
817 if ((me->bdb->open(me->bdb,
818 NULL,
819 buff,
820 NULL,
821 DB_BTREE, DB_CREATE,
822 0666)) ||
823 #else
824 if ((me->bdb->open(me->bdb,
825 buff,
826 NULL,
827 DB_BTREE, DB_CREATE,
828 0666)) ||
829 #endif
830 ((me->bdb->cursor) (me->bdb, NULL, &me->cursor, 0))) {
831 #endif /* Berkeley DB Version > 2 */
832
833 perror("backend_bdb_clear");
834 fprintf(stderr, "Could not open/create '%s' while clearing\n", buff);
835 return -1;
836 };
837 return 0;
838 };
839
840 rdfstore_flat_store_error_t
841 backend_bdb_from(
842 void *eme,
843 DBT closest_key,
844 DBT * key
845 )
846 {
847 backend_bdb_t *me = (backend_bdb_t *) eme;
848 int retval;
849 DBT val;
850
851 #ifdef RDFSTORE_FLAT_STORE_DEBUG
852 fprintf(stderr, "backend_bdb_from num=%d from '%s'\n", ++(me->num_from), me->filename);
853 #endif
854
855 memset(&val, 0, sizeof(val));
856
857 /* seek to closest_key and discard val */
858 memcpy(key, &closest_key, sizeof(closest_key));
859
860 #if DB_VERSION_MAJOR >= 2
861 retval = (me->cursor->c_get) (me->cursor, key, &val, DB_SET_RANGE);
862 #else
863 retval = (me->bdb->seq) (me->bdb, key, &val, R_CURSOR);
864 #endif
865
866 if (retval == 0) {
867 /*
868 * to ensure reentrancy we do a copy into caller space of what BDB layer returns
869 */
870 (*key) = backend_bdb_kvdup(me, *key);
871 };
872 return retval;
873 };
874
875 rdfstore_flat_store_error_t
876 backend_bdb_first(
877 void *eme,
878 DBT * first_key
879 )
880 {
881 backend_bdb_t *me = (backend_bdb_t *) eme;
882 int retval;
883 DBT val;
884
885 #ifdef RDFSTORE_FLAT_STORE_DEBUG
886 fprintf(stderr, "backend_bdb_first num=%d from '%s'\n", ++(me->num_first), me->filename);
887 #endif
888
889 memset(first_key, 0, sizeof(*first_key));
890 memset(&val, 0, sizeof(val));
891
892 #if DB_VERSION_MAJOR >= 2
893 retval = (me->cursor->c_get) (me->cursor, first_key, &val, DB_FIRST);
894 #else
895 retval = (me->bdb->seq) (me->bdb, first_key, &val, R_FIRST);
896 #endif
897
898 if (retval == 0) {
899 /*
900 * to ensure reentrancy we do a copy into caller space of what BDB layer returns
901 */
902 (*first_key) = backend_bdb_kvdup(me, *first_key);
903 };
904
905 return retval;
906 };
907
908 rdfstore_flat_store_error_t
909 backend_bdb_next(
910 void *eme,
911 DBT previous_key,
912 DBT * next_key
913 )
914 {
915 backend_bdb_t *me = (backend_bdb_t *) eme;
916 int retval;
917 DBT val;
918
919 #ifdef RDFSTORE_FLAT_STORE_DEBUG
920 fprintf(stderr, "backend_bdb_next num=%d from '%s'\n", ++(me->num_next), me->filename);
921 #endif
922
923 memset(next_key, 0, sizeof(*next_key));
924 memset(&val, 0, sizeof(val));
925
926 /* we really do not use/consider previous_key to carry out next_key - val is discarded */
927
928 #if DB_VERSION_MAJOR >= 2
929 retval = (me->cursor->c_get) (me->cursor, next_key, &val, DB_NEXT);
930 #else
931 retval = (me->bdb->seq) (me->bdb, next_key, &val, R_NEXT);
932 #endif
933
934 if (retval == 0) {
935 /*
936 * to ensure reentrancy we do a copy into caller space of what BDB layer returns
937 */
938 (*next_key) = backend_bdb_kvdup(me, *next_key);
939 };
940
941 return retval;
942 };
943
944 /* packed rdf_store_counter_t increment */
945 rdfstore_flat_store_error_t
946 backend_bdb_inc(
947 void *eme,
948 DBT key,
949 DBT * new_value
950 )
951 {
952 backend_bdb_t *me = (backend_bdb_t *) eme;
953 int retval;
954 rdf_store_counter_t l = 0;
955 unsigned char outbuf[256];
956
957 #ifdef RDFSTORE_FLAT_STORE_DEBUG
958 fprintf(stderr, "backend_bdb_inc num=%d in '%s'\n", ++(me->num_inc), me->filename);
959 #endif
960
961
962 /* it should be atomic with locking here... */
963 #if DB_VERSION_MAJOR >= 2
964 memset(new_value, 0, sizeof(*new_value));
965 (*new_value).flags = DB_DBT_MALLOC;
966 if ((((me->bdb)->get) (me->bdb, NULL, &key, new_value, 0)) != 0) {
967 return -1;
968 };
969 #else
970 if ((((me->bdb)->get) (me->bdb, &key, new_value, 0)) != 0) {
971 return -1;
972 };
973 #endif
974 unpackInt(new_value->data, &l);
975 l++;
976 #if DB_VERSION_MAJOR >= 2
977 if ((*new_value).data && (*new_value).size)
978 me->free((*new_value).data);
979 #endif
980 (*new_value).data = outbuf;
981 (*new_value).size = sizeof(rdf_store_counter_t);
982 packInt(l, new_value->data);
983
984 #ifdef DB_VERSION_MAJOR
985 retval = ((me->bdb)->put) (me->bdb, NULL, &key, new_value, 0);
986 #else
987 retval = ((me->bdb)->put) (me->bdb, &key, new_value, 0);
988 #endif
989
990 if (retval != 0) {
991 memset(new_value, 0, sizeof(*new_value));
992 (*new_value).data = NULL;
993 } else {
994 (*new_value) = backend_bdb_kvdup(me, *new_value);
995 };
996 return retval;
997 };
998
999 /* packed rdf_store_counter_t decrement */
1000 rdfstore_flat_store_error_t
1001 backend_bdb_dec(
1002 void *eme,
1003 DBT key,
1004 DBT * new_value
1005 )
1006 {
1007 backend_bdb_t *me = (backend_bdb_t *) eme;
1008 int retval;
1009 rdf_store_counter_t l = 0;
1010 unsigned char outbuf[256];
1011
1012 #ifdef RDFSTORE_FLAT_STORE_DEBUG
1013 fprintf(stderr, "backend_bdb_dec num=%d from '%s'\n", ++(me->num_dec), me->filename);
1014 #endif
1015
1016
1017 /* it should be atomic with locking here... */
1018 #if DB_VERSION_MAJOR >= 2
1019 memset(new_value, 0, sizeof(*new_value));
1020 (*new_value).flags = DB_DBT_MALLOC;
1021 if ((((me->bdb)->get) (me->bdb, NULL, &key, new_value, 0)) != 0) {
1022 return -1;
1023 };
1024 #else
1025 if ((((me->bdb)->get) (me->bdb, &key, new_value, 0)) != 0) {
1026 return -1;
1027 };
1028 #endif
1029 unpackInt(new_value->data, &l);
1030 assert(l > 0);
1031 l--;
1032 #if DB_VERSION_MAJOR >= 2
1033 if ((*new_value).data && (*new_value).size)
1034 me->free((*new_value).data);
1035 #endif
1036 (*new_value).data = outbuf;
1037 (*new_value).size = sizeof(rdf_store_counter_t);
1038 packInt(l, new_value->data);
1039
1040 #ifdef DB_VERSION_MAJOR
1041 retval = ((me->bdb)->put) (me->bdb, NULL, &key, new_value, 0);
1042 #else
1043 retval = ((me->bdb)->put) (me->bdb, &key, new_value, 0);
1044 #endif
1045
1046 if (retval != 0) {
1047 memset(new_value, 0, sizeof(*new_value));
1048 (*new_value).data = NULL;
1049 } else {
1050 (*new_value) = backend_bdb_kvdup(me, *new_value);
1051 };
1052 return retval;
1053 };
1054
1055 rdfstore_flat_store_error_t
1056 backend_bdb_sync(
1057 void *eme
1058 )
1059 {
1060 backend_bdb_t *me = (backend_bdb_t *) eme;
1061 int retval;
1062
1063 #ifdef RDFSTORE_FLAT_STORE_DEBUG
1064 fprintf(stderr, "backend_bdb_sync num=%d in '%s'\n", ++(me->num_sync), me->filename);
1065 #endif
1066
1067 retval = (me->bdb->sync) (me->bdb, 0);
1068 #ifdef DB_VERSION_MAJOR
1069 if (retval > 0)
1070 retval = -1;
1071 #endif
1072 return retval;
1073 }
1074
1075 int
1076 backend_bdb_isremote(
1077 void *eme
1078 )
1079 {
1080 return 0;
1081 }
1082
1083 /* misc subroutines */
1084
1085 /*
1086 * The following compare function are used for btree(s) for basic
1087 * XML-Schema data types xsd:integer, xsd:double (and will xsd:date)
1088 *
1089 * They return:
1090 * < 0 if a < b
1091 * = 0 if a = b
1092 * > 0 if a > b
1093 */
1094 #ifdef BERKELEY_DB_1_OR_2
1095 static int rdfstore_backend_bdb_compare_int(
1096 const DBT *a,
1097 const DBT *b ) {
1098 #else
1099 static int rdfstore_backend_bdb_compare_int(
1100 DB *file,
1101 const DBT *a,
1102 const DBT *b ) {
1103 #endif
1104 long ai, bi;
1105
1106 memcpy(&ai, a->data, sizeof(long));
1107 memcpy(&bi, b->data, sizeof(long));
1108
1109 return (ai - bi);
1110 };
1111
1112 #ifdef BERKELEY_DB_1_OR_2
1113 static int rdfstore_backend_bdb_compare_double(
1114 const DBT *a,
1115 const DBT *b ) {
1116 #else
1117 static int rdfstore_backend_bdb_compare_double(
1118 DB *file,
1119 const DBT *a,
1120 const DBT *b ) {
1121 #endif
1122 double ad,bd;
1123
1124 memcpy(&ad, a->data, sizeof(double));
1125 memcpy(&bd, b->data, sizeof(double));
1126
1127 if ( ad < bd ) {
1128 return -1;
1129 } else if ( ad > bd) {
1130 return 1;
1131 };
1132
1133 return 0;
1134 };
1135
1136 /*
1137 * returns null and/or full path to a hashed directory tree. the final
1138 * filename is hashed out within that three. Way to complex by now. Lifted
1139 * from another project which needed more.
1140 */
1141 static char *
1142 mkpath(char *base, char *infile)
1143 {
1144 char *file;
1145 int i, j;
1146 int last;
1147 struct stat s;
1148 char *slash, *dirname;
1149 char *inpath;
1150 static char tmp[MAXPATHLEN];
1151 char tmp2[MAXPATHLEN];
1152 #define MAXHASH 2
1153 static char hash[MAXHASH + 1];
1154
1155 tmp[0] = '\0';
1156
1157 strcpy(inpath = tmp2, infile);
1158
1159 memset(hash, '_', MAXHASH);
1160 hash[MAXHASH] = '\0';
1161
1162 if (base == NULL)
1163 base = "./";
1164
1165 if (inpath == NULL || inpath[0] == '\0') {
1166 fprintf(stderr, "No filename or path for the database specified\n");
1167 return NULL;
1168 };
1169
1170 /*
1171 * remove our standard docroot if present so we can work with
1172 * something relative. really a legacy thing from older perl DBMS.pm
1173 * versions. Can go now.
1174 */
1175 if (!(strncmp(base, inpath, strlen(base))))
1176 inpath += strlen(base);
1177
1178 /*
1179 * fetch the last leaf name
1180 */
1181 if ((file = strrchr(inpath, '/')) != NULL) {
1182 *file = '\0';
1183 file++;
1184 } else {
1185 file = inpath;
1186 inpath = "/";
1187 };
1188
1189 if (!strlen(file)) {
1190 fprintf(stderr, "No filename for the database specified\n");
1191 return NULL;
1192 };
1193
1194 strncpy(hash, file, MIN(strlen(file), MAXHASH));
1195
1196 /*
1197 strcpy(tmp,"./");
1198 */
1199 strcat(tmp, base);
1200 strcat(tmp, "/");
1201 strcat(tmp, inpath);
1202 strcat(tmp, "/");
1203 strcat(tmp, hash);
1204 strcat(tmp, "/");
1205 strcat(tmp, file);
1206
1207 if ((slash = strrchr(tmp, '.')) != NULL) {
1208 if ((!strcasecmp(slash + 1, "db")) ||
1209 (!strcasecmp(slash + 1, "dbm")) ||
1210 (!strcasecmp(slash + 1, "gdb"))
1211 )
1212 *slash = '\0';
1213 };
1214
1215 strcat(tmp, ".db");
1216
1217 for (i = 0, j = 0; tmp[i]; i++) {
1218 if (i && tmp[i] == '/' && tmp[i - 1] == '/')
1219 continue;
1220 if (i != j)
1221 tmp[j] = tmp[i];
1222 j++;
1223 };
1224 tmp[j] = '\0';
1225
1226 dirname = tmp;
1227
1228 /* Skip leading './'. */
1229 if (dirname[0] == '.')
1230 ++dirname;
1231 if (dirname[0] == '/')
1232 ++dirname;
1233
1234 for (last = 0; !last; ++dirname) {
1235 if (dirname[0] == '\0')
1236 break;
1237 else if (dirname[0] != '/')
1238 continue;
1239 *dirname = '\0';
1240 if (dirname[1] == '\0')
1241 last = 1;
1242
1243 /*
1244 * check if tmp exists and is a directory (or a link to one..
1245 * if not, create it, else give an error
1246 */
1247 if (stat(tmp, &s) == 0) {
1248 /*
1249 * something exists.. it must be a directory
1250 */
1251 if ((s.st_mode & S_IFDIR) == 0) {
1252 fprintf(stderr, "Creation of %s failed; path element not directory\n", tmp);
1253 return NULL;
1254 };
1255 } else if (errno == ENOENT) {
1256 if ((mkdir(tmp, (S_IRWXU | S_IRWXG | S_IRWXO))) != 0) {
1257 fprintf(stderr, "Creation of %s failed; %s\n", tmp, strerror(errno));
1258 return NULL;
1259 };
1260 } else {
1261 fprintf(stderr, "Path creation to failed at %s:%s\n", tmp, strerror(errno));
1262 return NULL;
1263 };
1264 if (!last)
1265 *dirname = '/';
1266 }
1267
1268 return tmp;
1269 }
1270
1271 #define BDB_VERSION (100)
1272 DECLARE_MODULE_BACKEND(backend_bdb, "BerkelyDB", BDB_VERSION)
1273