1 /* tools.c - tools for slap tools */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4 *
5 * Copyright 2011-2021 The OpenLDAP Foundation.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
10 * Public License.
11 *
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
15 */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21 #include <ac/errno.h>
22
23 #define AVL_INTERNAL
24 #include "back-mdb.h"
25 #include "idl.h"
26
27 #ifdef MDB_TOOL_IDL_CACHING
28 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
29
30 #define IDBLOCK 1024
31
32 typedef struct mdb_tool_idl_cache_entry {
33 struct mdb_tool_idl_cache_entry *next;
34 ID ids[IDBLOCK];
35 } mdb_tool_idl_cache_entry;
36
37 typedef struct mdb_tool_idl_cache {
38 struct berval kstr;
39 mdb_tool_idl_cache_entry *head, *tail;
40 ID first, last;
41 int count;
42 short offset;
43 short flags;
44 } mdb_tool_idl_cache;
45 #define WAS_FOUND 0x01
46 #define WAS_RANGE 0x02
47
48 #define MDB_TOOL_IDL_FLUSH(be, txn) mdb_tool_idl_flush(be, txn)
49 #else
50 #define MDB_TOOL_IDL_FLUSH(be, txn)
51 #endif /* MDB_TOOL_IDL_CACHING */
52
53 MDB_txn *mdb_tool_txn = NULL;
54
55 static MDB_txn *txi = NULL;
56 static MDB_cursor *cursor = NULL, *idcursor = NULL;
57 static MDB_cursor *mcp = NULL, *mcd = NULL;
58 static MDB_val key, data;
59 static ID previd = NOID;
60
61 typedef struct dn_id {
62 ID id;
63 struct berval dn;
64 } dn_id;
65
66 #define HOLE_SIZE 4096
67 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
68 static unsigned nhmax = HOLE_SIZE;
69 static unsigned nholes;
70
71 static struct berval *tool_base;
72 static int tool_scope;
73 static Filter *tool_filter;
74 static Entry *tool_next_entry;
75
76 static ID mdb_tool_ix_id;
77 static Operation *mdb_tool_ix_op;
78 static MDB_txn *mdb_tool_ix_txn;
79 static int mdb_tool_index_tcount, mdb_tool_threads;
80 static IndexRec *mdb_tool_index_rec;
81 static struct mdb_info *mdb_tool_info;
82 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
83 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
84 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
85 static void * mdb_tool_index_task( void *ctx, void *ptr );
86
87 static int mdb_writes, mdb_writes_per_commit;
88
89 /* Number of ops per commit in Quick mode.
90 * Batching speeds writes overall, but too large a
91 * batch will fail with MDB_TXN_FULL.
92 */
93 #ifndef MDB_WRITES_PER_COMMIT
94 #define MDB_WRITES_PER_COMMIT 500
95 #endif
96
97 static int
98 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
99
mdb_tool_entry_open(BackendDB * be,int mode)100 int mdb_tool_entry_open(
101 BackendDB *be, int mode )
102 {
103 /* In Quick mode, commit once per 500 entries */
104 mdb_writes = 0;
105 if ( slapMode & SLAP_TOOL_QUICK )
106 mdb_writes_per_commit = MDB_WRITES_PER_COMMIT;
107 else
108 mdb_writes_per_commit = 1;
109
110 /* Set up for threaded slapindex */
111 if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
112 if ( !mdb_tool_info ) {
113 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
114 ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
115 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
116 ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
117 if ( mdb->mi_nattrs ) {
118 int i;
119 #if 0 /* threaded indexing has no performance advantage */
120 mdb_tool_threads = slap_tool_thread_max - 1;
121 #endif
122 if ( mdb_tool_threads > 1 ) {
123 mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
124 mdb_tool_index_tcount = mdb_tool_threads - 1;
125 for (i=1; i<mdb_tool_threads; i++) {
126 int *ptr = ch_malloc( sizeof( int ));
127 *ptr = i;
128 ldap_pvt_thread_pool_submit( &connection_pool,
129 mdb_tool_index_task, ptr );
130 }
131 mdb_tool_info = mdb;
132 }
133 }
134 }
135 }
136
137 return 0;
138 }
139
mdb_tool_entry_close(BackendDB * be)140 int mdb_tool_entry_close(
141 BackendDB *be )
142 {
143 if ( mdb_tool_info ) {
144 slapd_shutdown = 1;
145 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
146
147 /* There might still be some threads starting */
148 while ( mdb_tool_index_tcount > 0 ) {
149 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
150 &mdb_tool_index_mutex );
151 }
152
153 mdb_tool_index_tcount = mdb_tool_threads - 1;
154 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
155
156 /* Make sure all threads are stopped */
157 while ( mdb_tool_index_tcount > 0 ) {
158 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
159 &mdb_tool_index_mutex );
160 }
161 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
162
163 mdb_tool_info = NULL;
164 slapd_shutdown = 0;
165 ch_free( mdb_tool_index_rec );
166 mdb_tool_index_tcount = mdb_tool_threads - 1;
167 }
168
169 if( idcursor ) {
170 mdb_cursor_close( idcursor );
171 idcursor = NULL;
172 }
173 if( cursor ) {
174 mdb_cursor_close( cursor );
175 cursor = NULL;
176 }
177 {
178 struct mdb_info *mdb = be->be_private;
179 if ( mdb ) {
180 int i;
181 for (i=0; i<mdb->mi_nattrs; i++)
182 mdb->mi_attrs[i]->ai_cursor = NULL;
183 }
184 }
185 if( mdb_tool_txn ) {
186 int rc;
187 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
188 if (( rc = mdb_txn_commit( mdb_tool_txn ))) {
189 Debug( LDAP_DEBUG_ANY,
190 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
191 "txn_commit failed: %s (%d)\n",
192 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
193 return -1;
194 }
195 mdb_tool_txn = NULL;
196 }
197 if( txi ) {
198 int rc;
199 if (( rc = mdb_txn_commit( txi ))) {
200 Debug( LDAP_DEBUG_ANY,
201 LDAP_XSTRING(mdb_tool_entry_close) ": database %s: "
202 "txn_commit failed: %s (%d)\n",
203 be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
204 return -1;
205 }
206 txi = NULL;
207 }
208
209 if( nholes ) {
210 unsigned i;
211 fprintf( stderr, "Error, entries missing!\n");
212 for (i=0; i<nholes; i++) {
213 fprintf(stderr, " entry %ld: %s\n",
214 holes[i].id, holes[i].dn.bv_val);
215 }
216 nholes = 0;
217 return -1;
218 }
219
220 return 0;
221 }
222
223 ID
mdb_tool_entry_first_x(BackendDB * be,struct berval * base,int scope,Filter * f)224 mdb_tool_entry_first_x(
225 BackendDB *be,
226 struct berval *base,
227 int scope,
228 Filter *f )
229 {
230 tool_base = base;
231 tool_scope = scope;
232 tool_filter = f;
233
234 return mdb_tool_entry_next( be );
235 }
236
mdb_tool_entry_next(BackendDB * be)237 ID mdb_tool_entry_next(
238 BackendDB *be )
239 {
240 int rc;
241 ID id;
242 struct mdb_info *mdb;
243
244 assert( be != NULL );
245 assert( slapMode & SLAP_TOOL_MODE );
246
247 mdb = (struct mdb_info *) be->be_private;
248 assert( mdb != NULL );
249
250 if ( !mdb_tool_txn ) {
251 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
252 if ( rc )
253 return NOID;
254 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
255 if ( rc ) {
256 mdb_txn_abort( mdb_tool_txn );
257 return NOID;
258 }
259 }
260
261 next:;
262 rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
263
264 if( rc ) {
265 return NOID;
266 }
267
268 previd = *(ID *)key.mv_data;
269 id = previd;
270
271 if ( !data.mv_size )
272 goto next;
273
274 if ( tool_filter || tool_base ) {
275 static Operation op = {0};
276 static Opheader ohdr = {0};
277
278 op.o_hdr = &ohdr;
279 op.o_bd = be;
280 op.o_tmpmemctx = NULL;
281 op.o_tmpmfuncs = &ch_mfuncs;
282
283 if ( tool_next_entry ) {
284 mdb_entry_release( &op, tool_next_entry, 0 );
285 tool_next_entry = NULL;
286 }
287
288 rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
289 if ( rc == LDAP_NO_SUCH_OBJECT ) {
290 goto next;
291 }
292
293 assert( tool_next_entry != NULL );
294
295 if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
296 {
297 mdb_entry_release( &op, tool_next_entry, 0 );
298 tool_next_entry = NULL;
299 goto next;
300 }
301 }
302
303 return id;
304 }
305
mdb_tool_dn2id_get(Backend * be,struct berval * dn)306 ID mdb_tool_dn2id_get(
307 Backend *be,
308 struct berval *dn
309 )
310 {
311 struct mdb_info *mdb;
312 Operation op = {0};
313 Opheader ohdr = {0};
314 ID id;
315 int rc;
316
317 if ( BER_BVISEMPTY(dn) )
318 return 0;
319
320 mdb = (struct mdb_info *) be->be_private;
321
322 if ( !mdb_tool_txn ) {
323 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
324 MDB_RDONLY : 0, &mdb_tool_txn );
325 if ( rc )
326 return NOID;
327 }
328
329 op.o_hdr = &ohdr;
330 op.o_bd = be;
331 op.o_tmpmemctx = NULL;
332 op.o_tmpmfuncs = &ch_mfuncs;
333
334 rc = mdb_dn2id( &op, mdb_tool_txn, NULL, dn, &id, NULL, NULL, NULL );
335 if ( rc == MDB_NOTFOUND )
336 return NOID;
337
338 return id;
339 }
340
341 static int
mdb_tool_entry_get_int(BackendDB * be,ID id,Entry ** ep)342 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
343 {
344 Operation op = {0};
345 Opheader ohdr = {0};
346
347 Entry *e = NULL;
348 struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
349 int rc;
350
351 assert( be != NULL );
352 assert( slapMode & SLAP_TOOL_MODE );
353
354 if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
355 *ep = tool_next_entry;
356 tool_next_entry = NULL;
357 return LDAP_SUCCESS;
358 }
359
360 if ( id != previd ) {
361 key.mv_size = sizeof(ID);
362 key.mv_data = &id;
363 rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
364 if ( rc ) {
365 rc = LDAP_OTHER;
366 goto done;
367 }
368 }
369 if ( !data.mv_size ) {
370 rc = LDAP_NO_SUCH_OBJECT;
371 goto done;
372 }
373
374 op.o_hdr = &ohdr;
375 op.o_bd = be;
376 op.o_tmpmemctx = NULL;
377 op.o_tmpmfuncs = &ch_mfuncs;
378 if ( slapMode & SLAP_TOOL_READONLY ) {
379 rc = mdb_id2name( &op, mdb_tool_txn, &idcursor, id, &dn, &ndn );
380 if ( rc ) {
381 rc = LDAP_OTHER;
382 goto done;
383 }
384 if ( tool_base != NULL ) {
385 if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
386 ch_free( dn.bv_val );
387 ch_free( ndn.bv_val );
388 rc = LDAP_NO_SUCH_OBJECT;
389 goto done;
390 }
391 }
392 }
393 rc = mdb_entry_decode( &op, mdb_tool_txn, &data, &e );
394 e->e_id = id;
395 if ( !BER_BVISNULL( &dn )) {
396 e->e_name = dn;
397 e->e_nname = ndn;
398 } else {
399 e->e_name.bv_val = NULL;
400 e->e_nname.bv_val = NULL;
401 }
402
403 done:
404 if ( e != NULL ) {
405 *ep = e;
406 }
407
408 return rc;
409 }
410
411 Entry*
mdb_tool_entry_get(BackendDB * be,ID id)412 mdb_tool_entry_get( BackendDB *be, ID id )
413 {
414 Entry *e = NULL;
415 int rc;
416
417 if ( !mdb_tool_txn ) {
418 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
419 rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
420 (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &mdb_tool_txn );
421 if ( rc )
422 return NULL;
423 }
424 if ( !cursor ) {
425 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
426 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &cursor );
427 if ( rc ) {
428 mdb_txn_abort( mdb_tool_txn );
429 mdb_tool_txn = NULL;
430 return NULL;
431 }
432 }
433 (void)mdb_tool_entry_get_int( be, id, &e );
434 return e;
435 }
436
mdb_tool_next_id(Operation * op,MDB_txn * tid,Entry * e,struct berval * text,int hole)437 static int mdb_tool_next_id(
438 Operation *op,
439 MDB_txn *tid,
440 Entry *e,
441 struct berval *text,
442 int hole )
443 {
444 struct berval dn = e->e_name;
445 struct berval ndn = e->e_nname;
446 struct berval pdn, npdn, nmatched;
447 ID id, pid = 0;
448 int rc;
449
450 if (ndn.bv_len == 0) {
451 e->e_id = 0;
452 return 0;
453 }
454
455 rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, NULL, &nmatched );
456 if ( rc == MDB_NOTFOUND ) {
457 if ( !be_issuffix( op->o_bd, &ndn ) ) {
458 ID eid = e->e_id;
459 dnParent( &ndn, &npdn );
460 if ( nmatched.bv_len != npdn.bv_len ) {
461 dnParent( &dn, &pdn );
462 e->e_name = pdn;
463 e->e_nname = npdn;
464 rc = mdb_tool_next_id( op, tid, e, text, 1 );
465 e->e_name = dn;
466 e->e_nname = ndn;
467 if ( rc ) {
468 return rc;
469 }
470 /* If parent didn't exist, it was created just now
471 * and its ID is now in e->e_id. Make sure the current
472 * entry gets added under the new parent ID.
473 */
474 if ( eid != e->e_id ) {
475 pid = e->e_id;
476 }
477 } else {
478 pid = id;
479 }
480 }
481 rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
482 if ( rc ) {
483 snprintf( text->bv_val, text->bv_len,
484 "next_id failed: %s (%d)",
485 mdb_strerror(rc), rc );
486 Debug( LDAP_DEBUG_ANY,
487 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
488 return rc;
489 }
490 rc = mdb_dn2id_add( op, mcp, mcd, pid, 1, 1, e );
491 if ( rc ) {
492 snprintf( text->bv_val, text->bv_len,
493 "dn2id_add failed: %s (%d)",
494 mdb_strerror(rc), rc );
495 Debug( LDAP_DEBUG_ANY,
496 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
497 } else if ( hole ) {
498 MDB_val key, data;
499 if ( nholes == nhmax - 1 ) {
500 if ( holes == hbuf ) {
501 holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
502 AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
503 } else {
504 holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
505 }
506 nhmax *= 2;
507 }
508 ber_dupbv( &holes[nholes].dn, &ndn );
509 holes[nholes++].id = e->e_id;
510 key.mv_size = sizeof(ID);
511 key.mv_data = &e->e_id;
512 data.mv_size = 0;
513 data.mv_data = NULL;
514 rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
515 if ( rc == MDB_KEYEXIST )
516 rc = 0;
517 if ( rc ) {
518 snprintf( text->bv_val, text->bv_len,
519 "dummy id2entry add failed: %s (%d)",
520 mdb_strerror(rc), rc );
521 Debug( LDAP_DEBUG_ANY,
522 "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
523 }
524 }
525 } else if ( !hole ) {
526 unsigned i, j;
527
528 e->e_id = id;
529
530 for ( i=0; i<nholes; i++) {
531 if ( holes[i].id == e->e_id ) {
532 free(holes[i].dn.bv_val);
533 for (j=i;j<nholes;j++) holes[j] = holes[j+1];
534 holes[j].id = 0;
535 nholes--;
536 break;
537 } else if ( holes[i].id > e->e_id ) {
538 break;
539 }
540 }
541 }
542 return rc;
543 }
544
545 static int
mdb_tool_index_add(Operation * op,MDB_txn * txn,Entry * e)546 mdb_tool_index_add(
547 Operation *op,
548 MDB_txn *txn,
549 Entry *e )
550 {
551 struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
552
553 if ( !mdb->mi_nattrs )
554 return 0;
555
556 if ( mdb_tool_threads > 1 ) {
557 IndexRec *ir;
558 int i, rc;
559 Attribute *a;
560
561 ir = mdb_tool_index_rec;
562 for (i=0; i<mdb->mi_nattrs; i++)
563 ir[i].ir_attrs = NULL;
564
565 for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
566 rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
567 &a->a_desc->ad_tags, ir );
568 if ( rc )
569 return rc;
570 }
571 for (i=0; i<mdb->mi_nattrs; i++) {
572 if ( !ir[i].ir_ai )
573 break;
574 rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
575 &ir[i].ir_ai->ai_cursor );
576 if ( rc )
577 return rc;
578 }
579 mdb_tool_ix_id = e->e_id;
580 mdb_tool_ix_op = op;
581 mdb_tool_ix_txn = txn;
582 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
583 /* Wait for all threads to be ready */
584 while ( mdb_tool_index_tcount ) {
585 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
586 &mdb_tool_index_mutex );
587 }
588
589 for ( i=1; i<mdb_tool_threads; i++ )
590 mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
591 mdb_tool_index_tcount = mdb_tool_threads - 1;
592 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
593 ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
594
595 rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
596 if ( rc )
597 return rc;
598 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
599 for ( i=1; i<mdb_tool_threads; i++ ) {
600 if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
601 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
602 &mdb_tool_index_mutex );
603 i--;
604 continue;
605 }
606 if ( mdb_tool_index_rec[i].ir_i ) {
607 rc = mdb_tool_index_rec[i].ir_i;
608 break;
609 }
610 }
611 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
612 return rc;
613 } else
614 {
615 return mdb_index_entry_add( op, txn, e );
616 }
617 }
618
mdb_tool_entry_put(BackendDB * be,Entry * e,struct berval * text)619 ID mdb_tool_entry_put(
620 BackendDB *be,
621 Entry *e,
622 struct berval *text )
623 {
624 int rc;
625 struct mdb_info *mdb;
626 Operation op = {0};
627 Opheader ohdr = {0};
628
629 assert( be != NULL );
630 assert( slapMode & SLAP_TOOL_MODE );
631
632 assert( text != NULL );
633 assert( text->bv_val != NULL );
634 assert( text->bv_val[0] == '\0' ); /* overconservative? */
635
636 Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
637 "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
638
639 mdb = (struct mdb_info *) be->be_private;
640
641 if ( !mdb_tool_txn ) {
642 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
643 if( rc != 0 ) {
644 snprintf( text->bv_val, text->bv_len,
645 "txn_begin failed: %s (%d)",
646 mdb_strerror(rc), rc );
647 Debug( LDAP_DEBUG_ANY,
648 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
649 text->bv_val, 0, 0 );
650 return NOID;
651 }
652 }
653 if ( !idcursor ) {
654 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_id2entry, &idcursor );
655 if( rc != 0 ) {
656 snprintf( text->bv_val, text->bv_len,
657 "cursor_open failed: %s (%d)",
658 mdb_strerror(rc), rc );
659 Debug( LDAP_DEBUG_ANY,
660 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
661 text->bv_val, 0, 0 );
662 return NOID;
663 }
664 if ( !mdb->mi_nextid ) {
665 ID dummy;
666 mdb_next_id( be, idcursor, &dummy );
667 }
668 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcp );
669 if( rc != 0 ) {
670 snprintf( text->bv_val, text->bv_len,
671 "cursor_open failed: %s (%d)",
672 mdb_strerror(rc), rc );
673 Debug( LDAP_DEBUG_ANY,
674 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
675 text->bv_val, 0, 0 );
676 return NOID;
677 }
678 rc = mdb_cursor_open( mdb_tool_txn, mdb->mi_dn2id, &mcd );
679 if( rc != 0 ) {
680 snprintf( text->bv_val, text->bv_len,
681 "cursor_open failed: %s (%d)",
682 mdb_strerror(rc), rc );
683 Debug( LDAP_DEBUG_ANY,
684 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
685 text->bv_val, 0, 0 );
686 return NOID;
687 }
688 }
689
690 op.o_hdr = &ohdr;
691 op.o_bd = be;
692 op.o_tmpmemctx = NULL;
693 op.o_tmpmfuncs = &ch_mfuncs;
694
695 /* add dn2id indices */
696 rc = mdb_tool_next_id( &op, mdb_tool_txn, e, text, 0 );
697 if( rc != 0 ) {
698 goto done;
699 }
700
701 rc = mdb_tool_index_add( &op, mdb_tool_txn, e );
702 if( rc != 0 ) {
703 snprintf( text->bv_val, text->bv_len,
704 "index_entry_add failed: err=%d", rc );
705 Debug( LDAP_DEBUG_ANY,
706 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
707 text->bv_val, 0, 0 );
708 goto done;
709 }
710
711
712 /* id2entry index */
713 rc = mdb_id2entry_add( &op, mdb_tool_txn, idcursor, e );
714 if( rc != 0 ) {
715 snprintf( text->bv_val, text->bv_len,
716 "id2entry_add failed: err=%d", rc );
717 Debug( LDAP_DEBUG_ANY,
718 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
719 text->bv_val, 0, 0 );
720 goto done;
721 }
722
723 done:
724 if( rc == 0 ) {
725 mdb_writes++;
726 if ( mdb_writes >= mdb_writes_per_commit ) {
727 unsigned i;
728 MDB_TOOL_IDL_FLUSH( be, mdb_tool_txn );
729 rc = mdb_txn_commit( mdb_tool_txn );
730 for ( i=0; i<mdb->mi_nattrs; i++ )
731 mdb->mi_attrs[i]->ai_cursor = NULL;
732 mdb_writes = 0;
733 mdb_tool_txn = NULL;
734 idcursor = NULL;
735 if( rc != 0 ) {
736 mdb->mi_numads = 0;
737 snprintf( text->bv_val, text->bv_len,
738 "txn_commit failed: %s (%d)",
739 mdb_strerror(rc), rc );
740 Debug( LDAP_DEBUG_ANY,
741 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
742 text->bv_val, 0, 0 );
743 e->e_id = NOID;
744 }
745 }
746
747 } else {
748 unsigned i;
749 mdb_txn_abort( mdb_tool_txn );
750 mdb_tool_txn = NULL;
751 idcursor = NULL;
752 for ( i=0; i<mdb->mi_nattrs; i++ )
753 mdb->mi_attrs[i]->ai_cursor = NULL;
754 mdb_writes = 0;
755 snprintf( text->bv_val, text->bv_len,
756 "txn_aborted! %s (%d)",
757 rc == LDAP_OTHER ? "Internal error" :
758 mdb_strerror(rc), rc );
759 Debug( LDAP_DEBUG_ANY,
760 "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
761 text->bv_val, 0, 0 );
762 e->e_id = NOID;
763 }
764
765 return e->e_id;
766 }
767
768 static int mdb_dn2id_upgrade( BackendDB *be );
769
mdb_tool_entry_reindex(BackendDB * be,ID id,AttributeDescription ** adv)770 int mdb_tool_entry_reindex(
771 BackendDB *be,
772 ID id,
773 AttributeDescription **adv )
774 {
775 struct mdb_info *mi = (struct mdb_info *) be->be_private;
776 int rc;
777 Entry *e;
778 Operation op = {0};
779 Opheader ohdr = {0};
780
781 Debug( LDAP_DEBUG_ARGS,
782 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
783 (long) id, 0, 0 );
784 assert( tool_base == NULL );
785 assert( tool_filter == NULL );
786
787 /* Special: do a dn2id upgrade */
788 if ( adv && adv[0] == slap_schema.si_ad_entryDN ) {
789 /* short-circuit tool_entry_next() */
790 mdb_cursor_get( cursor, &key, &data, MDB_LAST );
791 return mdb_dn2id_upgrade( be );
792 }
793
794 /* No indexes configured, nothing to do. Could return an
795 * error here to shortcut things.
796 */
797 if (!mi->mi_attrs) {
798 return 0;
799 }
800
801 /* Check for explicit list of attrs to index */
802 if ( adv ) {
803 int i, j, n;
804
805 if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
806 /* count */
807 for ( n = 0; adv[n]; n++ ) ;
808
809 /* insertion sort */
810 for ( i = 0; i < n; i++ ) {
811 AttributeDescription *ad = adv[i];
812 for ( j = i-1; j>=0; j--) {
813 if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
814 adv[j+1] = adv[j];
815 }
816 adv[j+1] = ad;
817 }
818 }
819
820 for ( i = 0; adv[i]; i++ ) {
821 if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
822 for ( j = i+1; j < mi->mi_nattrs; j++ ) {
823 if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
824 AttrInfo *ai = mi->mi_attrs[i];
825 mi->mi_attrs[i] = mi->mi_attrs[j];
826 mi->mi_attrs[j] = ai;
827 break;
828 }
829 }
830 if ( j == mi->mi_nattrs ) {
831 Debug( LDAP_DEBUG_ANY,
832 LDAP_XSTRING(mdb_tool_entry_reindex)
833 ": no index configured for %s\n",
834 adv[i]->ad_cname.bv_val, 0, 0 );
835 return -1;
836 }
837 }
838 }
839 mi->mi_nattrs = i;
840 }
841
842 e = mdb_tool_entry_get( be, id );
843
844 if( e == NULL ) {
845 Debug( LDAP_DEBUG_ANY,
846 LDAP_XSTRING(mdb_tool_entry_reindex)
847 ": could not locate id=%ld\n",
848 (long) id, 0, 0 );
849 return -1;
850 }
851
852 if ( !txi ) {
853 rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
854 if( rc != 0 ) {
855 Debug( LDAP_DEBUG_ANY,
856 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
857 "txn_begin failed: %s (%d)\n",
858 mdb_strerror(rc), rc, 0 );
859 goto done;
860 }
861 }
862
863 if ( slapMode & SLAP_TRUNCATE_MODE ) {
864 int i;
865 for ( i=0; i < mi->mi_nattrs; i++ ) {
866 rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
867 if ( rc ) {
868 Debug( LDAP_DEBUG_ANY,
869 LDAP_XSTRING(mdb_tool_entry_reindex)
870 ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
871 mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
872 mdb_strerror(rc), rc );
873 return -1;
874 }
875 }
876 slapMode ^= SLAP_TRUNCATE_MODE;
877 }
878
879 /*
880 * just (re)add them for now
881 * Use truncate mode to empty/reset index databases
882 */
883
884 Debug( LDAP_DEBUG_TRACE,
885 "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
886 (long) id, 0, 0 );
887
888 op.o_hdr = &ohdr;
889 op.o_bd = be;
890 op.o_tmpmemctx = NULL;
891 op.o_tmpmfuncs = &ch_mfuncs;
892
893 rc = mdb_tool_index_add( &op, txi, e );
894
895 done:
896 if( rc == 0 ) {
897 mdb_writes++;
898 if ( mdb_writes >= mdb_writes_per_commit ) {
899 MDB_val key;
900 unsigned i;
901 MDB_TOOL_IDL_FLUSH( be, txi );
902 rc = mdb_txn_commit( txi );
903 mdb_writes = 0;
904 for ( i=0; i<mi->mi_nattrs; i++ )
905 mi->mi_attrs[i]->ai_cursor = NULL;
906 if( rc != 0 ) {
907 Debug( LDAP_DEBUG_ANY,
908 "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
909 ": txn_commit failed: %s (%d)\n",
910 mdb_strerror(rc), rc, 0 );
911 e->e_id = NOID;
912 }
913 mdb_cursor_close( cursor );
914 txi = NULL;
915 /* Must close the read txn to allow old pages to be reclaimed. */
916 mdb_txn_abort( mdb_tool_txn );
917 /* and then reopen it so that tool_entry_next still works. */
918 mdb_txn_begin( mi->mi_dbenv, NULL, MDB_RDONLY, &mdb_tool_txn );
919 mdb_cursor_open( mdb_tool_txn, mi->mi_id2entry, &cursor );
920 key.mv_data = &id;
921 key.mv_size = sizeof(ID);
922 mdb_cursor_get( cursor, &key, NULL, MDB_SET );
923 }
924
925 } else {
926 unsigned i;
927 mdb_writes = 0;
928 mdb_cursor_close( cursor );
929 cursor = NULL;
930 mdb_txn_abort( txi );
931 for ( i=0; i<mi->mi_nattrs; i++ )
932 mi->mi_attrs[i]->ai_cursor = NULL;
933 Debug( LDAP_DEBUG_ANY,
934 "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
935 ": txn_aborted! err=%d\n",
936 rc, 0, 0 );
937 e->e_id = NOID;
938 txi = NULL;
939 }
940 mdb_entry_release( &op, e, 0 );
941
942 return rc;
943 }
944
mdb_tool_entry_modify(BackendDB * be,Entry * e,struct berval * text)945 ID mdb_tool_entry_modify(
946 BackendDB *be,
947 Entry *e,
948 struct berval *text )
949 {
950 int rc;
951 struct mdb_info *mdb;
952 Operation op = {0};
953 Opheader ohdr = {0};
954
955 assert( be != NULL );
956 assert( slapMode & SLAP_TOOL_MODE );
957
958 assert( text != NULL );
959 assert( text->bv_val != NULL );
960 assert( text->bv_val[0] == '\0' ); /* overconservative? */
961
962 assert ( e->e_id != NOID );
963
964 Debug( LDAP_DEBUG_TRACE,
965 "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
966 (long) e->e_id, e->e_dn, 0 );
967
968 mdb = (struct mdb_info *) be->be_private;
969
970 if( cursor ) {
971 mdb_cursor_close( cursor );
972 cursor = NULL;
973 }
974 if ( !mdb_tool_txn ) {
975 rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &mdb_tool_txn );
976 if( rc != 0 ) {
977 snprintf( text->bv_val, text->bv_len,
978 "txn_begin failed: %s (%d)",
979 mdb_strerror(rc), rc );
980 Debug( LDAP_DEBUG_ANY,
981 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
982 text->bv_val, 0, 0 );
983 return NOID;
984 }
985 }
986
987 op.o_hdr = &ohdr;
988 op.o_bd = be;
989 op.o_tmpmemctx = NULL;
990 op.o_tmpmfuncs = &ch_mfuncs;
991
992 /* id2entry index */
993 rc = mdb_id2entry_update( &op, mdb_tool_txn, NULL, e );
994 if( rc != 0 ) {
995 snprintf( text->bv_val, text->bv_len,
996 "id2entry_update failed: err=%d", rc );
997 Debug( LDAP_DEBUG_ANY,
998 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
999 text->bv_val, 0, 0 );
1000 goto done;
1001 }
1002
1003 done:
1004 if( rc == 0 ) {
1005 rc = mdb_txn_commit( mdb_tool_txn );
1006 if( rc != 0 ) {
1007 mdb->mi_numads = 0;
1008 snprintf( text->bv_val, text->bv_len,
1009 "txn_commit failed: %s (%d)",
1010 mdb_strerror(rc), rc );
1011 Debug( LDAP_DEBUG_ANY,
1012 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
1013 "%s\n", text->bv_val, 0, 0 );
1014 e->e_id = NOID;
1015 }
1016
1017 } else {
1018 mdb_txn_abort( mdb_tool_txn );
1019 snprintf( text->bv_val, text->bv_len,
1020 "txn_aborted! %s (%d)",
1021 mdb_strerror(rc), rc );
1022 Debug( LDAP_DEBUG_ANY,
1023 "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
1024 text->bv_val, 0, 0 );
1025 e->e_id = NOID;
1026 }
1027 mdb_tool_txn = NULL;
1028 idcursor = NULL;
1029
1030 return e->e_id;
1031 }
1032
1033 static void *
mdb_tool_index_task(void * ctx,void * ptr)1034 mdb_tool_index_task( void *ctx, void *ptr )
1035 {
1036 int base = *(int *)ptr;
1037
1038 free( ptr );
1039 while ( 1 ) {
1040 ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
1041 mdb_tool_index_tcount--;
1042 if ( !mdb_tool_index_tcount )
1043 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1044 ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
1045 &mdb_tool_index_mutex );
1046 if ( slapd_shutdown ) {
1047 mdb_tool_index_tcount--;
1048 if ( !mdb_tool_index_tcount )
1049 ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
1050 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1051 break;
1052 }
1053 ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
1054 mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op,
1055 mdb_tool_ix_txn,
1056 mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
1057 }
1058
1059 return NULL;
1060 }
1061
1062 #ifdef MDB_TOOL_IDL_CACHING
1063 static int
mdb_tool_idl_cmp(const void * v1,const void * v2)1064 mdb_tool_idl_cmp( const void *v1, const void *v2 )
1065 {
1066 const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
1067 int rc;
1068
1069 if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
1070 return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
1071 }
1072
1073 static int
mdb_tool_idl_flush_one(MDB_cursor * mc,AttrInfo * ai,mdb_tool_idl_cache * ic)1074 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic )
1075 {
1076 mdb_tool_idl_cache_entry *ice;
1077 MDB_val key, data[2];
1078 int i, rc;
1079 ID id, nid;
1080
1081 /* Freshly allocated, ignore it */
1082 if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
1083 return 0;
1084 }
1085
1086 key.mv_data = ic->kstr.bv_val;
1087 key.mv_size = ic->kstr.bv_len;
1088
1089 if ( ic->count > MDB_IDL_DB_SIZE ) {
1090 while ( ic->flags & WAS_FOUND ) {
1091 rc = mdb_cursor_get( mc, &key, data, MDB_SET );
1092 if ( rc ) {
1093 /* FIXME: find out why this happens */
1094 ic->flags = 0;
1095 break;
1096 }
1097 if ( ic->flags & WAS_RANGE ) {
1098 /* Skip lo */
1099 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1100
1101 /* Get hi */
1102 rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
1103
1104 /* Store range hi */
1105 data[0].mv_data = &ic->last;
1106 rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
1107 } else {
1108 /* Delete old data, replace with range */
1109 ic->first = *(ID *)data[0].mv_data;
1110 mdb_cursor_del( mc, MDB_NODUPDATA );
1111 }
1112 break;
1113 }
1114 if ( !(ic->flags & WAS_RANGE)) {
1115 /* range, didn't exist before */
1116 nid = 0;
1117 data[0].mv_size = sizeof(ID);
1118 data[0].mv_data = &nid;
1119 rc = mdb_cursor_put( mc, &key, data, 0 );
1120 if ( rc == 0 ) {
1121 data[0].mv_data = &ic->first;
1122 rc = mdb_cursor_put( mc, &key, data, 0 );
1123 if ( rc == 0 ) {
1124 data[0].mv_data = &ic->last;
1125 rc = mdb_cursor_put( mc, &key, data, 0 );
1126 }
1127 }
1128 if ( rc ) {
1129 rc = -1;
1130 }
1131 }
1132 } else {
1133 /* Normal write */
1134 int n;
1135
1136 data[0].mv_size = sizeof(ID);
1137 rc = 0;
1138 i = ic->offset;
1139 for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
1140 int end;
1141 if ( ice->next ) {
1142 end = IDBLOCK;
1143 } else {
1144 end = ic->count & (IDBLOCK-1);
1145 if ( !end )
1146 end = IDBLOCK;
1147 }
1148 data[1].mv_size = end - i;
1149 data[0].mv_data = &ice->ids[i];
1150 i = 0;
1151 rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
1152 if ( rc ) {
1153 if ( rc == MDB_KEYEXIST ) {
1154 rc = 0;
1155 continue;
1156 }
1157 rc = -1;
1158 break;
1159 }
1160 }
1161 if ( ic->head ) {
1162 ic->tail->next = ai->ai_flist;
1163 ai->ai_flist = ic->head;
1164 }
1165 }
1166 ic->head = ai->ai_clist;
1167 ai->ai_clist = ic;
1168 return rc;
1169 }
1170
1171 static int
mdb_tool_idl_flush_db(MDB_txn * txn,AttrInfo * ai)1172 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai )
1173 {
1174 MDB_cursor *mc;
1175 Avlnode *root;
1176 int rc;
1177
1178 mdb_cursor_open( txn, ai->ai_dbi, &mc );
1179 root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
1180 do {
1181 rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data );
1182 if ( rc != -1 )
1183 rc = 0;
1184 } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
1185 mdb_cursor_close( mc );
1186
1187 return rc;
1188 }
1189
1190 static int
mdb_tool_idl_flush(BackendDB * be,MDB_txn * txn)1191 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
1192 {
1193 struct mdb_info *mdb = (struct mdb_info *) be->be_private;
1194 int rc = 0;
1195 unsigned int i, dbi;
1196
1197 for ( i=0; i < mdb->mi_nattrs; i++ ) {
1198 if ( !mdb->mi_attrs[i]->ai_root ) continue;
1199 rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] );
1200 tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
1201 mdb->mi_attrs[i]->ai_root = NULL;
1202 if ( rc )
1203 break;
1204 }
1205 return rc;
1206 }
1207
mdb_tool_idl_add(BackendDB * be,MDB_cursor * mc,struct berval * keys,ID id)1208 int mdb_tool_idl_add(
1209 BackendDB *be,
1210 MDB_cursor *mc,
1211 struct berval *keys,
1212 ID id )
1213 {
1214 MDB_dbi dbi;
1215 mdb_tool_idl_cache *ic, itmp;
1216 mdb_tool_idl_cache_entry *ice;
1217 int i, rc, lcount;
1218 AttrInfo *ai = (AttrInfo *)mc;
1219 mc = ai->ai_cursor;
1220
1221 dbi = ai->ai_dbi;
1222 for (i=0; keys[i].bv_val; i++) {
1223 itmp.kstr = keys[i];
1224 ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
1225
1226 /* No entry yet, create one */
1227 if ( !ic ) {
1228 MDB_val key, data;
1229 ID nid;
1230 int rc;
1231
1232 if ( ai->ai_clist ) {
1233 ic = ai->ai_clist;
1234 ai->ai_clist = ic->head;
1235 } else {
1236 ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
1237 }
1238 ic->kstr.bv_len = itmp.kstr.bv_len;
1239 ic->kstr.bv_val = (char *)(ic+1);
1240 memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
1241 ic->head = ic->tail = NULL;
1242 ic->last = 0;
1243 ic->count = 0;
1244 ic->offset = 0;
1245 ic->flags = 0;
1246 tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
1247 avl_dup_error );
1248
1249 /* load existing key count here */
1250 key.mv_size = keys[i].bv_len;
1251 key.mv_data = keys[i].bv_val;
1252 rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
1253 if ( rc == 0 ) {
1254 ic->flags |= WAS_FOUND;
1255 nid = *(ID *)data.mv_data;
1256 if ( nid == 0 ) {
1257 ic->count = MDB_IDL_DB_SIZE+1;
1258 ic->flags |= WAS_RANGE;
1259 } else {
1260 size_t count;
1261
1262 mdb_cursor_count( mc, &count );
1263 ic->count = count;
1264 ic->first = nid;
1265 ic->offset = count & (IDBLOCK-1);
1266 }
1267 }
1268 }
1269 /* are we a range already? */
1270 if ( ic->count > MDB_IDL_DB_SIZE ) {
1271 ic->last = id;
1272 continue;
1273 /* Are we at the limit, and converting to a range? */
1274 } else if ( ic->count == MDB_IDL_DB_SIZE ) {
1275 if ( ic->head ) {
1276 ic->tail->next = ai->ai_flist;
1277 ai->ai_flist = ic->head;
1278 }
1279 ic->head = ic->tail = NULL;
1280 ic->last = id;
1281 ic->count++;
1282 continue;
1283 }
1284 /* No free block, create that too */
1285 lcount = ic->count & (IDBLOCK-1);
1286 if ( !ic->tail || lcount == 0) {
1287 if ( ai->ai_flist ) {
1288 ice = ai->ai_flist;
1289 ai->ai_flist = ice->next;
1290 } else {
1291 ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
1292 }
1293 ice->next = NULL;
1294 if ( !ic->head ) {
1295 ic->head = ice;
1296 } else {
1297 ic->tail->next = ice;
1298 }
1299 ic->tail = ice;
1300 if ( lcount )
1301 ice->ids[lcount-1] = 0;
1302 if ( !ic->count )
1303 ic->first = id;
1304 }
1305 ice = ic->tail;
1306 if (!lcount || ice->ids[lcount-1] != id)
1307 ice->ids[lcount] = id;
1308 ic->count++;
1309 }
1310
1311 return 0;
1312 }
1313 #endif /* MDB_TOOL_IDL_CACHING */
1314
1315 /* Upgrade from pre 2.4.34 dn2id format */
1316
1317 #include <ac/unistd.h>
1318 #include <lutil_meter.h>
1319
1320 #define STACKSIZ 2048
1321
1322 typedef struct rec {
1323 ID id;
1324 size_t len;
1325 char rdn[512];
1326 } rec;
1327
1328 static int
mdb_dn2id_upgrade(BackendDB * be)1329 mdb_dn2id_upgrade( BackendDB *be ) {
1330 struct mdb_info *mi = (struct mdb_info *) be->be_private;
1331 MDB_txn *mt;
1332 MDB_cursor *mc = NULL;
1333 MDB_val key, data;
1334 int rc, writes=0, depth=0;
1335 int enable_meter = 0;
1336 ID id = 0, *num, count = 0;
1337 rec *stack;
1338 lutil_meter_t meter;
1339
1340 if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
1341 Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
1342 be->be_suffix[0].bv_val, 0, 0 );
1343 return 0;
1344 }
1345
1346 {
1347 MDB_stat st;
1348
1349 mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
1350 if (!st.ms_entries) {
1351 /* Empty DB, nothing to upgrade? */
1352 return 0;
1353 }
1354 if (isatty(2))
1355 enable_meter = !lutil_meter_open(&meter,
1356 &lutil_meter_text_display,
1357 &lutil_meter_linear_estimator,
1358 st.ms_entries);
1359 }
1360
1361 num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
1362 stack = (rec *)(num + STACKSIZ);
1363
1364 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1365 if (rc) {
1366 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
1367 mdb_strerror(rc), rc, 0 );
1368 goto leave;
1369 }
1370 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1371 if (rc) {
1372 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
1373 mdb_strerror(rc), rc, 0 );
1374 goto leave;
1375 }
1376
1377 key.mv_size = sizeof(ID);
1378 /* post-order depth-first update */
1379 for(;;) {
1380 size_t dkids;
1381 unsigned char *ptr;
1382
1383 /* visit */
1384 key.mv_data = &id;
1385 stack[depth].id = id;
1386 rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
1387 if (rc) {
1388 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
1389 mdb_strerror(rc), rc, 0 );
1390 goto leave;
1391 }
1392 num[depth] = 1;
1393
1394 rc = mdb_cursor_count(mc, &dkids);
1395 if (rc) {
1396 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
1397 mdb_strerror(rc), rc, 0 );
1398 goto leave;
1399 }
1400 if (dkids > 1) {
1401 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1402 down:
1403 ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
1404 memcpy(&id, ptr, sizeof(ID));
1405 depth++;
1406 memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
1407 stack[depth].len = data.mv_size;
1408 continue;
1409 }
1410
1411
1412 /* pop: write updated count, advance to next node */
1413 pop:
1414 /* update superior counts */
1415 if (depth)
1416 num[depth-1] += num[depth];
1417
1418 key.mv_data = &id;
1419 id = stack[depth-1].id;
1420 data.mv_data = stack[depth].rdn;
1421 data.mv_size = stack[depth].len;
1422 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1423 if (rc) {
1424 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
1425 mdb_strerror(rc), rc, 0 );
1426 goto leave;
1427 }
1428 data.mv_data = stack[depth].rdn;
1429 ptr = (unsigned char *)data.mv_data + data.mv_size;
1430 memcpy(ptr, &num[depth], sizeof(ID));
1431 data.mv_size += sizeof(ID);
1432 rc = mdb_cursor_del(mc, 0);
1433 if (rc) {
1434 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
1435 mdb_strerror(rc), rc, 0 );
1436 goto leave;
1437 }
1438 rc = mdb_cursor_put(mc, &key, &data, 0);
1439 if (rc) {
1440 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
1441 mdb_strerror(rc), rc, 0 );
1442 goto leave;
1443 }
1444 count++;
1445 #if 1
1446 if (enable_meter)
1447 lutil_meter_update(&meter, count, 0);
1448 #else
1449 {
1450 int len;
1451 ptr = data.mv_data;
1452 len = (ptr[0] & 0x7f) << 8 | ptr[1];
1453 printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
1454 }
1455 #endif
1456 writes++;
1457 if (writes == 1000) {
1458 mdb_cursor_close(mc);
1459 rc = mdb_txn_commit(mt);
1460 if (rc) {
1461 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
1462 mdb_strerror(rc), rc, 0 );
1463 goto leave;
1464 }
1465 rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
1466 if (rc) {
1467 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
1468 mdb_strerror(rc), rc, 0 );
1469 goto leave;
1470 }
1471 rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
1472 if (rc) {
1473 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
1474 mdb_strerror(rc), rc, 0 );
1475 goto leave;
1476 }
1477 rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
1478 if (rc) {
1479 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
1480 mdb_strerror(rc), rc, 0 );
1481 goto leave;
1482 }
1483 writes = 0;
1484 }
1485 depth--;
1486
1487 rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
1488 if (rc == 0)
1489 goto down;
1490 rc = 0;
1491 if (depth)
1492 goto pop;
1493 else
1494 break;
1495 }
1496 leave:
1497 mdb_cursor_close(mc);
1498 if (mt) {
1499 int r2;
1500 r2 = mdb_txn_commit(mt);
1501 if (r2) {
1502 Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
1503 mdb_strerror(r2), r2, 0 );
1504 if (!rc)
1505 rc = r2;
1506 }
1507 }
1508 ch_free(num);
1509 if (enable_meter) {
1510 lutil_meter_update(&meter, count, 1);
1511 lutil_meter_close(&meter);
1512 }
1513 return rc;
1514 }
1515