1 /* This file is part of the Zebra server.
2 Copyright (C) 2004-2013 Index Data
3
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17
18 */
19
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <fcntl.h>
24 #ifdef WIN32
25 #include <io.h>
26 #endif
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #include <stdlib.h>
31 #include <string.h>
32 #include <stdio.h>
33 #include <assert.h>
34
35 #include "index.h"
36
37 #define KEY_SIZE (1+sizeof(struct it_key))
38 #define INP_NAME_MAX 768
39
40 struct key_file {
41 int no; /* file no */
42 off_t offset; /* file offset */
43 unsigned char *buf; /* buffer block */
44 size_t buf_size; /* number of read bytes in block */
45 size_t chunk; /* number of bytes allocated */
46 size_t buf_ptr; /* current position in buffer */
47 char *prev_name; /* last word read */
48 void *decode_handle;
49 off_t length; /* length of file */
50 /* handler invoked in each read */
51 void (*readHandler)(struct key_file *keyp, void *rinfo);
52 void *readInfo;
53 Res res;
54 };
55
56
57 #define PR_KEY_LOW 0
58 #define PR_KEY_TOP 0
59
60 #if PR_KEY_LOW || PR_KEY_TOP
61
pkey(const char * b,int mode)62 static void pkey(const char *b, int mode)
63 {
64 key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d");
65 }
66 #endif
67
68
getFnameTmp(Res res,char * fname,int no)69 void getFnameTmp(Res res, char *fname, int no)
70 {
71 const char *pre;
72
73 pre = res_get_def(res, "keyTmpDir", ".");
74 sprintf(fname, "%s/key%d.tmp", pre, no);
75 }
76
extract_get_fname_tmp(ZebraHandle zh,char * fname,int no)77 void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no)
78 {
79 const char *pre;
80
81 pre = res_get_def(zh->res, "keyTmpDir", ".");
82 sprintf(fname, "%s/key%d.tmp", pre, no);
83 }
84
key_file_chunk_read(struct key_file * f)85 void key_file_chunk_read(struct key_file *f)
86 {
87 int nr = 0, r = 0, fd;
88 char fname[1024];
89 getFnameTmp(f->res, fname, f->no);
90 fd = open(fname, O_BINARY|O_RDONLY);
91
92 f->buf_ptr = 0;
93 f->buf_size = 0;
94 if (fd == -1)
95 {
96 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname);
97 return ;
98 }
99 if (!f->length)
100 {
101 if ((f->length = lseek(fd, 0L, SEEK_END)) == (off_t) -1)
102 {
103 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
104 close(fd);
105 return ;
106 }
107 }
108 if (lseek(fd, f->offset, SEEK_SET) == -1)
109 {
110 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
111 close(fd);
112 return ;
113 }
114 while (f->chunk - nr > 0)
115 {
116 r = read(fd, f->buf + nr, f->chunk - nr);
117 if (r <= 0)
118 break;
119 nr += r;
120 }
121 if (r == -1)
122 {
123 yaz_log(YLOG_WARN|YLOG_ERRNO, "read of %s", fname);
124 close(fd);
125 return;
126 }
127 f->buf_size = nr;
128 if (f->readHandler)
129 (*f->readHandler)(f, f->readInfo);
130 close(fd);
131 }
132
key_file_destroy(struct key_file * f)133 void key_file_destroy(struct key_file *f)
134 {
135 iscz1_stop(f->decode_handle);
136 xfree(f->buf);
137 xfree(f->prev_name);
138 xfree(f);
139 }
140
key_file_init(int no,int chunk,Res res)141 struct key_file *key_file_init(int no, int chunk, Res res)
142 {
143 struct key_file *f;
144
145 f = (struct key_file *) xmalloc(sizeof(*f));
146 f->res = res;
147 f->decode_handle = iscz1_start();
148 f->no = no;
149 f->chunk = chunk;
150 f->offset = 0;
151 f->length = 0;
152 f->readHandler = NULL;
153 f->buf = (unsigned char *) xmalloc(f->chunk);
154 f->prev_name = (char *) xmalloc(INP_NAME_MAX);
155 *f->prev_name = '\0';
156 key_file_chunk_read(f);
157 return f;
158 }
159
key_file_getc(struct key_file * f)160 int key_file_getc(struct key_file *f)
161 {
162 if (f->buf_ptr < f->buf_size)
163 return f->buf[(f->buf_ptr)++];
164 if (f->buf_size < f->chunk)
165 return EOF;
166 f->offset += f->buf_size;
167 key_file_chunk_read(f);
168 if (f->buf_ptr < f->buf_size)
169 return f->buf[(f->buf_ptr)++];
170 else
171 return EOF;
172 }
173
key_file_read(struct key_file * f,char * key)174 int key_file_read(struct key_file *f, char *key)
175 {
176 int i, c;
177 char srcbuf[128];
178 const char *src = srcbuf;
179 char *dst;
180 int j;
181
182 c = key_file_getc(f);
183 if (c == 0)
184 {
185 strcpy(key, f->prev_name);
186 i = 1+strlen(key);
187 }
188 else if (c == EOF)
189 return 0;
190 else
191 {
192 i = 0;
193 key[i++] = c;
194 while ((c = key_file_getc(f)))
195 {
196 if (i < INP_NAME_MAX-2)
197 key[i++] = c;
198 }
199 key[i++] = '\0';
200 strcpy(f->prev_name, key);
201 iscz1_reset(f->decode_handle);
202 }
203 c = key_file_getc(f); /* length + insert/delete combined */
204 key[i++] = c & 128;
205 c = c & 127;
206 for (j = 0; j < c; j++)
207 srcbuf[j] = key_file_getc(f);
208 dst = key + i;
209 iscz1_decode(f->decode_handle, &dst, &src);
210
211 #if 0
212 /* debugging */
213 if (1)
214 {
215 struct it_key k;
216 memcpy(&k, key+i, sizeof(k));
217 if (!k.mem[1])
218 yaz_log(YLOG_LOG, "00 KEY");
219 }
220 #endif
221 return i + sizeof(struct it_key);
222 }
223
224 struct heap_info {
225 struct {
226 struct key_file **file;
227 char **buf;
228 } info;
229 int heapnum;
230 int *ptr;
231 int (*cmp)(const void *p1, const void *p2);
232 struct zebra_register *reg;
233 ZebraHandle zh;
234 zint no_diffs;
235 zint no_updates;
236 zint no_deletions;
237 zint no_insertions;
238 zint no_iterations;
239 };
240
key_heap_malloc(void)241 static struct heap_info *key_heap_malloc(void)
242 { /* malloc and clear it */
243 struct heap_info *hi;
244 hi = (struct heap_info *) xmalloc(sizeof(*hi));
245 hi->info.file = 0;
246 hi->info.buf = 0;
247 hi->heapnum = 0;
248 hi->ptr = 0;
249 hi->no_diffs = 0;
250 hi->no_diffs = 0;
251 hi->no_updates = 0;
252 hi->no_deletions = 0;
253 hi->no_insertions = 0;
254 hi->no_iterations = 0;
255 return hi;
256 }
257
key_heap_init_file(ZebraHandle zh,int nkeys,int (* cmp)(const void * p1,const void * p2))258 struct heap_info *key_heap_init_file(ZebraHandle zh,
259 int nkeys,
260 int (*cmp)(const void *p1, const void *p2))
261 {
262 struct heap_info *hi;
263 int i;
264
265 hi = key_heap_malloc();
266 hi->zh = zh;
267 hi->info.file = (struct key_file **)
268 xmalloc(sizeof(*hi->info.file) * (1+nkeys));
269 hi->info.buf = (char **) xmalloc(sizeof(*hi->info.buf) * (1+nkeys));
270 hi->ptr = (int *) xmalloc(sizeof(*hi->ptr) * (1+nkeys));
271 hi->cmp = cmp;
272 for (i = 0; i<= nkeys; i++)
273 {
274 hi->ptr[i] = i;
275 hi->info.buf[i] = (char *) xmalloc(INP_NAME_MAX);
276 }
277 return hi;
278 }
279
key_heap_destroy(struct heap_info * hi,int nkeys)280 void key_heap_destroy(struct heap_info *hi, int nkeys)
281 {
282 int i;
283 for (i = 0; i<=nkeys; i++)
284 xfree(hi->info.buf[i]);
285 xfree(hi->info.buf);
286 xfree(hi->ptr);
287 xfree(hi->info.file);
288 xfree(hi);
289 }
290
key_heap_swap(struct heap_info * hi,int i1,int i2)291 static void key_heap_swap(struct heap_info *hi, int i1, int i2)
292 {
293 int swap;
294
295 swap = hi->ptr[i1];
296 hi->ptr[i1] = hi->ptr[i2];
297 hi->ptr[i2] = swap;
298 }
299
300
key_heap_delete(struct heap_info * hi)301 static void key_heap_delete(struct heap_info *hi)
302 {
303 int cur = 1, child = 2;
304
305 assert(hi->heapnum > 0);
306
307 key_heap_swap(hi, 1, hi->heapnum);
308 hi->heapnum--;
309 while (child <= hi->heapnum) {
310 if (child < hi->heapnum &&
311 (*hi->cmp)(&hi->info.buf[hi->ptr[child]],
312 &hi->info.buf[hi->ptr[child+1]]) > 0)
313 child++;
314 if ((*hi->cmp)(&hi->info.buf[hi->ptr[cur]],
315 &hi->info.buf[hi->ptr[child]]) > 0)
316 {
317 key_heap_swap(hi, cur, child);
318 cur = child;
319 child = 2*cur;
320 }
321 else
322 break;
323 }
324 }
325
key_heap_insert(struct heap_info * hi,const char * buf,int nbytes,struct key_file * kf)326 static void key_heap_insert(struct heap_info *hi, const char *buf, int nbytes,
327 struct key_file *kf)
328 {
329 int cur, parent;
330
331 assert(nbytes < INP_NAME_MAX);
332
333 cur = ++(hi->heapnum);
334 memcpy(hi->info.buf[hi->ptr[cur]], buf, nbytes);
335 hi->info.file[hi->ptr[cur]] = kf;
336
337 parent = cur/2;
338 while (parent && (*hi->cmp)(&hi->info.buf[hi->ptr[parent]],
339 &hi->info.buf[hi->ptr[cur]]) > 0)
340 {
341 key_heap_swap(hi, cur, parent);
342 cur = parent;
343 parent = cur/2;
344 }
345 }
346
heap_read_one(struct heap_info * hi,char * name,char * key)347 static int heap_read_one(struct heap_info *hi, char *name, char *key)
348 {
349 int n, r;
350 char rbuf[INP_NAME_MAX];
351 struct key_file *kf;
352
353 if (!hi->heapnum)
354 return 0;
355 n = hi->ptr[1];
356 strcpy(name, hi->info.buf[n]);
357 kf = hi->info.file[n];
358 r = strlen(name);
359 memcpy(key, hi->info.buf[n] + r+1, KEY_SIZE);
360 key_heap_delete(hi);
361 if ((r = key_file_read(kf, rbuf)))
362 key_heap_insert(hi, rbuf, r, kf);
363 hi->no_iterations++;
364 return 1;
365 }
366
367
368 /* for debugging only */
zebra_log_dict_entry(ZebraHandle zh,const char * s)369 void zebra_log_dict_entry(ZebraHandle zh, const char *s)
370 {
371 char dst[INP_NAME_MAX+1];
372 int ord;
373 int len = key_SU_decode(&ord, (const unsigned char *) s);
374 const char *index_type;
375
376 if (!zh)
377 yaz_log(YLOG_LOG, "ord=%d", ord);
378 else
379 {
380 const char *string_index;
381 const char *db = 0;
382 zebraExplain_lookup_ord(zh->reg->zei,
383 ord, &index_type, &db, &string_index);
384
385 zebra_term_untrans(zh, index_type, dst, s + len);
386
387 yaz_log(YLOG_LOG, "ord=%d index_type=%s index=%s term=%s",
388 ord, index_type, string_index, dst);
389 }
390 }
391
392 struct heap_cread_info {
393 char prev_name[INP_NAME_MAX];
394 char cur_name[INP_NAME_MAX];
395 char *key;
396 char *key_1, *key_2;
397 int mode_1, mode_2;
398 int sz_1, sz_2;
399 struct heap_info *hi;
400 int first_in_list;
401 int more;
402 int ret;
403 int look_level;
404 };
405
406 static int heap_cread_item(void *vp, char **dst, int *insertMode);
407
heap_cread_item2(void * vp,char ** dst,int * insertMode)408 int heap_cread_item2(void *vp, char **dst, int *insertMode)
409 {
410 struct heap_cread_info *p = (struct heap_cread_info *) vp;
411 int level = 0;
412
413 if (p->look_level)
414 {
415 if (p->look_level > 0)
416 {
417 *insertMode = 1;
418 p->look_level--;
419 }
420 else
421 {
422 *insertMode = 0;
423 p->look_level++;
424 }
425 memcpy(*dst, p->key_1, p->sz_1);
426 #if PR_KEY_TOP
427 yaz_log(YLOG_LOG, "DUP level=%d", p->look_level);
428 pkey(*dst, *insertMode);
429 #endif
430 (*dst) += p->sz_1;
431 return 1;
432 }
433 if (p->ret == 0) /* lookahead was 0?. Return that in read next round */
434 {
435 p->ret = -1;
436 return 0;
437 }
438 else if (p->ret == -1) /* Must read new item ? */
439 {
440 char *dst_1 = p->key_1;
441 p->ret = heap_cread_item(vp, &dst_1, &p->mode_1);
442 p->sz_1 = dst_1 - p->key_1;
443 }
444 else
445 { /* lookahead in 2 . Now in 1. */
446 p->sz_1 = p->sz_2;
447 p->mode_1 = p->mode_2;
448 memcpy(p->key_1, p->key_2, p->sz_2);
449 }
450 if (p->mode_1)
451 level = 1; /* insert */
452 else
453 level = -1; /* delete */
454 while(1)
455 {
456 char *dst_2 = p->key_2;
457 p->ret = heap_cread_item(vp, &dst_2, &p->mode_2);
458 if (!p->ret)
459 {
460 if (level)
461 break;
462 p->ret = -1;
463 return 0;
464 }
465 p->sz_2 = dst_2 - p->key_2;
466
467 if (key_compare(p->key_1, p->key_2) == 0)
468 {
469 if (p->mode_2) /* adjust level according to deletes/inserts */
470 level++;
471 else
472 level--;
473 }
474 else
475 {
476 if (level)
477 break;
478 /* all the same. new round .. */
479 p->sz_1 = p->sz_2;
480 p->mode_1 = p->mode_2;
481 memcpy(p->key_1, p->key_2, p->sz_1);
482 if (p->mode_1)
483 level = 1; /* insert */
484 else
485 level = -1; /* delete */
486 }
487 }
488 /* outcome is insert (1) or delete (0) depending on final level */
489 if (level > 0)
490 {
491 *insertMode = 1;
492 level--;
493 }
494 else
495 {
496 *insertMode = 0;
497 level++;
498 }
499 p->look_level = level;
500 memcpy(*dst, p->key_1, p->sz_1);
501 #if PR_KEY_TOP
502 yaz_log(YLOG_LOG, "TOP");
503 pkey(*dst, *insertMode);
504 #endif
505 (*dst) += p->sz_1;
506 return 1;
507 }
508
heap_cread_item(void * vp,char ** dst,int * insertMode)509 int heap_cread_item(void *vp, char **dst, int *insertMode)
510 {
511 struct heap_cread_info *p = (struct heap_cread_info *) vp;
512 struct heap_info *hi = p->hi;
513
514 if (p->first_in_list)
515 {
516 *insertMode = p->key[0];
517 memcpy(*dst, p->key+1, sizeof(struct it_key));
518 #if PR_KEY_LOW
519 zebra_log_dict_entry(hi->zh, p->cur_name);
520 pkey(*dst, *insertMode);
521 #endif
522 (*dst) += sizeof(struct it_key);
523 p->first_in_list = 0;
524 return 1;
525 }
526 strcpy(p->prev_name, p->cur_name);
527 if (!(p->more = heap_read_one(hi, p->cur_name, p->key)))
528 return 0;
529 if (*p->cur_name && strcmp(p->cur_name, p->prev_name))
530 {
531 p->first_in_list = 1;
532 return 0;
533 }
534 *insertMode = p->key[0];
535 memcpy(*dst, p->key+1, sizeof(struct it_key));
536 #if PR_KEY_LOW
537 zebra_log_dict_entry(hi->zh, p->cur_name);
538 pkey(*dst, *insertMode);
539 #endif
540 (*dst) += sizeof(struct it_key);
541 return 1;
542 }
543
heap_inpc(struct heap_cread_info * hci,struct heap_info * hi)544 int heap_inpc(struct heap_cread_info *hci, struct heap_info *hi)
545 {
546 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
547
548 isamc_i->clientData = hci;
549 isamc_i->read_item = heap_cread_item2;
550
551 while (hci->more)
552 {
553 char this_name[INP_NAME_MAX];
554 ISAM_P isamc_p, isamc_p2;
555 char *dict_info;
556
557 strcpy(this_name, hci->cur_name);
558 assert(hci->cur_name[0]);
559 hi->no_diffs++;
560 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
561 {
562 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
563 isamc_p2 = isamc_p;
564 isamc_merge(hi->reg->isamc, &isamc_p2, isamc_i);
565 if (!isamc_p2)
566 {
567 hi->no_deletions++;
568 if (!dict_delete(hi->reg->dict, this_name))
569 abort();
570 }
571 else
572 {
573 hi->no_updates++;
574 if (isamc_p2 != isamc_p)
575 dict_insert(hi->reg->dict, this_name,
576 sizeof(ISAM_P), &isamc_p2);
577 }
578 }
579 else
580 {
581 isamc_p = 0;
582 isamc_merge(hi->reg->isamc, &isamc_p, isamc_i);
583 hi->no_insertions++;
584 if (isamc_p)
585 dict_insert(hi->reg->dict, this_name,
586 sizeof(ISAM_P), &isamc_p);
587 }
588 }
589 xfree(isamc_i);
590 return 0;
591 }
592
heap_inpb(struct heap_cread_info * hci,struct heap_info * hi)593 int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
594 {
595 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
596
597 isamc_i->clientData = hci;
598 isamc_i->read_item = heap_cread_item2;
599
600 while (hci->more)
601 {
602 char this_name[INP_NAME_MAX];
603 ISAM_P isamc_p, isamc_p2;
604 char *dict_info;
605
606 strcpy(this_name, hci->cur_name);
607 assert(hci->cur_name[0]);
608 hi->no_diffs++;
609
610 #if 0
611 assert(hi->zh);
612 zebra_log_dict_entry(hi->zh, hci->cur_name);
613 #endif
614 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
615 {
616 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
617 isamc_p2 = isamc_p;
618 isamb_merge(hi->reg->isamb, &isamc_p2, isamc_i);
619 if (!isamc_p2)
620 {
621 hi->no_deletions++;
622 if (!dict_delete(hi->reg->dict, this_name))
623 abort();
624 }
625 else
626 {
627 hi->no_updates++;
628 if (isamc_p2 != isamc_p)
629 dict_insert(hi->reg->dict, this_name,
630 sizeof(ISAM_P), &isamc_p2);
631 }
632 }
633 else
634 {
635 isamc_p = 0;
636 isamb_merge(hi->reg->isamb, &isamc_p, isamc_i);
637 hi->no_insertions++;
638 if (isamc_p)
639 dict_insert(hi->reg->dict, this_name,
640 sizeof(ISAM_P), &isamc_p);
641 }
642 }
643 xfree(isamc_i);
644 return 0;
645 }
646
heap_inps(struct heap_cread_info * hci,struct heap_info * hi)647 int heap_inps(struct heap_cread_info *hci, struct heap_info *hi)
648 {
649 ISAMS_I isams_i = (ISAMS_I) xmalloc(sizeof(*isams_i));
650
651 isams_i->clientData = hci;
652 isams_i->read_item = heap_cread_item;
653
654 while (hci->more)
655 {
656 char this_name[INP_NAME_MAX];
657 ISAM_P isams_p;
658 char *dict_info;
659
660 strcpy(this_name, hci->cur_name);
661 assert(hci->cur_name[0]);
662 hi->no_diffs++;
663 if (!(dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
664 {
665 isams_p = isams_merge(hi->reg->isams, isams_i);
666 hi->no_insertions++;
667 dict_insert(hi->reg->dict, this_name, sizeof(ISAM_P), &isams_p);
668 }
669 else
670 {
671 yaz_log(YLOG_FATAL, "isams doesn't support this kind of update");
672 break;
673 }
674 }
675 xfree(isams_i);
676 return 0;
677 }
678
679 struct progressInfo {
680 time_t startTime;
681 time_t lastTime;
682 off_t totalBytes;
683 off_t totalOffset;
684 };
685
progressFunc(struct key_file * keyp,void * info)686 void progressFunc(struct key_file *keyp, void *info)
687 {
688 struct progressInfo *p = (struct progressInfo *) info;
689 time_t now, remaining;
690
691 if (keyp->buf_size <= 0 || p->totalBytes <= 0)
692 return ;
693 time (&now);
694
695 if (now >= p->lastTime+10)
696 {
697 p->lastTime = now;
698 remaining = (time_t) ((now - p->startTime)*
699 ((double) p->totalBytes/p->totalOffset - 1.0));
700 if (remaining <= 130)
701 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining",
702 (100.0*p->totalOffset) / p->totalBytes, (long) remaining);
703 else
704 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining",
705 (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60);
706 }
707 p->totalOffset += keyp->buf_size;
708 }
709
710 #ifndef R_OK
711 #define R_OK 4
712 #endif
713
zebra_index_merge(ZebraHandle zh)714 void zebra_index_merge(ZebraHandle zh)
715 {
716 struct key_file **kf = 0;
717 char rbuf[1024];
718 int i, r;
719 struct heap_info *hi;
720 struct progressInfo progressInfo;
721 int nkeys = key_block_get_no_files(zh->reg->key_block);
722
723 if (nkeys == 0)
724 return;
725
726 if (nkeys < 0)
727 {
728 char fname[1024];
729 nkeys = 0;
730 while (1)
731 {
732 extract_get_fname_tmp (zh, fname, nkeys+1);
733 if (access(fname, R_OK) == -1)
734 break;
735 nkeys++;
736 }
737 if (!nkeys)
738 return ;
739 }
740 kf = (struct key_file **) xmalloc((1+nkeys) * sizeof(*kf));
741 progressInfo.totalBytes = 0;
742 progressInfo.totalOffset = 0;
743 time(&progressInfo.startTime);
744 time(&progressInfo.lastTime);
745 for (i = 1; i<=nkeys; i++)
746 {
747 kf[i] = key_file_init(i, 8192, zh->res);
748 kf[i]->readHandler = progressFunc;
749 kf[i]->readInfo = &progressInfo;
750 progressInfo.totalBytes += kf[i]->length;
751 progressInfo.totalOffset += kf[i]->buf_size;
752 }
753 hi = key_heap_init_file(zh, nkeys, key_qsort_compare);
754 hi->reg = zh->reg;
755
756 for (i = 1; i<=nkeys; i++)
757 if ((r = key_file_read(kf[i], rbuf)))
758 key_heap_insert(hi, rbuf, r, kf[i]);
759
760 if (1)
761 {
762 struct heap_cread_info hci;
763
764 hci.key = (char *) xmalloc(KEY_SIZE);
765 hci.key_1 = (char *) xmalloc(KEY_SIZE);
766 hci.key_2 = (char *) xmalloc(KEY_SIZE);
767 hci.ret = -1;
768 hci.first_in_list = 1;
769 hci.hi = hi;
770 hci.look_level = 0;
771 hci.more = heap_read_one(hi, hci.cur_name, hci.key);
772
773 if (zh->reg->isams)
774 heap_inps(&hci, hi);
775 if (zh->reg->isamc)
776 heap_inpc(&hci, hi);
777 if (zh->reg->isamb)
778 heap_inpb(&hci, hi);
779
780 xfree(hci.key);
781 xfree(hci.key_1);
782 xfree(hci.key_2);
783 }
784
785 for (i = 1; i<=nkeys; i++)
786 {
787 extract_get_fname_tmp (zh, rbuf, i);
788 unlink(rbuf);
789 }
790 for (i = 1; i<=nkeys; i++)
791 key_file_destroy(kf[i]);
792 xfree(kf);
793 if (hi->no_iterations)
794 { /* do not log if nothing happened */
795 yaz_log(YLOG_LOG, "Iterations: isam/dict "
796 ZINT_FORMAT "/" ZINT_FORMAT,
797 hi->no_iterations, hi->no_diffs);
798 yaz_log(YLOG_LOG, "Dict: inserts/updates/deletions: "
799 ZINT_FORMAT "/" ZINT_FORMAT "/" ZINT_FORMAT,
800 hi->no_insertions, hi->no_updates, hi->no_deletions);
801 }
802 key_block_destroy(&zh->reg->key_block);
803 key_heap_destroy(hi, nkeys);
804 }
805 /*
806 * Local variables:
807 * c-basic-offset: 4
808 * c-file-style: "Stroustrup"
809 * indent-tabs-mode: nil
810 * End:
811 * vim: shiftwidth=4 tabstop=8 expandtab
812 */
813
814