1 /* @source ajindex ************************************************************
2 **
3 ** B+ Tree Indexing plus Disc Cache.
4 **
5 ** @author Copyright (c) 2003 Alan Bleasby
6 ** @version $Revision: 1.112 $
7 ** @modified  subsequently heavily modified by Peter Rice
8 ** @modified $Date: 2013/01/24 15:31:04 $ by $Author: rice $
9 **
10 ** This library is free software; you can redistribute it and/or
11 ** modify it under the terms of the GNU Lesser General Public
12 ** License as published by the Free Software Foundation; either
13 ** version 2.1 of the License, or (at your option) any later version.
14 **
15 ** This library is distributed in the hope that it will be useful,
16 ** but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18 ** Lesser General Public License for more details.
19 **
20 ** You should have received a copy of the GNU Lesser General Public
21 ** License along with this library; if not, write to the Free Software
22 ** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
23 ** MA  02110-1301,  USA.
24 **
25 ******************************************************************************/
26 
27 
28 #include "ajlib.h"
29 
30 #include "ajindex.h"
31 #include "ajfileio.h"
32 #include "ajutil.h"
33 
34 #include <string.h>
35 #include <stdlib.h>
36 #include <errno.h>
37 #ifndef WIN32
38 #include <unistd.h>
39 #endif
40 
41 #include <sys/types.h>
42 
43 
44 #ifdef WIN32
45 #include <io.h>
46 #define fileno _fileno
47 #define ftruncate _chsize_s
48 #endif
49 
50 #if defined (AJ_IRIXLF)
51 #define usestat64 1
52 #endif
53 
54 #define AJINDEX_DEBUG 0
55 #define AJINDEX_DOSTATS 1
56 #define AJINDEX_STATIC 0
57 
58 static AjPBtpage btreeTestpage = NULL;
59 
60 static AjBool btreeDoRootSync = AJFALSE;
61 
62 static ajulong   statCallSync = 0UL;
63 static ajulong   statCallRootSync = 0UL;
64 static ajulong   statCallPriSplitroot = 0UL;
65 static ajulong   statCallSecSplitroot = 0UL;
66 static ajulong   statCallIdSplitroot = 0UL;
67 static ajulong   statCallNumSplitroot = 0UL;
68 static ajulong   statCallKeySplitleaf = 0UL;
69 static ajulong   statCallSecSplitleaf = 0UL;
70 static ajulong   statCallIdSplitleaf = 0UL;
71 static ajulong   statCallNumSplitleaf = 0UL;
72 static ajulong   statCallIdbucketsReorder = 0UL;
73 static ajulong   statCallPribucketsReorder = 0UL;
74 static ajulong   statCallSecbucketsReorder = 0UL;
75 /*static ajulong   statCallHybbucketsReorder = 0UL;*/
76 static ajulong   statCallNumbucketsReorder = 0UL;
77 static ajulong   statSyncLocked = 0UL;
78 static ajulong   statSyncWrite = 0UL;
79 static ajulong   statRootSyncLocked = 0UL;
80 static ajulong   statRootSyncUnlocked = 0UL;
81 static ajuint    statRootSyncMaxUnlocked = 0U;
82 static ajuint    statRootSyncMaxLocked = 0U;
83 
84 static AjPStr*   statSaveSecId = NULL;
85 static ajuint    statSaveSecIdNext = 0U;
86 static ajuint    statSaveSecIdMax = 0U;
87 
88 static AjPBtId*  statSaveBtreeId = NULL;
89 static ajuint    statSaveBtreeIdNext = 0U;
90 static ajuint    statSaveBtreeIdMax = 0U;
91 
92 static AjPBtHit* statSaveBtreeHit = NULL;
93 static ajuint    statSaveBtreeHitNext = 0U;
94 static ajuint    statSaveBtreeHitMax = 0U;
95 
96 static AjPBtHitref* statSaveBtreeHitref = NULL;
97 static ajuint       statSaveBtreeHitrefNext = 0U;
98 static ajuint       statSaveBtreeHitrefMax = 0U;
99 
100 static AjPBtPri* statSaveBtreePri = NULL;
101 static ajuint    statSaveBtreePriNext = 0U;
102 static ajuint    statSaveBtreePriMax = 0U;
103 
104 /* bucket cache */
105 static AjPIdbucket* statSaveIdbucket = NULL;
106 static AjPIdbucket* statSaveIdbucketEmpty = NULL;
107 static ajuint     statSaveIdbucketNext = 0U;
108 static ajuint     statSaveIdbucketMax = 0U;
109 static ajuint     statSaveIdbucketEmptyNext = 0U;
110 static ajuint     statSaveIdbucketEmptyMax = 0U;
111 
112 /* primary bucket cache */
113 static AjPPribucket* statSavePribucket = NULL;
114 static AjPPribucket* statSavePribucketEmpty = NULL;
115 static ajuint        statSavePribucketNext = 0U;
116 static ajuint        statSavePribucketMax = 0U;
117 static ajuint        statSavePribucketEmptyNext = 0U;
118 static ajuint        statSavePribucketEmptyMax = 0U;
119 
120 /* secondary bucket cache */
121 static AjPSecbucket* statSaveSecbucket = NULL;
122 static AjPSecbucket* statSaveSecbucketEmpty = NULL;
123 static ajuint        statSaveSecbucketNext = 0U;
124 static ajuint        statSaveSecbucketMax = 0U;
125 static ajuint        statSaveSecbucketEmptyNext = 0U;
126 static ajuint        statSaveSecbucketEmptyMax = 0U;
127 
128 /* number bucket cache */
129 static AjPNumbucket* statSaveNumbucket = NULL;
130 static ajuint        statSaveNumbucketNext = 0U;
131 static ajuint        statSaveNumbucketMax = 0U;
132 
133 static ajulong statCountAllocPriArrayNew = 0UL;
134 static ajulong statCountAllocPriArrayReuse = 0UL;
135 static ajulong statCountAllocPriArrayDel = 0UL;
136 static ajulong statCountAllocPriArrayFree = 0UL;
137 
138 static ajulong statCountAllocSecArrayNew = 0UL;
139 static ajulong statCountAllocSecArrayReuse = 0UL;
140 static ajulong statCountAllocSecArrayDel = 0UL;
141 static ajulong statCountAllocSecArrayFree = 0UL;
142 
143 static char const **btreeNodetypeNames = NULL;
144 
145 static AjPStr indexKeyword = NULL;
146 static AjPStr indexId = NULL;
147 
148 
149 /* @datastatic BtreeSField ****************************************************
150 **
151 ** Index field data
152 **
153 ** @alias BtreeOField
154 ** @alias BtreePField
155 **
156 ** @attr Name [AjPStr] Name
157 ** @attr Extension [AjPStr] Extension
158 ** @attr Secondary [AjBool] True for primary/secondary ID index
159 **                          False for Identifier/fileposition index
160 ** @attr Len [ajuint] Index field maximum length
161 **
162 ******************************************************************************/
163 
164 typedef struct BtreeSField
165 {
166     AjPStr Name;
167     AjPStr Extension;
168     AjBool Secondary;
169     ajuint Len;
170 } BtreeOField;
171 
172 #define BtreePField BtreeOField*
173 
174 
175 
176 
177 /* @datastatic BtreeSFielddef *************************************************
178 **
179 ** Known index fields
180 **
181 ** @alias BtreeOFielddef
182 ** @alias BtreePFielddef
183 **
184 ** @attr Name [const char*] Name
185 ** @attr Extension [const char*] Extension
186 ** @attr Secondary [AjBool] True for primary/secondary ID index
187 **                          False for Identifier/fileposition index
188 ** @attr Len [ajuint] Index field maximum length
189 **
190 ******************************************************************************/
191 
192 typedef struct BtreeSFielddef
193 {
194     const char* Name;
195     const char* Extension;
196     AjBool Secondary;
197     ajuint Len;
198 } BtreeOFielddef;
199 
200 #define BtreePFielddef BtreeOFielddef*
201 
202 
203 
204 BtreeOFielddef btreeFields[] =
205 {
206     {"acc",  "xac", AJFALSE, 15},
207     {"id",   "xid", AJFALSE, 15},
208     {"sv",   "xsv", AJFALSE, 15},
209     {"up",   "xup", AJFALSE, 8},
210     {"spc",  "xspc", AJFALSE, 15},
211     {"nam",  "xnm", AJTRUE,  15},
212     {"key",  "xkw", AJTRUE,  15},
213     {"des",  "xde", AJTRUE,  15},
214     {"org",  "xtx", AJTRUE,  15},
215     {"isa",  "xis", AJTRUE,  15},
216     {"xref", "xrf", AJTRUE,  15},
217     {"ns",   "xns", AJTRUE,  15},
218     {NULL,   NULL,  AJFALSE, 0}
219 };
220 
221 
222 
223 static AjPTable btreeFieldsTable = NULL;
224 static AjPStr   btreeFieldnameTmp = NULL;
225 
226 static AjBool btreeDoExtra = AJFALSE;
227 
228 
229 static void          btreePrirootCreate(AjPBtcache cache);
230 static void          btreeSecrootCreate(AjPBtcache cache, ajulong rootpage);
231 
232 static AjPBtpage     btreeIdentFind(AjPBtcache cache, const AjPStr key);
233 static AjBool        btreeKeyFind(AjPBtcache cache, const AjPStr key,
234                                   ajulong *treeblock);
235 static AjPBtId       btreeIdentQueryId(AjPBtcache cache, const AjPStr key);
236 static AjPBtHit      btreeIdentQueryHit(AjPBtcache cache, const AjPStr key);
237 static AjPBtHitref   btreeIdentQueryHitref(AjPBtcache cache, const AjPStr key);
238 static void          btreeIdentFetchMulti(AjPBtcache cache, const AjPStr idname,
239                                           ajulong rootblock,
240                                           AjPList list);
241 static void          btreeIdentFetchMultiHit(AjPBtcache cache,
242                                              ajulong rootblock,
243                                              AjPList list);
244 static void          btreeIdentFetchMultiHitref(AjPBtcache cache,
245                                                 ajulong rootblock,
246                                                 AjPList list);
247 
248 static AjBool        btreeKeyidMakeroot(AjPBtcache cache, AjPBtpage bucket);
249 
250 static AjBool        btreeKeyidExists(AjPBtcache cache, const AjPStr key);
251 static AjPBtpage     btreeKeyidFind(AjPBtcache cache, const AjPStr key);
252 static AjBool        btreeKeyidInsert(AjPBtcache cache, const AjPStr id);
253 
254 static AjPBtpage     btreePrimaryFetchFindleafWild(AjPBtcache cache,
255                                                    const AjPStr key);
256 
257 static void          btreeCacheSync(AjPBtcache cache, ajulong rootpage);
258 static void          btreeCacheRootSync(AjPBtcache cache, ajulong rootpage);
259 
260 static void          btreeCacheWriteCompress(AjPBtcache cache,
261                                              AjPBtpage cpage,
262                                              ajulong pagepos,
263                                              ajuint pagesize);
264 static void          btreeCacheWriteUncompress(AjPBtcache cache,
265                                                AjPBtpage cpage,
266                                                ajulong pagepos,
267                                                ajuint pagesize);
268 
269 static ajulong       btreeCacheCompress(AjPBtcache thys);
270 
271 static ajulong       btreeCacheUncompress(AjPBtcache thys);
272 
273 static ajulong       btreePageposCompress(ajulong oldpos,
274                                           const AjPTable newpagetable,
275                                           const char* desc);
276 
277 static ajulong       btreePageposUncompress(ajulong oldpos,
278                                             const AjPTable newpagetable,
279                                             const char* desc);
280 
281 static AjPBtpage     btreePripageNew(AjPBtcache cache);
282 static AjPBtpage     btreeSecpageNew(AjPBtcache cache);
283 
284 static ajuint        btreePageGetSizeIdbucket(const AjPBtpage page,
285                                             ajuint refcount);
286 static ajuint        btreePageGetSizeNode(const AjPBtpage page);
287 static ajuint        btreePageGetSizeNumbucket(const AjPBtpage page,
288                                                ajuint refcount);
289 static ajuint        btreePageGetSizeNumnode(const AjPBtpage page);
290 static ajuint        btreePageGetSizePribucket(const AjPBtpage page);
291 static ajuint        btreePageGetSizeSecbucket(const AjPBtpage page);
292 
293 static AjBool        btreePageCompress(AjPBtpage page,
294                                        const AjPTable newpagetable,
295                                        ajuint refcount);
296 static void          btreePageCompressIdbucket(AjPBtpage page,
297                                                const AjPTable newpagetable,
298                                                ajuint refcount);
299 static void          btreePageCompressNode(AjPBtpage page,
300                                            const AjPTable newpagetable);
301 static void          btreePageCompressNumnode(AjPBtpage page,
302                                               const AjPTable newpagetable);
303 static void          btreePageCompressPribucket(AjPBtpage page,
304                                                 const AjPTable newpagetable);
305 static AjBool        btreePageUncompress(AjPBtpage page,
306                                          const AjPTable newpagetable,
307                                          ajuint refcount);
308 static void          btreePageUncompressIdbucket(AjPBtpage page,
309                                                  const AjPTable newpagetable,
310                                                  ajuint refcount);
311 static void          btreePageUncompressNode(AjPBtpage page,
312                                              const AjPTable newpagetable);
313 static void          btreePageUncompressNumnode(AjPBtpage page,
314                                                 const AjPTable newpagetable);
315 static void          btreePageUncompressPribucket(AjPBtpage page,
316                                                   const AjPTable newpagetable);
317 
318 static void          btreeNocacheFetch(const AjPBtcache cache, AjPBtpage cpage,
319                                        ajulong pagepos);
320 static void          btreeCacheFetchSize(AjPBtcache cache, AjPBtpage cpage,
321                                          ajulong pagepos, ajuint pagesize);
322 
323 static AjPBtpage     btreePricacheBucketnew(AjPBtcache cache);
324 static AjPBtpage     btreePricacheControl(AjPBtcache cache, ajulong pagepos,
325                                           AjBool isread);
326 static void          btreePricacheDestage(AjPBtcache cache, AjPBtpage cpage);
327 static void          btreePricacheFetch(AjPBtcache cache, AjPBtpage cpage,
328                                         ajulong pagepos);
329 static AjPBtpage     btreePricacheLocate(AjPBtcache cache, ajulong page);
330 static AjPBtpage     btreePricacheLruUnlink(AjPBtcache cache);
331 static void          btreePricacheMruAdd(AjPBtcache cache, AjPBtpage cpage);
332 static AjPBtpage     btreePricacheNodenew(AjPBtcache cache);
333 static AjPBtpage     btreePricacheRead(AjPBtcache cache, ajulong pagepos);
334 static void          btreePricacheUnlink(AjPBtcache cache, AjPBtpage cpage);
335 static AjPBtpage     btreePricacheWrite(AjPBtcache cache, ajulong pagepos);
336 
337 static AjPBtpage     btreeSeccacheBucketnew(AjPBtcache cache);
338 static AjPBtpage     btreeSeccacheControl(AjPBtcache cache, ajulong pagepos,
339                                           AjBool isread);
340 static void          btreeSeccacheDestage(AjPBtcache cache, AjPBtpage cpage);
341 static void          btreeSeccacheFetch(AjPBtcache cache, AjPBtpage cpage,
342                                         ajulong pagepos);
343 static AjPBtpage     btreeSeccacheLocate(AjPBtcache cache, ajulong page);
344 static AjPBtpage     btreeSeccacheLruUnlink(AjPBtcache cache);
345 static void          btreeSeccacheMruAdd(AjPBtcache cache, AjPBtpage cpage);
346 static AjPBtpage     btreeSeccacheNodenew(AjPBtcache cache);
347 static AjPBtpage     btreeSeccacheRead(AjPBtcache cache, ajulong pagepos);
348 static void          btreeSeccacheUnlink(AjPBtcache cache, AjPBtpage cpage);
349 static AjPBtpage     btreeSeccacheWrite(AjPBtcache cache, ajulong pagepos);
350 
351 static AjPBtpage     btreePrimaryFindInode(AjPBtcache cache, AjPBtpage page,
352                                            const AjPStr item);
353 static AjPBtpage     btreePrimaryFindInodeWild(AjPBtcache cache, AjPBtpage page,
354                                                const AjPStr item);
355 static AjPBtpage     btreeKeyidFindINode(AjPBtcache cache, AjPBtpage page,
356                                          const AjPStr item);
357 static ajulong       btreeSecTreeCount(AjPBtcache cache, ajulong rootblock);
358 static AjPList       btreeSecTreeList(AjPBtcache cache, ajulong rootblock);
359 
360 
361 static AjPBtpage     btreePrimaryPageDown(AjPBtcache cache,
362                                           unsigned char *buf,
363                                           const AjPStr item);
364 static AjPBtpage     btreeSecondaryPageDown(AjPBtcache cache,
365                                             unsigned char *buf,
366                                             const AjPStr item);
367 static ajuint        btreeIdbucketCount(AjPBtcache cache, ajulong pagepos);
368 static AjPBtId       btreeIdbucketFindDupId(AjPBtcache cache, ajulong pagepos,
369                                             const AjPStr id, ajuint* ientry);
370 static ajulong       btreeIdbucketIdlist(AjPBtcache cache, ajulong pagepos,
371                                          AjPList idlist);
372 static void          btreeIdbucketSort(AjPIdbucket thys);
373 static AjPIdbucket   btreeReadIdbucket(AjPBtcache cache, ajulong pagepos);
374 static void          btreeWriteIdbucket(AjPBtcache cache,
375                                         const AjPIdbucket bucket,
376                                         ajulong pagepos);
377 static void          btreeWriteIdbucketEmpty(AjPBtcache cache, ajulong pagepos);
378 static void          btreeWriteIdbucketId(AjPBtcache cache, ajulong pagepos,
379                                           const AjPBtId btid, ajuint ientry);
380 static void          btreeIdbucketAdd(AjPBtcache cache, ajulong pagepos,
381                                       const AjPBtId id);
382 static void 	     btreeIdbucketDel(AjPIdbucket *thys);
383 static void          btreeIdbucketFree(AjPIdbucket *thys);
384 static AjBool        btreeIdbucketsReorder(AjPBtcache cache, AjPBtpage page);
385 static void          btreeGetKeys(AjPBtcache cache, unsigned char *buf,
386                                   AjPStr **keys, ajulong **ptrs);
387 static ajuint        btreeGetPointers(AjPBtcache cache, unsigned char *buf,
388                                       ajulong **ptrs);
389 static ajulong       btreeGetBlockC(AjPBtcache cache, unsigned char *buf,
390                                     const char* ckey);
391 static ajulong       btreeGetBlockN(AjPBtcache cache, unsigned char *buf,
392                                     ajulong numkey);
393 static ajulong       btreeGetBlockS(AjPBtcache cache, unsigned char *buf,
394                                     const AjPStr key);
395 static ajulong       btreeGetBlockFirstC(AjPBtcache cache, unsigned char *buf,
396                                          const char* ckey, ajuint clen);
397 static ajulong       btreeGetBlockFirstN(AjPBtcache cache, unsigned char *buf,
398                                          ajulong numkey);
399 static ajulong       btreeGetBlockFirstS(AjPBtcache cache, unsigned char *buf,
400                                          const AjPStr key);
401 static void          btreeIdFree(AjPBtId *thys);
402 static ajint         btreeIdCompare(const void *a, const void *b);
403 
404 static AjPIdbucket   btreeIdbucketNew(ajuint n, ajuint refcount);
405 static void          btreeWriteNode(AjPBtcache cache, AjPBtpage page,
406                                     AjPStr const *keys, const ajulong *ptrs,
407                                     ajuint nkeys);
408 static void          btreeWriteNodeSingle(AjPBtcache cache, AjPBtpage spage,
409                                           const AjPStr key, ajulong lptr,
410                                           ajulong rptr);
411 static AjBool        btreeNodeIsFull(const AjPBtcache cache, AjPBtpage page);
412 static AjBool        btreeNodeIsFullSec(const AjPBtcache cache,
413                                         AjPBtpage page);
414 static void          btreePrimaryInsertNonfull(AjPBtcache cache, AjPBtpage page,
415                                                const AjPStr key, ajulong less,
416                                                ajulong greater);
417 static void          btreePriSplitroot(AjPBtcache cache);
418 static void          btreePriInsertKey(AjPBtcache cache, AjPBtpage page,
419                                        const AjPStr key, ajulong less,
420                                        ajulong greater);
421 
422 static ajulong       btreeKeyInsertShift(AjPBtcache cache,
423                                          AjPBtpage *retpage,
424                                          const AjPStr key);
425 static void          btreePrimaryShift(AjPBtcache cache, AjPBtpage tpage);
426 
427 
428 #if 0
429 static AjPBtpage     btreeTraverseLeaves(AjPBtcache cache, AjPBtpage thys);
430 static void          btreeJoinLeaves(AjPBtcache cache);
431 #endif
432 
433 static AjPBtpage     btreePrimaryPageDownWild(AjPBtcache cache,
434                                               unsigned char *buf,
435                                               const AjPStr key);
436 static void          btreeIdleafFetch(AjPBtcache cache, AjPBtpage page,
437                                         AjPList list);
438 static void          btreePrileafFetch(AjPBtcache cache, AjPBtpage page,
439                                        AjPList list);
440 static AjPBtpage     btreeKeySplitleaf(AjPBtcache cache, AjPBtpage spage);
441 
442 
443 
444 
445 static void          btreePriFree(AjPBtPri *thys);
446 static AjPPribucket  btreePribucketNew(ajuint n);
447 static void          btreePribucketDel(AjPPribucket *thys);
448 static void          btreePribucketFree(AjPPribucket *thys);
449 static ajulong       btreePribucketIdlist(AjPBtcache cache, ajulong pagepos,
450                                           AjPList idlist);
451 static AjBool        btreePribucketFindId(AjPBtcache cache, ajulong pagepos,
452                                           const AjPStr id, ajulong* treeblock);
453 static void          btreePribucketSort(AjPPribucket thys);
454 static AjPPribucket  btreePribucketRead(AjPBtcache cache, ajulong pagepos);
455 static void          btreeWritePribucket(AjPBtcache cache,
456                                          const AjPPribucket bucket,
457                                          ajulong pagepos);
458 static void          btreeWritePribucketEmpty(AjPBtcache cache,
459                                               ajulong pagepos);
460 static void          btreePribucketAdd(AjPBtcache cache, ajulong pagepos,
461                                        const AjPStr keyword, const AjPStr id);
462 static ajuint        btreeNumInPribucket(AjPBtcache cache, ajulong pagepos);
463 static ajint         btreeKeywordCompare(const void *a, const void *b);
464 static AjBool        btreePribucketsReorder(AjPBtcache cache,
465                                             AjPBtpage leaf);
466 
467 
468 
469 
470 static AjBool        btreeSecbucketFindId(AjPBtcache cache, ajulong pagepos,
471                                           const AjPStr id);
472 static ajulong       btreeSecbucketIdcount(AjPBtcache cache, ajulong pagepos);
473 static ajulong       btreeSecbucketIdlist(AjPBtcache cache, ajulong pagepos,
474                                           AjPList idlist);
475 static AjPSecbucket  btreeReadSecbucket(AjPBtcache cache, ajulong pagepos);
476 static void          btreeWriteSecbucket(AjPBtcache cache,
477 					 const AjPSecbucket bucket,
478 					 ajulong pagepos);
479 static void          btreeWriteSecbucketEmpty(AjPBtcache cache,
480                                               ajulong pagepos);
481 static ajint         btreeKeywordIdCompare(const void *a, const void *b);
482 static AjPBtpage     btreeSecSplitleaf(AjPBtcache cache, AjPBtpage spage);
483 
484 static AjPSecbucket  btreeSecbucketNew(ajuint n, ajuint idlen);
485 static void          btreeSecbucketDel(AjPSecbucket *thys);
486 static void          btreeSecbucketFree(AjPSecbucket *thys);
487 static void          btreeSecLeftLeaf(AjPBtcache cache, AjPBtKeywild wild);
488 static AjBool        btreeKeywildNextList(AjPBtcache cache, AjPBtKeywild wild);
489 static void          btreeReadAllSecLeaves(AjPBtcache cache, AjPList list);
490 
491 static void          btreeSecbucketAdd(AjPBtcache cache, ajulong pagepos,
492                                        const AjPStr id);
493 static AjBool        btreeSecbucketsReorder(AjPBtcache cache, AjPBtpage leaf);
494 static void          btreeSecSplitroot(AjPBtcache cache);
495 static ajuint        btreeNumInSecbucket(AjPBtcache cache, ajulong pagepos);
496 static void          btreeInsertKeySec(AjPBtcache cache, AjPBtpage page,
497 			               const AjPStr key, ajulong less,
498 			               ajulong greater);
499 static ajulong       btreeKeyidInsertShift(AjPBtcache cache, AjPBtpage *retpage,
500                                            const AjPStr key);
501 static void          btreeKeyShiftSec(AjPBtcache cache, AjPBtpage tpage);
502 static void          btreeInsertNonfullSec(AjPBtcache cache, AjPBtpage page,
503 				           const AjPStr key, ajulong less,
504 				           ajulong greater);
505 
506 static void          btreeStrDel(void** pentry, void* cl);
507 
508 static void          btreeIdDelFromList(void** pentry, void* cl);
509 static void          btreeHitDelFromList(void** pentry, void* cl);
510 static void          btreeHitrefDelFromList(void** pentry, void* cl);
511 
512 static void          btreeKeyFullSearchId(AjPBtcache cache,
513                                           const AjPStr key,
514                                           AjPList idlist);
515 static void          btreeKeyFullSearchHit(AjPBtcache cache,
516                                            const AjPStr key,
517                                            AjPList idlist);
518 static void          btreeKeyFullSearchHitref(AjPBtcache cache,
519                                               const AjPStr key,
520                                               AjPList idlist);
521 static void          btreeKeywordFullSearchId(AjPBtcache cache,
522                                               const AjPStr key,
523                                               AjPBtcache idcache,
524                                               AjPList idlist);
525 static void          btreeKeywordFullSearchHit(AjPBtcache cache,
526                                                const AjPStr key,
527                                                AjPBtcache idcache,
528                                                AjPList idlist);
529 static void          btreeKeywordFullSearchHitref(AjPBtcache cache,
530                                                   const AjPStr key,
531                                                   AjPBtcache idcache,
532                                                   AjPList idlist);
533 static ajint         btreeIdOffsetCompare(const void *a, const void *b);
534 static ajint         btreeHitOffsetCompare(const void *a, const void *b);
535 static ajint         btreeHitrefOffsetCompare(const void *a, const void *b);
536 static ajint         btreeIdDbnoCompare(const void *a, const void *b);
537 static ajint         btreeHitDbnoCompare(const void *a, const void *b);
538 static ajint         btreeHitrefDbnoCompare(const void *a, const void *b);
539 
540 
541 static AjPBtMem      btreeAllocPriArray(AjPBtcache cache);
542 static void          btreeDeallocPriArray(AjPBtcache cache, AjPBtMem node);
543 static AjPBtMem      btreeAllocSecArray(AjPBtcache cache);
544 static void          btreeDeallocSecArray(AjPBtcache cache, AjPBtMem node);
545 static void          btreeFreePriArray(AjPBtcache cache);
546 static void          btreeFreeSecArray(AjPBtcache cache);
547 
548 
549 
550 static ajulong       btreeIdentInsertShift(AjPBtcache cache, AjPBtpage *retpage,
551                                            const AjPStr key);
552 static AjPBtpage     btreeIdSplitleaf(AjPBtcache cache, AjPBtpage spage);
553 static void          btreeIdInsertKey(AjPBtcache cache, AjPBtpage page,
554 				       const AjPStr key, ajulong less,
555 				       ajulong greater);
556 static void          btreeIdSplitroot(AjPBtcache cache);
557 static void          btreeIdentDupInsert(AjPBtcache cache, const AjPBtId newid,
558                                          AjPBtId curid);
559 static ajulong       btreeIdbucketIdlistAll(AjPBtcache cache, ajulong pagepos,
560                                             AjPList idlist);
561 
562 static void          btreeGetNumKeys(AjPBtcache cache, unsigned char *buf,
563 				     ajulong **keys, ajulong **ptrs);
564 static void          btreeGetNumPointers(AjPBtcache cache, unsigned char *buf,
565                                          ajulong **ptrs);
566 static void          btreeWriteNumNode(AjPBtcache cache, AjPBtpage spage,
567 				       const ajulong *keys, const ajulong *ptrs,
568 				       ajuint nkeys);
569 static ajulong       btreeNumbucketIdlist(AjPBtcache cache, ajulong pagepos,
570                                           AjPList idlist);
571 static ajulong       btreeNumbucketBtidlist(AjPBtcache cache,
572                                             ajulong pagepos,
573                                             const AjPStr idname,
574                                             AjPList idlist);
575 static ajulong       btreeNumbucketBthitlist(AjPBtcache cache,
576                                              ajulong pagepos,
577                                              AjPList idlist);
578 static ajulong       btreeNumbucketBthitreflist(AjPBtcache cache,
579                                                 ajulong pagepos,
580                                                 AjPList idlist);
581 static AjPNumbucket  btreeReadNumbucket(AjPBtcache cache, ajulong pagepos);
582 static void          btreeWriteNumbucket(AjPBtcache cache,
583                                          const AjPNumbucket bucket,
584                                          ajulong pagepos);
585 static void          btreeWriteNumbucketEmpty(AjPBtcache cache,
586                                               ajulong pagepos);
587 static void          btreeNumbucketDel(AjPNumbucket *thys);
588 static void          btreeNumbucketFree(AjPNumbucket *thys);
589 static void          btreeNumbucketAdd(AjPBtcache cache, ajulong pagepos,
590                                        const AjPBtNumId num);
591 static AjPBtpage     btreeNumFind(AjPBtcache cache, const ajulong key);
592 static AjPBtpage     btreeNumFindINode(AjPBtcache cache, AjPBtpage page,
593 				       ajulong item);
594 static AjPBtpage     btreeNumPageFromKey(AjPBtcache cache, unsigned char *buf,
595 					 ajulong key);
596 static ajuint        btreeNumInNumbucket(AjPBtcache cache, ajulong pagepos);
597 static AjBool        btreeReorderNumbuckets(AjPBtcache cache, AjPBtpage leaf);
598 static AjPNumbucket  btreeNumbucketNew(ajuint n, ajuint nref);
599 static ajint         btreeNumIdCompare(const void *a, const void *b);
600 static AjBool        btreeNumNodeIsFull(const AjPBtcache cache,
601 					AjPBtpage page);
602 static void          btreeNumInsert(AjPBtcache cache, const AjPBtNumId num,
603                                     AjPBtpage page);
604 static void          btreeNumInsertNonfull(AjPBtcache cache, AjPBtpage page,
605 					   ajulong key, ajulong less,
606 					   ajulong greater);
607 static void          btreeNumInsertKey(AjPBtcache cache, AjPBtpage page,
608 				       ajulong key, ajulong less,
609 				       ajulong greater);
610 static void          btreeNumSplitroot(AjPBtcache cache);
611 static void          btreeNumKeyShift(AjPBtcache cache, AjPBtpage tpage);
612 static ajulong       btreeNumInsertShift(AjPBtcache cache, AjPBtpage *retpage,
613 					 ajulong key);
614 static AjPBtpage     btreeNumSplitleaf(AjPBtcache cache, AjPBtpage spage);
615 
616 
617 static ajulong       btreeFindIdentBalanceOne(AjPBtcache cache,
618                                               ajulong thisNode,
619                                               ajulong leftNode,
620                                               ajulong rightNode,
621                                               ajulong lAnchor, ajulong rAnchor,
622                                               const AjPBtId btid);
623 static void          btreeFindHybMinOne(AjPBtcache cache, ajulong pagepos,
624                                         const AjPStr key);
625 
626 static AjBool        btreeRemoveIdentEntryOne(AjPBtcache cache,ajulong pagepos,
627                                               const AjPBtId btid);
628 
629 static void          btreeAdjustHybbucketsOne(AjPBtcache cache,
630                                               AjPBtpage leaf);
631 
632 static ajulong       btreeRebalanceHybOne(AjPBtcache cache, ajulong thisNode,
633                                           ajulong leftNode, ajulong rightNode,
634                                           ajulong lAnchor, ajulong rAnchor);
635 
636 static ajulong       btreeShiftHybOne(AjPBtcache cache, ajulong thisNode,
637                                       ajulong balanceNode, ajulong anchorNode);
638 
639 static ajulong       btreeMergeHybOne(AjPBtcache cache, ajulong thisNode,
640                                       ajulong mergeNode, ajulong anchorNode);
641 
642 static ajulong       btreeCollapseRootHybOne(AjPBtcache cache, ajulong pagepos);
643 
644 static AjBool        btreeDeleteIdentIdTwo(AjPBtcache cache,
645                                            const AjPBtId btid,
646                                            AjPBtId did);
647 
648 static ajulong       btreeFindHybBalanceTwo(AjPBtcache cache, ajulong thisNode,
649                                             ajulong leftNode, ajulong rightNode,
650                                             ajulong lAnchor, ajulong rAnchor,
651                                             ajulong key);
652 
653 static void          btreeFindHybMinTwo(AjPBtcache cache, ajulong pagepos,
654                                         ajulong key);
655 
656 static AjBool        btreeRemoveHybEntryTwo(AjPBtcache cache, ajulong pagepos,
657                                             ajulong key);
658 
659 static void          btreeAdjustHybbucketsTwo(AjPBtcache cache, AjPBtpage leaf);
660 
661 static ajulong       btreeRebalanceHybTwo(AjPBtcache cache, ajulong thisNode,
662                                           ajulong leftNode, ajulong rightNode,
663                                           ajulong lAnchor, ajulong rAnchor);
664 
665 static ajulong       btreeShiftHybTwo(AjPBtcache cache, ajulong thisNode,
666                                       ajulong balanceNode, ajulong anchorNode);
667 
668 static ajulong       btreeMergeHybTwo(AjPBtcache cache, ajulong thisNode,
669                                       ajulong mergeNode, ajulong anchorNode);
670 
671 static ajulong       btreeCollapseRootHybTwo(AjPBtcache cache, ajulong pagepos);
672 
673 static ajulong       btreeFindPriBalanceTwo(AjPBtcache cache, ajulong thisNode,
674                                             ajulong leftNode, ajulong rightNode,
675                                             ajulong lAnchor, ajulong rAnchor,
676                                             const AjPBtPri pri);
677 
678 static void          btreeFindPriMinTwo(AjPBtcache cache, ajulong pagepos,
679                                         const AjPStr key);
680 
681 static AjBool        btreeRemovePriEntryTwo(AjPBtcache cache, ajulong pagepos,
682                                             const AjPBtPri pri);
683 
684 static void          btreeAdjustPribucketsTwo(AjPBtcache cache, AjPBtpage leaf);
685 
686 static ajulong       btreeRebalancePriTwo(AjPBtcache cache, ajulong thisNode,
687                                           ajulong leftNode, ajulong rightNode,
688                                           ajulong lAnchor, ajulong rAnchor);
689 
690 static ajulong       btreeShiftPriTwo(AjPBtcache cache, ajulong thisNode,
691                                       ajulong balanceNode, ajulong anchorNode);
692 
693 static ajulong       btreeMergePriTwo(AjPBtcache cache, ajulong thisNode,
694                                       ajulong mergeNode, ajulong anchorNode);
695 
696 static ajulong       btreeCollapseRootPriTwo(AjPBtcache cache, ajulong pagepos);
697 
698 
699 static ajulong       btreeFindPriBalanceOne(AjPBtcache cache, ajulong thisNode,
700                                             ajulong leftNode, ajulong rightNode,
701                                             ajulong lAnchor, ajulong rAnchor,
702                                             const AjPBtPri pri);
703 
704 static void          btreeFindPriMinOne(AjPBtcache cache, ajulong pagepos,
705                                         const AjPStr key);
706 
707 static AjBool        btreeRemovePriEntryOne(AjPBtcache cache, ajulong pagepos,
708                                             const AjPBtPri pri);
709 
710 static void          btreeAdjustPribucketsOne(AjPBtcache cache, AjPBtpage leaf);
711 
712 static ajulong       btreeRebalancePriOne(AjPBtcache cache, ajulong thisNode,
713                                           ajulong leftNode, ajulong rightNode,
714                                           ajulong lAnchor, ajulong rAnchor);
715 
716 static ajulong       btreeShiftPriOne(AjPBtcache cache, ajulong thisNode,
717                                       ajulong balanceNode, ajulong anchorNode);
718 
719 static ajulong       btreeMergePriOne(AjPBtcache cache, ajulong thisNode,
720                                       ajulong mergeNode, ajulong anchorNode);
721 
722 static ajulong       btreeCollapseRootPriOne(AjPBtcache cache, ajulong pagepos);
723 
724 static AjBool        btreeIsSecEmpty(AjPBtcache cache);
725 
726 static AjBool        btreeBucketSplitCalc(ajuint totalkeys,
727                                           ajuint totalbickets,
728                                           ajuint maxbucketsize,
729                                           ajuint *leftbuckets,
730                                           ajuint *leftmax,
731                                           ajuint *leftkeys,
732                                           ajuint *rightbuckets,
733                                           ajuint *rightmax,
734                                           ajuint *rightkeys);
735 static AjBool        btreeBucketCalc(ajuint totalkeys, ajuint totalbuckets,
736                                      ajuint maxbucketsize,
737                                      ajuint *newbuckets, ajuint *newmax);
738 static const char*   btreeNodetype(const unsigned char* buf);
739 static void          btreeStatNumnode(AjPBtcache cache, const AjPBtpage page);
740 static AjBool        btreeCheckNode(AjPBtcache cache, const AjPBtpage page);
741 static AjBool        btreeCheckNumnode(AjPBtcache cache, const AjPBtpage page);
742 static AjBool        btreeCheckNodeHeader(AjPBtcache cache,
743                                           const AjPBtpage page,
744                                           const char* type);
745 static void          btreePripageClear(AjPBtcache cache, AjPBtpage page);
746 static void          btreeSecpageClear(AjPBtcache cache, AjPBtpage page);
747 static void          btreePripageSetfree(AjPBtcache cache, ajulong pagepos);
748 static void          btreeSecpageSetfree(AjPBtcache cache, ajulong pagepos);
749 static void          btreeFieldInit(void);
750 static BtreePField   btreeFieldNewField(const BtreePFielddef field);
751 static BtreePField   btreeFieldNewC(const char* nametxt);
752 static BtreePField   btreeFieldNewS(const AjPStr name);
753 static void          btreeFieldDel(BtreePField *Pfield);
754 static void          btreeFieldMapDel(void** key, void** value, void* cl);
755 
756 #if AJINDEX_STATIC
757 /*
758 ** debugging functions - define only if used in compiled code
759 */
760 static AjBool        btreeKeyidVerify(AjPBtcache cache, ajulong rootblock,
761                                       const AjPStr id);
762 static AjPList       btreeKeyidListDuplicates(AjPBtcache cache,
763                                               const AjPStr key);
764 static void          btreeLockTest(AjPBtcache cache);
765 static void          btreePageDump(const AjPBtcache cache,
766                                    const AjPBtpage page);
767 #endif
768 
769 #if 0
770 /*
771 ** functions only used in commented-out code
772 ** e.g. overflow pages
773 */
774 
775 static void          btreeInsertIdOnly(AjPBtcache cache,
776                                        const AjPBtPri pri);
777 static AjPBtpage     btreeCacheWriteIdbucket(AjPBtcache cache,
778                                              ajulong pagepos);
779 static AjBool        btreeHybbucketsReorder(AjPBtcache cache, AjPBtpage leaf);
780 static void          btreeHybbucketAdd(AjPBtcache cache, ajulong pagepos,
781                                        const AjPBtHybrid id);
782 static ajulong       btreeHybbucketIdlist(AjPBtcache cache, ajulong pagepos,
783                                           AjPList idlist);
784 static AjBool        btreeIdbucketSplitCalc(ajuint totalkeys,
785                                             ajuint totalbickets,
786                                             ajuint maxbucketsize,
787                                             ajuint *leftbuckets,
788                                             ajuint *leftmax,
789                                             ajuint *leftkeys,
790                                             ajuint *rightbuckets,
791                                             ajuint *rightmax,
792                                             ajuint *rightkeys);
793 static AjBool        btreeIdbucketCalc(ajuint totalkeys, ajuint totalbuckets,
794                                        ajuint maxbucketsize,
795                                        ajuint *newbuckets, ajuint *newmax);
796 static void          btreeHybbucketAddFull(AjPBtcache cache, ajulong pagepos,
797                                            const AjPBtHybrid id);
798 static void          btreeIdbucketAddFull(AjPBtcache cache, ajulong pagepos,
799                                           const AjPBtId id);
800 static void          btreeSecbucketAddFull(AjPBtcache cache, ajulong pagepos,
801                                            const AjPStr id);
802 static void          btreePribucketAddFull(AjPBtcache cache, ajulong pagepos,
803                                            const AjPStr keyword,
804                                            const AjPStr id);
805 static AjPBtpage     btreeCacheWriteOverflownew(AjPBtcache cache);
806 #endif
807 
808 
809 
810 
811 /* @funcstatic btreeFieldInit *************************************************
812 **
813 ** Initialises the table for named fields
814 **
815 ** @return [void]
816 **
817 ** @release 6.4.0
818 ******************************************************************************/
819 
btreeFieldInit(void)820 static void btreeFieldInit(void)
821 {
822     ajuint i;
823     BtreePField field = NULL;
824 
825     if(btreeFieldsTable)
826         return;
827 
828     btreeFieldsTable = ajTablestrNewConst(30);
829 
830     for(i=0; btreeFields[i].Name; i++)
831     {
832         field = btreeFieldNewField(&btreeFields[i]);
833         ajTablePut(btreeFieldsTable,
834                    field->Name,
835                    (void*) field);
836         field = btreeFieldNewField(&btreeFields[i]);
837         ajTablePut(btreeFieldsTable,
838                    field->Extension,
839                    (void*) field);
840     }
841 
842     return;
843 }
844 
845 
846 
847 
848 /* @funcstatic btreeFieldNewC *************************************************
849 **
850 ** Construct a new named field with default values
851 **
852 ** @param [r] nametxt [const char*] Field name
853 ** @return [BtreePField] New field object
854 **
855 ** @release 6.4.0
856 ******************************************************************************/
857 
btreeFieldNewC(const char * nametxt)858 static BtreePField btreeFieldNewC(const char* nametxt)
859 {
860     BtreePField ret;
861     ajuint ilen;
862 
863     ilen = strlen(nametxt);
864 
865     AJNEW0(ret);
866 
867     ret->Name = ajStrNewC(nametxt);
868     ret->Extension = ajStrNewRes(ilen+2);
869     ajStrAssignK(&ret->Extension, 'x');
870     ajStrAppendC(&ret->Extension, nametxt);
871     ret->Secondary = ajTrue;
872     ret->Len = 15;
873 
874     return ret;
875 }
876 
877 
878 
879 
880 /* @funcstatic btreeFieldNewS *************************************************
881 **
882 ** Construct a new named field with default values
883 **
884 ** @param [r] name [const AjPStr] Field name
885 ** @return [BtreePField] New field object
886 **
887 ** @release 6.4.0
888 ******************************************************************************/
889 
btreeFieldNewS(const AjPStr name)890 static BtreePField btreeFieldNewS(const AjPStr name)
891 {
892     BtreePField ret;
893     ajuint ilen;
894 
895     ilen = MAJSTRGETLEN(name);
896 
897     AJNEW0(ret);
898 
899     ret->Name = ajStrNewS(name);
900     ret->Extension = ajStrNewRes(ilen+2);
901     ajStrAssignK(&ret->Extension, 'x');
902     ajStrAppendS(&ret->Extension, name);
903     ret->Secondary = ajTrue;
904     ret->Len = 15;
905 
906     return ret;
907 }
908 
909 
910 
911 
912 /* @funcstatic btreeFieldNewField *********************************************
913 **
914 ** Construct copy of a known field
915 **
916 ** @param [r] field [const BtreePFielddef] Source field
917 ** @return [BtreePField] New field object
918 **
919 ** @release 6.4.0
920 ******************************************************************************/
921 
btreeFieldNewField(const BtreePFielddef field)922 static BtreePField btreeFieldNewField(const BtreePFielddef field)
923 {
924     BtreePField ret = NULL;
925 
926     AJNEW0(ret);
927 
928     ret->Name = ajStrNewC(field->Name);
929     ret->Extension = ajStrNewC(field->Extension);
930     ret->Secondary = field->Secondary;
931     ret->Len = field->Len;
932 
933     return ret;
934 }
935 
936 
937 
938 
939 /* @funcstatic btreeFieldDel **************************************************
940 **
941 ** Destructor for a named field
942 **
943 ** @param [d] Pfield [BtreePField*] Field object
944 ** @return [void]
945 **
946 ** @release 6.4.0
947 ******************************************************************************/
948 
btreeFieldDel(BtreePField * Pfield)949 static void btreeFieldDel(BtreePField *Pfield)
950 {
951     BtreePField field = *Pfield;
952 
953     ajStrDel(&field->Name);
954     ajStrDel(&field->Extension);
955     AJFREE(field);
956     *Pfield = NULL;
957 
958     return;
959 }
960 
961 
962 
963 
964 /* @funcstatic btreeFieldMapDel ***********************************************
965 **
966 ** Deletes entries from the MSF internal table. Called for each entry in turn.
967 **
968 ** @param [d] key [void**] Standard argument, table key.
969 ** @param [d] value [void**] Standard argument, table data item.
970 ** @param [r] cl [void*] Standard argument, usually NULL
971 ** @return [void]
972 **
973 ** @release 6.4.0
974 ** @@
975 ******************************************************************************/
976 
btreeFieldMapDel(void ** key,void ** value,void * cl)977 static void btreeFieldMapDel(void** key, void** value, void* cl)
978 {
979     BtreePField field;
980 
981     (void) cl;
982 
983     field = (BtreePField) *value;
984 
985     btreeFieldDel(&field);
986 
987     *key = NULL;
988     *value = NULL;
989 
990     return;
991 }
992 
993 
994 
995 
996 /* @func ajBtreeFieldGetLenC **************************************************
997 **
998 ** Returns the default indexed name length for a named field
999 **
1000 ** @param [r] nametxt [const char*] Field name
1001 ** @return [ajuint] Field name length
1002 **
1003 ** @release 6.4.0
1004 ******************************************************************************/
1005 
ajBtreeFieldGetLenC(const char * nametxt)1006 ajuint ajBtreeFieldGetLenC(const char* nametxt)
1007 {
1008     ajuint ret = 15;
1009     const BtreePField field;
1010 
1011     btreeFieldInit();
1012 
1013     field = ajTableFetchC(btreeFieldsTable, nametxt);
1014 
1015     if(field)
1016         return field->Len;
1017 
1018     return ret;
1019 }
1020 
1021 
1022 
1023 
1024 /* @func ajBtreeFieldGetLenS **************************************************
1025 **
1026 ** Returns the default indexed name length for a named field
1027 **
1028 ** @param [r] name [const AjPStr] Field name
1029 ** @return [ajuint] Field name length
1030 **
1031 ** @release 6.4.0
1032 ******************************************************************************/
1033 
ajBtreeFieldGetLenS(const AjPStr name)1034 ajuint ajBtreeFieldGetLenS(const AjPStr name)
1035 {
1036     ajuint ret = 15;
1037     const BtreePField field;
1038 
1039     btreeFieldInit();
1040 
1041     field = ajTableFetchS(btreeFieldsTable, name);
1042 
1043     if(field)
1044         return field->Len;
1045 
1046     return ret;
1047 }
1048 
1049 
1050 
1051 
1052 /* @func ajBtreeFieldGetExtensionC ********************************************
1053 **
1054 ** Returns the default index file extension for a named field
1055 **
1056 ** @param [r] nametxt [const char*] Field name
1057 ** @return [const AjPStr] Filename extension
1058 **
1059 ** @release 6.4.0
1060 ******************************************************************************/
1061 
ajBtreeFieldGetExtensionC(const char * nametxt)1062 const AjPStr ajBtreeFieldGetExtensionC(const char* nametxt)
1063 {
1064     const BtreePField field;
1065     BtreePField newfield;
1066 
1067     btreeFieldInit();
1068 
1069     field = ajTableFetchC(btreeFieldsTable, nametxt);
1070     if(field)
1071         return field->Extension;
1072 
1073     newfield = btreeFieldNewC(nametxt);
1074     ajTablePut(btreeFieldsTable,
1075                newfield->Name,
1076                (void*) newfield);
1077 
1078     return newfield->Extension;
1079 }
1080 
1081 
1082 
1083 
1084 /* @func ajBtreeFieldGetExtensionS ********************************************
1085 **
1086 ** Returns the default index file extension for a named field
1087 **
1088 ** @param [r] name [const AjPStr] Field name
1089 ** @return [const AjPStr] Filename extension
1090 **
1091 ** @release 6.4.0
1092 ******************************************************************************/
1093 
ajBtreeFieldGetExtensionS(const AjPStr name)1094 const AjPStr ajBtreeFieldGetExtensionS(const AjPStr name)
1095 {
1096     const BtreePField field;
1097     BtreePField newfield;
1098 
1099     btreeFieldInit();
1100 
1101     field = ajTableFetchS(btreeFieldsTable, name);
1102     if(field)
1103         return field->Extension;
1104 
1105     newfield = btreeFieldNewS(name);
1106     ajTablePut(btreeFieldsTable,
1107                newfield->Name,
1108                (void*) newfield);
1109 
1110     return newfield->Extension;
1111 }
1112 
1113 
1114 
1115 
1116 /* @func ajBtreeFieldGetSecondaryC ********************************************
1117 **
1118 ** Returns whether a named field uses a secondary index
1119 **
1120 ** @param [r] nametxt [const char*] Field name
1121 ** @return [AjBool] True for a secondary index where indexed terms treturn IDs
1122 **                  False for an ID index where terms return file positions.
1123 **
1124 ** @release 6.4.0
1125 ******************************************************************************/
1126 
ajBtreeFieldGetSecondaryC(const char * nametxt)1127 AjBool ajBtreeFieldGetSecondaryC(const char* nametxt)
1128 {
1129     AjBool ret = ajTrue;
1130     const BtreePField field;
1131 
1132     btreeFieldInit();
1133 
1134     ajStrAssignC(&btreeFieldnameTmp, nametxt);
1135     field = ajTableFetchC(btreeFieldsTable, nametxt);
1136 
1137     if(field)
1138         return field->Secondary;
1139 
1140     return ret;
1141 }
1142 
1143 
1144 
1145 
1146 /* @func ajBtreeFieldGetSecondaryS ********************************************
1147 **
1148 ** Returns whether a named field uses a secondary index
1149 **
1150 ** @param [r] name [const AjPStr] Field name
1151 ** @return [AjBool] True for a secondary index where indexed terms treturn IDs
1152 **                  False for an ID index where terms return file positions.
1153 **
1154 ** @release 6.4.0
1155 ******************************************************************************/
1156 
ajBtreeFieldGetSecondaryS(const AjPStr name)1157 AjBool ajBtreeFieldGetSecondaryS(const AjPStr name)
1158 {
1159     AjBool ret = ajTrue;
1160     const BtreePField field;
1161 
1162     btreeFieldInit();
1163 
1164     field = ajTableFetchS(btreeFieldsTable, name);
1165     if(field)
1166         return field->Secondary;
1167 
1168     return ret;
1169 }
1170 
1171 
1172 
1173 
1174 /* @func ajBtreeIdcacheNewC ***************************************************
1175 **
1176 s** Open a b+tree index file and initialise a cache object
1177 **
1178 ** @param [r] filetxt [const char *] name of file
1179 ** @param [r] exttxt [const char *] extension of file
1180 ** @param [r] idirtxt [const char *] index file directory
1181 ** @param [r] mode [const char *] opening mode
1182 ** @param [r] compressed [AjBool] Compressed index flag
1183 ** @param [r] kwlimit [ajuint] Max key size
1184 ** @param [r] refcount [ajuint] Extra files for each entry
1185 ** @param [r] pripagesize [ajuint] Primary pagesize
1186 ** @param [r] secpagesize [ajuint] Secondary pagesize
1187 ** @param [r] pricachesize [ajuint] size of primary cache
1188 ** @param [r] seccachesize [ajuint] size of secondary cache
1189 ** @param [r] pripagecount [ajulong] Primary page count
1190 ** @param [r] secpagecount [ajulong] Secondary page count
1191 ** @param [r] order [ajuint] Tree order
1192 ** @param [r] fill [ajuint] Number of entries per bucket
1193 ** @param [r] level [ajuint] level of tree
1194 ** @param [r] sorder [ajuint] order of secondary tree
1195 ** @param [r] sfill [ajuint] Number of entries per secondary bucket
1196 ** @param [r] count [ajulong] Number of entries in the index
1197 ** @param [r] countall [ajulong] Number of total entries in the index
1198 **
1199 ** @return [AjPBtcache] initialised disc block cache structure
1200 **
1201 ** @release 3.0.0
1202 ** @@
1203 ******************************************************************************/
1204 
ajBtreeIdcacheNewC(const char * filetxt,const char * exttxt,const char * idirtxt,const char * mode,AjBool compressed,ajuint kwlimit,ajuint refcount,ajuint pripagesize,ajuint secpagesize,ajuint pricachesize,ajuint seccachesize,ajulong pripagecount,ajulong secpagecount,ajuint order,ajuint fill,ajuint level,ajuint sorder,ajuint sfill,ajulong count,ajulong countall)1205 AjPBtcache ajBtreeIdcacheNewC(const char *filetxt, const char *exttxt,
1206                               const char *idirtxt, const char *mode,
1207                               AjBool compressed,
1208                               ajuint kwlimit, ajuint refcount,
1209                               ajuint pripagesize, ajuint secpagesize,
1210                               ajuint pricachesize, ajuint seccachesize,
1211                               ajulong pripagecount, ajulong secpagecount,
1212                               ajuint order, ajuint fill, ajuint level,
1213                               ajuint sorder, ajuint sfill,
1214                               ajulong count, ajulong countall)
1215 {
1216     AjPBtcache cache = NULL;
1217     AjPBtpage  page = NULL;
1218 #if defined (usestat64)
1219     struct stat64 buf;
1220 #else
1221     struct stat buf;
1222 #endif
1223 
1224     ajulong filelen = 0UL;
1225     AjBool douncompress = ajFalse;
1226     AjBool writemode = ajFalse;
1227     AjBool okcache = ajTrue;
1228 
1229     AJNEW0(cache);
1230 
1231     cache->prilistLength = 0;
1232     cache->seclistLength = 0;
1233 
1234     cache->plru   = NULL;
1235     cache->pmru   = NULL;
1236     cache->slru   = NULL;
1237     cache->smru   = NULL;
1238 
1239     cache->replace    = ajStrNew();
1240     cache->numreplace = 0UL;
1241 
1242     if(pripagesize>0)
1243 	cache->pripagesize = pripagesize;
1244     else
1245 	cache->pripagesize = BT_PAGESIZE;
1246 
1247     cache->plevel       = level;
1248     cache->porder       = order;
1249     cache->pnperbucket  = fill;
1250     cache->pricachesize = pricachesize;
1251     cache->refcount = refcount;
1252 
1253     cache->pripagecount = pripagecount;
1254     cache->secpagecount = secpagecount;
1255 
1256     cache->sorder = sorder;
1257     cache->slevel = 0;
1258     cache->snperbucket = sfill;
1259 
1260     if(secpagesize > 0)
1261 	cache->secpagesize = secpagesize;
1262     else
1263 	cache->secpagesize = cache->pripagesize;
1264 
1265     if(seccachesize > 0)
1266         cache->seccachesize = seccachesize;
1267     else
1268         cache->seccachesize = cache->pricachesize;
1269 
1270     cache->countunique = count;
1271     cache->countall = countall;
1272     cache->keylimit = kwlimit;
1273 
1274     cache->compressed = compressed;
1275 
1276 
1277     cache->bmem = NULL;
1278     cache->tmem = NULL;
1279 
1280     cache->bsmem = NULL;
1281     cache->tsmem = NULL;
1282 
1283     cache->pripagetable = ajTableulongNewConst(cache->pricachesize);
1284     cache->secpagetable = ajTableulongNewConst(cache->seccachesize);
1285 
1286     cache->basename = ajStrNewC(exttxt);
1287 
1288     cache->filename = ajStrNew();
1289     if(!*idirtxt)
1290         ajFmtPrintS(&cache->filename,"%s.%s",filetxt,exttxt);
1291     else if(idirtxt[strlen(idirtxt)-1] == SLASH_CHAR)
1292         ajFmtPrintS(&cache->filename,"%s%s.%s",idirtxt,filetxt,exttxt);
1293     else
1294         ajFmtPrintS(&cache->filename,"%s%s%s.%s",idirtxt,SLASH_STRING,
1295 		    filetxt,exttxt);
1296 
1297     if(cache->porder < 4)
1298     {
1299         ajErr("cache '%S' pagesize %u order %u too small, increase pagesize",
1300               cache->filename, cache->pripagesize, cache->porder);
1301         okcache = ajFalse;
1302     }
1303 
1304     if(cache->pnperbucket < 4)
1305     {
1306         ajErr("cache '%S' pagesize %u fill %u too small, increase pagesize",
1307               cache->filename, cache->pripagesize, cache->pnperbucket);
1308         okcache = ajFalse;
1309     }
1310 
1311     if(cache->sorder < 4)
1312     {
1313         ajErr("cache '%S' pagesize %u srder %u too small, increase pagesize",
1314               cache->filename, cache->pripagesize, cache->sorder);
1315         okcache = ajFalse;
1316     }
1317 
1318     if(cache->snperbucket < 4)
1319     {
1320         ajErr("cache '%S' pagesize %u sfill %u too small, increase pagesize",
1321               cache->filename, cache->secpagesize, cache->snperbucket);
1322         okcache = ajFalse;
1323     }
1324 
1325     if(!okcache)
1326         return NULL;
1327 
1328 
1329     cache->fp = fopen(MAJSTRGETPTR(cache->filename),mode);
1330 
1331     if(!cache->fp)
1332 	return NULL;
1333 
1334 
1335     if(ajCharMatchC(mode, "rb")) /* read-only */
1336     {
1337 #if defined (usestat64)
1338 	if(!stat64(MAJSTRGETPTR(cache->filename), &buf))
1339 #else
1340         if(!stat(MAJSTRGETPTR(cache->filename), &buf))
1341 #endif
1342             filelen = buf.st_size;
1343 
1344         cache->readonly = ajTrue;
1345     }
1346     else if(ajCharMatchC(mode, "rb+")) /* update */
1347     {
1348 #if defined (usestat64)
1349 	if(!stat64(MAJSTRGETPTR(cache->filename), &buf))
1350 #else
1351         if(!stat(MAJSTRGETPTR(cache->filename), &buf))
1352 #endif
1353             filelen = buf.st_size;
1354         if(compressed)
1355             douncompress = ajTrue;
1356     }
1357     else if(ajCharMatchC(mode, "wb+")) /* create */
1358     {
1359         writemode = ajTrue;
1360     }
1361     else
1362     {
1363         ajWarn("ajBtreeIdcacheNewC unknown mode '%s'", mode);
1364     }
1365 
1366     cache->totsize     = filelen;
1367     cache->filesize    = filelen;
1368     cache->maxsize = filelen;
1369 
1370     if(writemode)
1371     {
1372         if(cache->maxsize)
1373             cache->maxsize += cache->maxsize/2;
1374         else
1375             cache->maxsize = cache->pricachesize*cache->pripagesize +
1376                 cache->seccachesize * cache->secpagesize;
1377         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
1378         {
1379         }
1380     }
1381 
1382     if(douncompress)
1383         btreeCacheUncompress(cache);
1384 
1385     /* create or lock the root page */
1386 
1387     if(writemode)
1388         btreePrirootCreate(cache);
1389     else
1390     {
1391         page = btreePricacheRead(cache,0UL);
1392         page->dirty = BT_LOCK;
1393     }
1394 
1395     return cache;
1396 }
1397 
1398 
1399 
1400 
1401 /* @func ajBtreeIdcacheNewS ***************************************************
1402 **
1403 ** Open a b+tree index file and initialise a cache object
1404 **
1405 ** @param [r] file [const AjPStr] name of file
1406 ** @param [r] ext [const AjPStr] extension of file
1407 ** @param [r] idir [const AjPStr] index file directory
1408 ** @param [r] mode [const char *] opening mode
1409 ** @param [r] compressed [AjBool] Compressed index flag
1410 ** @param [r] refcount [ajuint] Extra files for each entry
1411 ** @param [r] kwlimit [ajuint] Max key size
1412 ** @param [r] pripagesize [ajuint] Primary pagesize
1413 ** @param [r] secpagesize [ajuint] Secondary pagesize
1414 ** @param [r] pricachesize [ajuint] size of primary cache
1415 ** @param [r] seccachesize [ajuint] size of secondary cache
1416 ** @param [r] pripagecount [ajulong] Primary page count
1417 ** @param [r] secpagecount [ajulong] Secondary page count
1418 ** @param [r] order [ajuint] Tree order
1419 ** @param [r] fill [ajuint] Number of entries per bucket
1420 ** @param [r] level [ajuint] level of tree
1421 ** @param [r] sorder [ajuint] order of secondary tree
1422 ** @param [r] sfill [ajuint] Number of entries per secondary bucket
1423 ** @param [r] count [ajulong] Number of entries in the index
1424 ** @param [r] countall [ajulong] Number of total entries in the index
1425 **
1426 ** @return [AjPBtcache] initialised disc block cache structure
1427 **
1428 ** @release 6.4.0
1429 ** @@
1430 ******************************************************************************/
1431 
ajBtreeIdcacheNewS(const AjPStr file,const AjPStr ext,const AjPStr idir,const char * mode,AjBool compressed,ajuint refcount,ajuint kwlimit,ajuint pripagesize,ajuint secpagesize,ajuint pricachesize,ajuint seccachesize,ajulong pripagecount,ajulong secpagecount,ajuint order,ajuint fill,ajuint level,ajuint sorder,ajuint sfill,ajulong count,ajulong countall)1432 AjPBtcache ajBtreeIdcacheNewS(const AjPStr file, const AjPStr ext,
1433                               const AjPStr idir, const char *mode,
1434                               AjBool compressed,
1435                               ajuint refcount,
1436                               ajuint kwlimit,
1437                               ajuint pripagesize, ajuint secpagesize,
1438                               ajuint pricachesize, ajuint seccachesize,
1439                               ajulong pripagecount, ajulong secpagecount,
1440                               ajuint order, ajuint fill, ajuint level,
1441                               ajuint sorder, ajuint sfill,
1442                               ajulong count, ajulong countall)
1443 {
1444     return ajBtreeIdcacheNewC(MAJSTRGETPTR(file), MAJSTRGETPTR(ext),
1445                               MAJSTRGETPTR(idir), mode,
1446                               compressed,  refcount, kwlimit,
1447                               pripagesize, secpagesize,
1448                               pricachesize, seccachesize,
1449                               pripagecount, secpagecount,
1450                               order, fill, level,
1451                               sorder, sfill,
1452                               count, countall);
1453 }
1454 
1455 
1456 
1457 
1458 /* @func ajBtreeCacheNewReadC *************************************************
1459 **
1460 ** Open an existing b+tree index file and initialise a cache object
1461 **
1462 ** @param [r] filetxt [const char *] name of file
1463 ** @param [r] exttxt [const char *] extension of file
1464 ** @param [r] idirtxt [const char *] index file directory
1465 **
1466 ** @return [AjPBtcache] initialised disc block cache structure
1467 **
1468 ** @release 6.4.0
1469 ** @@
1470 ******************************************************************************/
1471 
ajBtreeCacheNewReadC(const char * filetxt,const char * exttxt,const char * idirtxt)1472 AjPBtcache ajBtreeCacheNewReadC(const char *filetxt, const char *exttxt,
1473                                 const char *idirtxt)
1474 {
1475     AjPBtcache cache = NULL;
1476 
1477     ajuint pripagesize;
1478     ajuint pricachesize;
1479     ajuint secpagesize;
1480     ajuint seccachesize;
1481     ajuint order;
1482     ajuint sorder;
1483     ajuint fill;
1484     ajuint sfill;
1485     ajuint level;
1486     ajulong pripagecount;
1487     ajulong secpagecount;
1488     ajulong count;
1489     ajulong countall;
1490     ajuint kwlimit;
1491     ajuint idlimit;
1492     ajuint refcount;
1493     AjBool secondary = ajTrue;
1494     AjBool compressed = ajFalse;
1495 
1496 /* first read the parameter file */
1497 
1498     if(!ajBtreeReadParamsC(filetxt, exttxt, idirtxt,
1499                            &secondary, &compressed,
1500                            &kwlimit, &idlimit, &refcount,
1501                            &pripagesize, &secpagesize,
1502                            &pricachesize, &seccachesize,
1503                            &pripagecount, &secpagecount,
1504                            &order, &fill, &level,
1505                            &sorder, &sfill,
1506                            &count, &countall))
1507     {
1508          return ajFalse;
1509     }
1510 
1511     if(secondary)
1512         cache = ajBtreeSeccacheNewC(filetxt, exttxt, idirtxt, "rb",
1513                                     compressed,
1514                                     kwlimit, idlimit,
1515                                     pripagesize, secpagesize,
1516                                     pricachesize, seccachesize,
1517                                     pripagecount, secpagecount,
1518                                     order, fill, level,
1519                                     sorder, sfill,
1520                                     count, countall);
1521     else
1522         cache = ajBtreeIdcacheNewC(filetxt, exttxt, idirtxt, "rb",
1523                                    compressed,
1524                                    kwlimit, refcount,
1525                                    pripagesize, secpagesize,
1526                                    pricachesize, seccachesize,
1527                                    pripagecount, secpagecount,
1528                                    order, fill, level,
1529                                    sorder, sfill,
1530                                    count, countall);
1531 
1532     if(cache)
1533         cache->secondary = secondary;
1534 
1535     return cache;
1536 }
1537 
1538 
1539 
1540 
1541 /* @func ajBtreeCacheNewReadS *************************************************
1542 **
1543 ** Open an existing b+tree index file and initialise a cache object
1544 **
1545 ** @param [r] file [const AjPStr] name of file
1546 ** @param [r] ext  [const AjPStr] extension of file
1547 ** @param [r] idir [const AjPStr] index file directory
1548 **
1549 ** @return [AjPBtcache] initialised disc block cache structure
1550 **
1551 ** @release 6.4.0
1552 ** @@
1553 ******************************************************************************/
1554 
ajBtreeCacheNewReadS(const AjPStr file,const AjPStr ext,const AjPStr idir)1555 AjPBtcache ajBtreeCacheNewReadS(const AjPStr file, const AjPStr ext,
1556                                 const AjPStr idir)
1557 {
1558     return ajBtreeCacheNewReadC(MAJSTRGETPTR(file), MAJSTRGETPTR(ext),
1559                                 MAJSTRGETPTR(idir));
1560 }
1561 
1562 
1563 
1564 
1565 /* @func ajBtreeCacheNewUpdateC ***********************************************
1566 **
1567 ** Open an existing b+tree index file for update and initialise a cache object
1568 **
1569 ** @param [r] filetxt [const char *] name of file
1570 ** @param [r] exttxt [const char *] extension of file
1571 ** @param [r] idirtxt [const char *] index file directory
1572 **
1573 ** @return [AjPBtcache] initialised disc block cache structure
1574 **
1575 ** @release 6.4.0
1576 ** @@
1577 ******************************************************************************/
1578 
ajBtreeCacheNewUpdateC(const char * filetxt,const char * exttxt,const char * idirtxt)1579 AjPBtcache ajBtreeCacheNewUpdateC(const char *filetxt, const char *exttxt,
1580                                   const char *idirtxt)
1581 {
1582     AjPBtcache cache = NULL;
1583 
1584     ajuint pripagesize;
1585     ajuint secpagesize;
1586     ajuint pricachesize;
1587     ajuint seccachesize;
1588     ajuint order;
1589     ajuint sorder;
1590     ajuint fill;
1591     ajuint sfill;
1592     ajuint level;
1593     ajulong pripagecount;
1594     ajulong secpagecount;
1595     ajulong count;
1596     ajulong countall;
1597     ajuint kwlimit;
1598     ajuint idlimit;
1599     ajuint refcount;
1600     AjBool secondary = ajTrue;
1601     AjBool compressed = ajFalse;
1602 
1603 /* first read the parameter file */
1604 
1605     if(!ajBtreeReadParamsC(filetxt, exttxt, idirtxt,
1606                            &secondary, &compressed,
1607                            &kwlimit, &idlimit, &refcount,
1608                            &pripagesize, &secpagesize,
1609                            &pricachesize, &seccachesize,
1610                            &pripagecount, &secpagecount,
1611                            &order, &fill, &level,
1612                            &sorder, &sfill,
1613                            &count, &countall))
1614     {
1615          return ajFalse;
1616     }
1617 
1618     if(secondary)
1619         cache = ajBtreeSeccacheNewC(filetxt, exttxt, idirtxt, "rb+",
1620                                     compressed, kwlimit, idlimit,
1621                                     pripagesize, secpagesize,
1622                                     pricachesize, seccachesize,
1623                                     pripagecount, secpagecount,
1624                                     order, fill, level,
1625                                     sorder, sfill,
1626                                     count, countall);
1627     else
1628         cache = ajBtreeIdcacheNewC(filetxt, exttxt, idirtxt, "rb+",
1629                                    compressed, kwlimit, refcount,
1630                                    pripagesize, secpagesize,
1631                                    pricachesize, seccachesize,
1632                                    pripagecount, secpagecount,
1633                                    order, fill, level,
1634                                    sorder, sfill,
1635                                    count, countall);
1636 
1637     if(cache)
1638         cache->secondary = secondary;
1639 
1640     return cache;
1641 }
1642 
1643 
1644 
1645 
1646 /* @func ajBtreeCacheNewUpdateS ***********************************************
1647 **
1648 ** Open an existing b+tree index file for update and initialise a cache object
1649 **
1650 ** @param [r] file [const AjPStr] name of file
1651 ** @param [r] ext  [const AjPStr] extension of file
1652 ** @param [r] idir [const AjPStr] index file directory
1653 **
1654 ** @return [AjPBtcache] initialised disc block cache structure
1655 **
1656 ** @release 6.4.0
1657 ** @@
1658 ******************************************************************************/
1659 
ajBtreeCacheNewUpdateS(const AjPStr file,const AjPStr ext,const AjPStr idir)1660 AjPBtcache ajBtreeCacheNewUpdateS(const AjPStr file, const AjPStr ext,
1661                                   const AjPStr idir)
1662 {
1663     return ajBtreeCacheNewUpdateC(MAJSTRGETPTR(file), MAJSTRGETPTR(ext),
1664                                   MAJSTRGETPTR(idir));
1665 }
1666 
1667 
1668 
1669 
1670 /* @funcstatic btreePripageNew ************************************************
1671 **
1672 ** Construct a cache page object
1673 **
1674 ** @param [u] cache [AjPBtcache] cache
1675 **
1676 ** @return [AjPBtpage] initialised disc block cache structure
1677 **
1678 ** @release 2.8.0
1679 ** @@
1680 ******************************************************************************/
1681 
btreePripageNew(AjPBtcache cache)1682 static AjPBtpage btreePripageNew(AjPBtcache cache)
1683 {
1684     AjPBtpage thys = NULL;
1685     ajuint nodetype;
1686 
1687     /* ajDebug("In btreePripageNew\n"); */
1688 
1689     AJNEW0(thys);
1690     AJCNEW0(thys->buf,cache->pripagesize);
1691     nodetype     = BT_FREEPAGE;
1692     SBT_NODETYPE(thys->buf,nodetype);
1693 
1694     thys->next = NULL;
1695     thys->prev = NULL;
1696 
1697     ++cache->prilistLength;
1698 
1699 
1700     return thys;
1701 }
1702 
1703 
1704 
1705 
1706 /* @funcstatic btreeSecpageNew ************************************************
1707 **
1708 ** Construct a cache secondary page object
1709 **
1710 ** @param [u] cache [AjPBtcache] cache
1711 **
1712 ** @return [AjPBtpage] initialised disc block cache structure
1713 **
1714 ** @release 2.8.0
1715 ** @@
1716 ******************************************************************************/
1717 
btreeSecpageNew(AjPBtcache cache)1718 static AjPBtpage btreeSecpageNew(AjPBtcache cache)
1719 {
1720     AjPBtpage thys = NULL;
1721     ajuint nodetype;
1722 
1723     /* ajDebug("In btreeSecpageNew\n"); */
1724 
1725     AJNEW0(thys);
1726     AJCNEW0(thys->buf,cache->secpagesize);
1727 
1728     nodetype     = BT_SECFREEPAGE;
1729     SBT_NODETYPE(thys->buf,nodetype);
1730 
1731     thys->next = NULL;
1732     thys->prev = NULL;
1733 
1734     ++cache->seclistLength;
1735 
1736 
1737     return thys;
1738 }
1739 
1740 
1741 
1742 
1743 /* @funcstatic btreeIdbucketNew ***********************************************
1744 **
1745 ** Construct an id bucket object
1746 **
1747 ** @param [r] n [ajuint] Number of IDs
1748 ** @param [r] refcount [ajuint] Number pf reference offsets
1749 ** @return [AjPIdbucket] initialised id bucket
1750 **
1751 ** @release 2.8.0
1752 ** @@
1753 ******************************************************************************/
1754 
btreeIdbucketNew(ajuint n,ajuint refcount)1755 static AjPIdbucket btreeIdbucketNew(ajuint n, ajuint refcount)
1756 {
1757     AjPIdbucket bucket = NULL;
1758     AjPBtId Id = NULL;
1759     ajuint i;
1760 
1761     /* ajDebug("In btreeIdbucketNew\n"); */
1762 
1763     if(n)
1764     {
1765         if(statSaveIdbucketNext)
1766         {
1767             bucket = statSaveIdbucket[--statSaveIdbucketNext];
1768 
1769             for(i=0;i<bucket->Maxentries;++i)
1770             {
1771                 Id = bucket->Ids[i];
1772                 ajStrAssignClear(&Id->id);
1773                 if(refcount)
1774                     AJCRESIZE0(Id->refoffsets, Id->refcount, refcount);
1775                 else
1776                     AJFREE(Id->refoffsets);
1777             }
1778 
1779             if(n > bucket->Maxentries)
1780             {
1781                 AJCRESIZE0(bucket->keylen,bucket->Maxentries,n);
1782                 AJCRESIZE0(bucket->Ids,bucket->Maxentries,n);
1783                 for(i=bucket->Maxentries;i<n;++i)
1784                     bucket->Ids[i] = ajBtreeIdNew(refcount);
1785                 bucket->Maxentries = n;
1786             }
1787         }
1788         else
1789         {
1790             AJNEW0(bucket);
1791             AJCNEW0(bucket->Ids,n);
1792             AJCNEW0(bucket->keylen,n);
1793             for(i=0;i<n;++i)
1794                 bucket->Ids[i] = ajBtreeIdNew(refcount);
1795             bucket->Maxentries = n;
1796 
1797         }
1798     }
1799     else
1800     {
1801         if(statSaveIdbucketEmptyNext)
1802         {
1803             bucket = statSaveIdbucketEmpty[--statSaveIdbucketEmptyNext];
1804             for(i=0;i<bucket->Maxentries;++i)
1805             {
1806                 Id = bucket->Ids[i];
1807                 ajStrAssignClear(&Id->id);
1808                 if(refcount)
1809                     AJCRESIZE0(Id->refoffsets, Id->refcount, refcount);
1810                 else
1811                     AJFREE(Id->refoffsets);
1812             }
1813         }
1814         else
1815             AJNEW0(bucket);
1816     }
1817 
1818     bucket->NodeType = BT_IDBUCKET;
1819     bucket->Nentries = n;
1820     bucket->Overflow = 0UL;
1821 
1822     return bucket;
1823 }
1824 
1825 
1826 
1827 
1828 /* @funcstatic btreePricacheLocate ********************************************
1829 **
1830 ** Search for a page in the primary cache
1831 **
1832 ** @param [u] cache [AjPBtcache] cache structure
1833 ** @param [r] page [ajulong] page number to locate
1834 **
1835 ** @return [AjPBtpage]	pointer to page or NULL if not found
1836 **
1837 ** @release 6.5.0
1838 ** @@
1839 ******************************************************************************/
1840 
btreePricacheLocate(AjPBtcache cache,ajulong page)1841 static AjPBtpage btreePricacheLocate(AjPBtcache cache, ajulong page)
1842 {
1843     AjPBtpage cpage = NULL;
1844 
1845     /*ajDebug("In btreePricacheLocate %Lu\n", page);*/
1846 
1847 
1848     cpage = ajTableFetchmodV(cache->pripagetable, (const void*) &page);
1849 
1850     if(!cpage)
1851         return NULL;
1852 
1853     cache->pricachehits++;
1854 
1855     if(!ajBtreePageIsPrimary(cpage))
1856         ajWarn("btreePricacheLocate secondary page %Lu '%s'",
1857                cpage->pagepos, btreeNodetype(cpage->buf));
1858 
1859     return cpage;
1860 }
1861 
1862 
1863 
1864 
1865 /* @funcstatic btreeSeccacheLocate ********************************************
1866 **
1867 ** Search for a page in the secondary cache
1868 **
1869 ** @param [u] cache [AjPBtcache] cache structure
1870 ** @param [r] page [ajulong] page number to locate
1871 **
1872 ** @return [AjPBtpage]	pointer to page or NULL if not found
1873 **
1874 ** @release 6.5.0
1875 ** @@
1876 ******************************************************************************/
1877 
btreeSeccacheLocate(AjPBtcache cache,ajulong page)1878 static AjPBtpage btreeSeccacheLocate(AjPBtcache cache, ajulong page)
1879 {
1880     AjPBtpage cpage = NULL;
1881 
1882     /*ajDebug("In btreeSeccacheLocate %Lu\n", page);*/
1883 
1884 
1885     cpage = ajTableFetchmodV(cache->secpagetable, (const void*) &page);
1886 
1887     if(!cpage)
1888         return NULL;
1889 
1890     cache->seccachehits++;
1891 
1892     if(ajBtreePageIsPrimary(cpage))
1893         ajWarn("btreeSeccacheLocate primary page %Lu '%s'",
1894                cpage->pagepos, btreeNodetype(cpage->buf));
1895 
1896     return cpage;
1897 }
1898 
1899 
1900 
1901 
1902 /* @funcstatic btreePricacheUnlink ********************************************
1903 **
1904 ** Remove links to a primary cache page and return the address of the page
1905 **
1906 ** @param [w] cache [AjPBtcache] cache structure
1907 ** @param [u] cpage [AjPBtpage] cache page
1908 **
1909 ** @return [void]
1910 **
1911 ** @release 6.5.0
1912 ** @@
1913 ******************************************************************************/
1914 
btreePricacheUnlink(AjPBtcache cache,AjPBtpage cpage)1915 static void btreePricacheUnlink(AjPBtcache cache, AjPBtpage cpage)
1916 {
1917     /* ajDebug("In btreePricacheUnlink\n"); */
1918 
1919     if(!ajBtreePageIsPrimary(cpage))
1920         ajWarn("btreePricacheUnlink secondary page %Lu '%s'",
1921                cpage->pagepos, btreeNodetype(cpage->buf));
1922 
1923     ajTableRemove(cache->pripagetable, (void*) &cpage->pagepos);
1924 
1925     if(cache->pmru == cpage)
1926     {
1927 	cache->pmru = cpage->prev;
1928 
1929 	if(cpage->prev)
1930 	    cpage->prev->next = NULL;
1931 
1932 	if(cache->plru == cpage)
1933 	    cache->plru = NULL;
1934     }
1935     else if(cache->plru == cpage)
1936     {
1937 	cache->plru = cpage->next;
1938 
1939 	if(cpage->next)
1940 	    cpage->next->prev = NULL;
1941     }
1942     else
1943     {
1944 	cpage->prev->next = cpage->next;
1945 	cpage->next->prev = cpage->prev;
1946     }
1947 
1948     return;
1949 }
1950 
1951 
1952 
1953 
1954 /* @funcstatic btreeSeccacheUnlink ********************************************
1955 **
1956 ** Remove links to a secondary cache page and return the address of the page
1957 **
1958 ** @param [w] cache [AjPBtcache] cache structure
1959 ** @param [u] cpage [AjPBtpage] cache page
1960 **
1961 ** @return [void]
1962 **
1963 ** @release 6.5.0
1964 ** @@
1965 ******************************************************************************/
1966 
btreeSeccacheUnlink(AjPBtcache cache,AjPBtpage cpage)1967 static void btreeSeccacheUnlink(AjPBtcache cache, AjPBtpage cpage)
1968 {
1969     /* ajDebug("In btreeSeccacheUnlink\n"); */
1970 
1971     if(ajBtreePageIsPrimary(cpage))
1972         ajWarn("btreeSeccacheUnlink primary page %Lu '%s'",
1973                cpage->pagepos, btreeNodetype(cpage->buf));
1974 
1975     ajTableRemove(cache->secpagetable, (void*) &cpage->pagepos);
1976 
1977     if(cache->smru == cpage)
1978     {
1979 	cache->smru = cpage->prev;
1980 
1981 	if(cpage->prev)
1982 	    cpage->prev->next = NULL;
1983 
1984 	if(cache->slru == cpage)
1985 	    cache->slru = NULL;
1986     }
1987     else if(cache->slru == cpage)
1988     {
1989 	cache->slru = cpage->next;
1990 
1991 	if(cpage->next)
1992 	    cpage->next->prev = NULL;
1993     }
1994     else
1995     {
1996 	cpage->prev->next = cpage->next;
1997 	cpage->next->prev = cpage->prev;
1998     }
1999 
2000     return;
2001 }
2002 
2003 
2004 
2005 
2006 /* @funcstatic btreePricacheMruAdd ********************************************
2007 **
2008 ** Insert a primary cache page at the mru position
2009 **
2010 ** @param [w] cache [AjPBtcache] cache structure
2011 ** @param [u] cpage [AjPBtpage] cache page
2012 **
2013 ** @return [void]
2014 **
2015 ** @release 6.5.0
2016 ** @@
2017 ******************************************************************************/
2018 
btreePricacheMruAdd(AjPBtcache cache,AjPBtpage cpage)2019 static void btreePricacheMruAdd(AjPBtcache cache, AjPBtpage cpage)
2020 {
2021 
2022     /* ajDebug("In btreeCacheMruAdd\n"); */
2023 
2024     if(!ajBtreePageIsPrimary(cpage))
2025         ajWarn("btreePricacheMruAdd secondary page %Lu '%s'",
2026                cpage->pagepos, btreeNodetype(cpage->buf));
2027 
2028     ajTablePut(cache->pripagetable,
2029                (void*) &cpage->pagepos, (void*) cpage);
2030 
2031     cpage->prev = cache->pmru;
2032     cpage->next = NULL;
2033 
2034     if(cache->pmru)
2035 	cache->pmru->next = cpage;
2036 
2037     if(!cache->plru)
2038 	cache->plru = cpage;
2039 
2040     cache->pmru = cpage;
2041 
2042     return;
2043 }
2044 
2045 
2046 
2047 
2048 /* @funcstatic btreeSeccacheMruAdd ********************************************
2049 **
2050 ** Insert a secondary cache page at the mru position
2051 **
2052 ** @param [w] cache [AjPBtcache] cache structure
2053 ** @param [u] cpage [AjPBtpage] cache page
2054 **
2055 ** @return [void]
2056 **
2057 ** @release 6.5.0
2058 ** @@
2059 ******************************************************************************/
2060 
btreeSeccacheMruAdd(AjPBtcache cache,AjPBtpage cpage)2061 static void btreeSeccacheMruAdd(AjPBtcache cache, AjPBtpage cpage)
2062 {
2063 
2064     /* ajDebug("In btreeSeccacheMruAdd\n"); */
2065 
2066     if(ajBtreePageIsPrimary(cpage))
2067         ajWarn("btreeSeccacheMruAdd primary page %Lu '%s'",
2068                cpage->pagepos, btreeNodetype(cpage->buf));
2069 
2070     ajTablePut(cache->secpagetable,
2071                (void*) &cpage->pagepos, (void*) cpage);
2072 
2073     cpage->prev = cache->smru;
2074     cpage->next = NULL;
2075 
2076     if(cache->smru)
2077 	cache->smru->next = cpage;
2078 
2079     if(!cache->slru)
2080 	cache->slru = cpage;
2081 
2082     cache->smru = cpage;
2083 
2084     return;
2085 }
2086 
2087 
2088 
2089 
2090 /* @funcstatic btreePricacheLruUnlink *****************************************
2091 **
2092 ** Remove links to an LRU primary cache page
2093 **
2094 ** @param [w] cache [AjPBtcache] cache structure
2095 **
2096 ** @return [AjPBtpage]	pointer to unlinked cache page
2097 **
2098 ** @release 6.5.0
2099 ** @@
2100 ******************************************************************************/
2101 
btreePricacheLruUnlink(AjPBtcache cache)2102 static AjPBtpage btreePricacheLruUnlink(AjPBtcache cache)
2103 {
2104     AjPBtpage ret;
2105 
2106     /* ajDebug("In btreePricacheLruUnlink\n"); */
2107 
2108     if(cache->plru->dirty != BT_LOCK)
2109     {
2110 	if(!cache->plru)
2111 	    ajFatal("btreePricacheLruUnlink: No pages nodes found in cache %S",
2112                     cache->filename);
2113 
2114 	ret = cache->plru;
2115 	ret->next->prev = NULL;
2116 	cache->plru = ret->next;
2117         ajTableRemove(cache->pripagetable, (void*) &ret->pagepos);
2118 
2119         if(!ajBtreePageIsPrimary(ret))
2120             ajWarn("btreePricacheLruUnlink secondary page %Lu '%s'",
2121                    ret->pagepos, btreeNodetype(ret->buf));
2122 
2123         return ret;
2124     }
2125 
2126     for(ret=cache->plru; ret; ret=ret->next)
2127 	if(ret->dirty != BT_LOCK)
2128 	    break;
2129 
2130     if(!ret)
2131 	ajFatal("Too many locked cache page in primary cache %S. "
2132                 "Try increasing cachesize (%u)",
2133                 cache->filename, cache->pricachesize);
2134 
2135     btreePricacheUnlink(cache,ret);
2136 
2137     return ret;
2138 }
2139 
2140 
2141 
2142 
2143 /* @funcstatic btreeSeccacheLruUnlink *****************************************
2144 **
2145 ** Remove links to an LRU secondary cache page
2146 **
2147 ** @param [w] cache [AjPBtcache] cache structure
2148 **
2149 ** @return [AjPBtpage]	pointer to unlinked cache page
2150 **
2151 ** @release 6.5.0
2152 ** @@
2153 ******************************************************************************/
2154 
btreeSeccacheLruUnlink(AjPBtcache cache)2155 static AjPBtpage btreeSeccacheLruUnlink(AjPBtcache cache)
2156 {
2157     AjPBtpage ret;
2158 
2159     /* ajDebug("In btreeSeccacheLruUnlink\n"); */
2160 
2161     if(cache->slru->dirty != BT_LOCK)
2162     {
2163 	if(!cache->slru)
2164 	    ajFatal("btreeSeccacheLruUnlink: No pages nodes found in cache %S",
2165                     cache->filename);
2166 
2167 	ret = cache->slru;
2168 	ret->next->prev = NULL;
2169 	cache->slru = ret->next;
2170 
2171         ajTableRemove(cache->secpagetable, (void*) &ret->pagepos);
2172 
2173         if(ajBtreePageIsPrimary(ret))
2174             ajWarn("btreeSeccacheLruUnlink primary page %Lu '%s'",
2175                    ret->pagepos, btreeNodetype(ret->buf));
2176 
2177         return ret;
2178     }
2179 
2180     for(ret=cache->slru; ret; ret=ret->next)
2181 	if(ret->dirty != BT_LOCK)
2182 	    break;
2183 
2184     if(!ret)
2185 	ajFatal("Too many locked cache page in secondary cache %S. "
2186                 "Try increasing cachesize (%u)",
2187                 cache->filename, cache->seccachesize);
2188 
2189     btreeSeccacheUnlink(cache,ret);
2190 
2191     return ret;
2192 }
2193 
2194 
2195 
2196 
2197 /* @funcstatic btreePricacheDestage *******************************************
2198 **
2199 ** Destage a primary cache page
2200 **
2201 ** @param [u] cache [AjPBtcache] cache
2202 ** @param [u] cpage [AjPBtpage] cache page
2203 **
2204 ** @return [void]
2205 **
2206 ** @release 6.5.0
2207 ** @@
2208 ******************************************************************************/
2209 
btreePricacheDestage(AjPBtcache cache,AjPBtpage cpage)2210 static void btreePricacheDestage(AjPBtcache cache, AjPBtpage cpage)
2211 {
2212     ajuint written = 0;
2213     ajuint retries = 0;
2214 
2215     /* ajDebug("In btreePricacheDestage\n");*/
2216 
2217     if(!ajBtreePageIsPrimary(cpage))
2218         ajWarn("btreePricacheDestage secondary page %Lu '%s'",
2219                cpage->pagepos, btreeNodetype(cpage->buf));
2220 
2221     if(fseek(cache->fp, cpage->pagepos,SEEK_SET))
2222     {
2223         ajWarn("fseek %Lu failed for cache '%S' %d: '%s'",
2224                cpage->pagepos, cache->filename,
2225                ferror(cache->fp), strerror(ferror(cache->fp)));
2226 	fseek(cache->fp,0L,SEEK_END);
2227     }
2228 
2229     while(written != cache->pripagesize && retries != BT_MAXRETRIES)
2230     {
2231 	written += fwrite((void *)(cpage->buf+written),1,
2232                           cache->pripagesize-written,
2233 			  cache->fp);
2234 	++retries;
2235     }
2236 
2237     if(retries == BT_MAXRETRIES)
2238 	ajFatal("Maximum retries (%u) reached in btreePricacheDestage "
2239                 "cache %S error: '%s'",
2240 		BT_MAXRETRIES, cache->filename, strerror(ferror(cache->fp)));
2241 
2242     cpage->dirty = BT_CLEAN;
2243 
2244     cache->priwrites++;
2245 
2246     return;
2247 }
2248 
2249 
2250 
2251 
2252 /* @funcstatic btreeSeccacheDestage *******************************************
2253 **
2254 ** Destage a secondary cache page
2255 **
2256 ** @param [u] cache [AjPBtcache] cache
2257 ** @param [u] cpage [AjPBtpage] cache page
2258 **
2259 ** @return [void]
2260 **
2261 ** @release 6.5.0
2262 ** @@
2263 ******************************************************************************/
2264 
btreeSeccacheDestage(AjPBtcache cache,AjPBtpage cpage)2265 static void btreeSeccacheDestage(AjPBtcache cache, AjPBtpage cpage)
2266 {
2267     ajuint written = 0;
2268     ajuint retries = 0;
2269 
2270     /* ajDebug("In btreeSeccacheDestage\n");*/
2271 
2272     if(ajBtreePageIsPrimary(cpage))
2273         ajWarn("btreeSeccacheDestage primary page %Lu '%s'",
2274                cpage->pagepos, btreeNodetype(cpage->buf));
2275 
2276     if(fseek(cache->fp, cpage->pagepos,SEEK_SET))
2277     {
2278         ajWarn("fseek %Lu failed for cache '%S' %d: '%s'",
2279                cpage->pagepos, cache->filename,
2280                ferror(cache->fp), strerror(ferror(cache->fp)));
2281 	fseek(cache->fp,0L,SEEK_END);
2282     }
2283 
2284     while(written != cache->secpagesize && retries != BT_MAXRETRIES)
2285     {
2286 	written += fwrite((void *)(cpage->buf+written),1,
2287                           cache->secpagesize-written,
2288 			  cache->fp);
2289 	++retries;
2290     }
2291 
2292     if(retries == BT_MAXRETRIES)
2293 	ajFatal("Maximum retries (%u) reached in btreeSeccacheDestage "
2294                 "cache %S error: '%s'",
2295 		BT_MAXRETRIES, cache->filename, strerror(ferror(cache->fp)));
2296 
2297     cpage->dirty = BT_CLEAN;
2298 
2299     cache->secwrites++;
2300 
2301     return;
2302 }
2303 
2304 
2305 
2306 
2307 /* @funcstatic btreeCacheFetchSize ********************************************
2308 **
2309 ** Fetch a cache page from disc
2310 **
2311 ** @param [u] cache [AjPBtcache] cache
2312 ** @param [w] cpage [AjPBtpage] cache page
2313 ** @param [r] pagepos [ajulong] page number
2314 ** @param [r] pagesize [ajuint] page size
2315 **
2316 ** @return [void]
2317 **
2318 ** @release 2.8.0
2319 ** @@
2320 ******************************************************************************/
2321 
btreeCacheFetchSize(AjPBtcache cache,AjPBtpage cpage,ajulong pagepos,ajuint pagesize)2322 static void btreeCacheFetchSize(AjPBtcache cache, AjPBtpage cpage,
2323                                 ajulong pagepos, ajuint pagesize)
2324 {
2325     ajuint sum = 0;
2326     ajuint retries = 0;
2327 
2328     /* ajDebug("In btreeCacheFetchSize\n"); */
2329 
2330     if(fseek(cache->fp,pagepos,SEEK_SET))
2331 	ajFatal("Seek error %d: '%s' in ajBtreeCacheFetchSize file %S",
2332                 ferror(cache->fp),
2333                 strerror(ferror(cache->fp)),
2334                 cache->filename);
2335 
2336     while(sum != pagesize && retries != BT_MAXRETRIES)
2337     {
2338 	sum += fread((void *)(cpage->buf+sum),1,pagesize-sum,
2339 		     cache->fp);
2340 
2341 	++retries;
2342     }
2343 
2344     if(retries == BT_MAXRETRIES)
2345 	ajFatal("Maximum retries (%u) reached in btreeCacheFetchSize "
2346                 "for page %Lu cache '%S'",
2347 		BT_MAXRETRIES,pagepos, cache->filename);
2348 
2349     cpage->pagepos = pagepos;
2350 
2351     return;
2352 }
2353 
2354 
2355 
2356 
2357 /* @funcstatic btreePricacheFetch *********************************************
2358 **
2359 ** Fetch a cache primary page from disc
2360 **
2361 ** @param [u] cache [AjPBtcache] cache
2362 ** @param [w] cpage [AjPBtpage] cache page
2363 ** @param [r] pagepos [ajulong] page number
2364 **
2365 ** @return [void]
2366 **
2367 ** @release 6.5.0
2368 ** @@
2369 ******************************************************************************/
2370 
btreePricacheFetch(AjPBtcache cache,AjPBtpage cpage,ajulong pagepos)2371 static void btreePricacheFetch(AjPBtcache cache, AjPBtpage cpage,
2372                                ajulong pagepos)
2373 {
2374     ajuint sum = 0;
2375     ajuint retries = 0;
2376 
2377     /* ajDebug("In btreePricacheFetch\n"); */
2378 
2379     if(fseek(cache->fp,pagepos,SEEK_SET))
2380 	ajFatal("Seek error %d: '%s' in ajBtreePricacheFetch file %S",
2381                 ferror(cache->fp),
2382                 strerror(ferror(cache->fp)),
2383                 cache->filename);
2384 
2385     while(sum != cache->pripagesize && retries != BT_MAXRETRIES)
2386     {
2387 	sum += fread((void *)(cpage->buf+sum),1,cache->pripagesize-sum,
2388 		     cache->fp);
2389 
2390 	++retries;
2391     }
2392 
2393     if(retries == BT_MAXRETRIES)
2394 	ajFatal("Maximum retries (%u) reached in btreePricacheFetch "
2395                 "for page %Lu cache '%S'",
2396 		BT_MAXRETRIES,pagepos, cache->filename);
2397 
2398     cpage->pagepos = pagepos;
2399     cache->prireads++;
2400 
2401     return;
2402 }
2403 
2404 
2405 
2406 
2407 /* @funcstatic btreeSeccacheFetch *********************************************
2408 **
2409 ** Fetch a cache secondary page from disc
2410 **
2411 ** @param [u] cache [AjPBtcache] cache
2412 ** @param [w] cpage [AjPBtpage] cache page
2413 ** @param [r] pagepos [ajulong] page number
2414 **
2415 ** @return [void]
2416 **
2417 ** @release 6.5.0
2418 ** @@
2419 ******************************************************************************/
2420 
btreeSeccacheFetch(AjPBtcache cache,AjPBtpage cpage,ajulong pagepos)2421 static void btreeSeccacheFetch(AjPBtcache cache, AjPBtpage cpage,
2422                                ajulong pagepos)
2423 {
2424     ajuint sum = 0;
2425     ajuint retries = 0;
2426 
2427     /* ajDebug("In btreeSeccacheFetch\n"); */
2428 
2429     if(fseek(cache->fp,pagepos,SEEK_SET))
2430 	ajFatal("Seek error %d: '%s' in ajBtreeSeccacheFetch file %S",
2431                 ferror(cache->fp),
2432                 strerror(ferror(cache->fp)),
2433                 cache->filename);
2434 
2435     while(sum != cache->secpagesize && retries != BT_MAXRETRIES)
2436     {
2437 	sum += fread((void *)(cpage->buf+sum),1,cache->secpagesize-sum,
2438 		     cache->fp);
2439 
2440 	++retries;
2441     }
2442 
2443     if(retries == BT_MAXRETRIES)
2444 	ajFatal("Maximum retries (%u) reached in btreeSeccacheFetch "
2445                 "for page %Lu cache '%S'",
2446 		BT_MAXRETRIES,pagepos, cache->filename);
2447 
2448     cpage->pagepos = pagepos;
2449     cache->secreads++;
2450 
2451     return;
2452 }
2453 
2454 
2455 
2456 
2457 /* @funcstatic btreeCacheWriteCompress ****************************************
2458 **
2459 ** Write a compressed cache page
2460 **
2461 ** @param [u] cache [AjPBtcache] cache
2462 ** @param [u] cpage [AjPBtpage] cache page
2463 ** @param [r] pagepos [ajulong] Cache file position
2464 ** @param [r] pagesize [ajuint] Compressed page size
2465 **
2466 ** @return [void]
2467 **
2468 ** @release 6.4.0
2469 ** @@
2470 ******************************************************************************/
2471 
btreeCacheWriteCompress(AjPBtcache cache,AjPBtpage cpage,ajulong pagepos,ajuint pagesize)2472 static void btreeCacheWriteCompress(AjPBtcache cache, AjPBtpage cpage,
2473                                     ajulong pagepos, ajuint pagesize)
2474 {
2475     ajuint written = 0;
2476     ajuint retries = 0;
2477 
2478     /* ajDebug("In btreeCacheWriteCompress\n");*/
2479 
2480     if(fseek(cache->fp, pagepos,SEEK_SET))
2481     {
2482         ajWarn("fseek %Lu failed for cache '%S' %d: '%s'",
2483                pagepos, cache->filename, errno, strerror(errno));
2484 	fseek(cache->fp,0L,SEEK_END);
2485     }
2486 
2487     while(written != pagesize && retries != BT_MAXRETRIES)
2488     {
2489 	written += fwrite((void *)(cpage->buf+written),1,pagesize-written,
2490 			  cache->fp);
2491 	++retries;
2492     }
2493 
2494     if(retries == BT_MAXRETRIES)
2495 	ajFatal("Maximum retries (%u) reached in btreeCacheWriteCompress "
2496                 "cache %S error: '%s'",
2497 		BT_MAXRETRIES, cache->filename, strerror(ferror(cache->fp)));
2498 
2499     return;
2500 }
2501 
2502 
2503 
2504 
2505 /* @funcstatic btreeCacheWriteUncompress **************************************
2506 **
2507 ** Rewrite an uncompressed cache page
2508 **
2509 ** @param [u] cache [AjPBtcache] cache
2510 ** @param [u] cpage [AjPBtpage] cache page
2511 ** @param [r] pagepos [ajulong] Cache file position
2512 ** @param [r] pagesize [ajuint] Page size
2513 **
2514 ** @return [void]
2515 **
2516 ** @release 6.4.0
2517 ** @@
2518 ******************************************************************************/
2519 
btreeCacheWriteUncompress(AjPBtcache cache,AjPBtpage cpage,ajulong pagepos,ajuint pagesize)2520 static void btreeCacheWriteUncompress(AjPBtcache cache, AjPBtpage cpage,
2521                                       ajulong pagepos, ajuint pagesize)
2522 {
2523     ajuint written = 0;
2524     ajuint retries = 0;
2525 
2526     /* ajDebug("In btreeCacheWriteCompress\n");*/
2527 
2528     if(fseek(cache->fp, pagepos,SEEK_SET))
2529     {
2530         ajWarn("fseek %Lu failed for cache '%S' %d: '%s'",
2531                pagepos, cache->filename, errno, strerror(errno));
2532 	fseek(cache->fp,0L,SEEK_END);
2533     }
2534 
2535     while(written != pagesize && retries != BT_MAXRETRIES)
2536     {
2537 	written += fwrite((void *)(cpage->buf+written),1,pagesize-written,
2538 			  cache->fp);
2539 	++retries;
2540     }
2541 
2542     if(retries == BT_MAXRETRIES)
2543 	ajFatal("Maximum retries (%u) reached in btreeCacheWriteUncompress "
2544                 "cache %S error: '%s'",
2545 		BT_MAXRETRIES, cache->filename, strerror(ferror(cache->fp)));
2546 
2547     return;
2548 }
2549 
2550 
2551 
2552 
2553 /* @funcstatic btreeCacheCompress *********************************************
2554 **
2555 ** Compress a b+tree cache on closing
2556 **
2557 ** @param [u] thys [AjPBtcache] Expanded cache to be compressed
2558 **
2559 ** @return [ajulong] File size after compression
2560 **
2561 ** @release 6.4.0
2562 ** @@
2563 ******************************************************************************/
2564 
btreeCacheCompress(AjPBtcache thys)2565 static ajulong btreeCacheCompress(AjPBtcache thys)
2566 {
2567     AjPTable  newpostable;
2568     ajulong  *oldpagepos = NULL;
2569     ajulong  *newpagepos = NULL;
2570     ajuint    oldpagesize;
2571     ajuint    newpagesize;
2572     AjPBtpage page = NULL;
2573     ajuint    freepages = 0;
2574     ajulong   oldpos = 0UL;
2575     ajulong   newpos = 0UL;
2576     ajulong   newsize = 0UL;
2577 
2578     ajuint   *newpagesizes = NULL;
2579     ajulong   i;
2580     ajuint    maxpagesize = 0U;
2581     ajuint    minpagesize = 0U;
2582 
2583     ajulong   pagecount = thys->pripagecount + thys->secpagecount;
2584 
2585     if(thys->secpagesize > thys->pripagesize)
2586     {
2587         page = btreeSecpageNew(thys);
2588         maxpagesize = thys->secpagesize;
2589         minpagesize = thys->pripagesize;
2590     }
2591     else
2592     {
2593         page = btreePripageNew(thys);
2594         maxpagesize = thys->pripagesize;
2595         minpagesize = thys->secpagesize;
2596     }
2597 
2598     AJCNEW(oldpagepos, pagecount);
2599     AJCNEW(newpagepos, pagecount);
2600     AJCNEW(newpagesizes, pagecount);
2601 
2602     newpostable = ajTableulongNewConst((ajint) pagecount);
2603 
2604     oldpos = 0UL;
2605     newpos = 0UL;
2606     newsize = 0UL;
2607 
2608     for(i=0; i < pagecount; i++)
2609     {
2610         btreeCacheFetchSize(thys, page, oldpos, minpagesize);
2611 
2612         if(ajBtreePageIsPrimary(page))
2613             oldpagesize = thys->pripagesize;
2614         else
2615             oldpagesize = thys->secpagesize;
2616 
2617         btreeCacheFetchSize(thys, page, oldpos, oldpagesize);
2618 
2619         newpagesize = ajBtreePageGetSize(page, thys->refcount);
2620 
2621         oldpagepos[i] = oldpos;
2622         newpagepos[i] = newpos;
2623         newpagesizes[i] = newpagesize;
2624 
2625         ajTablePut(newpostable, (void*) &oldpagepos[i],
2626                    (void*) &newpagepos[i]);
2627 
2628         oldpos += oldpagesize;
2629         newpos += newpagesize;
2630     }
2631 
2632     newsize = newpos;
2633     newsize += maxpagesize - newpagesizes[pagecount-1];
2634     newpagesizes[pagecount-1] = maxpagesize;
2635 
2636     oldpos = 0UL;
2637 
2638     for(i=0; i < pagecount; i++)
2639     {
2640         btreeCacheFetchSize(thys, page, oldpos, minpagesize);
2641         if(ajBtreePageIsPrimary(page))
2642             oldpagesize = thys->pripagesize;
2643         else
2644             oldpagesize = thys->secpagesize;
2645 
2646         btreeCacheFetchSize(thys, page, oldpos, oldpagesize);
2647 
2648         oldpos += oldpagesize;
2649 
2650         if(!btreePageCompress(page, newpostable, thys->refcount))
2651         {
2652             freepages++;
2653             continue;
2654         }
2655 
2656         btreeCacheWriteCompress(thys, page, newpagepos[i], newpagesizes[i]);
2657     }
2658 
2659     thys->filesize = newsize;
2660 
2661     AJFREE(oldpagepos);
2662     AJFREE(newpagepos);
2663     AJFREE(newpagesizes);
2664 
2665     ajTableFree(&newpostable);
2666 
2667     AJFREE(page->buf);
2668     AJFREE(page);
2669 
2670     return newsize;
2671 }
2672 
2673 
2674 
2675 
2676 /* @funcstatic btreeCacheUncompress *******************************************
2677 **
2678 ** Uncompress a b+tree cache on opening
2679 **
2680 ** @param [u] thys [AjPBtcache] Compressed cache to be expanded
2681 **
2682 ** @return [ajulong] File size compressed
2683 **
2684 ** @release 6.4.0
2685 ** @@
2686 *******************************************************************************/
2687 
btreeCacheUncompress(AjPBtcache thys)2688 static ajulong btreeCacheUncompress(AjPBtcache thys)
2689 {
2690     AjPTable  newpostable;
2691     ajulong  *oldpagepos = NULL;
2692     ajulong  *newpagepos = NULL;
2693     ajuint    oldpagesize;
2694     ajuint    newpagesize;
2695     AjPBtpage page = NULL;
2696     ajuint    freepages = 0U;
2697     ajulong   newsize = 0UL;
2698     ajulong   oldpos = 0UL;
2699     ajulong   newpos = 0UL;
2700 
2701     ajulong   i;
2702     ajuint    icnt;
2703     ajulong   writepos;
2704     ajulong   oldsize;
2705     ajuint    pagesize;
2706     ajuint    writepagesize;
2707     AjBool    isprimary;
2708 
2709     ajulong   pagecount = thys->pripagecount + thys->secpagecount;
2710 
2711     ajDebug("btreeCacheUncompress %S size: %Lu pages: %Lu + %Lu\n",
2712             thys->filename, thys->totsize,
2713             thys->pripagecount, thys->secpagecount);
2714 
2715     if(thys->secpagesize > thys->pripagesize)
2716     {
2717         page = btreeSecpageNew(thys);
2718         pagesize = thys->secpagesize;
2719     }
2720     else
2721     {
2722         page = btreePripageNew(thys);
2723         pagesize = thys->pripagesize;
2724     }
2725 
2726     oldsize = thys->totsize;
2727 
2728     AJCNEW0(oldpagepos, pagecount);
2729     AJCNEW0(newpagepos, pagecount);
2730     newpostable = ajTableulongNewConst((ajint) pagecount);
2731 
2732     oldpos = 0UL;
2733     newpos = 0UL;
2734 
2735     for(i=0; i < pagecount; i++)
2736     {
2737         btreeCacheFetchSize(thys, page, oldpos, pagesize);
2738         oldpagesize = ajBtreePageGetSize(page, thys->refcount);
2739         if(ajBtreePageIsPrimary(page))
2740             newpagesize = thys->pripagesize;
2741         else
2742             newpagesize = thys->secpagesize;
2743 
2744         oldpagepos[i] = oldpos;
2745         newpagepos[i] = newpos;
2746 
2747         ajTablePut(newpostable, (void*) &oldpagepos[i],
2748                    (void*) &newpagepos[i]);
2749 
2750         oldpos += oldpagesize;
2751         newpos += newpagesize;
2752     }
2753 
2754     thys->totsize = (ajulong) (thys->pripagesize * thys->pripagecount) +
2755         (ajulong) (thys->secpagesize * thys->secpagecount);
2756 
2757     writepos = thys->totsize;
2758 
2759     oldpos = 0UL;
2760     newpos = 0UL;
2761 
2762     for(i=pagecount; i > 0; i--)
2763     {
2764         btreeCacheFetchSize(thys, page, oldpagepos[i-1], pagesize);
2765 
2766         if(!btreePageUncompress(page, newpostable,
2767                                 thys->refcount))
2768         {
2769             freepages++;
2770             continue;
2771         }
2772 
2773         oldpagesize = ajBtreePageGetSize(page, thys->refcount);
2774 
2775         isprimary = ajBtreePageIsPrimary(page);
2776         if(isprimary)
2777             writepagesize = thys->pripagesize;
2778         else
2779             writepagesize = thys->secpagesize;
2780 
2781         icnt=writepagesize - oldpagesize;
2782         if(icnt)
2783             memset(page->buf+oldpagesize, 0, icnt);
2784 
2785         if(writepos < writepagesize)
2786             ajErr("in btreeCacheWriteUncompress writepos %Lu writepagesize %u",
2787                   writepos, writepagesize);
2788 
2789         writepos -= writepagesize;
2790 
2791         btreeCacheWriteUncompress(thys, page,
2792                                   writepos, writepagesize);
2793 
2794     }
2795 
2796     thys->filesize = newsize;
2797 
2798     AJFREE(oldpagepos);
2799     AJFREE(newpagepos);
2800 
2801     ajTableFree(&newpostable);
2802 
2803     AJFREE(page->buf);
2804     AJFREE(page);
2805 
2806     return oldsize;
2807 }
2808 
2809 
2810 
2811 
2812 /* @func ajBtreeCacheDel ******************************************************
2813 **
2814 ** Close a b+tree cache
2815 **
2816 ** @param [w] Pthis [AjPBtcache*] Cache object
2817 **
2818 ** @return [ajulong] Index file size
2819 **
2820 ** @release 3.0.0
2821 ** @@
2822 ******************************************************************************/
2823 
ajBtreeCacheDel(AjPBtcache * Pthis)2824 ajulong ajBtreeCacheDel(AjPBtcache *Pthis)
2825 {
2826     AjPBtcache thys;
2827     AjPBtpage  page  = NULL;
2828     AjPBtpage  temp  = NULL;
2829     ajulong ret = 0UL;
2830 
2831     if(!Pthis || !*Pthis) return ret;
2832     thys = *Pthis;
2833 
2834     /* ajDebug("In ajBtreeCacheDel\n"); */
2835 
2836     /* clear the dirty / locked pages */
2837 
2838     if(!thys->readonly)
2839         btreeCacheSync(thys, 0UL);
2840 
2841     if(thys->bmem)
2842         btreeFreePriArray(thys);
2843     if(thys->bsmem)
2844         btreeFreeSecArray(thys);
2845 
2846     /* tidy up clean (read not written or locked) cached pages */
2847 
2848     for(page=thys->plru;page;page=temp)
2849     {
2850 	temp = page->next;
2851         ajTableRemove(thys->pripagetable, (void*) &page->pagepos);
2852 	AJFREE(page->buf);
2853 	AJFREE(page);
2854     }
2855 
2856     for(page=thys->slru;page;page=temp)
2857     {
2858 	temp = page->next;
2859         ajTableRemove(thys->secpagetable, (void*) &page->pagepos);
2860 	AJFREE(page->buf);
2861 	AJFREE(page);
2862     }
2863 
2864     if(!thys->readonly)
2865     {
2866         /* Ubuntu warns if ftruncate return is not used */
2867         if(ftruncate(fileno(thys->fp), thys->totsize) == -1)
2868         {
2869         }
2870 
2871         if(thys->compressed)
2872         {
2873             ret = btreeCacheCompress(thys);
2874             /* Ubuntu warns if ftruncate return is not used */
2875             if(ftruncate(fileno(thys->fp), ret) == -1)
2876             {
2877             }
2878         }
2879     }
2880     else
2881         ret = thys->totsize;
2882 
2883     ajStrDel(&thys->filename);
2884     ajStrDel(&thys->basename);
2885     ajStrDel(&thys->replace);
2886 
2887     fclose(thys->fp);
2888 
2889     ajTableFree(&thys->pripagetable);
2890     ajTableFree(&thys->secpagetable);
2891 
2892     AJFREE(*Pthis);
2893     *Pthis = NULL;
2894 
2895     return ret;
2896 }
2897 
2898 
2899 
2900 
2901 /* @func ajBtreeCacheIsCompressed *********************************************
2902 **
2903 ** Returns true if the index is compressed
2904 **
2905 ** @param [r] cache [const AjPBtcache] cache
2906 **
2907 ** @return [AjBool] True if index is compressed
2908 **
2909 ** @release 6.5.0
2910 ** @@
2911 ******************************************************************************/
2912 
ajBtreeCacheIsCompressed(const AjPBtcache cache)2913 AjBool ajBtreeCacheIsCompressed(const AjPBtcache cache)
2914 {
2915     return cache->compressed;
2916 }
2917 
2918 
2919 
2920 
2921 /* @func ajBtreeCacheIsSecondary **********************************************
2922 **
2923 ** Test whether a cache is a secondary index
2924 **
2925 ** @param [r] thys [const AjPBtcache] Cache object
2926 **
2927 ** @return [AjBool] True if cache is secondary
2928 **
2929 ** @release 6.4.0
2930 ******************************************************************************/
2931 
ajBtreeCacheIsSecondary(const AjPBtcache thys)2932 AjBool ajBtreeCacheIsSecondary(const AjPBtcache thys)
2933 {
2934     return thys->secondary;
2935 }
2936 
2937 
2938 
2939 
2940 /* @func ajBtreeCacheStatsOut *************************************************
2941 **
2942 ** Cache statistics for writing a new index
2943 **
2944 ** @param [u] outf [AjPFile] output file file
2945 ** @param [r] cache [const AjPBtcache] cache object
2946 ** @param [u] Ppricache [ajulong*] Number of cache reads to date
2947 ** @param [u] Pseccache [ajulong*] Number of cache reads to date
2948 ** @param [u] Pprireads [ajulong*] Number of disk reads to date
2949 ** @param [u] Psecreads [ajulong*] Number of disk reads to date
2950 ** @param [u] Ppriwrites [ajulong*] Number of disk writes to date
2951 ** @param [u] Psecwrites [ajulong*] Number of disk writes to date
2952 ** @param [u] Pprisize [ajulong*] Number of primary cache pages to date
2953 ** @param [u] Psecsize [ajulong*] Number of secondary cache pages to date
2954 **
2955 ** @return [void]
2956 **
2957 **
2958 ** @release 6.4.0
2959 ******************************************************************************/
2960 
ajBtreeCacheStatsOut(AjPFile outf,const AjPBtcache cache,ajulong * Ppricache,ajulong * Pseccache,ajulong * Pprireads,ajulong * Psecreads,ajulong * Ppriwrites,ajulong * Psecwrites,ajulong * Pprisize,ajulong * Psecsize)2961 void ajBtreeCacheStatsOut(AjPFile outf, const AjPBtcache cache,
2962                           ajulong* Ppricache, ajulong* Pseccache,
2963                           ajulong* Pprireads, ajulong* Psecreads,
2964                           ajulong* Ppriwrites, ajulong* Psecwrites,
2965                           ajulong* Pprisize, ajulong* Psecsize)
2966 {
2967     ajulong lpriread;
2968     ajulong lsecread;
2969     ajulong lpricache;
2970     ajulong lseccache;
2971     ajulong lpriwrite;
2972     ajulong lsecwrite;
2973     ajulong lprinew;
2974     ajulong lsecnew;
2975     ajulong lpripages;
2976     ajulong lsecpages;
2977 
2978     ajulong lprirewrite = 0UL;
2979     ajulong lsecrewrite = 0UL;
2980 
2981     if(!cache) return;
2982 
2983     lpripages = cache->pripagecount;
2984     lsecpages = cache->secpagecount;
2985     lpricache = cache->pricachehits - *Ppricache;
2986     lseccache = cache->seccachehits - *Pseccache;
2987     lpriread = cache->prireads - *Pprireads;
2988     lsecread = cache->secreads - *Psecreads;
2989     lpriwrite = cache->priwrites - *Ppriwrites;
2990     lsecwrite = cache->secwrites - *Psecwrites;
2991 
2992     if(lpripages > *Pprisize)
2993         lprinew = lpripages - *Pprisize;
2994     else
2995         lprinew = 0UL;
2996 
2997     if(lsecpages > *Psecsize)
2998         lsecnew = lsecpages - *Psecsize;
2999     else
3000         lsecnew = 0UL;
3001 
3002     /*
3003     ** writes counts those committed to disk
3004     ** newpages counts growth in index size
3005     ** pending writes may give a negative count for rewrites
3006     */
3007     if(lpriwrite >= lprinew)
3008         lprirewrite = (ajlong)lpriwrite - (ajlong)lprinew;
3009     if(lsecwrite >= lsecnew)
3010         lsecrewrite = (ajlong)lsecwrite - (ajlong)lsecnew;
3011 
3012     ajFmtPrintF(outf,
3013                 "ajBtreeCacheStatsOut '%S' cached: %8Lu/%-8Lu "
3014                 "reads: %8Lu/%-8Lu writes: %8Lu/%-8Lu "
3015                 "rewrite: %8Lu/%-8Lu pages: %8Lu/%-8Lu\n",
3016                 cache->basename,  lpricache,  lseccache,
3017                 lpriread, lsecread, lpriwrite, lsecwrite,
3018                 lprirewrite, lsecrewrite, lprinew, lsecnew);
3019 
3020     *Ppricache = cache->pricachehits;
3021     *Pseccache = cache->seccachehits;
3022     *Pprireads = cache->prireads;
3023     *Psecreads = cache->secreads;
3024     *Ppriwrites = cache->priwrites;
3025     *Psecwrites = cache->secwrites;
3026     *Pprisize = lpripages;
3027     *Psecsize = lsecpages;
3028 
3029     return;
3030 }
3031 
3032 
3033 
3034 
3035 /* @funcstatic btreePricacheControl *******************************************
3036 **
3037 ** Master control function for primary cache read/write
3038 **
3039 ** @param [w] cache [AjPBtcache] name of file
3040 ** @param [r] pagepos [ajulong] page number
3041 ** @param [r] isread [AjBool] is this a read operation?
3042 **
3043 ** @return [AjPBtpage] disc cache page pointer
3044 **
3045 ** @release 6.5.0
3046 ** @@
3047 ******************************************************************************/
3048 
btreePricacheControl(AjPBtcache cache,ajulong pagepos,AjBool isread)3049 static AjPBtpage btreePricacheControl(AjPBtcache cache, ajulong pagepos,
3050                                       AjBool isread)
3051 {
3052     AjPBtpage ret      = NULL;
3053 
3054     /* ajDebug("In btreePricacheControl\n"); */
3055 
3056     if(pagepos < cache->totsize)
3057         ret = btreePricacheLocate(cache,pagepos);
3058 
3059     if(ret)
3060     {
3061 	btreePricacheUnlink(cache,ret);
3062     }
3063     else
3064     {
3065 	if(cache->prilistLength == cache->pricachesize)
3066 	{
3067 	    ret = btreePricacheLruUnlink(cache);
3068 
3069 	    if(ret->dirty == BT_DIRTY)
3070 		btreePricacheDestage(cache,ret);
3071 
3072 	    if(isread || pagepos!=cache->totsize)
3073 		btreePricacheFetch(cache,ret,pagepos);
3074             else
3075                 btreePripageClear(cache, ret);
3076 	}
3077 	else
3078 	{
3079 	    ret = btreePripageNew(cache);
3080 
3081 	    if(isread || pagepos!=cache->totsize)
3082 		btreePricacheFetch(cache,ret,pagepos);
3083 	}
3084 
3085 	if(!isread)
3086 	    ret->pagepos = pagepos;
3087 	else
3088 	    ret->dirty = BT_CLEAN;
3089     }
3090 
3091     btreePricacheMruAdd(cache,ret);
3092 
3093     if(!ajBtreePageIsPrimary(ret))
3094         ajWarn("btreePricacheControl secondary page %Lu '%s'",
3095                ret->pagepos, btreeNodetype(ret->buf));
3096 
3097     return ret;
3098 }
3099 
3100 
3101 
3102 
3103 /* @funcstatic btreeSeccacheControl *******************************************
3104 **
3105 ** Master control function for secondary cache read/write
3106 **
3107 ** @param [w] cache [AjPBtcache] name of file
3108 ** @param [r] pagepos [ajulong] page number
3109 ** @param [r] isread [AjBool] is this a read operation?
3110 **
3111 ** @return [AjPBtpage] disc cache page pointer
3112 **
3113 ** @release 6.5.0
3114 ** @@
3115 ******************************************************************************/
3116 
btreeSeccacheControl(AjPBtcache cache,ajulong pagepos,AjBool isread)3117 static AjPBtpage btreeSeccacheControl(AjPBtcache cache, ajulong pagepos,
3118                                       AjBool isread)
3119 {
3120     AjPBtpage ret      = NULL;
3121 
3122     /* ajDebug("In btreeSeccacheControl\n"); */
3123 
3124     if(pagepos < cache->totsize)
3125         ret = btreeSeccacheLocate(cache,pagepos);
3126 
3127     if(ret)
3128     {
3129 	btreeSeccacheUnlink(cache,ret);
3130     }
3131     else
3132     {
3133 	if(cache->seclistLength == cache->seccachesize)
3134 	{
3135 	    ret = btreeSeccacheLruUnlink(cache);
3136 
3137 	    if(ret->dirty == BT_DIRTY)
3138 		btreeSeccacheDestage(cache,ret);
3139 
3140 	    if(isread || pagepos!=cache->totsize)
3141 		btreeSeccacheFetch(cache,ret,pagepos);
3142             else
3143                 btreeSecpageClear(cache, ret);
3144 	}
3145 	else
3146 	{
3147 	    ret = btreeSecpageNew(cache);
3148 
3149 	    if(isread || pagepos!=cache->totsize)
3150 		btreeSeccacheFetch(cache,ret,pagepos);
3151 	}
3152 
3153 	if(!isread)
3154 	    ret->pagepos = pagepos;
3155 	else
3156 	    ret->dirty = BT_CLEAN;
3157     }
3158 
3159     btreeSeccacheMruAdd(cache,ret);
3160 
3161     if(ajBtreePageIsPrimary(ret))
3162         ajWarn("btreeSeccacheControl primary page %Lu '%s'",
3163                ret->pagepos, btreeNodetype(ret->buf));
3164 
3165     return ret;
3166 }
3167 
3168 
3169 
3170 
3171 /* @func ajBtreeCacheRead **************************************************
3172 **
3173 ** Get a pointer to a disc cache page
3174 **
3175 ** @param [w] cache [AjPBtcache] cache
3176 ** @param [r] pagepos [ajulong] page number
3177 **
3178 ** @return [AjPBtpage] disc cache page pointer
3179 **
3180 ** @release 6.5.0
3181 ** @@
3182 ******************************************************************************/
3183 
ajBtreeCacheRead(AjPBtcache cache,ajulong pagepos)3184 AjPBtpage ajBtreeCacheRead(AjPBtcache cache, ajulong pagepos)
3185 {
3186     AjPBtpage ret = NULL;
3187 
3188     if(!btreeTestpage)
3189     {
3190         if(cache->secpagesize > cache->pripagesize)
3191             btreeTestpage = btreeSecpageNew(cache);
3192         else
3193             btreeTestpage = btreePripageNew(cache);
3194     }
3195 
3196     /* ajDebug("In ajBtreeCacheRead\n"); */
3197 
3198     btreeNocacheFetch(cache, btreeTestpage, pagepos);
3199 
3200     if(ajBtreePageIsPrimary(btreeTestpage))
3201         ret = btreePricacheControl(cache,pagepos,BT_READ);
3202     else
3203         ret = btreeSeccacheControl(cache,pagepos,BT_READ);
3204 
3205     return ret;
3206 }
3207 
3208 
3209 
3210 
3211 /* @funcstatic btreePricacheRead **********************************************
3212 **
3213 ** Get a pointer to a disc primary cache page
3214 **
3215 ** @param [w] cache [AjPBtcache] cache
3216 ** @param [r] pagepos [ajulong] page number
3217 **
3218 ** @return [AjPBtpage] disc cache page pointer
3219 **
3220 ** @release 6.5.0
3221 ** @@
3222 ******************************************************************************/
3223 
btreePricacheRead(AjPBtcache cache,ajulong pagepos)3224 static AjPBtpage btreePricacheRead(AjPBtcache cache, ajulong pagepos)
3225 {
3226     AjPBtpage ret = NULL;
3227 
3228     /* ajDebug("In btreePricacheRead\n"); */
3229 
3230     ret = btreePricacheControl(cache,pagepos,BT_READ);
3231 
3232     return ret;
3233 }
3234 
3235 
3236 
3237 
3238 /* @funcstatic btreeSeccacheRead **********************************************
3239 **
3240 ** Get a pointer to a disc secondary cache page
3241 **
3242 ** @param [w] cache [AjPBtcache] cache
3243 ** @param [r] pagepos [ajulong] page number
3244 **
3245 ** @return [AjPBtpage] disc cache page pointer
3246 **
3247 ** @release 6.5.0
3248 ** @@
3249 ******************************************************************************/
3250 
btreeSeccacheRead(AjPBtcache cache,ajulong pagepos)3251 static AjPBtpage btreeSeccacheRead(AjPBtcache cache, ajulong pagepos)
3252 {
3253     AjPBtpage ret = NULL;
3254 
3255     /* ajDebug("In btreeSeccacheRead\n"); */
3256 
3257     ret = btreeSeccacheControl(cache,pagepos,BT_READ);
3258 
3259     return ret;
3260 }
3261 
3262 
3263 
3264 
3265 /* @funcstatic btreeCacheSync *************************************************
3266 **
3267 ** Sync all dirty cache pages
3268 **
3269 ** @param [u] cache [AjPBtcache] cache
3270 ** @param [r] rootpage [ajulong] root page
3271 **
3272 ** @return [void]
3273 **
3274 ** @release 3.0.0
3275 ** @@
3276 ******************************************************************************/
3277 
btreeCacheSync(AjPBtcache cache,ajulong rootpage)3278 static void btreeCacheSync(AjPBtcache cache, ajulong rootpage)
3279 {
3280     AjPBtpage page = NULL;
3281     ajuint numlocked = 0;
3282     ajuint numwrite = 0;
3283 
3284     /* ajDebug("In btreeCacheSync\n");*/
3285 
3286     if(!cache->pmru && !cache->smru) /* cache unused*/
3287         return;
3288 
3289     statCallSync++;
3290 
3291     if(cache->pmru)
3292     {
3293         for(page=cache->plru;page;page=page->next)
3294         {
3295             if(page->dirty)         /* BT_LOCK or BT_DIRTY */
3296             {
3297                 numwrite++;
3298                 if(page->dirty == BT_LOCK)
3299                     numlocked++;
3300                 btreePricacheDestage(cache,page);
3301             }
3302         }
3303     }
3304 
3305     if(cache->smru)
3306     {
3307         for(page=cache->slru;page;page=page->next)
3308         {
3309             if(page->dirty)         /* BT_LOCK or BT_DIRTY */
3310             {
3311                 numwrite++;
3312                 if(page->dirty == BT_LOCK)
3313                     numlocked++;
3314                 btreeSeccacheDestage(cache,page);
3315             }
3316         }
3317     }
3318 
3319     if(rootpage)
3320     {
3321         if(numlocked > 2)
3322             ajWarn("CacheSync cache '%S' root: %Lu locked: %u",
3323                    cache->filename, rootpage, numlocked);
3324     }
3325     else
3326     {
3327         if(numlocked > 1)
3328             ajWarn("CacheSync cache '%S' root: zero locked: %u",
3329                    cache->filename, numlocked);
3330     }
3331 
3332 
3333     page = btreePricacheLocate(cache,rootpage);
3334     page->dirty = BT_LOCK;
3335     page->lockfor = 1001;
3336 
3337     if(rootpage)
3338     {
3339         page = btreePricacheLocate(cache,0UL);
3340         page->dirty = BT_LOCK;
3341         page->lockfor = 1002;
3342     }
3343 
3344     statSyncLocked += numlocked;
3345     statSyncWrite += numwrite;
3346     return;
3347 }
3348 
3349 
3350 
3351 
3352 /* @funcstatic btreeCacheRootSync *********************************************
3353 **
3354 ** Reset locked pages to dirty and relock the specified secondary root page.
3355 **
3356 ** Pages will be written only when they are removed from the cache.
3357 **
3358 ** This function clears all locks.
3359 ** The caller must relock the root page (zero) if needed.
3360 **
3361 ** @param [u] cache [AjPBtcache] cache
3362 ** @param [r] rootpage [ajulong] secondary (locked) root page
3363 **
3364 ** @return [void]
3365 **
3366 ** @release 6.4.0
3367 ** @@
3368 ******************************************************************************/
3369 
btreeCacheRootSync(AjPBtcache cache,ajulong rootpage)3370 static void btreeCacheRootSync(AjPBtcache cache, ajulong rootpage)
3371 {
3372     AjPBtpage page = NULL;
3373     ajuint numlocked = 0;
3374     ajuint numunlocked = 0;
3375 
3376     /*ajDebug("In btreeCacheRootSync '%S' rootpage:%Lu\n",
3377       cache->filename, rootpage);*/
3378 
3379     statCallRootSync++;
3380 
3381     for(page=cache->plru;page;page=page->next)
3382     {
3383         if(page->dirty == BT_LOCK)
3384         {
3385             numlocked++;
3386             if(page->pagepos)
3387             {
3388                 if(page->pagepos != rootpage)
3389                 {
3390                     numunlocked++;
3391                     page->dirty = BT_DIRTY;
3392                     ajWarn("btreeCacheRootSync locked '%Lu' lockfor: %u",
3393                             page->pagepos, page->lockfor);
3394 #if AJINDEX_DEBUG
3395                     ajDebug("btreeCacheRootSync locked '%Lu' lockfor: %u\n",
3396                             page->pagepos, page->lockfor);
3397 #endif
3398                 }
3399             }
3400         }
3401     }
3402 
3403     for(page=cache->slru;page;page=page->next)
3404     {
3405         if(page->dirty == BT_LOCK)
3406         {
3407             numlocked++;
3408             if(page->pagepos)
3409             {
3410                 if(page->pagepos != rootpage)
3411                 {
3412                     numunlocked++;
3413                     page->dirty = BT_DIRTY;
3414                     ajWarn("btreeCacheRootSync locked '%Lu' lockfor: %u",
3415                             page->pagepos, page->lockfor);
3416 #if AJINDEX_DEBUG
3417                     ajDebug("btreeCacheRootSync locked '%Lu' lockfor: %u\n",
3418                             page->pagepos, page->lockfor);
3419 #endif
3420                 }
3421             }
3422         }
3423     }
3424 
3425     if(numunlocked > statRootSyncMaxUnlocked)
3426         statRootSyncMaxUnlocked = numunlocked;
3427 
3428     if(numlocked > statRootSyncMaxLocked)
3429         statRootSyncMaxLocked = numlocked;
3430 
3431     statRootSyncLocked += numlocked;
3432     statRootSyncUnlocked += numunlocked;
3433 
3434     page = btreeSeccacheLocate(cache,rootpage);
3435     page->dirty = BT_LOCK;      /* (re)lock the secondary root page */
3436     page->lockfor = 1011;
3437 
3438     return;
3439 }
3440 
3441 
3442 
3443 
3444 /* @funcstatic btreePricacheWrite *********************************************
3445 **
3446 ** Get a pointer to a disc primary cache page for writing
3447 **
3448 ** @param [w] cache [AjPBtcache] cache
3449 ** @param [r] pagepos [ajulong] page number
3450 **
3451 ** @return [AjPBtpage] disc cache page pointer
3452 **
3453 ** @release 6.5.0
3454 ** @@
3455 ******************************************************************************/
3456 
btreePricacheWrite(AjPBtcache cache,ajulong pagepos)3457 static AjPBtpage btreePricacheWrite(AjPBtcache cache, ajulong pagepos)
3458 {
3459     AjPBtpage ret = NULL;
3460 
3461 #if AJINDEX_DEBUG
3462     ajDebug("btreePricacheWrite %Lu\n", pagepos);
3463 #endif
3464 
3465     ret = btreePricacheControl(cache,pagepos,BT_WRITE);
3466 
3467     return ret;
3468 }
3469 
3470 
3471 
3472 
3473 /* @funcstatic btreeSeccacheWrite *********************************************
3474 **
3475 ** Get a pointer to a disc secondary cache page for writing
3476 **
3477 ** @param [w] cache [AjPBtcache] cache
3478 ** @param [r] pagepos [ajulong] page number
3479 **
3480 ** @return [AjPBtpage] disc cache page pointer
3481 **
3482 ** @release 6.5.0
3483 ** @@
3484 ******************************************************************************/
3485 
btreeSeccacheWrite(AjPBtcache cache,ajulong pagepos)3486 static AjPBtpage btreeSeccacheWrite(AjPBtcache cache, ajulong pagepos)
3487 {
3488     AjPBtpage ret = NULL;
3489 
3490 #if AJINDEX_DEBUG
3491     ajDebug("btreeSeccacheWrite %Lu\n", pagepos);
3492 #endif
3493 
3494     ret = btreeSeccacheControl(cache,pagepos,BT_WRITE);
3495 
3496     return ret;
3497 }
3498 
3499 
3500 
3501 
3502 #if 0
3503 /* #funcstatic btreeCacheWriteIdbucket ****************************************
3504 **
3505 ** Get a pointer to a disc cache page for writing
3506 **
3507 ** #param [w] cache [AjPBtcache] cache
3508 ** #param [r] pagepos [ajulong] page number
3509 **
3510 ** #return [AjPBtpage] disc cache page pointer
3511 **
3512 ** #release 6.4.0
3513 ** ##
3514 ******************************************************************************/
3515 /*
3516 static AjPBtpage btreeCacheWriteIdbucket(AjPBtcache cache, ajulong pagepos)
3517 {
3518     AjPBtpage ret = NULL;
3519 
3520     /# ajDebug("In btreeCacheWriteIdbucket\n");#/
3521 
3522     ret = btreePricacheControl(cache,pagepos,BT_WRITE);
3523 
3524     return ret;
3525 }
3526 */
3527 #endif
3528 
3529 
3530 
3531 
3532 /* @funcstatic btreePricacheNodenew *******************************************
3533 **
3534 ** Get a pointer to a new disc primary cache page for writing a bucket.
3535 **
3536 ** Clears the page and sets the block number.
3537 **
3538 ** @param [w] cache [AjPBtcache] cache
3539 **
3540 ** @return [AjPBtpage] disc cache page pointer
3541 **
3542 ** @release 6.5.0
3543 ** @@
3544 ******************************************************************************/
3545 
btreePricacheNodenew(AjPBtcache cache)3546 static AjPBtpage btreePricacheNodenew(AjPBtcache cache)
3547 {
3548     AjPBtpage ret = NULL;
3549     unsigned char *p;
3550     ajuint nodetype;
3551     ajulong newsize;
3552 
3553 #if AJINDEX_DEBUG
3554     ajDebug("btreePricacheNodeenew pagepos:%Lu\n", cache->totsize);
3555 #endif
3556 
3557     ret = btreePricacheControl(cache,cache->totsize,BT_WRITE);
3558     p = ret->buf;
3559     AJCSET0(p, cache->pripagesize); /* clear buffer to zeros */
3560 
3561     /* keep pointers to other cached pages */
3562 /*
3563     ret->next = NULL;
3564     ret->prev = NULL;
3565 */
3566 
3567     ret->dirty = BT_DIRTY;
3568 
3569     ret->pagepos = cache->totsize;
3570 
3571     nodetype     = BT_FREEPAGE;
3572     SBT_NODETYPE(p,nodetype);
3573 
3574     SBT_BLOCKNUMBER(p,ret->pagepos);
3575 
3576     newsize = cache->totsize + cache->pripagesize;
3577     if(newsize > cache->maxsize)
3578     {
3579         cache->maxsize += cache->maxsize/2;
3580         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
3581         {
3582         }
3583     }
3584 
3585     cache->totsize = newsize;
3586     cache->pripagecount++;
3587 
3588     return ret;
3589 }
3590 
3591 
3592 
3593 
3594 /* @funcstatic btreeSeccacheNodenew *******************************************
3595 **
3596 ** Get a pointer to a new disc secondary cache page for writing
3597 **
3598 ** Clears the page and sets the block number.
3599 **
3600 ** @param [w] cache [AjPBtcache] cache
3601 **
3602 ** @return [AjPBtpage] disc cache page pointer
3603 **
3604 ** @release 6.5.0
3605 ** @@
3606 ******************************************************************************/
3607 
btreeSeccacheNodenew(AjPBtcache cache)3608 static AjPBtpage btreeSeccacheNodenew(AjPBtcache cache)
3609 {
3610     AjPBtpage ret = NULL;
3611     unsigned char *p;
3612     ajuint nodetype;
3613     ajulong newsize;
3614 
3615 #if AJINDEX_DEBUG
3616     ajDebug("btreeSeccacheNodenew pagepos:%Lu\n", cache->totsize);
3617 #endif
3618 
3619     ret = btreeSeccacheControl(cache,cache->totsize,BT_WRITE);
3620     p = ret->buf;
3621     AJCSET0(p, cache->secpagesize); /* clear buffer to zeros */
3622 
3623     /* keep pointers to other cached pages */
3624 /*
3625     ret->next = NULL;
3626     ret->prev = NULL;
3627 */
3628 
3629     ret->dirty = BT_DIRTY;
3630 
3631     ret->pagepos = cache->totsize;
3632 
3633     nodetype     = BT_SECFREEPAGE;
3634     SBT_NODETYPE(p,nodetype);
3635 
3636     SBT_BLOCKNUMBER(p,ret->pagepos);
3637 
3638     newsize = cache->totsize + cache->secpagesize;
3639     if(newsize > cache->maxsize)
3640     {
3641         cache->maxsize += cache->maxsize/2;
3642         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
3643         {
3644         }
3645     }
3646 
3647     cache->totsize = newsize;
3648     cache->secpagecount++;
3649 
3650     return ret;
3651 }
3652 
3653 
3654 
3655 
3656 /* @funcstatic btreePricacheBucketnew *****************************************
3657 **
3658 ** Get a pointer to a new disc primary cache page for writing a bucket.
3659 **
3660 ** Clears the page.
3661 **
3662 ** @param [w] cache [AjPBtcache] cache
3663 **
3664 ** @return [AjPBtpage] disc cache page pointer
3665 **
3666 ** @release 6.5.0
3667 ** @@
3668 ******************************************************************************/
3669 
btreePricacheBucketnew(AjPBtcache cache)3670 static AjPBtpage btreePricacheBucketnew(AjPBtcache cache)
3671 {
3672     AjPBtpage ret = NULL;
3673     unsigned char *p;
3674     ajuint nodetype;
3675     ajulong newsize;
3676 
3677 #if AJINDEX_DEBUG
3678     ajDebug("btreePricacheBucketnew pagepos:%Lu\n", cache->totsize);
3679 #endif
3680 
3681     ret = btreePricacheControl(cache,cache->totsize,BT_WRITE);
3682     p = ret->buf;
3683     AJCSET0(p, cache->pripagesize); /* clear buffer to zeros */
3684 
3685     /* keep pointers to other cached pages */
3686 /*
3687     ret->next = NULL;
3688     ret->prev = NULL;
3689 */
3690 
3691     ret->dirty = BT_DIRTY;
3692 
3693     ret->pagepos = cache->totsize;
3694 
3695     nodetype     = BT_FREEPAGE;
3696     SBT_NODETYPE(p,nodetype);
3697 
3698     newsize = cache->totsize + cache->pripagesize;
3699     if(newsize > cache->maxsize)
3700     {
3701         cache->maxsize += cache->maxsize/2;
3702         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
3703         {
3704         }
3705     }
3706 
3707     cache->totsize = newsize;
3708     cache->pripagecount++;
3709 
3710     return ret;
3711 }
3712 
3713 
3714 
3715 
3716 /* @funcstatic btreeSeccacheBucketnew ******************************************
3717 **
3718 ** Get a pointer to a new disc secondary cache page for writing a bucket
3719 **
3720 ** Clears the page.
3721 **
3722 ** @param [w] cache [AjPBtcache] cache
3723 **
3724 ** @return [AjPBtpage] disc cache page pointer
3725 **
3726 ** @release 6.5.0
3727 ** @@
3728 ******************************************************************************/
3729 
btreeSeccacheBucketnew(AjPBtcache cache)3730 static AjPBtpage btreeSeccacheBucketnew(AjPBtcache cache)
3731 {
3732     AjPBtpage ret = NULL;
3733     unsigned char *p;
3734     ajuint nodetype;
3735     ajulong newsize;
3736 
3737 #if AJINDEX_DEBUG
3738     ajDebug("btreeSeccacheBucketnew pagepos:%Lu\n", cache->totsize);
3739 #endif
3740 
3741     ret = btreeSeccacheControl(cache,cache->totsize,BT_WRITE);
3742     p = ret->buf;
3743     AJCSET0(p, cache->secpagesize); /* clear buffer to zeros */
3744 
3745     /* keep pointers to other cached pages */
3746 /*
3747     ret->next = NULL;
3748     ret->prev = NULL;
3749 */
3750 
3751     ret->dirty = BT_DIRTY;
3752 
3753     ret->pagepos = cache->totsize;
3754 
3755     nodetype     = BT_SECFREEPAGE;
3756     SBT_NODETYPE(p,nodetype);
3757 
3758     newsize = cache->totsize + cache->secpagesize;
3759     if(newsize > cache->maxsize)
3760     {
3761         cache->maxsize += cache->maxsize/2;
3762         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
3763         {
3764         }
3765     }
3766 
3767     cache->totsize = newsize;
3768     cache->secpagecount++;
3769 
3770     return ret;
3771 }
3772 
3773 
3774 
3775 #if 0
3776 /* #funcstatic btreePricacheWriteOverflownew **********************************
3777 **
3778 ** Get a pointer to a new disc cache page for writing an overflow node
3779 **
3780 ** Clears the page and sets the block number and nodetype
3781 **
3782 ** #param [w] cache [AjPBtcache] cache
3783 **
3784 ** #return [AjPBtpage] disc cache page pointer
3785 **
3786 ** #release 6.5.0
3787 ** ##
3788 ******************************************************************************/
3789 
3790 /*
3791 //static AjPBtpage btreePricacheWriteOverflownew(AjPBtcache cache)
3792 //{
3793 //    AjPBtpage ret = NULL;
3794 //    unsigned char *p;
3795 //    ajuint v;
3796 //    ajulong newsize;
3797 //
3798 //#if AJINDEX_DEBUG
3799 //    ajDebug("btreePricacheWriteOverflownew pagepos:%Lu\n", cache->totsize);
3800 //#endif
3801 //
3802 //    ret = btreePricacheControl(cache,cache->totsize,BT_WRITE);
3803 //    p = ret->buf;
3804 //    AJCSET0(p, cache->pripagesize); /# clear buffer to zeros #/
3805 //
3806 //    ret->next = NULL;
3807 //    ret->prev = NULL;
3808 //
3809 //    ret->dirty = BT_DIRTY;
3810 //
3811 //    ret->pagepos = cache->totsize;
3812 //
3813 //    SBT_BLOCKNUMBER(p,ret->pagepos);
3814 //
3815 //    v = BT_OVERFLOW;
3816 //    SBT_NODETYPE(p,v);
3817 //
3818 //    newsize = cache->totsize + cache->pripagesize;
3819 //    if(newsize > cache->maxsize)
3820 //    {
3821 //        cache->maxsize += cache->maxsize/2;
3822 //        if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
3823 //        {
3824 //        }
3825 //    }
3826 //
3827 //    cache->totsize = newsize;
3828 //    cache->pripagecount++;
3829 //
3830 //    return ret;
3831 //}
3832 */
3833 #endif
3834 
3835 
3836 
3837 
3838 /* @funcstatic btreePrirootCreate *********************************************
3839 **
3840 ** Create and write an empty primary root node. Set it as root, write it to
3841 ** disc and then lock the page in the disc cache.
3842 **
3843 ** The root node is at block 0UL
3844 **
3845 ** @param [w] cache [AjPBtcache] cache
3846 **
3847 ** @return [void]
3848 **
3849 ** @release 6.5.0
3850 ** @@
3851 ******************************************************************************/
3852 
btreePrirootCreate(AjPBtcache cache)3853 static void btreePrirootCreate(AjPBtcache cache)
3854 {
3855     AjPBtpage page = NULL;
3856     unsigned char *p;
3857     ajuint nodetype;
3858     ajulong blocknumber;
3859     ajuint nkeys;
3860     ajuint totlen;
3861     ajulong left;
3862     ajulong right;
3863     ajulong overflow;
3864     ajulong prev;
3865     ajulong rootpage = 0UL;
3866 
3867 #if AJINDEX_DEBUG
3868     ajDebug("btreePrirootCreate\n");
3869 #endif
3870 
3871 
3872     if(!cache->totsize)
3873         page = btreePricacheNodenew(cache); /* new index */
3874     else
3875         page = btreePricacheWrite(cache, rootpage); /* existing index? */
3876 
3877     page->pagepos    = rootpage;
3878 
3879     p = page->buf;
3880 
3881     nodetype    = BT_ROOT;
3882     blocknumber = rootpage;
3883     nkeys       = 0U;
3884     totlen      = 0U;
3885     left        = 0UL;
3886     right       = 0UL;
3887     prev        = 0UL;
3888     overflow    = 0UL;
3889 
3890     /* Don't reuse the variables. Endian-ness may be changed */
3891     SBT_NODETYPE(p,nodetype);
3892     SBT_BLOCKNUMBER(p,blocknumber);
3893     SBT_NKEYS(p,nkeys);
3894     SBT_TOTLEN(p,totlen);
3895     SBT_LEFT(p,left);
3896     SBT_RIGHT(p,right);
3897     SBT_PREV(p,prev);
3898     SBT_OVERFLOW(p,overflow);
3899 
3900     page->dirty = BT_DIRTY;
3901 
3902     if(btreeDoRootSync)
3903         btreeCacheRootSync(cache,rootpage);
3904     page->dirty = BT_LOCK;
3905     page->lockfor = 1021;
3906 
3907     return;
3908 }
3909 
3910 
3911 
3912 
3913 /* @funcstatic btreeSecrootCreate *********************************************
3914 **
3915 ** Create and write an empty secondary root node. Set it as root, write it to
3916 ** disc and then lock the page in the disc cache.
3917 **
3918 ** The root node is at block 0UL, secondary roots are on other pages
3919 **
3920 ** @param [w] cache [AjPBtcache] cache
3921 ** @param [r] rootpage [ajulong] root block
3922 **
3923 ** @return [void]
3924 **
3925 ** @release 6.5.0
3926 ** @@
3927 ******************************************************************************/
3928 
btreeSecrootCreate(AjPBtcache cache,ajulong rootpage)3929 static void btreeSecrootCreate(AjPBtcache cache, ajulong rootpage)
3930 {
3931     AjPBtpage page = NULL;
3932     unsigned char *p;
3933     ajuint nodetype;
3934     ajulong blocknumber;
3935     ajuint nkeys;
3936     ajuint totlen;
3937     ajulong left;
3938     ajulong right;
3939     ajulong overflow;
3940     ajulong prev;
3941 
3942 #if AJINDEX_DEBUG
3943     ajDebug("btreeSecrootCreate %Lu\n", rootpage);
3944 #endif
3945 
3946 
3947     if(!rootpage)
3948     {
3949         if(rootpage == cache->totsize)
3950             page = btreeSeccacheNodenew(cache);
3951         else
3952             page = btreeSeccacheWrite(cache, rootpage);
3953     }
3954     else
3955     {
3956         if(rootpage == cache->totsize)
3957             page = btreeSeccacheNodenew(cache);
3958         else
3959             page = btreeSeccacheWrite(cache, rootpage);
3960     }
3961 
3962     page->pagepos    = rootpage;
3963 
3964     p = page->buf;
3965 
3966     nodetype    = BT_SECROOT;
3967     blocknumber = rootpage;
3968     nkeys       = 0U;
3969     totlen      = 0U;
3970     left        = 0UL;
3971     right       = 0UL;
3972     prev        = 0UL;
3973     overflow    = 0UL;
3974 
3975     /* Don't reuse the variables. Endian-ness may be changed */
3976     SBT_NODETYPE(p,nodetype);
3977     SBT_BLOCKNUMBER(p,blocknumber);
3978     SBT_NKEYS(p,nkeys);
3979     SBT_TOTLEN(p,totlen);
3980     SBT_LEFT(p,left);
3981     SBT_RIGHT(p,right);
3982     SBT_PREV(p,prev);
3983     SBT_OVERFLOW(p,overflow);
3984 
3985     page->dirty = BT_DIRTY;
3986 
3987     if(btreeDoRootSync)
3988         btreeCacheRootSync(cache,rootpage);
3989     page->dirty = BT_LOCK;
3990     page->lockfor = 1021;
3991 
3992     return;
3993 }
3994 
3995 
3996 
3997 
3998 /* @funcstatic btreePrimaryFindInode ******************************************
3999 **
4000 ** Recursive search for insert node
4001 **
4002 ** @param [u] cache [AjPBtcache] cache
4003 ** @param [u] page [AjPBtpage] page
4004 ** @param [r] item [const AjPStr] key to search for
4005 **
4006 ** @return [AjPBtpage] leaf node where item should be inserted
4007 **
4008 ** @release 6.5.0
4009 ** @@
4010 ******************************************************************************/
4011 
btreePrimaryFindInode(AjPBtcache cache,AjPBtpage page,const AjPStr item)4012 static AjPBtpage btreePrimaryFindInode(AjPBtcache cache, AjPBtpage page,
4013                                        const AjPStr item)
4014 {
4015     AjPBtpage ret = NULL;
4016     AjPBtpage pg  = NULL;
4017 
4018     unsigned char *buf = NULL;
4019     ajuint status       = 0;
4020     ajuint ival         = 0;
4021 
4022     /* ajDebug("In btreePrimaryFindInode\n"); */
4023 
4024     ret = page;
4025     buf = page->buf;
4026     GBT_NODETYPE(buf,&ival);
4027 
4028     if(ival != BT_LEAF)
4029     {
4030 	status = ret->dirty;
4031 	ret->dirty = BT_LOCK;	/* Lock in case of lots of overflow pages */
4032         ret->lockfor = 1031;
4033 	pg = btreePrimaryPageDown(cache,buf,item);
4034 	ret->dirty = status;
4035 	ret = btreePrimaryFindInode(cache,pg,item);
4036     }
4037 
4038     return ret;
4039 }
4040 
4041 
4042 
4043 
4044 /* @funcstatic btreeKeyidFindINode ********************************************
4045 **
4046 ** Recursive search for insert node in a secondary tree
4047 **
4048 ** @param [u] cache [AjPBtcache] cache
4049 ** @param [u] page [AjPBtpage] page
4050 ** @param [r] item [const AjPStr] key to search for
4051 **
4052 ** @return [AjPBtpage] leaf node where item should be inserted
4053 **
4054 ** @release 4.0.0
4055 ** @@
4056 ******************************************************************************/
4057 
btreeKeyidFindINode(AjPBtcache cache,AjPBtpage page,const AjPStr item)4058 static AjPBtpage btreeKeyidFindINode(AjPBtcache cache, AjPBtpage page,
4059                                    const AjPStr item)
4060 {
4061     AjPBtpage ret = NULL;
4062     AjPBtpage pg  = NULL;
4063 
4064     unsigned char *buf = NULL;
4065     ajuint status       = 0;
4066     ajuint ival         = 0;
4067 
4068     /* ajDebug("In btreeKeyidFindINode\n"); */
4069 
4070     ret = page;
4071     buf = page->buf;
4072     GBT_NODETYPE(buf,&ival);
4073 
4074     if(ival != BT_SECLEAF)
4075     {
4076 	status = ret->dirty;
4077 	ret->dirty = BT_LOCK;	/* Lock in case of lots of overflow pages */
4078         ret->lockfor = 1041;
4079 	pg = btreeSecondaryPageDown(cache,buf,item);
4080 	ret->dirty = status;
4081 	ret = btreeKeyidFindINode(cache,pg,item);
4082     }
4083 
4084     return ret;
4085 }
4086 
4087 
4088 
4089 
4090 /* @funcstatic btreePrimaryPageDown *******************************************
4091 **
4092 ** Return next lower index page given a key
4093 **
4094 ** @param [u] cache [AjPBtcache] cache
4095 ** @param [u] buf [unsigned char *] page buffer
4096 ** @param [r] key [const AjPStr] key to search for
4097 **
4098 ** @return [AjPBtpage] pointer to a page
4099 **
4100 ** @release 6.5.0
4101 ** @@
4102 ******************************************************************************/
4103 
btreePrimaryPageDown(AjPBtcache cache,unsigned char * buf,const AjPStr key)4104 static AjPBtpage btreePrimaryPageDown(AjPBtcache cache, unsigned char *buf,
4105                                          const AjPStr key)
4106 {
4107     ajulong blockno = 0UL;
4108     AjPBtpage page = NULL;
4109 
4110     /* ajDebug("In btreePrimaryPageDown\n"); */
4111 
4112     blockno = btreeGetBlockS(cache,buf,key);
4113     page = btreePricacheRead(cache,blockno);
4114 
4115     return page;
4116 }
4117 
4118 
4119 
4120 
4121 /* @funcstatic btreeSecondaryPageDown *****************************************
4122 **
4123 ** Return next lower index page given a key in a secondary tree
4124 **
4125 ** @param [u] cache [AjPBtcache] cache
4126 ** @param [u] buf [unsigned char *] page buffer
4127 ** @param [r] key [const AjPStr] key to search for
4128 **
4129 ** @return [AjPBtpage] pointer to a page
4130 **
4131 ** @release 6.5.0
4132 ** @@
4133 ******************************************************************************/
4134 
btreeSecondaryPageDown(AjPBtcache cache,unsigned char * buf,const AjPStr key)4135 static AjPBtpage btreeSecondaryPageDown(AjPBtcache cache, unsigned char *buf,
4136                                         const AjPStr key)
4137 {
4138     ajulong blockno = 0UL;
4139     AjPBtpage page = NULL;
4140 
4141     /* ajDebug("In btreeSecondaryPageDown\n"); */
4142 
4143     blockno = btreeGetBlockS(cache, buf, key);
4144 
4145     page =  btreeSeccacheRead(cache,blockno);
4146 
4147     return page;
4148 }
4149 
4150 
4151 
4152 
4153 /* @func ajBtreeIdNew *********************************************************
4154 **
4155 ** Constructor for index bucket ID information
4156 **
4157 ** @param [r] refcount [ajuint] Number of reference files for each entry
4158 ** @return [AjPBtId] Index ID object
4159 **
4160 ** @release 3.0.0
4161 ** @@
4162 ******************************************************************************/
4163 
ajBtreeIdNew(ajuint refcount)4164 AjPBtId ajBtreeIdNew(ajuint refcount)
4165 {
4166     AjPBtId Id = NULL;
4167 
4168     /* ajDebug("In ajBtreeIdNew\n"); */
4169 
4170     if(statSaveBtreeIdNext)
4171     {
4172         Id = statSaveBtreeId[--statSaveBtreeIdNext];
4173         MAJSTRASSIGNCLEAR(&Id->id);
4174         if(Id->refcount != refcount)
4175         {
4176             if(refcount)
4177                 AJCRESIZE0(Id->refoffsets, Id->refcount, refcount);
4178             else
4179                 AJFREE(Id->refoffsets);
4180 
4181             Id->refcount = refcount;
4182         }
4183     }
4184     else
4185     {
4186         AJNEW0(Id);
4187         Id->id = ajStrNew();
4188         Id->refcount = refcount;
4189 
4190         if(refcount)
4191             AJCNEW(Id->refoffsets, refcount);
4192     }
4193 
4194     Id->dbno = 0U;
4195     Id->dups = 0U;
4196     Id->offset = 0UL;
4197 
4198     return Id;
4199 }
4200 
4201 
4202 
4203 
4204 /* @func ajBtreeIdDel *********************************************************
4205 **
4206 ** Destructor for index bucket ID information
4207 **
4208 ** @param [w] thys [AjPBtId*] index ID object
4209 **
4210 ** @return [void]
4211 **
4212 ** @release 3.0.0
4213 ** @@
4214 ******************************************************************************/
4215 
ajBtreeIdDel(AjPBtId * thys)4216 void ajBtreeIdDel(AjPBtId *thys)
4217 {
4218     ajuint newmax;
4219 
4220     /* ajDebug("In ajBtreeIdDel\n"); */
4221 
4222     if(!statSaveBtreeId)
4223     {
4224         statSaveBtreeIdMax = 2048;
4225         AJCNEW0(statSaveBtreeId, statSaveBtreeIdMax);
4226         statSaveBtreeIdNext = 0;
4227     }
4228 
4229     if(!thys || !*thys)
4230 	return;
4231 
4232     if(statSaveBtreeIdNext >= statSaveBtreeIdMax)
4233     {
4234         newmax = statSaveBtreeIdMax + statSaveBtreeIdMax;
4235         AJCRESIZE0(statSaveBtreeId,statSaveBtreeIdMax,newmax);
4236         statSaveBtreeIdMax = newmax;
4237     }
4238 
4239     statSaveBtreeId[statSaveBtreeIdNext++] = *thys;
4240 
4241     *thys = NULL;
4242 
4243     return;
4244 }
4245 
4246 
4247 
4248 
4249 /* @func ajBtreeIdDelVoid *****************************************************
4250 **
4251 ** Destructor for index bucket ID information
4252 **
4253 ** @param [w] voidarg [void**] Index ID object, passed as void for use by
4254 **                             list and table destructors.
4255 ** @return [void]
4256 **
4257 ** @release 6.4.0
4258 ** @@
4259 ******************************************************************************/
4260 
ajBtreeIdDelVoid(void ** voidarg)4261 void ajBtreeIdDelVoid(void **voidarg)
4262 {
4263     AjPBtId *thys = (AjPBtId *)voidarg;
4264 
4265     ajBtreeIdDel(thys);
4266 
4267     return;
4268 }
4269 
4270 
4271 
4272 
4273 /* @func ajBtreeIdIsMulti *****************************************************
4274 **
4275 ** Tests whether an ID object links to multiple entries
4276 **
4277 ** @param [r] thys [const AjPBtId] index ID object
4278 **
4279 ** @return [AjBool] True if ID object points to duplicated identifiers
4280 **
4281 ** @release 6.5.0
4282 ** @@
4283 ******************************************************************************/
4284 
ajBtreeIdIsMulti(const AjPBtId thys)4285 AjBool ajBtreeIdIsMulti(const AjPBtId thys)
4286 {
4287     if(!thys)
4288 	return ajFalse;
4289 
4290     if(thys->dups)
4291         return ajTrue;
4292 
4293     return ajFalse;
4294 }
4295 
4296 
4297 
4298 
4299 /* @funcstatic btreeIdFree ****************************************************
4300 **
4301 ** Destructor for index bucket ID information
4302 **
4303 ** @param [w] thys [AjPBtId*] index ID object
4304 **
4305 ** @return [void]
4306 **
4307 ** @release 6.4.0
4308 ** @@
4309 ******************************************************************************/
4310 
btreeIdFree(AjPBtId * thys)4311 static void btreeIdFree(AjPBtId *thys)
4312 {
4313     AjPBtId Id = NULL;
4314 
4315     /* ajDebug("In btreeIdFree\n"); */
4316 
4317     if(!thys || !*thys)
4318 	return;
4319 
4320     Id = *thys;
4321 
4322     ajStrDel(&Id->id);
4323 
4324     if(Id->refoffsets)
4325         AJFREE(Id->refoffsets);
4326 
4327     AJFREE(Id);
4328     *thys = NULL;
4329 
4330     return;
4331 }
4332 
4333 
4334 
4335 
4336 /* @func ajBtreeHitNew ********************************************************
4337 **
4338 ** Constructor for index hit information
4339 **
4340 ** @return [AjPBtHit] Index hit object
4341 **
4342 ** @release 6.5.0
4343 ** @@
4344 ******************************************************************************/
4345 
ajBtreeHitNew(void)4346 AjPBtHit ajBtreeHitNew(void)
4347 {
4348     AjPBtHit Hit = NULL;
4349 
4350     /* ajDebug("In ajBtreeHitNew\n"); */
4351 
4352     if(statSaveBtreeHitNext)
4353     {
4354         Hit = statSaveBtreeHit[--statSaveBtreeHitNext];
4355         Hit->dbno = 0U;
4356         Hit->refcount = 0U;
4357         Hit->offset = 0UL;
4358     }
4359     else
4360     {
4361         AJNEW0(Hit);
4362     }
4363 
4364     return Hit;
4365 }
4366 
4367 
4368 
4369 
4370 /* @func ajBtreeHitNewId ******************************************************
4371 **
4372 ** Constructor for index hit information from an existing Btid object
4373 **
4374 ** @param [r] btid [const AjPBtId] Index id object
4375 ** @return [AjPBtHit] Index hit object
4376 **
4377 ** @release 6.5.0
4378 ** @@
4379 ******************************************************************************/
4380 
ajBtreeHitNewId(const AjPBtId btid)4381 AjPBtHit ajBtreeHitNewId(const AjPBtId btid)
4382 {
4383     AjPBtHit Hit = NULL;
4384 
4385     /* ajDebug("In ajBtreeHitNewId\n"); */
4386 
4387     if(statSaveBtreeHitNext)
4388     {
4389         Hit = statSaveBtreeHit[--statSaveBtreeHitNext];
4390     }
4391     else
4392     {
4393         AJNEW0(Hit);
4394     }
4395 
4396     Hit->dbno = btid->dbno;
4397     Hit->refcount = 0;
4398     Hit->offset = btid->offset;
4399 
4400     return Hit;
4401 }
4402 
4403 
4404 
4405 
4406 /* @func ajBtreeHitDel ********************************************************
4407 **
4408 ** Destructor for index hit information
4409 **
4410 ** @param [w] thys [AjPBtHit*] index hit object
4411 **
4412 ** @return [void]
4413 **
4414 ** @release 6.5.0
4415 ** @@
4416 ******************************************************************************/
4417 
ajBtreeHitDel(AjPBtHit * thys)4418 void ajBtreeHitDel(AjPBtHit *thys)
4419 {
4420     ajuint newmax;
4421 
4422     /* ajDebug("In ajBtreeHitDel\n"); */
4423 
4424     if(!statSaveBtreeHit)
4425     {
4426         statSaveBtreeHitMax = 2048;
4427         AJCNEW0(statSaveBtreeHit, statSaveBtreeHitMax);
4428         statSaveBtreeHitNext = 0;
4429     }
4430 
4431     if(!thys || !*thys)
4432 	return;
4433 
4434     if(statSaveBtreeHitNext >= statSaveBtreeHitMax)
4435     {
4436         newmax = statSaveBtreeHitMax + statSaveBtreeHitMax;
4437         AJCRESIZE0(statSaveBtreeHit,statSaveBtreeHitMax,newmax);
4438         statSaveBtreeHitMax = newmax;
4439     }
4440 
4441     statSaveBtreeHit[statSaveBtreeHitNext++] = *thys;
4442 
4443     *thys = NULL;
4444 
4445     return;
4446 }
4447 
4448 
4449 
4450 
4451 /* @func ajBtreeHitDelVoid ****************************************************
4452 **
4453 ** Destructor for index hit information
4454 **
4455 ** @param [w] voidarg [void**] Index hit object, passed as void for use by
4456 **                             list and table destructors.
4457 ** @return [void]
4458 **
4459 ** @release 6.5.0
4460 ** @@
4461 ******************************************************************************/
4462 
ajBtreeHitDelVoid(void ** voidarg)4463 void ajBtreeHitDelVoid(void **voidarg)
4464 {
4465     AjPBtHit *thys = (AjPBtHit *)voidarg;
4466 
4467     ajBtreeHitDel(thys);
4468 
4469     return;
4470 }
4471 
4472 
4473 
4474 
4475 /* @funcstatic btreeHitFree ***************************************************
4476 **
4477 ** Destructor for index hit information
4478 **
4479 ** @param [w] thys [AjPBtHit*] index hit object
4480 **
4481 ** @return [void]
4482 **
4483 ** @release 6.5.0
4484 ** @@
4485 ******************************************************************************/
4486 
btreeHitFree(AjPBtHit * thys)4487 static void btreeHitFree(AjPBtHit *thys)
4488 {
4489     AjPBtHit Hit = NULL;
4490 
4491     /* ajDebug("In btreeHitFree\n"); */
4492 
4493     if(!thys || !*thys)
4494 	return;
4495 
4496     Hit = *thys;
4497 
4498     AJFREE(Hit);
4499     *thys = NULL;
4500 
4501     return;
4502 }
4503 
4504 
4505 
4506 
4507 /* @func ajBtreeHitrefNew *****************************************************
4508 **
4509 ** Constructor for index reference hit information
4510 **
4511 ** @return [AjPBtHitref] Index reference hit object
4512 **
4513 ** @release 6.5.0
4514 ** @@
4515 ******************************************************************************/
4516 
ajBtreeHitrefNew(void)4517 AjPBtHitref ajBtreeHitrefNew(void)
4518 {
4519     AjPBtHitref Hitref = NULL;
4520 
4521     /* ajDebug("In ajBtreeHitrefNew\n"); */
4522 
4523     if(statSaveBtreeHitrefNext)
4524     {
4525         Hitref = statSaveBtreeHitref[--statSaveBtreeHitrefNext];
4526         Hitref->dbno = 0U;
4527         Hitref->refcount = 0U;
4528         Hitref->offset = 0UL;
4529     }
4530     else
4531     {
4532         AJNEW0(Hitref);
4533     }
4534 
4535     return Hitref;
4536 }
4537 
4538 
4539 
4540 
4541 /* @func ajBtreeHitrefNewId ****************************************************
4542 **
4543 ** Constructor for index reference hit information from an existing
4544 ** Btid object
4545 **
4546 ** @param [r] btid [const AjPBtId] Index id object
4547 ** @return [AjPBtHitref] Index hit object
4548 **
4549 ** @release 6.5.0
4550 ** @@
4551 ******************************************************************************/
4552 
ajBtreeHitrefNewId(const AjPBtId btid)4553 AjPBtHitref ajBtreeHitrefNewId(const AjPBtId btid)
4554 {
4555     AjPBtHitref Hitref = NULL;
4556 
4557     /* ajDebug("In ajBtreeHitrefNewId\n"); */
4558 
4559     if(statSaveBtreeHitrefNext)
4560     {
4561         Hitref = statSaveBtreeHitref[--statSaveBtreeHitrefNext];
4562     }
4563     else
4564     {
4565         AJNEW0(Hitref);
4566     }
4567 
4568     Hitref->dbno = btid->dbno;
4569     Hitref->refcount = 0U;
4570     Hitref->offset = btid->offset;
4571     if(btid->refcount)
4572         Hitref->refoffset = btid->refoffsets[0];
4573     else
4574         Hitref->refoffset = 0UL;
4575 
4576     return Hitref;
4577 }
4578 
4579 
4580 
4581 
4582 /* @func ajBtreeHitrefDel *****************************************************
4583 **
4584 ** Destructor for index reference hit information
4585 **
4586 ** @param [w] thys [AjPBtHitref*] index reference hit object
4587 **
4588 ** @return [void]
4589 **
4590 ** @release 6.5.0
4591 ** @@
4592 ******************************************************************************/
4593 
ajBtreeHitrefDel(AjPBtHitref * thys)4594 void ajBtreeHitrefDel(AjPBtHitref *thys)
4595 {
4596     ajuint newmax;
4597 
4598     /* ajDebug("In ajBtreeHitrefDel\n"); */
4599 
4600     if(!statSaveBtreeHitref)
4601     {
4602         statSaveBtreeHitrefMax = 2048;
4603         AJCNEW0(statSaveBtreeHitref, statSaveBtreeHitrefMax);
4604         statSaveBtreeHitrefNext = 0;
4605     }
4606 
4607     if(!thys || !*thys)
4608 	return;
4609 
4610     if(statSaveBtreeHitrefNext >= statSaveBtreeHitrefMax)
4611     {
4612         newmax = statSaveBtreeHitrefMax + statSaveBtreeHitrefMax;
4613         AJCRESIZE0(statSaveBtreeHitref,statSaveBtreeHitrefMax,newmax);
4614         statSaveBtreeHitrefMax = newmax;
4615     }
4616 
4617     statSaveBtreeHitref[statSaveBtreeHitrefNext++] = *thys;
4618 
4619     *thys = NULL;
4620 
4621     return;
4622 }
4623 
4624 
4625 
4626 
4627 /* @func ajBtreeHitrefDelVoid *************************************************
4628 **
4629 ** Destructor for index reference hit information
4630 **
4631 ** @param [w] voidarg [void**] Index reference hit object, passed as
4632 **                             void for use by list and table
4633 **                             destructors.  @return [void]
4634 **
4635 ** @release 6.5.0
4636 ** @@
4637 ******************************************************************************/
4638 
ajBtreeHitrefDelVoid(void ** voidarg)4639 void ajBtreeHitrefDelVoid(void **voidarg)
4640 {
4641     AjPBtHitref *thys = (AjPBtHitref *)voidarg;
4642 
4643     ajBtreeHitrefDel(thys);
4644 
4645     return;
4646 }
4647 
4648 
4649 
4650 
4651 /* @funcstatic btreeHitrefFree ************************************************
4652 **
4653 ** Destructor for index reference hit information
4654 **
4655 ** @param [w] thys [AjPBtHitref*] index reference hit object
4656 **
4657 ** @return [void]
4658 **
4659 ** @release 6.5.0
4660 ** @@
4661 ******************************************************************************/
4662 
btreeHitrefFree(AjPBtHitref * thys)4663 static void btreeHitrefFree(AjPBtHitref *thys)
4664 {
4665     AjPBtHitref Hitref = NULL;
4666 
4667     /* ajDebug("In btreeHitrefFree\n"); */
4668 
4669     if(!thys || !*thys)
4670 	return;
4671 
4672     Hitref = *thys;
4673 
4674     AJFREE(Hitref);
4675     *thys = NULL;
4676 
4677     return;
4678 }
4679 
4680 
4681 
4682 
4683 /* @funcstatic btreeIdbucketIdlist ********************************************
4684 **
4685 ** Copies all IDs into a list
4686 **
4687 ** @param [u] cache [AjPBtcache] cache
4688 ** @param [r] pagepos [ajulong] page number
4689 ** @param [u] idlist [AjPList] list to hold IDs
4690 **
4691 ** @return [ajulong] Overflow
4692 **
4693 ** @release 6.4.0
4694 ** @@
4695 ******************************************************************************/
4696 
btreeIdbucketIdlist(AjPBtcache cache,ajulong pagepos,AjPList idlist)4697 static ajulong btreeIdbucketIdlist(AjPBtcache cache, ajulong pagepos,
4698                                    AjPList idlist)
4699 {
4700     AjPBtpage page      = NULL;
4701     AjPBtpage lpage     = NULL;
4702     unsigned char *buf  = NULL;
4703     unsigned char *kptr = NULL;
4704     AjPBtId id          = NULL;
4705 
4706     unsigned char *idptr = NULL;
4707 
4708     ajuint  nodetype  = 0U;
4709     ajuint  nentries  = 0U;
4710     ajulong overflow  = 0UL;
4711     ajulong pageoverflow = 0UL;
4712     ajuint  dirtysave = 0U;
4713 
4714     ajuint  i;
4715     ajuint  iref;
4716     ajuint  len = 0U;
4717     ajuint idlen;
4718     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
4719 
4720     /* ajDebug("In btreeIdbucketIdlist\n"); */
4721 
4722     if(!pagepos)
4723 	ajFatal("IdbucketIdlist: cannot read bucket from root page cache %S",
4724                 cache->filename);
4725 
4726     page  = btreePricacheRead(cache,pagepos);
4727     lpage = page;
4728     dirtysave = lpage->dirty;
4729     lpage->dirty = BT_LOCK;     /* reset on return */
4730     lpage->lockfor = 1051;
4731 
4732     buf = lpage->buf;
4733 
4734     GBT_BUCKNODETYPE(buf,&nodetype);
4735     if(nodetype != BT_IDBUCKET)
4736 	ajFatal("BucketIdlist: NodeType mismatch. "
4737                 "Not bucket (%u) cache %S",
4738                 nodetype, cache->filename);
4739 
4740     GBT_BUCKNENTRIES(buf,&nentries);
4741     if(nentries > cache->pnperbucket)
4742 	ajFatal("BucketIdlist: Bucket too full page: %Lu "
4743                 "entries: %u max: %u cache %S",
4744                 pagepos, nentries, cache->pnperbucket, cache->filename);
4745 
4746 
4747     GBT_BUCKOVERFLOW(buf,&overflow);
4748     pageoverflow = overflow;
4749 
4750     kptr  = PBT_BUCKKEYLEN(buf);
4751     idptr = kptr + (nentries * sizeof(ajuint));
4752 
4753     for(i=0;i<nentries;++i)
4754     {
4755 	BT_GETAJUINT(kptr,&len);
4756         idlen = len - keyskip;
4757 
4758 /*
4759 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
4760 //	{
4761 //	    /# ajDebug("IdbucketIdlist: Overflow\n"); #/
4762 //	    page  = btreePricacheRead(cache,overflow);
4763 //	    buf = page->buf;
4764 //	    GBT_BUCKNODETYPE(buf,&nodetype);
4765 //	    if(nodetype != BT_IDBUCKET)
4766 //		ajFatal("BucketIdlist: NodeType mismatch. "
4767 //                        "Not bucket (%u) cache %S",
4768 //			nodetype, cache->filename);
4769 //	    GBT_BUCKOVERFLOW(buf,&overflow);
4770 //	    /# overflow bucket ids start at the keylen position #/
4771 //	    idptr = PBT_BUCKKEYLEN(buf);
4772 //	}
4773 */
4774 
4775 	id = ajBtreeIdNew(cache->refcount);
4776 
4777 	/* Fill ID objects */
4778 	ajStrAssignLenC(&id->id,(const char *)idptr,idlen);
4779 	idptr += (idlen + 1);
4780 	BT_GETAJUINT(idptr,&id->dbno);
4781 	idptr += sizeof(ajuint);
4782 	BT_GETAJUINT(idptr,&id->dups);
4783 	idptr += sizeof(ajuint);
4784 	BT_GETAJULONG(idptr,&id->offset);
4785 	idptr += sizeof(ajulong);
4786         if(cache->refcount)
4787         {
4788             for(iref=0; iref< cache->refcount; iref++)
4789             {
4790                 BT_GETAJULONG(idptr,&id->refoffsets[iref]);
4791                 idptr += sizeof(ajulong);
4792             }
4793         }
4794 
4795 	kptr += sizeof(ajuint);
4796         ajListPushAppend(idlist, id);
4797     }
4798 
4799     lpage->dirty = dirtysave;
4800 
4801     return pageoverflow;
4802 }
4803 
4804 
4805 
4806 
4807 /* @funcstatic btreeReadIdbucket **********************************************
4808 **
4809 ** Constructor for id bucket given a disc page number
4810 ** Creates one empty key slot for possible addition
4811 **
4812 ** @param [u] cache [AjPBtcache] cache
4813 ** @param [r] pagepos [ajulong] page number
4814 **
4815 ** @return [AjPIdbucket] Id bucket
4816 **
4817 ** @release 2.8.0
4818 ** @@
4819 ******************************************************************************/
4820 
btreeReadIdbucket(AjPBtcache cache,ajulong pagepos)4821 static AjPIdbucket btreeReadIdbucket(AjPBtcache cache, ajulong pagepos)
4822 {
4823     AjPIdbucket bucket    = NULL;
4824     AjPBtpage page      = NULL;
4825     AjPBtpage lpage     = NULL;
4826     unsigned char *buf  = NULL;
4827     unsigned char *kptr = NULL;
4828     AjPBtId id          = NULL;
4829 
4830     unsigned char *idptr = NULL;
4831 
4832     ajuint nodetype  = 0U;
4833     ajuint  nentries  = 0U;
4834     ajulong overflow  = 0UL;
4835     ajuint  dirtysave = 0U;
4836 
4837     ajuint  i;
4838     ajuint  iref;
4839     ajuint  len = 0U;
4840     ajuint idlen;
4841     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
4842 
4843     /* ajDebug("In btreeReadIdbucket\n"); */
4844 
4845     if(!pagepos)
4846 	ajFatal("BucketRead: cannot read bucket from root page");
4847 
4848     page  = btreePricacheRead(cache,pagepos);
4849     lpage = page;
4850     dirtysave = lpage->dirty;
4851     lpage->dirty = BT_LOCK;
4852     lpage->lockfor = 1061;
4853 
4854     buf = lpage->buf;
4855 
4856     GBT_BUCKNODETYPE(buf,&nodetype);
4857 
4858     if(nodetype != BT_IDBUCKET)
4859 	ajFatal("ReadBucket: NodeType mismatch. Not bucket (%u)", nodetype);
4860 
4861     GBT_BUCKNENTRIES(buf,&nentries);
4862 
4863     if(nentries > cache->pnperbucket)
4864 	ajFatal("ReadBucket: Bucket too full page: %Lu "
4865                 "entries: %u max: %u cache %S",
4866                 pagepos, nentries, cache->pnperbucket, cache->filename);
4867 
4868 
4869     GBT_BUCKOVERFLOW(buf,&overflow);
4870 
4871     bucket = btreeIdbucketNew(cache->pnperbucket, cache->refcount);
4872     bucket->Nentries = nentries;
4873 
4874     kptr  = PBT_BUCKKEYLEN(buf);
4875     idptr = kptr + (nentries * sizeof(ajuint));
4876 
4877     for(i=0;i<nentries;++i)
4878     {
4879 	BT_GETAJUINT(kptr,&len);
4880         idlen = len - keyskip;
4881 
4882 /*
4883 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
4884 //	{
4885 //#if AJINDEX_DEBUG
4886 //	    ajDebug("ReadIdbucket: Overflow\n");
4887 //#endif
4888 //	    page  = btreePricacheRead(cache,overflow);
4889 //	    buf = page->buf;
4890 //	    GBT_BUCKNODETYPE(buf,&nodetype);
4891 //	    if(nodetype != BT_IDBUCKET)
4892 //		ajFatal("ReadBucket: NodeType mismatch. Not bucket (%u)",
4893 //			nodetype);
4894 //	    GBT_BUCKOVERFLOW(buf,&overflow);
4895 //	    /# overflow bucket ids start at the keylen position #/
4896 //	    idptr = PBT_BUCKKEYLEN(buf);
4897 //	}
4898 */
4899 
4900         id = bucket->Ids[i];
4901 
4902 	/* Fill ID objects */
4903 	ajStrAssignLenC(&id->id,(const char *)idptr, idlen);
4904 	idptr += (idlen + 1);
4905 	BT_GETAJUINT(idptr,&id->dbno);
4906 	idptr += sizeof(ajuint);
4907 	BT_GETAJUINT(idptr,&id->dups);
4908 	idptr += sizeof(ajuint);
4909 	BT_GETAJULONG(idptr,&id->offset);
4910 	idptr += sizeof(ajulong);
4911         if(cache->refcount)
4912         {
4913             for(iref=0; iref < cache->refcount; iref++)
4914             {
4915                 BT_GETAJULONG(idptr,&id->refoffsets[iref]);
4916                 idptr += sizeof(ajulong);
4917             }
4918         }
4919 
4920 	kptr += sizeof(ajuint);
4921     }
4922 
4923     lpage->dirty = dirtysave;
4924 
4925     return bucket;
4926 }
4927 
4928 
4929 
4930 
4931 /* @funcstatic btreeIdbucketFindDupId *****************************************
4932 **
4933 ** Tests for an ID in an id bucket.
4934 ** If found, returns the ID and its count in the bucket.
4935 **
4936 ** @param [u] cache [AjPBtcache] cache
4937 ** @param [r] pagepos [ajulong] page number
4938 ** @param [r] id [const AjPStr] id to search for
4939 ** @param [w] ientry [ajuint*] entry number matched
4940 **
4941 ** @return [AjPBtId] ID found
4942 **
4943 ** @release 6.4.0
4944 ** @@
4945 ******************************************************************************/
4946 
btreeIdbucketFindDupId(AjPBtcache cache,ajulong pagepos,const AjPStr id,ajuint * ientry)4947 static AjPBtId btreeIdbucketFindDupId(AjPBtcache cache, ajulong pagepos,
4948                                       const AjPStr id, ajuint* ientry)
4949 {
4950     AjPBtId bid = NULL;
4951     AjPBtpage page      = NULL;
4952     AjPBtpage lpage     = NULL;
4953     unsigned char *buf  = NULL;
4954     unsigned char *lbuf  = NULL;
4955     unsigned char *kptr = NULL;
4956 
4957     unsigned char *idptr = NULL;
4958 
4959     ajuint  nodetype  = 0U;
4960     ajuint  nentries  = 0U;
4961     ajulong overflow  = 0UL;
4962     ajuint  dirtysave = 0U;
4963 
4964     ajuint  i;
4965     ajuint  iref;
4966     ajuint  len = 0U;
4967     ajuint  idlen;
4968     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
4969 
4970     /* ajDebug("In btreeIdbucketFindDupId\n"); */
4971 
4972     if(!pagepos)
4973 	ajFatal("btreeIdbucketFindDupId: "
4974                 "cannot read bucket from root page cache %S",
4975                 cache->filename);
4976 
4977     page  = btreePricacheRead(cache,pagepos);
4978     lpage = page;
4979     dirtysave = lpage->dirty;
4980     lpage->dirty = BT_LOCK;     /* reset on return */
4981     lpage->lockfor = 1071;
4982 
4983     buf = lpage->buf;
4984     lbuf = buf;
4985 
4986     GBT_BUCKNODETYPE(lbuf,&nodetype);
4987     if(nodetype != BT_IDBUCKET)
4988 	ajFatal("BucketIdFindDupId: NodeType mismatch. "
4989                 "Not bucket (%u) cache %S",
4990                 nodetype, cache->filename);
4991 
4992     GBT_BUCKNENTRIES(lbuf,&nentries);
4993     if(nentries > cache->pnperbucket)
4994 	ajFatal("BucketIdFindDupId: Bucket too full page: %Lu "
4995                 "entries: %u max: %u cache %S",
4996                 pagepos, nentries, cache->pnperbucket, cache->filename);
4997 
4998 
4999     GBT_BUCKOVERFLOW(lbuf,&overflow);
5000 
5001     kptr  = PBT_BUCKKEYLEN(buf);
5002     idptr = kptr + (nentries * sizeof(ajuint));
5003 
5004     for(i=0;i<nentries;++i)
5005     {
5006 	BT_GETAJUINT(kptr,&len);
5007 
5008 /*
5009 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
5010 //	{
5011 //	    /# ajDebug("BucketFindId: Overflow\n"); #/
5012 //	    page  = btreePricacheRead(cache,overflow);
5013 //	    buf = page->buf;
5014 //	    GBT_BUCKNODETYPE(buf,&nodetype);
5015 //	    if(nodetype != BT_IDBUCKET)
5016 //		ajFatal("BucketFindId: NodeType mismatch. "
5017 //                        "Not bucket (%u) cache %S",
5018 //			nodetype, cache->filename);
5019 //	    GBT_BUCKOVERFLOW(buf,&overflow);
5020 //	    /# overflow bucket ids start at the keylen position #/
5021 //	    idptr = PBT_BUCKKEYLEN(buf);
5022 //	}
5023 */
5024 
5025 	if(!MAJSTRCMPC(id,(const char*)idptr))
5026         {
5027             bid = ajBtreeIdNew(cache->refcount);
5028             idlen = len - keyskip;
5029             ajStrAssignLenC(&bid->id,(const char *)idptr, idlen);
5030             idptr += (idlen + 1);
5031             BT_GETAJUINT(idptr,&bid->dbno);
5032             idptr += sizeof(ajuint);
5033             BT_GETAJUINT(idptr,&bid->dups);
5034             idptr += sizeof(ajuint);
5035             BT_GETAJULONG(idptr,&bid->offset);
5036             idptr += sizeof(ajulong);
5037 
5038             if(cache->refcount)
5039             {
5040                 for(iref=0; iref < cache->refcount; iref++)
5041                 {
5042                     BT_GETAJULONG(idptr,&bid->refoffsets[iref]);
5043                     idptr += sizeof(ajulong);
5044                 }
5045             }
5046 
5047             *ientry = i;
5048             lpage->dirty = dirtysave;
5049             return bid;
5050         }
5051 
5052 /* Step through ID object data */
5053 
5054 	idptr += len;
5055         kptr += sizeof(ajuint);
5056     }
5057 
5058     lpage->dirty = dirtysave;
5059 
5060     *ientry = nentries;
5061     return NULL;
5062 }
5063 
5064 
5065 
5066 
5067 /* @funcstatic btreeIdbucketSort **********************************************
5068 **
5069 ** Sorts IDs in an id bucket
5070 **
5071 ** @param [u] thys [AjPIdbucket] Id bucket
5072 **
5073 ** @return [void]
5074 ** @@
5075 ******************************************************************************/
5076 
btreeIdbucketSort(AjPIdbucket thys)5077 static void btreeIdbucketSort(AjPIdbucket thys)
5078 {
5079     qsort(thys->Ids, thys->Nentries, sizeof(AjPBtId), btreeIdCompare);
5080 
5081     return;
5082 }
5083 
5084 
5085 
5086 
5087 /* @funcstatic btreeWriteIdbucketId *******************************************
5088 **
5089 ** Tests for an ID in a bucket.
5090 ** If found, returns the ID and its count in the bucket.
5091 **
5092 ** @param [u] cache [AjPBtcache] cache
5093 ** @param [r] pagepos [ajulong] page number
5094 ** @param [r] btid [const AjPBtId] Bucket id to update
5095 ** @param [r] ientry [ajuint] entry number for this ID
5096 **
5097 ** @return [void]
5098 **
5099 ** @release 6.4.0
5100 ** @@
5101 ******************************************************************************/
5102 
btreeWriteIdbucketId(AjPBtcache cache,ajulong pagepos,const AjPBtId btid,ajuint ientry)5103 static void btreeWriteIdbucketId(AjPBtcache cache, ajulong pagepos,
5104                                  const AjPBtId btid, ajuint ientry)
5105 {
5106     AjPBtpage page      = NULL;
5107     AjPBtpage lpage     = NULL;
5108     unsigned char *buf  = NULL;
5109     unsigned char *lbuf = NULL;
5110     unsigned char *kptr = NULL;
5111 
5112     unsigned char *idptr = NULL;
5113 
5114     ajuint  nodetype  = 0U;
5115     ajuint  nentries  = 0U;
5116     ajulong overflow  = 0UL;
5117 
5118     ajuint  i;
5119     ajuint  len = 0U;
5120     ajuint  idlen;
5121     ajuint  v;
5122     ajulong lv;
5123     ajuint iref;
5124     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
5125 
5126     /* ajDebug("In btreeWriteIdbucketId\n"); */
5127 
5128     if(!pagepos)
5129 	ajFatal("WriteIdbucketId: cannot read bucket from root page cache %S",
5130                 cache->filename);
5131 
5132     page  = btreePricacheRead(cache,pagepos);
5133     lpage = page;
5134     lpage->dirty = BT_LOCK;     /* reset on return */
5135     lpage->lockfor = 1081;
5136 
5137     buf = lpage->buf;
5138     lbuf= buf;
5139 
5140     GBT_BUCKNODETYPE(lbuf,&nodetype);
5141     if(nodetype != BT_IDBUCKET)
5142 	ajFatal("WriteIdbucketId: NodeType mismatch. "
5143                 "Not bucket (%u) cache %S",
5144                 nodetype, cache->filename);
5145 
5146     GBT_BUCKNENTRIES(lbuf,&nentries);
5147     if(nentries > cache->pnperbucket)
5148 	ajFatal("WriteIdbucketId: Bucket too full page: %Lu "
5149                 "entries: %u max: %u cache %S",
5150                 pagepos, nentries, cache->pnperbucket, cache->filename);
5151 
5152 
5153     GBT_BUCKOVERFLOW(lbuf,&overflow);
5154 
5155     kptr  = PBT_BUCKKEYLEN(lbuf);
5156     idptr = kptr + (nentries * sizeof(ajuint));
5157 
5158     for(i=0;i<ientry;++i)
5159     {
5160 	BT_GETAJUINT(kptr,&len);
5161 
5162 /*
5163 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
5164 //	{
5165 //	    /# ajDebug("BucketFindId: Overflow\n"); #/
5166 //	    page  = btreePricacheRead(cache,overflow);
5167 //	    buf = page->buf;
5168 //	    GBT_BUCKNODETYPE(buf,&nodetype);
5169 //	    if(nodetype != BT_IDBUCKET)
5170 //		ajFatal("WriteIdbucketId: NodeType mismatch. "
5171 //                        "Not bucket (%u) cache %S",
5172 //			nodetype, cache->filename);
5173 //	    GBT_BUCKOVERFLOW(buf,&overflow);
5174 //	    /# overflow bucket ids start at the keylen position #/
5175 //	    idptr = PBT_BUCKKEYLEN(buf);
5176 //	}
5177 */
5178 
5179 /* Step through ID object data */
5180 	idptr += len;
5181         kptr += sizeof(ajuint);
5182     }
5183 
5184     BT_GETAJUINT(kptr,&len);
5185     idlen = len - keyskip;
5186     idptr += (idlen + 1);
5187 
5188     v = btid->dbno;
5189     BT_SETAJUINT(idptr,v);
5190     idptr += sizeof(ajuint);
5191 
5192     v = btid->dups;
5193     BT_SETAJUINT(idptr,v);
5194     idptr += sizeof(ajuint);
5195 
5196     lv = btid->offset;
5197     BT_SETAJULONG(idptr,lv);
5198     idptr += sizeof(ajulong);
5199 
5200     if(cache->refcount)
5201     {
5202         for(iref=0; iref < cache->refcount; iref++)
5203         {
5204             lv = btid->refoffsets[iref];
5205             BT_SETAJULONG(idptr,lv);
5206             idptr += sizeof(ajulong);
5207         }
5208 
5209     }
5210 
5211     lpage->dirty = BT_DIRTY;
5212 
5213     return;
5214 }
5215 
5216 
5217 
5218 
5219 /* @funcstatic btreeWriteIdbucket *********************************************
5220 **
5221 ** Write index bucket object to the cache given a disc page number
5222 **
5223 ** @param [u] cache [AjPBtcache] cache
5224 ** @param [r] bucket [const AjPIdbucket] Id bucket
5225 ** @param [r] pagepos [ajulong] page number
5226 **
5227 ** @return [void]
5228 **
5229 ** @release 6.5.0
5230 ** @@
5231 ******************************************************************************/
5232 
btreeWriteIdbucket(AjPBtcache cache,const AjPIdbucket bucket,ajulong pagepos)5233 static void btreeWriteIdbucket(AjPBtcache cache, const AjPIdbucket bucket,
5234                                ajulong pagepos)
5235 {
5236     AjPBtpage page      = NULL;
5237     AjPBtpage lpage     = NULL;
5238     unsigned char *buf  = NULL;
5239     unsigned char *lbuf = NULL;
5240     ajuint  uv  = 0U;
5241     ajuint   v   = 0U;
5242     ajuint  i    = 0U;
5243     ajuint  iref = 0U;
5244     ajuint   len = 0U;
5245     ajulong lv   = 0UL;
5246     AjPBtId id  = NULL;
5247 
5248     ajuint nentries = 0U;
5249     ajulong overflow = 0UL;
5250     unsigned char *keyptr = NULL;
5251     unsigned char *lptr   = NULL;
5252     ajuint refskip = cache->refcount*BT_EXTRA;
5253 
5254 /*    ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1;*/
5255 
5256     if(pagepos == cache->totsize)	/* Create a new page */
5257     {
5258 	page = btreePricacheBucketnew(cache);
5259 	buf = page->buf;
5260     }
5261     else
5262     {
5263 	page = btreePricacheRead(cache,pagepos);
5264 	buf = page->buf;
5265 	GBT_BUCKOVERFLOW(buf,&overflow);
5266     }
5267 
5268     v = BT_IDBUCKET;
5269     SBT_BUCKNODETYPE(buf,v);
5270 
5271     lbuf = buf;
5272     page->dirty = BT_LOCK;
5273     page->lockfor = 1091;
5274     lpage = page;
5275 
5276     nentries = bucket->Nentries;
5277     v = nentries;
5278     SBT_BUCKNENTRIES(buf,v);
5279 
5280     /* Write out key lengths */
5281     keyptr = PBT_BUCKKEYLEN(lbuf);
5282 
5283     for(i=0;i<nentries;++i)
5284     {
5285         /* drop check - tested before call - pagesize dependency */
5286 /*
5287 //	if((ajuint)((keyptr-lbuf+1)+sizeof(ajuint)) > cache->pagesize)
5288 //	    ajFatal("WriteIdbucket: Bucket cannot hold more than %u keys",
5289 //		    i-1);
5290 */
5291 
5292 	id = bucket->Ids[i];
5293 	/* Need to alter this if bucket ID structure changes */
5294 	len = BT_BUCKIDLEN(id->id) + refskip;
5295         v = len;
5296 	BT_SETAJUINT(keyptr,v);
5297 	keyptr += sizeof(ajuint);
5298     }
5299 
5300 
5301     /* Write out IDs using overflow if necessary */
5302     lptr = keyptr;
5303 
5304     for(i=0;i<nentries;++i)
5305     {
5306 	id = bucket->Ids[i];
5307 	len = BT_BUCKIDLEN(id->id) + refskip;
5308 
5309 /*
5310 //        /# overflow #/
5311 //	if((lptr-buf+1) + (len+keyskip)) > (ajuint) cache->pagesize)
5312 //	{
5313 //	    if(!overflow)		/# No overflow buckets yet #/
5314 //	    {
5315 //                page = btreePricacheBucketnew(cache);
5316 //		buf = page->buf;
5317 //		v = BT_IDBUCKET;
5318 //		SBT_BUCKNODETYPE(buf,v);
5319 //	    }
5320 //	    else
5321 //	    {
5322 //		page = btreePricacheRead(cache,overflow);
5323 //		buf  = page->buf;
5324 //		GBT_BUCKOVERFLOW(buf,&overflow);
5325 //	    }
5326 //
5327 //	    page->dirty = BT_DIRTY;
5328 //
5329 //	    lptr = PBT_BUCKKEYLEN(buf);
5330 //	}
5331 */
5332 
5333 	sprintf((char *)lptr,"%s",ajStrGetPtr(id->id));
5334 	lptr += (ajStrGetLen(id->id) + 1);
5335         uv = id->dbno;
5336 	BT_SETAJUINT(lptr,uv);
5337 	lptr += sizeof(ajuint);
5338         uv = id->dups;
5339 	BT_SETAJUINT(lptr,uv);
5340 	lptr += sizeof(ajuint);
5341         lv = id->offset;
5342 	BT_SETAJULONG(lptr,lv);
5343 	lptr += sizeof(ajulong);
5344 
5345         if(cache->refcount)
5346         {
5347             for(iref=0; iref < cache->refcount; iref++)
5348             {
5349                 lv = id->refoffsets[iref];
5350                 BT_SETAJULONG(lptr,lv);
5351                 lptr += sizeof(ajulong);
5352             }
5353         }
5354     }
5355 
5356     lv = 0UL;
5357     SBT_BUCKOVERFLOW(buf,lv);
5358 
5359     lpage->dirty = BT_DIRTY;
5360 
5361     return;
5362 }
5363 
5364 
5365 
5366 
5367 /* @funcstatic btreeWriteIdbucketEmpty ****************************************
5368 **
5369 ** Write empty index id bucket object to the cache given a disc page number
5370 **
5371 ** @param [u] cache [AjPBtcache] cache
5372 ** @param [r] pagepos [ajulong] page number
5373 **
5374 ** @return [void]
5375 **
5376 ** @release 6.4.0
5377 ** @@
5378 ******************************************************************************/
5379 
btreeWriteIdbucketEmpty(AjPBtcache cache,ajulong pagepos)5380 static void btreeWriteIdbucketEmpty(AjPBtcache cache, ajulong pagepos)
5381 {
5382     AjPBtpage page      = NULL;
5383     unsigned char *buf  = NULL;
5384     ajuint   v   = 0U;
5385     ajulong lv   = 0UL;
5386 
5387     ajuint  nentries = 0U;
5388     ajulong overflow = 0UL;
5389 
5390     /* ajDebug("In btreeWriteIdbucketEmpty\n"); */
5391 
5392     if(pagepos == cache->totsize)	/* Create a new page */
5393     {
5394 	page = btreePricacheBucketnew(cache);
5395 	buf = page->buf;
5396 	overflow = 0UL;
5397 	lv = overflow;
5398 	SBT_BUCKOVERFLOW(buf,lv);
5399     }
5400     else
5401     {
5402 	page = btreePricacheRead(cache,pagepos);
5403 	buf = page->buf;
5404 	GBT_BUCKOVERFLOW(buf,&overflow);
5405     }
5406 
5407     v = BT_IDBUCKET;
5408     SBT_BUCKNODETYPE(buf,v);
5409 
5410     nentries = 0;
5411     v = nentries;
5412     SBT_BUCKNENTRIES(buf,v);
5413 
5414     lv = 0UL;
5415     SBT_BUCKOVERFLOW(buf,lv);
5416 
5417     page->dirty = BT_DIRTY;
5418 
5419     return;
5420 }
5421 
5422 
5423 
5424 
5425 /* @funcstatic btreeIdbucketDel ***********************************************
5426 **
5427 ** Delete an id bucket object
5428 **
5429 ** @param [w] thys [AjPIdbucket*] bucket
5430 **
5431 ** @return [void] bucket
5432 **
5433 ** @release 2.8.0
5434 ** @@
5435 ******************************************************************************/
5436 
btreeIdbucketDel(AjPIdbucket * thys)5437 static void btreeIdbucketDel(AjPIdbucket *thys)
5438 {
5439     AjPIdbucket pthis = NULL;
5440     ajuint newmax;
5441 
5442     /* ajDebug("In btreeIdbucketDel\n"); */
5443 
5444     if(!thys || !*thys)
5445 	return;
5446 
5447 
5448     pthis = *thys;
5449 
5450     if(!statSaveIdbucket)
5451     {
5452         statSaveIdbucketMax=2048;
5453         statSaveIdbucketNext=0;
5454         AJCNEW0(statSaveIdbucket,statSaveIdbucketMax);
5455     }
5456 
5457     if(!statSaveIdbucketEmpty)
5458     {
5459         statSaveIdbucketEmptyMax=2048;
5460         statSaveIdbucketEmptyNext=0;
5461         AJCNEW0(statSaveIdbucketEmpty,statSaveIdbucketEmptyMax);
5462     }
5463 
5464     if(pthis->Maxentries)
5465     {
5466         if(statSaveIdbucketNext >= statSaveIdbucketMax)
5467         {
5468             newmax = statSaveIdbucketMax + statSaveIdbucketMax;
5469             AJCRESIZE0(statSaveIdbucket,statSaveIdbucketMax,newmax);
5470             statSaveIdbucketMax = newmax;
5471         }
5472 
5473         statSaveIdbucket[statSaveIdbucketNext++] = pthis;
5474     }
5475     else
5476     {
5477         if(statSaveIdbucketEmptyNext >= statSaveIdbucketEmptyMax)
5478         {
5479             newmax = statSaveIdbucketEmptyMax + statSaveIdbucketEmptyMax;
5480             AJCRESIZE0(statSaveIdbucketEmpty,statSaveIdbucketEmptyMax,newmax);
5481             statSaveIdbucketEmptyMax = newmax;
5482         }
5483         statSaveIdbucketEmpty[statSaveIdbucketEmptyNext++] = pthis;
5484     }
5485 
5486     *thys = NULL;
5487 
5488     return;
5489 }
5490 
5491 
5492 
5493 
5494 /* @funcstatic btreeIdbucketFree **********************************************
5495 **
5496 ** Delete a keyword id bucket object
5497 **
5498 ** @param [w] thys [AjPIdbucket*] bucket
5499 **
5500 ** @return [void]
5501 **
5502 ** @release 6.5.0
5503 ** @@
5504 ******************************************************************************/
5505 
btreeIdbucketFree(AjPIdbucket * thys)5506 static void btreeIdbucketFree(AjPIdbucket *thys)
5507 {
5508     AjPIdbucket pthis = NULL;
5509     ajuint n;
5510     ajuint i;
5511 
5512     /*ajDebug("In btreeIdbucketFree\n"); */
5513 
5514     if(!thys || !*thys)
5515 	return;
5516 
5517     pthis = *thys;
5518 
5519     n = pthis->Maxentries;
5520 
5521     for(i=0;i<n;++i)
5522 	btreeIdFree(&pthis->Ids[i]);
5523 
5524     AJFREE(pthis->keylen);
5525     AJFREE(pthis->Ids);
5526 
5527     AJFREE(pthis);
5528 
5529     *thys = NULL;
5530 
5531     return;
5532 }
5533 
5534 
5535 
5536 
5537 /* @funcstatic btreeIdbucketAdd ***********************************************
5538 **
5539 ** Add an ID to an id bucket
5540 ** Only called if there is room in the bucket
5541 **
5542 ** @param [u] cache [AjPBtcache] cache
5543 ** @param [r] pagepos [ajulong] page number of bucket
5544 ** @param [r] id [const AjPBtId] ID info
5545 **
5546 ** @return [void]
5547 **
5548 ** @release 2.8.0
5549 ** @@
5550 ******************************************************************************/
5551 
btreeIdbucketAdd(AjPBtcache cache,ajulong pagepos,const AjPBtId id)5552 static void btreeIdbucketAdd(AjPBtcache cache, ajulong pagepos,
5553                              const AjPBtId id)
5554 {
5555     unsigned char *buf  = NULL;
5556     unsigned char *kptr = NULL;
5557     unsigned char *src  = NULL;
5558     unsigned char *dest = NULL;
5559 
5560     unsigned char *endptr  = NULL;
5561 
5562     ajuint nentries = 0;
5563     ajuint nodetype = 0;
5564 
5565     ajuint sum = 0;
5566     ajuint len = 0;
5567     ajuint i;
5568     ajuint v;
5569     ajuint uv;
5570     ajulong lv;
5571 
5572     ajuint iref;
5573 
5574     AjPBtpage page = NULL;
5575     static ajuint calls = 0;
5576     ajuint refskip = cache->refcount*BT_EXTRA;
5577 
5578 /*  ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1;*/
5579 
5580 /*    unsigned char *lastptr = NULL;*/
5581 /*    ajuint idlen    = 0;*/
5582 /*    static ajuint overflowcalls=0;*/
5583 
5584     calls++;
5585 
5586     page = btreePricacheRead(cache,pagepos);
5587     buf  = page->buf;
5588 
5589     GBT_BUCKNODETYPE(buf,&nodetype);
5590     if(nodetype != BT_IDBUCKET)
5591         ajFatal("Wrong nodetype in IdbucketAdd cache %S", cache->filename);
5592 
5593     GBT_BUCKNENTRIES(buf,&nentries);
5594     if(nentries == cache->pnperbucket)
5595         ajFatal("Bucket too full in IdbucketAdd page: %Lu "
5596                 "entries: %u max: %u cache %S",
5597                 pagepos, nentries, cache->snperbucket, cache->filename);
5598 
5599     kptr = PBT_BUCKKEYLEN(buf);
5600     src  = kptr + (nentries * sizeof(ajuint));
5601 
5602     sum = 0;
5603     for(i=0;i<nentries;++i)
5604     {
5605         BT_GETAJUINT(kptr,&len);
5606         sum += len;
5607         kptr += sizeof(ajuint);
5608     }
5609     /*sum += nentries;*/
5610 
5611     endptr  = src + sum - 1;
5612 
5613 /*
5614 //    idlen   = MAJSTRGETLEN(id->id);
5615 //    lastptr = endptr + sizeof(ajuint) + idlen + keyskip;
5616 //
5617 //    if((ajuint) (lastptr - buf) >= cache->pripagesize)
5618 //    {
5619 //        overflowcalls++;
5620 //        ajWarn("\nOverflow in IdbucketAdd nentries:%u fails %u/%u cache %S",
5621 //               nentries, overflowcalls,calls, cache->filename);
5622 //        btreeIdbucketAddFull(cache,pagepos,id);
5623 //        return;
5624 //    }
5625 */
5626 
5627     dest = src + sizeof(ajuint);
5628     memmove((void *)dest, (void *)src, sum);
5629 
5630     v = BT_BUCKIDLEN(id->id) + refskip;
5631     BT_SETAJUINT(src,v);
5632 
5633     endptr += sizeof(ajuint) + 1;
5634     strcpy((char *)endptr,MAJSTRGETPTR(id->id));
5635     endptr += (MAJSTRGETLEN(id->id) + 1);
5636     uv = id->dbno;
5637     BT_SETAJUINT(endptr,uv);
5638     endptr += sizeof(ajuint);
5639     uv = id->dups;
5640     BT_SETAJUINT(endptr,uv);
5641     endptr += sizeof(ajuint);
5642     lv = id->offset;
5643     BT_SETAJULONG(endptr,lv);
5644     endptr += sizeof(ajulong);
5645 
5646     if(cache->refcount)
5647     {
5648         for(iref=0; iref < cache->refcount; iref++)
5649         {
5650             lv = id->refoffsets[iref];
5651             BT_SETAJULONG(endptr,lv);
5652             endptr += sizeof(ajulong);
5653         }
5654     }
5655 
5656     v = nentries + 1;
5657     SBT_BUCKNENTRIES(buf,v);
5658 
5659     page->dirty = BT_DIRTY;
5660 
5661     return;
5662 }
5663 
5664 
5665 
5666 
5667 #if 0
5668 /* #funcstatic btreeIdbucketAddFull *******************************************
5669 **
5670 ** Add an ID to an id bucket
5671 ** Only called if there is no room in the bucket
5672 **
5673 ** #param [u] cache [AjPBtcache] cache
5674 ** #param [r] pagepos [ajulong] page number of bucket
5675 ** #param [r] id [const AjPBtId] ID info
5676 **
5677 ** #return [void]
5678 **
5679 ** #release 6.5.0
5680 ** ##
5681 ******************************************************************************/
5682 /*
5683 //static void btreeIdbucketAddFull(AjPBtcache cache, ajulong pagepos,
5684 //                                 const AjPBtId id)
5685 //{
5686 //    AjPIdbucket bucket = NULL;
5687 //    AjPBtId   destid = NULL;
5688 //
5689 //    ajuint nentries;
5690 //    ajuint iref;
5691 //
5692 //    /# ajDebug("In btreeIdbucketAddFull\n"); #/
5693 //
5694 //    bucket   = btreeReadIdbucket(cache,pagepos);
5695 //    nentries = bucket->Nentries;
5696 //
5697 //
5698 //    /# Reading a bucket always gives one extra ID position #/
5699 //    destid = bucket->Ids[nentries];
5700 //
5701 //    ajStrAssignS(&destid->id,id->id);
5702 //    destid->dbno      = id->dbno;
5703 //    destid->dups      = id->dups;
5704 //    destid->offset    = id->offset;
5705 //
5706 //    if(cache->refcount)
5707 //    {
5708 //        for(iref=0; iref < cache->refcount; iref++)
5709 //            destid->refoffsets[iref] = id->refoffsets[iref];
5710 //    }
5711 //
5712 //    ++bucket->Nentries;
5713 //
5714 //    btreeWriteIdbucket(cache,bucket,pagepos);
5715 //
5716 //    btreeIdbucketDel(&bucket);
5717 //
5718 //    return;
5719 //}
5720 */
5721 #endif
5722 
5723 
5724 
5725 
5726 /* @funcstatic btreeIdbucketCount *********************************************
5727 **
5728 ** Return number of entries in an id bucket
5729 **
5730 ** @param [u] cache [AjPBtcache] cache
5731 ** @param [r] pagepos [ajulong] page number
5732 **
5733 ** @return [ajuint] Number of entries in bucket
5734 **
5735 ** @release 2.8.0
5736 ** @@
5737 ******************************************************************************/
5738 
btreeIdbucketCount(AjPBtcache cache,ajulong pagepos)5739 static ajuint btreeIdbucketCount(AjPBtcache cache, ajulong pagepos)
5740 {
5741     AjPBtpage page     = NULL;
5742     unsigned char *buf = NULL;
5743     ajuint nodetype    = 0;
5744     ajuint nentries    = 0;
5745 
5746     /* ajDebug("In btreeIdbucketCount\n"); */
5747 
5748     if(!pagepos)
5749 	ajFatal("btreeIdbucketCount: Attempt to read bucket from root page\n");
5750 
5751     page  = btreePricacheRead(cache,pagepos);
5752 
5753     buf = page->buf;
5754 
5755     GBT_BUCKNODETYPE(buf,&nodetype);
5756 
5757     if(nodetype != BT_IDBUCKET)
5758 	ajFatal("btreeIdbucketCount: NodeType mismatch. Not id bucket (%u)",
5759                 nodetype);
5760 
5761     GBT_BUCKNENTRIES(buf,&nentries);
5762 
5763     return nentries;
5764 }
5765 
5766 
5767 
5768 
5769 /* @funcstatic btreeIdbucketsReorder ******************************************
5770 **
5771 ** Re-order leaf id buckets
5772 ** Must only be called if one of the buckets is full
5773 **
5774 ** @param [u] cache [AjPBtcache] cache
5775 ** @param [u] leaf [AjPBtpage] leaf page
5776 **
5777 ** @return [AjBool] true if reorder was successful i.e. leaf not full
5778 **
5779 ** @release 6.5.0
5780 ** @@
5781 ******************************************************************************/
5782 
btreeIdbucketsReorder(AjPBtcache cache,AjPBtpage leaf)5783 static AjBool btreeIdbucketsReorder(AjPBtcache cache, AjPBtpage leaf)
5784 {
5785     ajuint nkeys = 0;
5786     unsigned char *lbuf = NULL;
5787 
5788     ajulong *ptrs        = NULL;
5789     ajulong *overflows   = NULL;
5790     AjPStr *newkeys     = NULL;
5791     ajulong *newptrs     = NULL;
5792     AjPBtMem arrays = NULL;
5793     AjPBtMem newarrays = NULL;
5794 
5795     ajuint i = 0;
5796     ajuint iref = 0;
5797 
5798     ajuint order;
5799     ajuint totalkeys     = 0;
5800     ajuint maxnperbucket = 0;
5801     ajuint count         = 0;
5802     ajuint keylimit      = 0;
5803     ajuint bucketlimit   = 0;
5804     ajuint nodetype      = 0;
5805 
5806     AjPList idlist    = NULL;
5807     ajuint  dirtysave = 0;
5808     AjPBtId bid       = NULL;
5809     AjPIdbucket cbucket = NULL;
5810     AjPBtId cid       = NULL;
5811 
5812     ajuint iold = 0;
5813     ajuint refskip = cache->refcount*BT_EXTRA;
5814 
5815     /* ajDebug("In btreeIdbucketsReorder\n"); */
5816 
5817     dirtysave = leaf->dirty;
5818 
5819     leaf->dirty = BT_LOCK;
5820     leaf->lockfor = 1101;
5821     lbuf = leaf->buf;
5822 
5823     GBT_NODETYPE(lbuf,&nodetype);
5824 
5825     order = cache->porder;
5826 
5827     /* Read keys/ptrs */
5828     arrays = btreeAllocPriArray(cache);
5829     ptrs = arrays->parray;
5830 
5831     btreeGetPointers(cache,lbuf,&ptrs);
5832 
5833     GBT_NKEYS(lbuf,&nkeys);
5834 
5835 
5836     if(!nkeys)
5837 	ajFatal("IdbucketsReorder: Attempt to reorder empty leaf");
5838 
5839     keylimit = nkeys + 1;
5840     for(i=0;i<=nkeys;++i)
5841 	totalkeys += btreeIdbucketCount(cache,ptrs[i]);
5842 
5843     btreeBucketCalc(totalkeys, keylimit, cache->pnperbucket,
5844                     &bucketlimit, &maxnperbucket);
5845 
5846     if(bucketlimit >= order)
5847     {
5848         btreeDeallocPriArray(cache, arrays);
5849 
5850 	leaf->dirty = dirtysave;
5851 	return ajFalse;
5852     }
5853 
5854 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
5855     ajDebug("btreeIdbucketsReorder '%S' %Lu id '%S' key '%S'\n",
5856             cache->basename, leaf->pagepos, indexId, indexKeyword);
5857 #endif
5858     ++statCallIdbucketsReorder;
5859 
5860     newarrays = btreeAllocPriArray(cache);
5861     newkeys = newarrays->karray;
5862     newptrs = newarrays->parray;
5863     overflows = newarrays->overflows;
5864 
5865     /* Read IDs from all buckets and push to list and sort (increasing id) */
5866     idlist  = ajListNew();
5867 
5868     for(i=0;i<keylimit;++i)
5869 	overflows[i] = btreeIdbucketIdlist(cache,ptrs[i],idlist);
5870 
5871     ajListSort(idlist, &btreeIdCompare);
5872 
5873 
5874     cbucket = btreeIdbucketNew(maxnperbucket, cache->refcount);
5875     iold=0;
5876     for(i=0;i<bucketlimit;++i)
5877     {
5878 	cbucket->Overflow = overflows[i];
5879 	cbucket->Nentries = 0;
5880 
5881 	count = 0;
5882 
5883 	while(count!=maxnperbucket)
5884 	{
5885 	    ajListPop(idlist,(void **)&bid);
5886 
5887 	    cid = cbucket->Ids[count];
5888 	    ajStrAssignS(&cid->id,bid->id);
5889 	    cid->dbno = bid->dbno;
5890 	    cid->dups = bid->dups;
5891 	    cid->offset = bid->offset;
5892 
5893             if(cache->refcount)
5894             {
5895                 for(iref=0; iref < cache->refcount; iref++)
5896                     cid->refoffsets[iref] = bid->refoffsets[iref];
5897 	    }
5898 
5899 	    cbucket->keylen[count] = BT_BUCKIDLEN(bid->id) + refskip;
5900 	    ++cbucket->Nentries;
5901 	    ++count;
5902 	    ajBtreeIdDel(&bid);
5903 	}
5904 
5905 
5906 	ajListPeek(idlist,(void **)&bid);
5907 	ajStrAssignS(&newkeys[i],bid->id);
5908 
5909 	if((iold < order) && ptrs[iold])
5910             newptrs[i] = ptrs[iold++];
5911         else
5912 	    newptrs[i] = cache->totsize;
5913 	btreeWriteIdbucket(cache,cbucket,newptrs[i]);
5914     }
5915 
5916 
5917     /* Deal with greater-than bucket */
5918 
5919     cbucket->Overflow = overflows[i];
5920     cbucket->Nentries = 0;
5921 
5922     count = 0;
5923 
5924     while(ajListPop(idlist,(void **)&bid))
5925     {
5926 	cid = cbucket->Ids[count];
5927 	ajStrAssignS(&cid->id,bid->id);
5928 	cid->dbno = bid->dbno;
5929 	cid->dups = bid->dups;
5930 	cid->offset = bid->offset;
5931 
5932         if(cache->refcount)
5933         {
5934             for(iref=0; iref < cache->refcount; iref++)
5935                 cid->refoffsets[iref] = bid->refoffsets[iref];
5936 	}
5937 
5938 	++cbucket->Nentries;
5939 	++count;
5940 	ajBtreeIdDel(&bid);
5941     }
5942 
5943     if((iold < order) && ptrs[iold])
5944         newptrs[i] = ptrs[iold++];
5945     else
5946         newptrs[i] = cache->totsize;
5947      btreeWriteIdbucket(cache,cbucket,newptrs[i]);
5948 
5949     btreeIdbucketDel(&cbucket);
5950 
5951 #if AJINDEX_DEBUG
5952     if(iold < keylimit)
5953         ajDebug("btreeIdbucketsReorder '%S' %u -> %u",
5954                 cache->filename, keylimit, iold);
5955 #endif
5956 
5957     for(i = iold+1; i <= keylimit; i++)
5958     {
5959         btreePripageSetfree(cache, ptrs[i]);
5960     }
5961 
5962     /* Now write out a modified leaf with new keys/ptrs */
5963 
5964     nkeys = bucketlimit;
5965     btreeWriteNode(cache,leaf,newkeys,newptrs,nkeys);
5966 
5967     leaf->dirty = BT_DIRTY;
5968 
5969     if(nodetype == BT_ROOT)
5970     {
5971 	leaf->dirty = BT_LOCK;
5972         leaf->lockfor = 1102;
5973     }
5974 
5975     btreeDeallocPriArray(cache, arrays);
5976     btreeDeallocPriArray(cache, newarrays);
5977 
5978     btreeIdbucketDel(&cbucket);
5979     ajListFree(&idlist);
5980 
5981     return ajTrue;
5982 }
5983 
5984 
5985 
5986 
5987 /* @funcstatic btreeNodeIsFull ************************************************
5988 **
5989 ** Tests whether a node is full of keys
5990 **
5991 ** @param [r] cache [const AjPBtcache] cache
5992 ** @param [u] page [AjPBtpage] original page
5993 **
5994 ** @return [AjBool] true if full
5995 **
5996 ** @release 2.8.0
5997 ** @@
5998 ******************************************************************************/
5999 
btreeNodeIsFull(const AjPBtcache cache,AjPBtpage page)6000 static AjBool btreeNodeIsFull(const AjPBtcache cache, AjPBtpage page)
6001 {
6002     unsigned char *buf = NULL;
6003     ajuint nkeys = 0;
6004 
6005     /* ajDebug("In btreeNodeIsFull\n"); */
6006 
6007     buf = page->buf;
6008     GBT_NKEYS(buf,&nkeys);
6009 
6010     if(nkeys == cache->porder - 1)
6011 	return ajTrue;
6012 
6013     return ajFalse;
6014 }
6015 
6016 
6017 
6018 
6019 
6020 /* @func ajBtreeDumpKeywords **************************************************
6021 **
6022 ** Read the leaves of a complete keywords tree, returning all identifiers
6023 ** found within a range of total occurrences
6024 **
6025 ** @param [u] cache [AjPBtcache] cache
6026 ** @param [r] dmin [ajuint] minimum number of times the key should appear
6027 ** @param [r] dmax [ajuint] maximum number of times the key should appear
6028 ** @param [u] outf [AjPFile] output file
6029 **
6030 ** @return [void]
6031 **
6032 ** @release 6.5.0
6033 ** @@
6034 ******************************************************************************/
6035 
ajBtreeDumpKeywords(AjPBtcache cache,ajuint dmin,ajuint dmax,AjPFile outf)6036 void ajBtreeDumpKeywords(AjPBtcache cache,
6037                          ajuint dmin, ajuint dmax, AjPFile outf)
6038 {
6039     ajulong *parray;
6040     AjPBtpage page;
6041     unsigned char *buf;
6042     ajuint nodetype;
6043     ajuint i;
6044     ajuint j;
6045     ajulong kdups;
6046 
6047     AjPPribucket bucket;
6048     ajuint nkeys;
6049     ajulong right;
6050     AjPBtMem array = NULL;
6051     AjPList tlist  = NULL;
6052     ajuint dirtysave = 0U;
6053     ajulong totkeys = 0UL;
6054     ajulong totdups = 0UL;
6055 
6056     if(!cache->secondary)
6057     {
6058         ajBtreeDumpIdentifiers(cache, dmin, dmax, outf);
6059         return;
6060     }
6061 
6062     array = btreeAllocPriArray(cache);
6063     parray = array->parray;
6064 
6065     page = btreePricacheRead(cache, 0);
6066     buf = page->buf;
6067     dirtysave  = page->dirty;
6068     page->dirty = BT_LOCK;
6069     page->lockfor = 1111;
6070 
6071     btreeGetPointers(cache,buf,&parray);
6072     GBT_NODETYPE(buf,&nodetype);
6073 
6074     while(nodetype != BT_LEAF && cache->plevel != 0)
6075     {
6076         page->dirty = dirtysave;
6077 	page = btreePricacheRead(cache,parray[0]);
6078         dirtysave  =page->dirty;
6079         page->dirty = BT_LOCK;
6080         page->lockfor = 1112;
6081 	buf = page->buf;
6082 	btreeGetPointers(cache,buf,&parray);
6083 	GBT_NODETYPE(buf,&nodetype);
6084     }
6085 
6086     do
6087     {
6088 	GBT_NKEYS(buf,&nkeys);
6089 	for(i=0;i<=nkeys;++i)
6090 	{
6091 	    bucket = btreePribucketRead(cache,parray[i]);
6092             btreePribucketSort(bucket);
6093 
6094 	    for(j=0;j<bucket->Nentries;++j)
6095 	    {
6096                 totkeys++;
6097 
6098 		kdups = btreeSecTreeCount(cache, bucket->codes[j]->treeblock);
6099                 totdups += kdups;
6100 
6101 		ajListstrFreeData(&tlist);
6102 
6103                 if(kdups < dmin)
6104                     continue;
6105                 if(dmax && kdups > dmax)
6106                     continue;
6107                 ajFmtPrintF(outf,"%10Ld %S\n",
6108                             kdups, bucket->codes[j]->keyword);
6109 	    }
6110 	    btreePribucketDel(&bucket);
6111 	}
6112 
6113 	right = 0UL;
6114 	if(cache->plevel)
6115 	{
6116 	    GBT_RIGHT(buf,&right);
6117 	    if(right)
6118 	    {
6119                 page->dirty = dirtysave;
6120 		page = btreePricacheRead(cache,right);
6121                 dirtysave  =page->dirty;
6122                 page->dirty = BT_LOCK;
6123                 page->lockfor = 1113;
6124 		buf = page->buf;
6125 		btreeGetPointers(cache,buf,&parray);
6126 	    }
6127 	}
6128     } while(right);
6129 
6130     btreeDeallocPriArray(cache,array);
6131 
6132     ajFmtPrintF(outf,"\nTotal keywords: %Lu IDs: %Lu\n", totkeys, totdups);
6133     return;
6134 }
6135 
6136 
6137 
6138 
6139 /* @funcstatic btreeNodeIsFullSec *********************************************
6140 **
6141 ** Tests whether a secondary node is full of keys
6142 **
6143 ** @param [r] cache [const AjPBtcache] cache
6144 ** @param [u] page [AjPBtpage] original page
6145 **
6146 ** @return [AjBool] true if full
6147 **
6148 ** @release 4.0.0
6149 ** @@
6150 ******************************************************************************/
6151 
btreeNodeIsFullSec(const AjPBtcache cache,AjPBtpage page)6152 static AjBool btreeNodeIsFullSec(const AjPBtcache cache, AjPBtpage page)
6153 {
6154     unsigned char *buf = NULL;
6155     ajuint nkeys = 0;
6156 
6157     /* ajDebug("In btreeNodeIsFull\n"); */
6158 
6159     buf = page->buf;
6160     GBT_NKEYS(buf,&nkeys);
6161 
6162     if(nkeys == cache->sorder - 1)
6163 	return ajTrue;
6164 
6165     return ajFalse;
6166 }
6167 
6168 
6169 
6170 
6171 /* @funcstatic btreePrimaryInsertNonfull **************************************
6172 **
6173 ** Insert a key into a non-full primary node (identifiers or keywords)
6174 **
6175 ** @param [u] cache [AjPBtcache] cache
6176 ** @param [u] page [AjPBtpage] original page
6177 ** @param [r] key [const AjPStr] key to insert
6178 ** @param [r] less [ajulong] less-than pointer
6179 ** @param [r] greater [ajulong] greater-than pointer
6180 **
6181 ** @return [void]
6182 **
6183 ** @release 6.5.0
6184 ** @@
6185 ******************************************************************************/
6186 
btreePrimaryInsertNonfull(AjPBtcache cache,AjPBtpage page,const AjPStr key,ajulong less,ajulong greater)6187 static void btreePrimaryInsertNonfull(AjPBtcache cache, AjPBtpage page,
6188                                       const AjPStr key,
6189                                       ajulong less, ajulong greater)
6190 {
6191     unsigned char *buf = NULL;
6192 
6193     AjPStr *karray     = NULL;
6194     ajulong *parray     = NULL;
6195     AjPBtMem arrays1 = NULL;
6196 
6197     ajuint nkeys  = 0U;
6198     ajuint ipos   = 0U;
6199     ajuint i;
6200     ajint  ii;
6201     ajuint count  = 0U;
6202 
6203     ajulong lv = 0UL;
6204 
6205     AjPBtpage ppage = NULL;
6206     ajulong pagepos   = 0UL;
6207 
6208     ajuint nodetype = 0U;
6209 
6210     /* ajDebug("In btreeInsertNonfull\n"); */
6211 
6212     arrays1 = btreeAllocPriArray(cache);
6213     karray = arrays1->karray;
6214     parray = arrays1->parray;
6215 
6216     buf = page->buf;
6217     GBT_NKEYS(buf,&nkeys);
6218     GBT_NODETYPE(buf,&nodetype);
6219 
6220     btreeGetKeys(cache,buf,&karray,&parray);
6221 
6222     i = 0U;
6223 
6224     while(i!=nkeys && MAJSTRCMPS(key,karray[i]) >= 0)
6225 	++i;
6226 
6227     ipos = i;
6228 
6229     count = nkeys - ipos;
6230 
6231 
6232     if(ipos == nkeys)
6233     {
6234 	ajStrAssignS(&karray[ipos],key);
6235 	parray[ipos+1] = greater;
6236 	parray[ipos]   = less;
6237     }
6238     else
6239     {
6240 	parray[nkeys+1] = parray[nkeys];
6241 
6242 	for(ii=nkeys-1; count>0; --count, --ii)
6243 	{
6244 	    ajStrAssignS(&karray[ii+1],karray[ii]);
6245 	    parray[ii+1] = parray[ii];
6246 	}
6247 
6248 	ajStrAssignS(&karray[ipos],key);
6249 	parray[ipos] = less;
6250 	parray[ipos+1] = greater;
6251     }
6252 
6253     ++nkeys;
6254     btreeWriteNode(cache,page,karray,parray,nkeys);
6255 
6256     if(nodetype == BT_ROOT)
6257     {
6258 	page->dirty = BT_LOCK;
6259         page->lockfor = 1121;
6260     }
6261 
6262     pagepos = page->pagepos;
6263     ppage = btreePricacheRead(cache,less);
6264     lv = pagepos;
6265     SBT_PREV(ppage->buf,lv);
6266     ppage->dirty = BT_DIRTY;
6267 
6268     ppage = btreePricacheRead(cache,greater);
6269     lv = pagepos;
6270     SBT_PREV(ppage->buf,lv);
6271     ppage->dirty = BT_DIRTY;
6272 
6273     btreeDeallocPriArray(cache,arrays1);
6274 
6275     if(nodetype != BT_ROOT)
6276 	btreePrimaryShift(cache,page);
6277 
6278     return;
6279 }
6280 
6281 
6282 
6283 
6284 /* @funcstatic btreePriInsertKey **********************************************
6285 **
6286 ** Insert a key into a potentially full primary node
6287 **
6288 ** @param [u] cache [AjPBtcache] cache
6289 ** @param [u] page [AjPBtpage] original page
6290 ** @param [r] key [const AjPStr] key to insert
6291 ** @param [r] less [ajulong] less-than pointer
6292 ** @param [r] greater [ajulong] greater-than pointer
6293 **
6294 ** @return [void]
6295 **
6296 ** @release 6.5.0
6297 ** @@
6298 ******************************************************************************/
6299 
btreePriInsertKey(AjPBtcache cache,AjPBtpage page,const AjPStr key,ajulong less,ajulong greater)6300 static void btreePriInsertKey(AjPBtcache cache, AjPBtpage page,
6301                               const AjPStr key, ajulong less, ajulong greater)
6302 {
6303     unsigned char *lbuf = NULL;
6304     unsigned char *rbuf = NULL;
6305     unsigned char *tbuf = NULL;
6306 
6307     AjPStr *karray      = NULL;
6308     ajulong *parray      = NULL;
6309     AjPStr *tkarray     = NULL;
6310     ajulong *tparray     = NULL;
6311     AjPBtMem savekeyarrays = NULL;
6312     AjPBtMem arrays2 = NULL;
6313 
6314     ajuint nkeys    = 0U;
6315     ajuint order    = 0U;
6316     ajuint keypos   = 0U;
6317     ajuint rkeyno   = 0U;
6318 
6319     ajuint i = 0U;
6320     ajuint n = 0U;
6321 
6322     ajuint nodetype  = 0;
6323     AjPBtpage ipage = NULL;
6324     AjPBtpage lpage = NULL;
6325     AjPBtpage rpage = NULL;
6326     AjPBtpage tpage = NULL;
6327 
6328     ajulong blockno  = 0UL;
6329     ajulong rblockno = 0UL;
6330     ajulong lblockno = 0UL;
6331     ajulong ibn      = 0UL;
6332 
6333 
6334     AjPStr mediankey  = NULL;
6335     ajulong medianless = 0UL;
6336     ajulong mediangtr  = 0UL;
6337     ajulong overflow   = 0UL;
6338     ajulong prev       = 0UL;
6339     ajuint totlen     = 0U;
6340 
6341     ajulong lv = 0UL;
6342     ajuint  v  = 0U;
6343 
6344     if(!btreeNodeIsFull(cache,page))
6345     {
6346 	btreePrimaryInsertNonfull(cache,page,key,less,greater);
6347 	return;
6348     }
6349 
6350     order = cache->porder;
6351     lbuf = page->buf;
6352     GBT_NODETYPE(lbuf,&nodetype);
6353     page->dirty = BT_LOCK;
6354     page->lockfor = 1131;
6355 
6356     if(nodetype == BT_ROOT)
6357     {
6358 	btreePriSplitroot(cache);
6359 
6360 	if(page->pagepos)
6361 	    page->dirty = BT_DIRTY;
6362 
6363         blockno = btreeGetBlockFirstS(cache,lbuf,key);
6364 
6365 	ipage = btreePricacheRead(cache,blockno);
6366 	btreePrimaryInsertNonfull(cache,ipage,key,less,greater);
6367 
6368 	return;
6369     }
6370 
6371 
6372     savekeyarrays = btreeAllocPriArray(cache);
6373     arrays2 = btreeAllocPriArray(cache);
6374     karray = savekeyarrays->karray;
6375     parray = savekeyarrays->parray;
6376     tkarray = arrays2->karray;
6377     tparray = arrays2->parray;
6378 
6379     lpage = page;
6380     lbuf = lpage->buf;
6381 
6382     btreeGetKeys(cache,lbuf,&karray,&parray);
6383 
6384     GBT_BLOCKNUMBER(lbuf,&lblockno);
6385     rblockno = cache->totsize;
6386     rpage = btreePricacheNodenew(cache);
6387     rpage->dirty = BT_LOCK;
6388     rpage->lockfor = 1132;
6389     rbuf = rpage->buf;
6390 
6391 
6392     GBT_PREV(lbuf,&prev);
6393     lv = prev;
6394     SBT_PREV(rbuf,lv);
6395 
6396     nkeys = order - 1;
6397     keypos = nkeys / 2;
6398 
6399     if(!(nkeys % 2))
6400 	--keypos;
6401 
6402     mediankey = ajStrNewS(karray[keypos]);
6403     medianless = lblockno;
6404     mediangtr  = rblockno;
6405 
6406 
6407     GBT_NODETYPE(lbuf,&nodetype);
6408     v = nodetype;
6409     SBT_NODETYPE(rbuf,v);
6410     lv = overflow;
6411     SBT_OVERFLOW(rbuf,lv);
6412 
6413 
6414     totlen = 0;
6415 
6416     for(i=0;i<keypos;++i)
6417     {
6418 	ajStrAssignS(&tkarray[i],karray[i]);
6419 	totlen += ajStrGetLen(karray[i]);
6420 	tparray[i] = parray[i];
6421     }
6422 
6423     tparray[i] = parray[i];
6424     v = totlen;
6425     SBT_TOTLEN(lbuf,v);
6426     n = i;
6427     v = n;
6428     SBT_NKEYS(lbuf,v);
6429     btreeWriteNode(cache,lpage,tkarray,tparray,i);
6430 
6431 
6432 
6433     for(i=0;i<n+1;++i)
6434     {
6435 	tpage = btreePricacheRead(cache,tparray[i]);
6436 	tbuf = tpage->buf;
6437 	lv = lblockno;
6438 	SBT_PREV(tbuf,lv);
6439 	tpage->dirty = BT_DIRTY;
6440     }
6441 
6442 
6443     totlen = 0;
6444 
6445     for(i=keypos+1;i<nkeys;++i)
6446     {
6447 	ajStrAssignS(&tkarray[i-(keypos+1)],karray[i]);
6448 	totlen += ajStrGetLen(karray[i]);
6449 	tparray[i-(keypos+1)] = parray[i];
6450     }
6451 
6452     tparray[i-(keypos+1)] = parray[i];
6453     v = totlen;
6454     SBT_TOTLEN(rbuf,v);
6455     rkeyno = (nkeys-keypos) - 1;
6456     v = rkeyno;
6457     SBT_NKEYS(rbuf,v);
6458     rpage->dirty = BT_DIRTY;
6459 
6460     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
6461 
6462     for(i=0;i<rkeyno+1;++i)
6463     {
6464 	tpage = btreePricacheRead(cache,tparray[i]);
6465 	tbuf = tpage->buf;
6466 	lv = rblockno;
6467 	SBT_PREV(tbuf,lv);
6468 	tpage->dirty = BT_DIRTY;
6469     }
6470 
6471 
6472     ibn = rblockno;
6473 
6474     if(MAJSTRCMPS(key,mediankey)<0)
6475 	ibn = lblockno;
6476 
6477     ipage = btreePricacheRead(cache,ibn);
6478 
6479     btreePrimaryInsertNonfull(cache,ipage,key,less,greater);
6480 
6481     btreeDeallocPriArray(cache, savekeyarrays);
6482     btreeDeallocPriArray(cache, arrays2);
6483 
6484     ipage = btreePricacheRead(cache,prev);
6485 
6486     btreePriInsertKey(cache,ipage,mediankey,medianless,mediangtr);
6487     ajStrDel(&mediankey);
6488 
6489     return;
6490 }
6491 
6492 
6493 
6494 
6495 /* @funcstatic btreePriSplitroot **********************************************
6496 **
6497 ** Split the root node
6498 **
6499 ** @param [u] cache [AjPBtcache] cache
6500 **
6501 ** @return [void]
6502 **
6503 ** @release 6.5.0
6504 ** @@
6505 ******************************************************************************/
6506 
btreePriSplitroot(AjPBtcache cache)6507 static void btreePriSplitroot(AjPBtcache cache)
6508 {
6509     AjPBtpage rootpage = NULL;
6510     AjPBtpage rpage    = NULL;
6511     AjPBtpage lpage    = NULL;
6512     AjPBtpage tpage    = NULL;
6513 
6514     AjPBtMem arrays1 = NULL;
6515     AjPBtMem arrays2 = NULL;
6516     AjPStr *karray     = NULL;
6517     AjPStr *tkarray    = NULL;
6518     ajulong *parray     = NULL;
6519     ajulong *tparray    = NULL;
6520 
6521     ajuint order     = 0U;
6522     ajuint nkeys     = 0U;
6523     ajuint keypos    = 0U;
6524 
6525     ajulong rblockno = 0UL;
6526     ajulong lblockno = 0UL;
6527 
6528     AjPStr key = NULL;
6529     ajuint  i;
6530     ajuint  j;
6531 
6532     unsigned char *rootbuf = NULL;
6533     unsigned char *rbuf    = NULL;
6534     unsigned char *lbuf    = NULL;
6535     unsigned char *tbuf    = NULL;
6536 
6537     ajuint nodetype  = 0U;
6538     ajulong overflow = 0UL;
6539     ajulong zero     = 0UL;
6540     ajuint rkeyno    = 0U;
6541 
6542     ajulong lv = 0UL;
6543     ajuint  v  = 0U;
6544 
6545 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
6546     ajDebug("btreePriSplitroot '%S' zero id '%S' key '%S'\n",
6547             cache->basename, indexId, indexKeyword);
6548 #endif
6549     ++statCallPriSplitroot;
6550 
6551     /* ajDebug("In btreePriSplitroot\n"); */
6552 
6553     order = cache->porder;
6554 
6555     arrays1 = btreeAllocPriArray(cache);
6556     arrays2 = btreeAllocPriArray(cache);
6557     karray = arrays1->karray;
6558     parray = arrays1->parray;
6559     tkarray = arrays2->karray;
6560     tparray = arrays2->parray;
6561 
6562     key = ajStrNew();
6563 
6564     rootpage = btreePricacheLocate(cache,0UL);
6565 
6566     if(!rootpage)
6567 	ajFatal("Root page has been unlocked 1 cache %S", cache->filename);
6568 
6569     rootbuf = rootpage->buf;
6570 
6571     nkeys = order - 1;
6572 
6573     keypos = nkeys / 2;
6574 
6575     if(!(nkeys % 2))
6576 	--keypos;
6577 
6578     rblockno = cache->totsize;
6579     rpage = btreePricacheNodenew(cache);
6580     rpage->dirty = BT_LOCK;
6581     rpage->lockfor = 1141;
6582 
6583     lblockno = cache->totsize;
6584     lpage = btreePricacheNodenew(cache);
6585     rpage->dirty = BT_DIRTY;
6586     rpage->lockfor = 1142;
6587 
6588     if(!cache->plevel)
6589     {
6590 	lv = zero;
6591 	SBT_LEFT(lpage->buf,lv);
6592 	lv = rblockno;
6593 	SBT_RIGHT(lpage->buf,lv);
6594 	lv = lblockno;
6595 	SBT_LEFT(rpage->buf,lv);
6596 	lv = zero;
6597 	SBT_RIGHT(rpage->buf,lv);
6598     }
6599 
6600     btreeGetKeys(cache,rootbuf,&karray,&parray);
6601 
6602     btreeWriteNodeSingle(cache,rootpage,karray[keypos],lblockno,rblockno);
6603 
6604     rootpage->dirty = BT_LOCK;
6605     rootpage->lockfor = 1143;
6606 
6607     rbuf = rpage->buf;
6608     lbuf = lpage->buf;
6609 
6610     if(cache->plevel)
6611 	nodetype = BT_INTERNAL;
6612     else
6613 	nodetype = BT_LEAF;
6614 
6615     v = nodetype;
6616     SBT_NODETYPE(rbuf,v);
6617     v = nodetype;
6618     SBT_NODETYPE(lbuf,v);
6619     lv = overflow;
6620     SBT_OVERFLOW(rbuf,lv);
6621     lv = zero;
6622     SBT_PREV(rbuf,lv);
6623     lv = overflow;
6624     SBT_OVERFLOW(lbuf,lv);
6625     lv = zero;
6626     SBT_PREV(lbuf,lv);
6627 
6628     for(i=0;i<keypos;++i)
6629     {
6630 	ajStrAssignS(&tkarray[i],karray[i]);
6631 	tparray[i] = parray[i];
6632     }
6633 
6634     tparray[i] = parray[i];
6635     btreeWriteNode(cache,lpage,tkarray,tparray,keypos);
6636 
6637     for(i=0;i<=keypos;++i)
6638     {
6639 	tpage = btreePricacheRead(cache,tparray[i]);
6640 	tbuf = tpage->buf;
6641 	lv = lblockno;
6642 	SBT_PREV(tbuf,lv);
6643 	tpage->dirty = BT_DIRTY;
6644     }
6645 
6646     j = 0;
6647 
6648     for(i=keypos+1;i<nkeys;++i)
6649     {
6650 	ajStrAssignS(&tkarray[j],karray[i]);
6651 	tparray[j++] = parray[i];
6652     }
6653 
6654     tparray[j] = parray[i];
6655     rkeyno = (nkeys-keypos) - 1;
6656     rpage->dirty = BT_DIRTY;
6657 
6658     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
6659 
6660     for(i=0;i<rkeyno+1;++i)
6661     {
6662 	tpage = btreePricacheRead(cache,tparray[i]);
6663 	tbuf = tpage->buf;
6664 	lv = rblockno;
6665 	SBT_PREV(tbuf,lv);
6666 	tpage->dirty = BT_DIRTY;
6667     }
6668 
6669 
6670     for(i=0;i<order;++i)
6671     {
6672 	ajStrDel(&karray[i]);
6673 	ajStrDel(&tkarray[i]);
6674     }
6675 
6676     btreeDeallocPriArray(cache,arrays1);
6677     btreeDeallocPriArray(cache,arrays2);
6678     ++cache->plevel;
6679 
6680     ajStrDel(&key);
6681 
6682     return;
6683 }
6684 
6685 
6686 
6687 
6688 /* @funcstatic btreeGetKeys ***************************************************
6689 **
6690 ** Get Keys and Pointers from an internal node
6691 **
6692 ** @param [u] cache [AjPBtcache] cache
6693 ** @param [u] buf [unsigned char *] page buffer
6694 ** @param [w] keys [AjPStr **] keys
6695 ** @param [w] ptrs [ajulong**] ptrs
6696 **
6697 ** @return [void]
6698 **
6699 ** @release 2.8.0
6700 ** @@
6701 ******************************************************************************/
6702 
btreeGetKeys(AjPBtcache cache,unsigned char * buf,AjPStr ** keys,ajulong ** ptrs)6703 static void btreeGetKeys(AjPBtcache cache, unsigned char *buf,
6704 			 AjPStr **keys, ajulong **ptrs)
6705 {
6706     AjPStr *karray = NULL;
6707     ajulong *parray = NULL;
6708 
6709     ajuint  nkeys;
6710     unsigned char *lenptr = NULL;
6711     unsigned char *keyptr = NULL;
6712     unsigned char *tbuf   = NULL;
6713 
6714     ajuint    i;
6715     ajuint    ival = 0U;
6716 
6717     ajuint    len;
6718 
6719 /*    ajulong   overflow = 0UL;*/
6720 /*    ajuint    pagesize = 0U;*/
6721 /*    AjPBtpage page = NULL;*/
6722 
6723     karray = *keys;
6724     parray = *ptrs;
6725 
6726     tbuf    = buf;
6727 
6728 /*    pagesize = cache->pagesize;*/
6729 
6730     GBT_NKEYS(tbuf,&ival);
6731     nkeys = ival;
6732     if(!nkeys)
6733 	ajFatal("GetKeys: No keys in node cache %S", cache->filename);
6734 
6735     lenptr =  PBT_KEYLEN(tbuf);
6736     keyptr = lenptr + nkeys * sizeof(ajuint);
6737 
6738 
6739     for(i=0U; i<nkeys; ++i)
6740     {
6741 	BT_GETAJUINT(lenptr,&ival);
6742 	len = ival+1;
6743 
6744 /*
6745 //	if((ajuint)((keyptr-tbuf+1) + len + sizeof(ajulong)) > pagesize)
6746 //	{
6747 //#if AJINDEX_DEBUG
6748 //    	    ajDebug("GetKeys: Overflow\n");
6749 //#endif
6750 //	    GBT_OVERFLOW(tbuf,&overflow);
6751 //	    page = btreePricacheRead(cache,overflow);
6752 //	    tbuf = page->buf;
6753 //	    GBT_NODETYPE(tbuf,&ival);
6754 //
6755 //	    if(ival != BT_OVERFLOW)
6756 //		ajFatal("Overflow node expected but not found for cache %S",
6757 //                        cache->filename);
6758 //	    /#
6759 //	     ** The length pointer is restricted to the initial page.
6760 //	     ** The keyptr in overflow pages starts at the Key Lengths
6761 //	     ** position!
6762 //	     #/
6763 //	    keyptr = PBT_KEYLEN(tbuf);
6764 //	}
6765 */
6766 
6767 	ajStrAssignC(&karray[i],(const char *)keyptr);
6768 	keyptr += len;
6769 
6770 	BT_GETAJULONG(keyptr,&parray[i]);
6771 	keyptr += sizeof(ajulong);
6772 	lenptr += sizeof(ajuint);
6773     }
6774 
6775 /*
6776 //    if((ajuint)((keyptr-tbuf+1) + sizeof(ajulong)) > pagesize)
6777 //    {
6778 //	GBT_OVERFLOW(tbuf,&overflow);
6779 //	page = btreePricacheRead(cache,overflow);
6780 //	tbuf = page->buf;
6781 //	GBT_NODETYPE(tbuf,&ival);
6782 //
6783 //	if(ival != BT_OVERFLOW)
6784 //	    ajFatal("Overflow node expected but not found at end for cache %S",
6785 //                    cache->filename);
6786 //	/#
6787 //	 ** The length pointer is restricted to the initial page.
6788 //	 ** The keyptr in overflow pages starts at the Key Lengths
6789 //	 ** position!
6790 //	 #/
6791 //	keyptr = PBT_KEYLEN(tbuf);
6792 //    }
6793 */
6794 
6795     BT_GETAJULONG(keyptr,&parray[i]);
6796 
6797     return;
6798 }
6799 
6800 
6801 
6802 
6803 /* @funcstatic btreeGetPointers ***********************************************
6804 **
6805 ** Get Pointers from an internal node
6806 **
6807 ** @param [u] cache [AjPBtcache] cache
6808 ** @param [u] buf [unsigned char *] page buffer
6809 ** @param [w] ptrs [ajulong**] ptrs
6810 **
6811 ** @return [ajuint] Number of pointers
6812 **
6813 ** @release 6.4.0
6814 ** @@
6815 ******************************************************************************/
6816 
btreeGetPointers(AjPBtcache cache,unsigned char * buf,ajulong ** ptrs)6817 static ajuint btreeGetPointers(AjPBtcache cache, unsigned char *buf,
6818                                ajulong **ptrs)
6819 {
6820     ajulong *parray = NULL;
6821 
6822     ajuint  nkeys;
6823     unsigned char *lenptr = NULL;
6824     unsigned char *keyptr = NULL;
6825     unsigned char *tbuf   = NULL;
6826 
6827     ajuint    i;
6828     ajuint    ival = 0U;
6829 
6830     ajuint    len;
6831 /*    ajuint    pagesize = 0U;*/
6832 /*    ajulong   overflow = 0UL;*/
6833 /*    AjPBtpage page = NULL;*/
6834 
6835     parray = *ptrs;
6836 
6837     tbuf    = buf;
6838 
6839 /*    pagesize = cache->pagesize;*/
6840 
6841     GBT_NKEYS(tbuf,&ival);
6842     nkeys = ival;
6843     if(!nkeys)
6844 	ajFatal("btreeGetPointers: No keys in node cache %S",
6845                 cache->filename);
6846 
6847     lenptr =  PBT_KEYLEN(tbuf);
6848     keyptr = lenptr + nkeys * sizeof(ajuint);
6849 
6850     GBT_NODETYPE(tbuf,&ival);
6851 
6852     for(i=0U; i<nkeys; ++i)
6853     {
6854 	BT_GETAJUINT(lenptr,&ival);
6855 	len = ival+1;
6856 
6857 /*
6858 //        if((ajuint)((keyptr-tbuf+1) + len + sizeof(ajulong)) > pagesize)
6859 //	{
6860 //	    GBT_OVERFLOW(tbuf,&overflow);
6861 //	    page = ajBtreeCacheRead(cache,overflow); /# pri or sec possible #/
6862 //	    tbuf = page->buf;
6863 //	    GBT_NODETYPE(tbuf,&ival);
6864 //	    if(ival != BT_OVERFLOW)
6865 //		ajFatal("Overflow node expected but not found cache %S",
6866 //                        cache->filename);
6867 //	    /#
6868 //	     ** The length pointer is restricted to the initial page.
6869 //	     ** The keyptr in overflow pages starts at the Key Lengths
6870 //	     ** position!
6871 //	     #/
6872 //	    keyptr = PBT_KEYLEN(tbuf);
6873 //	}
6874 */
6875 
6876 	keyptr += len;
6877 
6878 	BT_GETAJULONG(keyptr,&parray[i]);
6879 	keyptr += sizeof(ajulong);
6880 	lenptr += sizeof(ajuint);
6881     }
6882 
6883 /*
6884 //    if((ajuint)((keyptr-tbuf+1) + sizeof(ajulong)) > pagesize)
6885 //    {
6886 //	/# ajDebug("GetPointers: Overflow\n"); #/
6887 //	GBT_OVERFLOW(tbuf,&overflow);
6888 //	page = ajBtreeCacheRead(cache,overflow); /# pri or sec possible #/
6889 //	tbuf = page->buf;
6890 //	GBT_NODETYPE(tbuf,&ival);
6891 //	if(ival != BT_OVERFLOW)
6892 //	    ajFatal("Overflow node expected but not found cache %S",
6893 //                    cache->filename);
6894 //	/#
6895 //	 ** The length pointer is restricted to the initial page.
6896 //	 ** The keyptr in overflow pages starts at the Key Lengths
6897 //	 ** position!
6898 //	 #/
6899 //	keyptr = PBT_KEYLEN(tbuf);
6900 //    }
6901 */
6902 
6903     BT_GETAJULONG(keyptr,&parray[i]);
6904 
6905     return i;
6906 }
6907 
6908 
6909 
6910 
6911 
6912 /* @funcstatic btreeGetBlockC *************************************************
6913 **
6914 ** Get Block number matching a key
6915 **
6916 ** @param [u] cache [AjPBtcache] cache
6917 ** @param [u] buf [unsigned char *] page buffer
6918 ** @param [r] ckey [const char*] key
6919 **
6920 ** @return [ajulong] Block number
6921 **
6922 ** @release 6.4.0
6923 ** @@
6924 ******************************************************************************/
6925 
btreeGetBlockC(AjPBtcache cache,unsigned char * buf,const char * ckey)6926 static ajulong btreeGetBlockC(AjPBtcache cache, unsigned char *buf,
6927                               const char* ckey)
6928 {
6929     ajulong blockno = 0UL;
6930 
6931     ajuint m;
6932     unsigned char *lenptr = NULL;
6933     unsigned char *keyptr = NULL;
6934     unsigned char *tbuf   = NULL;
6935 
6936     ajuint    i;
6937     ajuint    ival = 0U;
6938     ajuint    len;
6939 
6940 /*    AjPBtpage page = NULL;*/
6941 /*    ajuint    pagesize = 0U;*/
6942 /*    ajulong   overflow = 0UL;*/
6943 
6944 /*    pagesize = cache->pagesize;*/
6945 
6946     tbuf = buf;
6947 
6948     GBT_NKEYS(tbuf,&m);
6949     if(!m)
6950 	ajFatal("btreeGetBlockC: No keys in node cache %S",
6951                 cache->filename);
6952 
6953     lenptr =  PBT_KEYLEN(tbuf);
6954     keyptr = lenptr + m * sizeof(ajuint);
6955 
6956     for(i=0U; i<m; ++i)
6957     {
6958 	BT_GETAJUINT(lenptr,&ival);
6959 	len = ival+1;
6960 
6961 /*
6962 //	if((ajuint)((keyptr-tbuf+1) + len + sizeof(ajulong)) > pagesize)
6963 //	{
6964 //    	    /# ajDebug("btreeGetBlockC: Overflow\n"); #/
6965 //	    GBT_OVERFLOW(tbuf,&overflow);
6966 //	    page = ajBtreeCacheRead(cache,overflow); /# pri or sec possible #/
6967 //	    tbuf = page->buf;
6968 //	    GBT_NODETYPE(tbuf,&ival);
6969 //	    if(ival != BT_OVERFLOW)
6970 //		ajFatal("Overflow node expected but not found cache %S",
6971 //                        cache->filename);
6972 //	    /#
6973 //	     ** The length pointer is restricted to the initial page.
6974 //	     ** The keyptr in overflow pages starts at the Key Lengths
6975 //	     ** position!
6976 //	     #/
6977 //	    keyptr = PBT_KEYLEN(tbuf);
6978 //	}
6979 */
6980 
6981         if(strcmp((const char*)keyptr, ckey) > 0)
6982         {
6983             keyptr += len;
6984 
6985             BT_GETAJULONG(keyptr,&blockno);
6986             return blockno;
6987         }
6988 
6989         keyptr += len;
6990 	keyptr += sizeof(ajulong);
6991 	lenptr += sizeof(ajuint);
6992     }
6993 
6994 /*
6995 //    if((ajuint)((keyptr-tbuf+1) + sizeof(ajulong)) > pagesize)
6996 //    {
6997 //	/# ajDebug("btreeGetBlockC: Overflow\n"); #/
6998 //	GBT_OVERFLOW(tbuf,&overflow);
6999 //	page = ajBtreeCacheRead(cache,overflow); /# pri or sec possible #/
7000 //	tbuf = page->buf;
7001 //	GBT_NODETYPE(tbuf,&ival);
7002 //	if(ival != BT_OVERFLOW)
7003 //	    ajFatal("Overflow node expected but not found cache %S",
7004 //                    cache->filename);
7005 //	/#
7006 //	 ** The length pointer is restricted to the initial page.
7007 //	 ** The keyptr in overflow pages starts at the Key Lengths
7008 //	 ** position!
7009 //	 #/
7010 //	keyptr = PBT_KEYLEN(tbuf);
7011 //    }
7012 */
7013 
7014     BT_GETAJULONG(keyptr,&blockno);
7015 
7016     return blockno;
7017 }
7018 
7019 
7020 
7021 
7022 /* @funcstatic btreeGetBlockN *************************************************
7023 **
7024 ** Get Block number matching a numeric key
7025 **
7026 ** @param [u] cache [AjPBtcache] cache
7027 ** @param [u] buf [unsigned char *] page buffer
7028 ** @param [r] numkey [ajulong] key
7029 **
7030 ** @return [ajulong] Block number
7031 **
7032 ** @release 6.4.0
7033 ** @@
7034 ******************************************************************************/
7035 
btreeGetBlockN(AjPBtcache cache,unsigned char * buf,ajulong numkey)7036 static ajulong btreeGetBlockN(AjPBtcache cache, unsigned char *buf,
7037                               ajulong numkey)
7038 {
7039     ajulong blockno = 0UL;
7040 
7041     ajuint m;
7042     unsigned char *pptr = NULL;
7043     unsigned char *kptr = NULL;
7044 
7045     ajuint    i;
7046     ajulong   kval = 0UL;
7047 
7048     GBT_NKEYS(buf,&m);
7049     if(!m)
7050 	ajFatal("btreeGetBlockN: No keys in node cache %S", cache->filename);
7051 
7052     kptr = PBT_KEYLEN(buf);
7053     pptr = kptr + m * sizeof(ajulong);
7054 
7055     for(i=0U; i<m; ++i)
7056     {
7057 	BT_GETAJULONG(kptr,&kval);
7058 
7059         if(kval > numkey)
7060         {
7061             BT_GETAJULONG(pptr,&blockno);
7062             return blockno;
7063         }
7064 
7065 	kptr += sizeof(ajulong);
7066 	pptr += sizeof(ajulong);
7067     }
7068 
7069 
7070     BT_GETAJULONG(pptr,&blockno);
7071 
7072     return blockno;
7073 }
7074 
7075 
7076 
7077 
7078 /* @funcstatic btreeGetBlockFirstN ********************************************
7079 **
7080 ** Get Block number of first block if more than numeric key
7081 ** or second block
7082 **
7083 ** @param [u] cache [AjPBtcache] cache
7084 ** @param [u] buf [unsigned char *] page buffer
7085 ** @param [r] numkey [ajulong] key
7086 **
7087 ** @return [ajulong] Block number
7088 **
7089 ** @release 6.4.0
7090 ** @@
7091 ******************************************************************************/
7092 
btreeGetBlockFirstN(AjPBtcache cache,unsigned char * buf,ajulong numkey)7093 static ajulong btreeGetBlockFirstN(AjPBtcache cache, unsigned char *buf,
7094                                    ajulong numkey)
7095 {
7096     ajulong blockno = 0UL;
7097 
7098     ajuint m;
7099     unsigned char *pptr = NULL;
7100     unsigned char *kptr = NULL;
7101 
7102     ajulong   kval = 0UL;
7103 
7104     GBT_NKEYS(buf,&m);
7105     if(!m)
7106 	ajFatal("btreeGetBlockFirstN: No keys in node cache %S",
7107                 cache->filename);
7108 
7109     kptr = PBT_KEYLEN(buf);
7110     pptr = kptr + m * sizeof(ajulong);
7111 
7112     BT_GETAJULONG(kptr,&kval);
7113 
7114     if(kval > numkey)
7115     {
7116         BT_GETAJULONG(pptr,&blockno);
7117         return blockno;
7118     }
7119 
7120     kptr += sizeof(ajulong);
7121     pptr += sizeof(ajulong);
7122 
7123 
7124     BT_GETAJULONG(pptr,&blockno);
7125 
7126     return blockno;
7127 }
7128 
7129 
7130 
7131 
7132 /* @funcstatic btreeGetBlockS *************************************************
7133 **
7134 ** Get Block number matching a key
7135 **
7136 ** @param [u] cache [AjPBtcache] cache
7137 ** @param [u] buf [unsigned char *] page buffer
7138 ** @param [r] key [const AjPStr] key
7139 **
7140 ** @return [ajulong] Block number
7141 **
7142 ** @release 6.4.0
7143 ** @@
7144 ******************************************************************************/
7145 
btreeGetBlockS(AjPBtcache cache,unsigned char * buf,const AjPStr key)7146 static ajulong btreeGetBlockS(AjPBtcache cache, unsigned char *buf,
7147                              const AjPStr key)
7148 {
7149     return btreeGetBlockC(cache, buf, MAJSTRGETPTR(key));
7150 }
7151 
7152 
7153 
7154 
7155 /* @funcstatic btreeGetBlockFirstC ********************************************
7156 **
7157 ** Get Block number of first block if more than key
7158 ** or second block
7159 **
7160 ** @param [u] cache [AjPBtcache] cache
7161 ** @param [u] buf [unsigned char *] page buffer
7162 ** @param [r] ckey [const char*] key
7163 ** @param [r] clen [ajuint] Maximum length of key to test (e.g. for wildcards)
7164 **                         or zero to test full key
7165 **
7166 ** @return [ajulong] Block number
7167 **
7168 ** @release 6.4.0
7169 ** @@
7170 ******************************************************************************/
7171 
btreeGetBlockFirstC(AjPBtcache cache,unsigned char * buf,const char * ckey,ajuint clen)7172 static ajulong btreeGetBlockFirstC(AjPBtcache cache, unsigned char *buf,
7173                                   const char* ckey, ajuint clen)
7174 {
7175     ajulong blockno = 0UL;
7176 
7177     ajuint m;
7178     unsigned char *lenptr = NULL;
7179     unsigned char *keyptr = NULL;
7180     unsigned char *tbuf   = NULL;
7181 
7182     ajuint    ival = 0U;
7183     ajuint    len;
7184     ajuint    klen = 0U;
7185 
7186     tbuf    = buf;
7187 
7188     GBT_NKEYS(tbuf,&m);
7189     if(!m)
7190 	ajFatal("btreeGetBlockFirstC: No keys in node cache %S",
7191                 cache->filename);
7192 
7193     lenptr =  PBT_KEYLEN(tbuf);
7194     keyptr = lenptr + m * sizeof(ajuint);
7195 
7196     BT_GETAJUINT(lenptr,&ival);
7197     len = ival+1;
7198     klen = ival;
7199     if(clen && (klen > clen))
7200         klen = clen;
7201 
7202     if(strncmp((const char*)keyptr, ckey, klen) > 0)
7203     {
7204         keyptr += len;
7205 
7206         BT_GETAJULONG(keyptr,&blockno);
7207         return blockno;
7208     }
7209 
7210     keyptr += len;
7211     keyptr += sizeof(ajulong);
7212     lenptr += sizeof(ajuint);
7213 
7214     BT_GETAJULONG(keyptr,&blockno);
7215 
7216     return blockno;
7217 }
7218 
7219 
7220 
7221 
7222 /* @funcstatic btreeGetBlockFirstS ********************************************
7223 **
7224 ** Get Block number matching a key
7225 **
7226 ** @param [u] cache [AjPBtcache] cache
7227 ** @param [u] buf [unsigned char *] page buffer
7228 ** @param [r] key [const AjPStr] key
7229 **
7230 ** @return [ajulong] Block number
7231 **
7232 ** @release 6.4.0
7233 ** @@
7234 ******************************************************************************/
7235 
btreeGetBlockFirstS(AjPBtcache cache,unsigned char * buf,const AjPStr key)7236 static ajulong btreeGetBlockFirstS(AjPBtcache cache, unsigned char *buf,
7237                                   const AjPStr key)
7238 {
7239     return btreeGetBlockFirstC(cache, buf, MAJSTRGETPTR(key),0);
7240 }
7241 
7242 
7243 
7244 
7245 /* @funcstatic btreeIdCompare *************************************************
7246 **
7247 ** Comparison function for ajListSort
7248 **
7249 ** @param [r] a [const void*] ID 1
7250 ** @param [r] b [const void*] ID 2
7251 **
7252 ** @return [ajint] 0 = bases match
7253 **
7254 ** @release 2.8.0
7255 ** @@
7256 ******************************************************************************/
7257 
btreeIdCompare(const void * a,const void * b)7258 static ajint btreeIdCompare(const void *a, const void *b)
7259 {
7260     return MAJSTRCMPS((*(AjPBtId const *)a)->id,
7261                       (*(AjPBtId const *)b)->id);
7262 }
7263 
7264 
7265 
7266 
7267 /* @funcstatic btreeNumIdCompare **********************************************
7268 **
7269 ** Comparison function for ajListSort
7270 **
7271 ** @param [r] a [const void*] ID 1
7272 ** @param [r] b [const void*] ID 2
7273 **
7274 ** @return [ajint] 0 = bases match
7275 **
7276 ** @release 4.0.0
7277 ** @@
7278 ******************************************************************************/
7279 
btreeNumIdCompare(const void * a,const void * b)7280 static ajint btreeNumIdCompare(const void *a, const void *b)
7281 {
7282     ajlong val;
7283 
7284     val = (ajlong) ((*(AjPBtNumId const *)a)->offset -
7285                     (*(AjPBtNumId const *)b)->offset);
7286 
7287     if(!val)
7288       return 0;
7289 
7290     return (val < 0L) ? -1 : 1;
7291 }
7292 
7293 
7294 
7295 
7296 /* @funcstatic btreeWriteNode *************************************************
7297 **
7298 ** Write an internal node
7299 **
7300 ** @param [u] cache [AjPBtcache] cache
7301 ** @param [u] spage [AjPBtpage] buffer
7302 ** @param [r] keys [AjPStr const *] keys
7303 ** @param [r] ptrs [const ajulong*] page pointers
7304 ** @param [r] nkeys [ajuint] number of keys
7305 
7306 **
7307 ** @return [void]
7308 **
7309 ** @release 2.8.0
7310 ** @@
7311 ******************************************************************************/
7312 
btreeWriteNode(AjPBtcache cache,AjPBtpage spage,AjPStr const * keys,const ajulong * ptrs,ajuint nkeys)7313 static void btreeWriteNode(AjPBtcache cache, AjPBtpage spage,
7314 			   AjPStr const *keys, const ajulong *ptrs,
7315                            ajuint nkeys)
7316 {
7317     unsigned char *lbuf   = NULL;
7318     unsigned char *tbuf   = NULL;
7319     unsigned char *lenptr = NULL;
7320     unsigned char *keyptr = NULL;
7321     AjPBtpage page        = NULL;
7322 
7323     ajulong   overflow = 0UL;
7324     ajulong   blockno  = 0UL;
7325 
7326     ajuint   i;
7327     ajuint   len;
7328     ajuint    v  = 0U;
7329     ajulong   lv = 0UL;
7330     ajuint totlen = 0U;
7331 
7332     (void) cache;               /* make it used */
7333 
7334     lbuf = spage->buf;
7335     tbuf = lbuf;
7336     page = spage;
7337 
7338     v = nkeys;
7339     SBT_NKEYS(lbuf,v);
7340 
7341     lenptr = PBT_KEYLEN(lbuf);
7342     keyptr = lenptr + nkeys * sizeof(ajuint);
7343 
7344     for(i=0;i<nkeys;++i)
7345     {
7346 
7347 /* not the right check - lenptr gets longer than this anyway */
7348 /*
7349 //	if((ajuint) (lenptr-lbuf+1) > cache->pagesize)
7350 //	    ajFatal("WriteNode: Too many key lengths for available pagesize");
7351 */
7352 
7353 	len = ajStrGetLen(keys[i]);
7354 	v = len;
7355 	BT_SETAJUINT(lenptr,v);
7356         totlen += len;
7357 	lenptr += sizeof(ajuint);
7358     }
7359 
7360     v = totlen;
7361     SBT_TOTLEN(lbuf,v);
7362 
7363     GBT_OVERFLOW(lbuf,&overflow);
7364     GBT_BLOCKNUMBER(lbuf, &blockno);
7365 
7366     for(i=0;i<nkeys;++i)
7367     {
7368 	len = ajStrGetLen(keys[i]) + 1;
7369 
7370 /*
7371 //	if((ajuint)((keyptr-tbuf+1) + len + sizeof(ajulong)) > cache->pagesize)
7372 //	{
7373 //	    if(!overflow)		/# No overflow buckets yet #/
7374 //	    {
7375 //		page->dirty = BT_DIRTY;
7376 //		blockno = cache->totsize;
7377 //		lv = blockno;
7378 //		SBT_OVERFLOW(tbuf,lv);
7379 //		page = ajBtreeCacheWriteNodenew(cache); /# pri/sec possible #/
7380 //		tbuf = page->buf;
7381 //		v = BT_OVERFLOW;
7382 //		SBT_NODETYPE(tbuf,v);
7383 //		lv = blockno;
7384 //		SBT_BLOCKNUMBER(tbuf,lv);
7385 //	    }
7386 //	    else
7387 //	    {
7388 //		page = ajBtreeCacheRead(cache,overflow); /# pri/sec possible #/
7389 //		tbuf = page->buf;
7390 //		GBT_OVERFLOW(tbuf,&overflow);
7391 //	    }
7392 //
7393 //	    keyptr = PBT_KEYLEN(tbuf);
7394 //	    page->dirty = BT_DIRTY;
7395 //	}
7396 */
7397 
7398 	sprintf((char *)keyptr,"%s",ajStrGetPtr(keys[i]));
7399 	keyptr += len;
7400 	lv = ptrs[i];
7401 	BT_SETAJULONG(keyptr,lv);
7402 	keyptr += sizeof(ajulong);
7403     }
7404 
7405 
7406 
7407 /*
7408 //    if((ajuint)((keyptr-tbuf+1) + sizeof(ajulong)) > cache->pagesize)
7409 //    {
7410 //	page->dirty = BT_DIRTY;
7411 //
7412 //	if(!overflow)			/# No overflow buckets yet #/
7413 //	{
7414 //	    blockno = cache->totsize;
7415 //	    lv = blockno;
7416 //	    SBT_OVERFLOW(tbuf,lv);
7417 //	    page = ajBtreeCacheWriteNodenew(cache); /# pri or sec possible #/
7418 //	    tbuf = page->buf;
7419 //	    v = BT_OVERFLOW;
7420 //	    SBT_NODETYPE(tbuf,v);
7421 //	}
7422 //	else
7423 //	{
7424 //	    page = ajBtreeCacheRead(cache,overflow); /# pri or sec possible #/
7425 //	    tbuf = page->buf;
7426 //	}
7427 //
7428 //	keyptr = PBT_KEYLEN(tbuf);
7429 //    }
7430 */
7431 
7432     page->dirty = BT_DIRTY;
7433 
7434     overflow = 0UL;
7435     SBT_OVERFLOW(tbuf,overflow);
7436 
7437     lv = ptrs[i];
7438     BT_SETAJULONG(keyptr,lv);
7439     keyptr += sizeof(ajulong);
7440 
7441     return;
7442 }
7443 
7444 
7445 
7446 
7447 /* @funcstatic btreeWriteNodeSingle *******************************************
7448 **
7449 ** Write an internal node with a single key
7450 **
7451 ** @param [u] cache [AjPBtcache] cache
7452 ** @param [u] spage [AjPBtpage] buffer
7453 ** @param [r] key [const AjPStr] key
7454 ** @param [r] lptr [ajulong] left page pointer
7455 ** @param [r] rptr [ajulong] right page pointer
7456 
7457 **
7458 ** @return [void]
7459 **
7460 ** @release 6.4.0
7461 ** @@
7462 ******************************************************************************/
7463 
btreeWriteNodeSingle(AjPBtcache cache,AjPBtpage spage,const AjPStr key,ajulong lptr,ajulong rptr)7464 static void btreeWriteNodeSingle(AjPBtcache cache, AjPBtpage spage,
7465                                  const AjPStr key, ajulong lptr,
7466                                  ajulong rptr)
7467 {
7468     unsigned char *lbuf   = NULL;
7469     unsigned char *tbuf   = NULL;
7470     unsigned char *lenptr = NULL;
7471     unsigned char *keyptr = NULL;
7472     AjPBtpage page        = NULL;
7473 
7474     ajulong   overflow = 0UL;
7475 
7476     ajuint   len;
7477     ajuint    v  = 0U;
7478     ajulong   lv = 0UL;
7479 
7480     ajuint nkeys = 1U;
7481 
7482     /* ajDebug("In btreeWriteNodeSingle\n"); */
7483 
7484     (void) cache;               /* make it used */
7485 
7486     lbuf = spage->buf;
7487     tbuf = lbuf;
7488     page = spage;
7489 
7490     v = 1;
7491     SBT_NKEYS(lbuf,v);
7492     lenptr = PBT_KEYLEN(lbuf);
7493     keyptr = lenptr + nkeys * sizeof(ajuint);
7494 
7495     len = MAJSTRGETLEN(key);
7496     v = len;
7497     BT_SETAJUINT(lenptr,v);
7498     lenptr += sizeof(ajuint);
7499     v = len;
7500     SBT_TOTLEN(lbuf,v);
7501 
7502     GBT_OVERFLOW(lbuf,&overflow);
7503 
7504     len = MAJSTRGETLEN(key) + 1;
7505 
7506     sprintf((char *)keyptr,"%s",MAJSTRGETPTR(key));
7507     keyptr += len;
7508     lv = lptr;
7509     BT_SETAJULONG(keyptr,lv);
7510     keyptr += sizeof(ajulong);
7511 
7512     overflow = 0UL;
7513     SBT_OVERFLOW(tbuf,overflow);
7514 
7515     lv = rptr;
7516     BT_SETAJULONG(keyptr,lv);
7517     keyptr += sizeof(ajulong);
7518 
7519     page->dirty = BT_DIRTY;
7520 
7521     return;
7522 }
7523 
7524 
7525 
7526 
7527 /* @funcstatic btreeIdentQueryId **********************************************
7528 **
7529 ** Get an ID structure from a leaf node given an identifier
7530 **
7531 ** @param [u] cache [AjPBtcache] cache
7532 ** @param [r] key [const AjPStr] key
7533 **
7534 ** @return [AjPBtId] Btree ID object
7535 **
7536 ** @release 6.5.0
7537 ** @@
7538 ******************************************************************************/
7539 
btreeIdentQueryId(AjPBtcache cache,const AjPStr key)7540 static AjPBtId btreeIdentQueryId(AjPBtcache cache, const AjPStr key)
7541 {
7542     AjPBtpage page   = NULL;
7543     AjPIdbucket bucket = NULL;
7544     AjPBtId   id     = NULL;
7545     AjPBtId   tid    = NULL;
7546 
7547     unsigned char *buf = NULL;
7548 
7549     ajuint nentries = 0U;
7550 
7551     ajuint i;
7552     ajuint iref;
7553 
7554     ajulong blockno = 0UL;
7555     AjBool found   = ajFalse;
7556 
7557     AjPStr keystr = NULL;
7558 
7559     /* ajDebug("In btreeIdentQueryId\n"); */
7560 
7561     if(!cache->countunique && !cache->countall)
7562         return NULL;
7563 
7564     keystr = ajStrNewS(key);
7565     ajStrFmtQuery(&keystr);
7566 
7567     page = btreeIdentFind(cache,keystr);
7568     buf = page->buf;
7569 
7570     blockno = btreeGetBlockS(cache,buf,keystr);
7571 
7572     bucket = btreeReadIdbucket(cache,blockno);
7573 
7574     nentries = bucket->Nentries;
7575 
7576     found = ajFalse;
7577 
7578     for(i=0;i<nentries;++i)
7579 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
7580 	{
7581 	    found = ajTrue;
7582 	    break;
7583 	}
7584 
7585     if(found)
7586     {
7587 	id  = ajBtreeIdNew(cache->refcount);
7588 	tid = bucket->Ids[i];
7589 	ajStrAssignS(&id->id,tid->id);
7590 	id->dups = tid->dups;
7591 	id->dbno = tid->dbno;
7592 	id->offset = tid->offset;
7593 
7594         if(cache->refcount)
7595 	{
7596             for(iref=0; iref < cache->refcount; iref++)
7597                 id->refoffsets[iref] = tid->refoffsets[iref];
7598         }
7599     }
7600 
7601     btreeIdbucketDel(&bucket);
7602     ajStrDel(&keystr);
7603 
7604     if(!found)
7605 	return NULL;
7606 
7607     return id;
7608 }
7609 
7610 
7611 
7612 
7613 /* @funcstatic btreeIdentQueryHit *********************************************
7614 **
7615 ** Get a hit structure from a leaf node given an identifier
7616 **
7617 ** @param [u] cache [AjPBtcache] cache
7618 ** @param [r] key [const AjPStr] key
7619 **
7620 ** @return [AjPBtHit] Btree hit object
7621 **
7622 ** @release 6.5.0
7623 ** @@
7624 ******************************************************************************/
7625 
btreeIdentQueryHit(AjPBtcache cache,const AjPStr key)7626 static AjPBtHit btreeIdentQueryHit(AjPBtcache cache, const AjPStr key)
7627 {
7628     AjPBtpage page   = NULL;
7629     AjPIdbucket bucket = NULL;
7630     AjPBtHit  hit    = NULL;
7631     AjPBtId   tid    = NULL;
7632 
7633     unsigned char *buf = NULL;
7634 
7635     ajuint nentries = 0U;
7636 
7637     ajuint i;
7638 
7639     ajulong blockno = 0UL;
7640     AjBool found   = ajFalse;
7641 
7642     AjPStr keystr = NULL;
7643 
7644     /* ajDebug("In btreeIdentQueryHit\n"); */
7645 
7646     if(!cache->countunique && !cache->countall)
7647         return NULL;
7648 
7649     keystr = ajStrNewS(key);
7650     ajStrFmtQuery(&keystr);
7651 
7652     page = btreeIdentFind(cache,keystr);
7653     buf = page->buf;
7654 
7655     blockno = btreeGetBlockS(cache,buf,keystr);
7656 
7657     bucket = btreeReadIdbucket(cache,blockno);
7658 
7659     nentries = bucket->Nentries;
7660 
7661     found = ajFalse;
7662 
7663     for(i=0;i<nentries;++i)
7664 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
7665 	{
7666 	    found = ajTrue;
7667 	    break;
7668 	}
7669 
7670     if(found)
7671     {
7672 	tid = bucket->Ids[i];
7673 	hit = ajBtreeHitNewId(tid);
7674     }
7675 
7676     btreeIdbucketDel(&bucket);
7677     ajStrDel(&keystr);
7678 
7679     if(!found)
7680 	return NULL;
7681 
7682     return hit;
7683 }
7684 
7685 
7686 
7687 
7688 /* @funcstatic btreeIdentQueryHitref ******************************************
7689 **
7690 ** Get a reference hit structure from a leaf node given an identifier
7691 **
7692 ** @param [u] cache [AjPBtcache] cache
7693 ** @param [r] key [const AjPStr] key
7694 **
7695 ** @return [AjPBtHitref] Btree hit object
7696 **
7697 ** @release 6.5.0
7698 ** @@
7699 ******************************************************************************/
7700 
btreeIdentQueryHitref(AjPBtcache cache,const AjPStr key)7701 static AjPBtHitref btreeIdentQueryHitref(AjPBtcache cache, const AjPStr key)
7702 {
7703     AjPBtpage page   = NULL;
7704     AjPIdbucket bucket = NULL;
7705     AjPBtHitref hitref = NULL;
7706     AjPBtId   tid    = NULL;
7707 
7708     unsigned char *buf = NULL;
7709 
7710     ajuint nentries = 0U;
7711 
7712     ajuint i;
7713 
7714     ajulong blockno = 0UL;
7715     AjBool found   = ajFalse;
7716 
7717     AjPStr keystr = NULL;
7718 
7719     /* ajDebug("In btreeIdentQueryHit\n"); */
7720 
7721     if(!cache->countunique && !cache->countall)
7722         return NULL;
7723 
7724     keystr = ajStrNewS(key);
7725     ajStrFmtQuery(&keystr);
7726 
7727     page = btreeIdentFind(cache,keystr);
7728     buf = page->buf;
7729 
7730     blockno = btreeGetBlockS(cache,buf,keystr);
7731 
7732     bucket = btreeReadIdbucket(cache,blockno);
7733 
7734     nentries = bucket->Nentries;
7735 
7736     found = ajFalse;
7737 
7738     for(i=0;i<nentries;++i)
7739 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
7740 	{
7741 	    found = ajTrue;
7742 	    break;
7743 	}
7744 
7745     if(found)
7746     {
7747 	tid = bucket->Ids[i];
7748 	hitref = ajBtreeHitrefNewId(tid);
7749     }
7750 
7751     btreeIdbucketDel(&bucket);
7752     ajStrDel(&keystr);
7753 
7754     if(!found)
7755 	return NULL;
7756 
7757     return hitref;
7758 }
7759 
7760 
7761 
7762 
7763 /* @func ajBtreeIdentFetchId **************************************************
7764 **
7765 ** Get an ID structure from a leaf node given an identifier
7766 **
7767 ** @param [u] cache [AjPBtcache] cache
7768 ** @param [r] key [const AjPStr] key
7769 ** @param [w] btidlist [AjPList] List of ID objects
7770 **
7771 ** @return [void]
7772 **
7773 ** @release 6.5.0
7774 ** @@
7775 ******************************************************************************/
7776 
ajBtreeIdentFetchId(AjPBtcache cache,const AjPStr key,AjPList btidlist)7777 void ajBtreeIdentFetchId(AjPBtcache cache, const AjPStr key,
7778                          AjPList btidlist)
7779 {
7780     AjPBtpage page   = NULL;
7781     AjPIdbucket bucket = NULL;
7782     AjPBtId   id     = NULL;
7783     AjPBtId   tid    = NULL;
7784 
7785     unsigned char *buf = NULL;
7786 
7787     ajuint nentries = 0U;
7788 
7789     ajuint i;
7790     ajuint iref;
7791 
7792     ajulong blockno = 0UL;
7793     AjBool found   = ajFalse;
7794 
7795     AjPStr keystr = NULL;
7796 
7797     /* ajDebug("In ajBtreeIdentFetch\n"); */
7798 
7799     if(!cache->countunique && !cache->countall)
7800         return;
7801 
7802     keystr = ajStrNewS(key);
7803     ajStrFmtQuery(&keystr);
7804 
7805     page = btreeIdentFind(cache,keystr);
7806     buf = page->buf;
7807 
7808     blockno = btreeGetBlockS(cache,buf,keystr);
7809 
7810     bucket = btreeReadIdbucket(cache,blockno);
7811 
7812     nentries = bucket->Nentries;
7813 
7814     found = ajFalse;
7815 
7816     for(i=0;i<nentries;++i)
7817 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
7818 	{
7819 	    found = ajTrue;
7820 	    break;
7821 	}
7822 
7823     if(found)
7824     {
7825 	id  = ajBtreeIdNew(cache->refcount);
7826 	tid = bucket->Ids[i];
7827 	ajStrAssignS(&id->id,tid->id);
7828 	id->dups = tid->dups;
7829 	id->dbno = tid->dbno;
7830 	id->offset = tid->offset;
7831 
7832         if(cache->refcount)
7833 	{
7834             for(iref=0; iref < cache->refcount; iref++)
7835                 id->refoffsets[iref] = tid->refoffsets[iref];
7836         }
7837     }
7838 
7839     btreeIdbucketDel(&bucket);
7840     ajStrDel(&keystr);
7841 
7842     if(!found)
7843 	return;
7844 
7845     if(!id->dups)
7846     {
7847         ajListPushAppend(btidlist,(void *)id);
7848         id = NULL;
7849     }
7850     else
7851     {
7852         btreeIdentFetchMulti(cache, id->id, id->offset,
7853                              btidlist);
7854         ajBtreeIdDel(&id);
7855     }
7856 
7857     return;
7858 }
7859 
7860 
7861 
7862 
7863 /* @func ajBtreeIdentFetchHit *************************************************
7864 **
7865 ** Get an ID hit structure from a leaf node given an identifier
7866 **
7867 ** @param [u] cache [AjPBtcache] cache
7868 ** @param [r] key [const AjPStr] key
7869 ** @param [w] hitlist [AjPList] List of hit objects
7870 **
7871 ** @return [void]
7872 **
7873 ** @release 6.5.0
7874 ** @@
7875 ******************************************************************************/
7876 
ajBtreeIdentFetchHit(AjPBtcache cache,const AjPStr key,AjPList hitlist)7877 void ajBtreeIdentFetchHit(AjPBtcache cache, const AjPStr key,
7878                           AjPList hitlist)
7879 {
7880     AjPBtpage page   = NULL;
7881     AjPIdbucket bucket = NULL;
7882     AjPBtHit  hit    = NULL;
7883     AjPBtId   tid    = NULL;
7884 
7885     unsigned char *buf = NULL;
7886 
7887     ajuint nentries = 0U;
7888 
7889     ajuint i;
7890 
7891     ajulong blockno = 0UL;
7892     AjBool found   = ajFalse;
7893 
7894     AjPStr keystr = NULL;
7895 
7896     /* ajDebug("In ajBtreeIdentFetch\n"); */
7897 
7898     if(!cache->countunique && !cache->countall)
7899         return;
7900 
7901     if(cache->refcount)
7902         ajWarn("ajBtreeIdentFetchHit called for cache '%S' "
7903                "with %u reference files",
7904                cache->filename, cache->refcount);
7905 
7906     keystr = ajStrNewS(key);
7907     ajStrFmtQuery(&keystr);
7908 
7909     page = btreeIdentFind(cache,keystr);
7910     buf = page->buf;
7911 
7912     blockno = btreeGetBlockS(cache,buf,keystr);
7913 
7914     bucket = btreeReadIdbucket(cache,blockno);
7915 
7916     nentries = bucket->Nentries;
7917 
7918     found = ajFalse;
7919 
7920     for(i=0;i<nentries;++i)
7921 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
7922 	{
7923 	    found = ajTrue;
7924 	    break;
7925 	}
7926 
7927     if(found)
7928     {
7929 	tid = bucket->Ids[i];
7930 	hit = ajBtreeHitNewId(tid);
7931     }
7932 
7933     ajStrDel(&keystr);
7934 
7935     if(!found)
7936     {
7937         btreeIdbucketDel(&bucket);
7938 	return;
7939     }
7940 
7941     if(!tid->dups)
7942     {
7943         ajListPushAppend(hitlist,(void *)hit);
7944         hit = NULL;
7945     }
7946     else
7947     {
7948         btreeIdentFetchMultiHit(cache, tid->offset,
7949                                 hitlist);
7950         ajBtreeHitDel(&hit);
7951     }
7952 
7953     btreeIdbucketDel(&bucket);
7954 
7955     return;
7956 }
7957 
7958 
7959 
7960 
7961 /* @func ajBtreeIdentFetchHitref **********************************************
7962 **
7963 ** Get an ID reference hit structure from a leaf node given an identifier
7964 **
7965 ** @param [u] cache [AjPBtcache] cache
7966 ** @param [r] key [const AjPStr] key
7967 ** @param [w] hitlist [AjPList] List of reference hit objects
7968 **
7969 ** @return [void]
7970 **
7971 ** @release 6.5.0
7972 ** @@
7973 ******************************************************************************/
7974 
ajBtreeIdentFetchHitref(AjPBtcache cache,const AjPStr key,AjPList hitlist)7975 void ajBtreeIdentFetchHitref(AjPBtcache cache, const AjPStr key,
7976                           AjPList hitlist)
7977 {
7978     AjPBtpage page   = NULL;
7979     AjPIdbucket bucket = NULL;
7980     AjPBtHitref hitref = NULL;
7981     AjPBtId   tid    = NULL;
7982 
7983     unsigned char *buf = NULL;
7984 
7985     ajuint nentries = 0U;
7986 
7987     ajuint i;
7988 
7989     ajulong blockno = 0UL;
7990     AjBool found   = ajFalse;
7991 
7992     AjPStr keystr = NULL;
7993 
7994     /* ajDebug("In ajBtreeIdentFetch\n"); */
7995 
7996     if(!cache->countunique && !cache->countall)
7997         return;
7998 
7999     if(cache->refcount != 1)
8000         ajWarn("ajBtreeIdentFetchHitref called for cache '%S' "
8001                "with %u reference files",
8002                cache->filename, cache->refcount);
8003 
8004     keystr = ajStrNewS(key);
8005     ajStrFmtQuery(&keystr);
8006 
8007     page = btreeIdentFind(cache,keystr);
8008     buf = page->buf;
8009 
8010     blockno = btreeGetBlockS(cache,buf,keystr);
8011 
8012     bucket = btreeReadIdbucket(cache,blockno);
8013 
8014     nentries = bucket->Nentries;
8015 
8016     found = ajFalse;
8017 
8018     for(i=0;i<nentries;++i)
8019 	if(!MAJSTRCMPS(bucket->Ids[i]->id, keystr))
8020 	{
8021 	    found = ajTrue;
8022 	    break;
8023 	}
8024 
8025     if(found)
8026     {
8027 	tid = bucket->Ids[i];
8028 	hitref = ajBtreeHitrefNewId(tid);
8029     }
8030 
8031     ajStrDel(&keystr);
8032 
8033     if(!found)
8034     {
8035         btreeIdbucketDel(&bucket);
8036 	return;
8037     }
8038 
8039     if(!tid->dups)
8040     {
8041         ajListPushAppend(hitlist,(void *)hitref);
8042         hitref = NULL;
8043     }
8044     else
8045     {
8046         btreeIdentFetchMultiHitref(cache, tid->offset,
8047                                 hitlist);
8048         ajBtreeHitrefDel(&hitref);
8049     }
8050 
8051     btreeIdbucketDel(&bucket);
8052 
8053     return;
8054 }
8055 
8056 
8057 
8058 
8059 /* @func ajBtreeWriteParamsC **************************************************
8060 **
8061 ** Write B+ tree parameters to file
8062 **
8063 ** @param [r] cache [const AjPBtcache] cache
8064 ** @param [r] fntxt [const char *] file name
8065 ** @param [r] exttxt [const char *] index file extension name
8066 ** @param [r] idirtxt [const char *] index file directory
8067 **
8068 ** @return [void]
8069 **
8070 ** @release 6.4.0
8071 ** @@
8072 ******************************************************************************/
8073 
ajBtreeWriteParamsC(const AjPBtcache cache,const char * fntxt,const char * exttxt,const char * idirtxt)8074 void ajBtreeWriteParamsC(const AjPBtcache cache, const char *fntxt,
8075 			 const char *exttxt, const char *idirtxt)
8076 {
8077     AjPStr  fname = NULL;
8078     AjPFile outf  = NULL;
8079 
8080     fname = ajStrNewRes(128);
8081 
8082     if(!*idirtxt)
8083       ajFmtPrintS(&fname,"%s.p%s",fntxt,exttxt);
8084     else
8085       ajFmtPrintS(&fname,"%s%s%s.p%s",idirtxt,SLASH_STRING,fntxt,exttxt);
8086 
8087     if(!(outf = ajFileNewOutNameS(fname)))
8088 	ajFatal("Cannot open param file %S\n",fname);
8089 
8090     if(cache->secondary)
8091         ajFmtPrintF(outf,"Type      %s\n","Secondary");
8092     else
8093         ajFmtPrintF(outf,"Type         %s\n","Identifier");
8094     ajFmtPrintF(outf,"Compress     %B\n",cache->compressed);
8095     ajFmtPrintF(outf,"Pages        %Lu\n",cache->pripagecount);
8096     ajFmtPrintF(outf,"Secpages     %Lu\n",cache->secpagecount);
8097     ajFmtPrintF(outf,"Order        %u\n",cache->porder);
8098     ajFmtPrintF(outf,"Fill         %u\n",cache->pnperbucket);
8099     ajFmtPrintF(outf,"Level        %u\n",cache->plevel);
8100     ajFmtPrintF(outf,"Pagesize     %u\n",cache->pripagesize);
8101     ajFmtPrintF(outf,"Cachesize    %u\n",cache->pricachesize);
8102     ajFmtPrintF(outf,"Order2       %u\n",cache->sorder);
8103     ajFmtPrintF(outf,"Fill2        %u\n",cache->snperbucket);
8104     ajFmtPrintF(outf,"Secpagesize  %u\n",cache->secpagesize);
8105     ajFmtPrintF(outf,"Seccachesize %u\n",cache->seccachesize);
8106     ajFmtPrintF(outf,"Count        %Lu\n",cache->countunique);
8107     ajFmtPrintF(outf,"Fullcount    %Lu\n",cache->countall);
8108     ajFmtPrintF(outf,"Kwlimit      %u\n",cache->keylimit);
8109     if(cache->secondary)
8110         ajFmtPrintF(outf,"Idlimit      %u\n",cache->idlimit);
8111     else
8112         ajFmtPrintF(outf,"Reffiles     %u\n",cache->refcount);
8113 
8114     ajFileClose(&outf);
8115     ajStrDel(&fname);
8116 
8117     return;
8118 }
8119 
8120 
8121 
8122 
8123 /* @func ajBtreeWriteParamsS **************************************************
8124 **
8125 ** Write B+ tree parameters to file
8126 **
8127 ** @param [r] cache [const AjPBtcache] cache
8128 ** @param [r] fn [const AjPStr] file name
8129 ** @param [r] ext [const AjPStr] index file extension name
8130 ** @param [r] idir [const AjPStr] index file directory
8131 **
8132 ** @return [void]
8133 **
8134 ** @release 6.4.0
8135 ** @@
8136 ******************************************************************************/
8137 
ajBtreeWriteParamsS(const AjPBtcache cache,const AjPStr fn,const AjPStr ext,const AjPStr idir)8138 void ajBtreeWriteParamsS(const AjPBtcache cache, const AjPStr fn,
8139                          const AjPStr ext, const AjPStr idir)
8140 {
8141     AjPStr  fname = NULL;
8142     AjPFile outf  = NULL;
8143 
8144     fname = ajStrNewRes(128);
8145 
8146     if(ajStrGetLen(idir))
8147       ajFmtPrintS(&fname,"%S%s%S.p%S",idir,SLASH_STRING,fn,ext);
8148     else
8149         ajFmtPrintS(&fname,"%S.p%S",fn,ext);
8150 
8151     if(!(outf = ajFileNewOutNameS(fname)))
8152 	ajFatal("Cannot open param file %S\n",fname);
8153 
8154     if(cache->secondary)
8155         ajFmtPrintF(outf,"Type         %s\n","Secondary");
8156     else
8157         ajFmtPrintF(outf,"Type         %s\n","Identifier");
8158     ajFmtPrintF(outf,"Compress     %B\n",cache->compressed);
8159     ajFmtPrintF(outf,"Pages        %Lu\n",cache->pripagecount);
8160     ajFmtPrintF(outf,"Secpages     %Lu\n",cache->secpagecount);
8161     ajFmtPrintF(outf,"Order        %u\n",cache->porder);
8162     ajFmtPrintF(outf,"Fill         %u\n",cache->pnperbucket);
8163     ajFmtPrintF(outf,"Level        %u\n",cache->plevel);
8164     ajFmtPrintF(outf,"Pagesize     %u\n",cache->pripagesize);
8165     ajFmtPrintF(outf,"Cachesize    %u\n",cache->pricachesize);
8166     ajFmtPrintF(outf,"Order2       %u\n",cache->sorder);
8167     ajFmtPrintF(outf,"Fill2        %u\n",cache->snperbucket);
8168     ajFmtPrintF(outf,"Secpagesize  %u\n",cache->secpagesize);
8169     ajFmtPrintF(outf,"Seccachesize %u\n",cache->seccachesize);
8170     ajFmtPrintF(outf,"Count        %Lu\n",cache->countunique);
8171     ajFmtPrintF(outf,"Fullcount    %Lu\n",cache->countall);
8172     ajFmtPrintF(outf,"Kwlimit      %u\n",cache->keylimit);
8173     if(cache->secondary)
8174         ajFmtPrintF(outf,"Idlimit      %u\n",cache->idlimit);
8175     else
8176         ajFmtPrintF(outf,"Reffiles     %u\n",cache->refcount);
8177 
8178     ajFileClose(&outf);
8179     ajStrDel(&fname);
8180 
8181     return;
8182 }
8183 
8184 
8185 
8186 
8187 /* @func ajBtreeReadParamsC ***************************************************
8188 **
8189 ** Read B+ tree parameters from file
8190 **
8191 ** @param [r] filetxt [const char *] file
8192 ** @param [r] exttxt [const char *] file extension
8193 ** @param [r] idirtxt [const char *] index directory
8194 ** @param [w] secondary [AjBool*] true for a secondary index
8195 ** @param [w] compressed [AjBool*] true for a compressed index
8196 ** @param [w] kwlimit [ajuint*] maximum length of a keyword
8197 ** @param [w] idlimit [ajuint*] maximum length of a secondary id
8198 ** @param [w] refcount [ajuint*] reference file(s) per entry
8199 ** @param [w] pripagesize [ajuint*] size of primary pages
8200 ** @param [w] secpagesize [ajuint*] size of secondary pages
8201 ** @param [w] pricachesize [ajuint*] Primary cachesize
8202 ** @param [w] seccachesize [ajuint*] Secondary cachesize
8203 ** @param [w] pripagecount [ajulong*] Primary page count
8204 ** @param [w] secpagecount [ajulong*] Secondary page count
8205 ** @param [w] order [ajuint*] tree order
8206 ** @param [w] nperbucket [ajuint*] bucket fill
8207 ** @param [w] level [ajuint*] depth of tree (0 = root leaf)
8208 ** @param [w] sorder [ajuint*] secondary tree order
8209 ** @param [w] snperbucket [ajuint*] secondary bucket fill
8210 ** @param [w] count [ajulong*] number of primary keywords in the index
8211 ** @param [w] countall [ajulong*] number of total keywords in the index
8212 **
8213 ** @return [AjBool] True on success
8214 **
8215 ** @release 6.4.0
8216 ** @@
8217 ******************************************************************************/
8218 
ajBtreeReadParamsC(const char * filetxt,const char * exttxt,const char * idirtxt,AjBool * secondary,AjBool * compressed,ajuint * kwlimit,ajuint * idlimit,ajuint * refcount,ajuint * pripagesize,ajuint * secpagesize,ajuint * pricachesize,ajuint * seccachesize,ajulong * pripagecount,ajulong * secpagecount,ajuint * order,ajuint * nperbucket,ajuint * level,ajuint * sorder,ajuint * snperbucket,ajulong * count,ajulong * countall)8219 AjBool ajBtreeReadParamsC(const char *filetxt, const char *exttxt,
8220                           const char *idirtxt,
8221                           AjBool *secondary, AjBool *compressed,
8222                           ajuint *kwlimit, ajuint *idlimit,
8223                           ajuint *refcount,
8224                           ajuint *pripagesize, ajuint *secpagesize,
8225                           ajuint *pricachesize, ajuint *seccachesize,
8226                           ajulong *pripagecount, ajulong *secpagecount,
8227                           ajuint *order, ajuint *nperbucket, ajuint *level,
8228                           ajuint *sorder, ajuint *snperbucket,
8229                           ajulong *count, ajulong *countall)
8230 {
8231     AjPStr fname = NULL;
8232     AjPStr line  = NULL;
8233     AjPFile inf  = NULL;
8234     AjPStr type = NULL;
8235     AjBool setsecond = ajFalse;
8236 
8237     *countall = 0UL;
8238     *compressed = ajFalse;
8239     *pripagecount = 0UL;
8240     *secpagecount = 0UL;
8241     *refcount = 1U;
8242     *secpagesize = 0U;
8243     *seccachesize = 0U;
8244     *idlimit = 0U;
8245 
8246     line  = ajStrNew();
8247 
8248     fname = ajStrNew();
8249 
8250     if(!*idirtxt)
8251         ajFmtPrintS(&fname,"%s.p%s",filetxt,exttxt);
8252     else if(idirtxt[strlen(idirtxt)-1] == SLASH_CHAR)
8253         ajFmtPrintS(&fname,"%s%s.p%s",idirtxt,filetxt,exttxt);
8254     else
8255       ajFmtPrintS(&fname,"%s%s%s.p%s",idirtxt,SLASH_STRING,filetxt,exttxt);
8256 
8257     if(!(inf = ajFileNewInNameS(fname)))
8258     {
8259 	ajErr("Cannot open param file %S %d: '%s'\n",
8260               fname, errno, strerror(errno));
8261         return ajFalse;
8262     }
8263 
8264     while(ajReadlineTrim(inf,&line))
8265     {
8266 	if(ajStrPrefixC(line,"Order2 "))
8267 	{
8268 	    ajFmtScanS(line,"%*s%u",sorder);
8269 	    continue;
8270 	}
8271 
8272 	if(ajStrPrefixC(line,"Fill2 "))
8273 	{
8274 	    ajFmtScanS(line,"%*s%u",snperbucket);
8275 	    continue;
8276 	}
8277 
8278 	if(ajStrPrefixC(line,"Type "))
8279         {
8280 	    ajFmtScanS(line,"%*s%S",&type);
8281             if(ajStrMatchC(type, "Secondary"))
8282                 *secondary = ajTrue;
8283             else if(ajStrMatchC(type, "Identifier"))
8284                 *secondary = ajFalse;
8285             else
8286                 ajErr("param file %S unknown Type value '%S'",
8287                       fname, type);
8288             setsecond = ajTrue;
8289         }
8290 
8291 	if(ajStrPrefixC(line,"Compress "))
8292 	    ajFmtScanS(line,"%*s%b",compressed);
8293 
8294 	if(ajStrPrefixC(line,"Pages "))
8295 	    ajFmtScanS(line,"%*s%Lu",pripagecount);
8296 
8297 	if(ajStrPrefixC(line,"Secpages "))
8298 	    ajFmtScanS(line,"%*s%Lu",secpagecount);
8299 
8300 	if(ajStrPrefixC(line,"Order "))
8301 	    ajFmtScanS(line,"%*s%u",order);
8302 
8303 	if(ajStrPrefixC(line,"Fill "))
8304 	    ajFmtScanS(line,"%*s%u",nperbucket);
8305 
8306 	if(ajStrPrefixC(line,"Pagesize "))
8307 	    ajFmtScanS(line,"%*s%u",pripagesize);
8308 
8309 	if(ajStrPrefixC(line,"Secpagesize "))
8310 	    ajFmtScanS(line,"%*s%u",secpagesize);
8311 
8312 	if(ajStrPrefixC(line,"Level "))
8313 	    ajFmtScanS(line,"%*s%u",level);
8314 
8315 	if(ajStrPrefixC(line,"Cachesize "))
8316 	    ajFmtScanS(line,"%*s%u",pricachesize);
8317 
8318 	if(ajStrPrefixC(line,"Seccachesize "))
8319 	    ajFmtScanS(line,"%*s%u",seccachesize);
8320 
8321 	if(ajStrPrefixC(line,"Count "))
8322 	    ajFmtScanS(line,"%*s%Lu",count);
8323 
8324 	if(ajStrPrefixC(line,"Fullcount "))
8325 	    ajFmtScanS(line,"%*s%Lu",countall);
8326 
8327 	if(ajStrPrefixC(line,"Kwlimit "))
8328 	    ajFmtScanS(line,"%*s%u",kwlimit);
8329 
8330 	if(ajStrPrefixC(line,"Idlimit "))
8331             ajFmtScanS(line,"%*s%u",idlimit);
8332 
8333 	if(ajStrPrefixC(line,"Reffiles "))
8334 	    ajFmtScanS(line,"%*s%u",refcount);
8335     }
8336 
8337     if(!setsecond)
8338     {
8339         *secondary = ajBtreeFieldGetSecondaryC(exttxt);
8340     }
8341 
8342     if(!*pripagecount && !*compressed)
8343     {
8344         if(!*idirtxt)
8345             ajFmtPrintS(&fname,"%s.%s",filetxt,exttxt);
8346         else if(idirtxt[strlen(idirtxt)-1] == SLASH_CHAR)
8347             ajFmtPrintS(&fname,"%s%s.%s",idirtxt,filetxt,exttxt);
8348         else
8349 	    ajFmtPrintS(&fname,"%s%s%s.%s",
8350 			idirtxt,SLASH_STRING, filetxt,exttxt);
8351         *pripagecount = ajFilenameGetSize(fname) / *pripagesize;
8352         *secpagecount = 0;
8353     }
8354 
8355     if(!*secpagesize)
8356         *secpagesize = *pripagesize;
8357 
8358     if(!*seccachesize)
8359         *seccachesize = *pricachesize;
8360 
8361     if(*secondary && !*idlimit)
8362         *idlimit = *kwlimit;
8363 
8364     ajFileClose(&inf);
8365     ajStrDel(&fname);
8366     ajStrDel(&line);
8367     ajStrDel(&type);
8368 
8369     return ajTrue;
8370 }
8371 
8372 
8373 
8374 
8375 /* @func ajBtreeReadParamsS ***************************************************
8376 **
8377 ** Read B+ tree parameters from file
8378 **
8379 ** @param [r] file [const AjPStr] file
8380 ** @param [r] ext [const AjPStr] file extension
8381 ** @param [r] idir [const AjPStr] index directory
8382 ** @param [w] secondary [AjBool*] true for a secondary index
8383 ** @param [w] compressed [AjBool*] true for a compressed index
8384 ** @param [w] kwlimit [ajuint*] maximum length of a keyword
8385 ** @param [w] idlimit [ajuint*] maximum length of a secondary id
8386 ** @param [w] refcount [ajuint*] reference file(s) per entry
8387 ** @param [w] pripagesize [ajuint*] size of primary pages
8388 ** @param [w] secpagesize [ajuint*] size of secondary pages
8389 ** @param [w] pricachesize [ajuint*] Primary cachesize
8390 ** @param [w] seccachesize [ajuint*] Secondary cachesize
8391 ** @param [w] pripagecount [ajulong*] Primary page count
8392 ** @param [w] secpagecount [ajulong*] Secondary page count
8393 ** @param [w] order [ajuint*] tree order
8394 ** @param [w] nperbucket [ajuint*] bucket fill
8395 ** @param [w] level [ajuint*] depth of tree (0 = root leaf)
8396 ** @param [w] sorder [ajuint*] secondary tree order
8397 ** @param [w] snperbucket [ajuint*] secondary bucket fill
8398 ** @param [w] count [ajulong*] number of primary keywords in the index
8399 ** @param [w] countall [ajulong*] number of total keywords in the index
8400 **
8401 ** @return [AjBool] True on success
8402 **
8403 ** @release 6.4.0
8404 ** @@
8405 ******************************************************************************/
8406 
ajBtreeReadParamsS(const AjPStr file,const AjPStr ext,const AjPStr idir,AjBool * secondary,AjBool * compressed,ajuint * kwlimit,ajuint * idlimit,ajuint * refcount,ajuint * pripagesize,ajuint * secpagesize,ajuint * pricachesize,ajuint * seccachesize,ajulong * pripagecount,ajulong * secpagecount,ajuint * order,ajuint * nperbucket,ajuint * level,ajuint * sorder,ajuint * snperbucket,ajulong * count,ajulong * countall)8407 AjBool ajBtreeReadParamsS(const AjPStr file, const AjPStr ext,
8408                           const AjPStr idir,
8409                           AjBool *secondary, AjBool *compressed,
8410                           ajuint *kwlimit, ajuint *idlimit,
8411                           ajuint *refcount,
8412                           ajuint *pripagesize, ajuint *secpagesize,
8413                           ajuint *pricachesize, ajuint *seccachesize,
8414                           ajulong *pripagecount, ajulong *secpagecount,
8415                           ajuint *order, ajuint *nperbucket, ajuint *level,
8416                           ajuint *sorder, ajuint *snperbucket,
8417                           ajulong *count, ajulong *countall)
8418 {
8419     return ajBtreeReadParamsC(MAJSTRGETPTR(file), MAJSTRGETPTR(ext),
8420                               MAJSTRGETPTR(idir), secondary, compressed,
8421                               kwlimit, idlimit, refcount,
8422                               pripagesize, secpagesize,
8423                               pricachesize, seccachesize,
8424                               pripagecount, secpagecount,
8425                               order, nperbucket, level,
8426                               sorder, snperbucket,
8427                               count, countall);
8428 }
8429 
8430 
8431 
8432 
8433 #if 0
8434 /* #funcstatic btreeIdentSplitleaf ********************************************
8435 **
8436 ** Split an identifier primary leaf and propagate up if necessary
8437 **
8438 ** #param [u] cache [AjPBtcache] cache
8439 ** #param [u] spage [AjPBtpage] page
8440 **
8441 ** #return [AjPBtpage] pointer to a parent page
8442 **
8443 ** #release 6.5.0
8444 ** ##
8445 ******************************************************************************/
8446 /*
8447 static AjPBtpage btreeIdentSplitleaf(AjPBtcache cache, AjPBtpage spage)
8448 {
8449     ajuint nkeys     = 0U;
8450     ajuint order     = 0U;
8451     ajuint totalkeys = 0U;
8452     ajuint keylimit  = 0U;
8453     ajuint nodetype  = 0U;
8454 
8455     ajuint rootnodetype  = 0U;
8456 
8457     ajuint i;
8458     ajuint j;
8459     ajuint iref;
8460 
8461     AjPBtpage lpage = NULL;
8462     AjPBtpage rpage = NULL;
8463     AjPBtpage page  = NULL;
8464 
8465     AjPStr mediankey  = NULL;
8466     ajulong mediangtr  = 0UL;
8467     ajulong medianless = 0UL;
8468 
8469 
8470     AjPBtId bid = NULL;
8471     AjPBtId cid = NULL;
8472 
8473     unsigned char *buf  = NULL;
8474     unsigned char *lbuf = NULL;
8475     unsigned char *rbuf = NULL;
8476 
8477     AjPList idlist = NULL;
8478 
8479     AjPIdbucket cbucket  = NULL;
8480 
8481     AjPBtMem arrays1 = NULL;
8482     AjPBtMem arrays2 = NULL;
8483     ajulong *parray = NULL;
8484     AjPStr *newkarray = NULL;
8485     ajulong *newparray = NULL;
8486 
8487     ajuint lno    = 0U;
8488     ajuint rno    = 0U;
8489 
8490     ajuint lbucketlimit   = 0U;
8491     ajuint rbucketlimit   = 0U;
8492     ajuint lmaxnperbucket = 0U;
8493     ajuint rmaxnperbucket = 0U;
8494 
8495     ajuint count         = 0U;
8496 
8497     ajulong lblockno = 0UL;
8498     ajulong rblockno = 0UL;
8499     ajulong prev     = 0UL;
8500     ajulong overflow = 0UL;
8501     ajulong prevsave = 0UL;
8502 
8503     ajulong zero = 0UL;
8504     ajulong join = 0UL;
8505 
8506     ajulong lv = 0UL;
8507     ajuint  v  = 0U;
8508     ajuint iold = 0U;
8509     ajuint refskip = cache->refcount*BT_EXTRA;
8510 
8511 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
8512     ajDebug("btreeIdentSplitleaf '%S' %Lu id '%S' key '%S'\n",
8513             cache->basename, spage->pagepos, indexId, indexKeyword);
8514 #endif
8515     ++statCallIdentSplitleaf;
8516 
8517     /# ajDebug("In btreeIdentSplitleaf\n"); #/
8518 
8519     order = cache->porder;
8520 
8521     mediankey = ajStrNew();
8522 
8523     arrays1 = btreeAllocPriArray(cache);
8524     parray = arrays1->parray;
8525 
8526     arrays2 = btreeAllocPriArray(cache);
8527     newkarray = arrays2->karray;
8528     newparray = arrays2->parray;
8529 
8530     buf = spage->buf;
8531     lbuf = buf;
8532 
8533     GBT_NKEYS(buf,&nkeys);
8534     GBT_NODETYPE(buf,&rootnodetype);
8535 
8536     if(rootnodetype == BT_ROOT)
8537     {
8538 	/# ajDebug("Splitting root node\n"); #/
8539 	lblockno = cache->totsize;
8540 	lpage = btreePricacheNodenew(cache);
8541 	lbuf = lpage->buf;
8542 	lv = prev;
8543 	SBT_PREV(lbuf,lv);
8544     }
8545     else
8546     {
8547 	lpage = spage;
8548     }
8549 
8550     lblockno = lpage->pagepos;
8551     lpage->dirty = BT_LOCK;
8552     lpage->lockfor = 1151;
8553 
8554 
8555     rblockno = cache->totsize;
8556     rpage = btreePricacheNodenew(cache);
8557     rpage->dirty = BT_LOCK;
8558     rpage->lockfor = 1152;
8559     rbuf = rpage->buf;
8560 
8561     if(rootnodetype == BT_ROOT)
8562     {
8563 	lv = zero;
8564 	SBT_RIGHT(rbuf,lv);
8565 	lv = zero;
8566 	SBT_LEFT(lbuf,lv);
8567     }
8568     else
8569     {
8570 	GBT_RIGHT(lbuf,&join);
8571 	lv = join;
8572 	SBT_RIGHT(rbuf,lv);
8573     }
8574 
8575     lv = lblockno;
8576     SBT_LEFT(rbuf,lv);
8577     lv = rblockno;
8578     SBT_RIGHT(lbuf,lv);
8579 
8580 
8581     btreeGetPointers(cache,buf,&parray);
8582 
8583 
8584     keylimit = nkeys+1;
8585 
8586     idlist = ajListNew();
8587 
8588     for(i=0;i<keylimit;++i)
8589 	btreeIdbucketIdlist(cache,parray[i],idlist);
8590 
8591     ajListSort(idlist, &btreeIdCompare);
8592 
8593     totalkeys = ajListGetLength(idlist);
8594 
8595     btreeBucketSplitCalc(totalkeys, keylimit, cache->pnperbucket,
8596                          &lbucketlimit,&lmaxnperbucket,&lno,
8597                          &rbucketlimit,&rmaxnperbucket,&rno);
8598 
8599     cbucket = btreeIdbucketNew(cache->pnperbucket, cache->refcount);
8600 
8601     count = 0;
8602     iold = 0;
8603 
8604     for(i=0;i<lbucketlimit;++i)
8605     {
8606 	cbucket->Nentries = 0;
8607 
8608 	for(j=0;j<lmaxnperbucket;++j)
8609 	{
8610 	    ajListPop(idlist,(void **)&bid);
8611 
8612 	    cid = cbucket->Ids[j];
8613 	    ajStrAssignS(&cid->id,bid->id);
8614 	    cid->dbno = bid->dbno;
8615 	    cid->dups = bid->dups;
8616 	    cid->offset = bid->offset;
8617 
8618             if(cache->refcount)
8619             {
8620                 for(iref=0; iref < cache->refcount; iref++)
8621                     cid->refoffsets[iref] = bid->refoffsets[iref];
8622 	    }
8623 
8624 	    cbucket->keylen[j] =
8625                 BT_BUCKIDLEN(bid->id) + refskip;
8626 	    ++count;
8627 	    ++cbucket->Nentries;
8628 	    ajBtreeIdDel(&bid);
8629 	}
8630 
8631 	ajListPeek(idlist,(void **)&bid);
8632 
8633 	ajStrAssignS(&newkarray[i],bid->id);
8634 
8635 	if((iold < order) && parray[iold])
8636             newparray[i] = parray[iold++];
8637         else
8638 	    newparray[i] = cache->totsize;
8639 
8640 	btreeWriteIdbucket(cache,cbucket,newparray[i]);
8641     }
8642 
8643     cbucket->Nentries = 0;
8644 
8645     j = 0;
8646 
8647     while(count != lno)
8648     {
8649 	ajListPop(idlist,(void **)&bid);
8650 	cid = cbucket->Ids[j];
8651 	++j;
8652 	++count;
8653 
8654 	ajStrAssignS(&cid->id,bid->id);
8655 	cid->dbno = bid->dbno;
8656 	cid->dups = bid->dups;
8657 	cid->offset = bid->offset;
8658 
8659         if(cache->refcount)
8660         {
8661             for(iref=0; iref < cache->refcount; iref++)
8662                 cid->refoffsets[iref] = bid->refoffsets[iref];
8663         }
8664 
8665 	++cbucket->Nentries;
8666 	ajBtreeIdDel(&bid);
8667     }
8668 
8669     if((iold < order) && parray[iold])
8670         newparray[i] = parray[iold++];
8671     else
8672         newparray[i] = cache->totsize;
8673     btreeWriteIdbucket(cache,cbucket,newparray[i]);
8674 
8675     nkeys = lbucketlimit;
8676     nodetype = BT_LEAF;
8677     v = nodetype;
8678     SBT_NODETYPE(lbuf,v);
8679     lpage->dirty = BT_DIRTY;
8680 
8681     GBT_PREV(lbuf,&prevsave);
8682 
8683     btreeWriteNode(cache,lpage,newkarray,newparray,nkeys);
8684 
8685     ajListPeek(idlist,(void **)&bid);
8686     ajStrAssignS(&mediankey,bid->id);
8687 
8688     for(i=0;i<rbucketlimit;++i)
8689     {
8690 	cbucket->Nentries = 0;
8691 
8692 	for(j=0;j<rmaxnperbucket;++j)
8693 	{
8694 	    ajListPop(idlist,(void **)&bid);
8695 
8696 	    cid = cbucket->Ids[j];
8697 	    ajStrAssignS(&cid->id,bid->id);
8698 	    cid->dbno = bid->dbno;
8699 	    cid->dups = bid->dups;
8700 	    cid->offset = bid->offset;
8701 
8702             if(cache->refcount)
8703             {
8704                 for(iref=0; iref < cache->refcount; iref++)
8705                     cid->refoffsets[iref] = bid->refoffsets[iref];
8706             }
8707 
8708 	    cbucket->keylen[j] =
8709                 BT_BUCKIDLEN(bid->id)+ refskip;
8710 	    ++cbucket->Nentries;
8711 	    ajBtreeIdDel(&bid);
8712 	}
8713 
8714 	ajListPeek(idlist,(void **)&bid);
8715 	ajStrAssignS(&newkarray[i],bid->id);
8716 
8717         if((iold < order) && parray[iold])
8718             newparray[i] = parray[iold++];
8719         else
8720             newparray[i] = cache->totsize;
8721 	btreeWriteIdbucket(cache,cbucket,newparray[i]);
8722     }
8723 
8724     cbucket->Nentries = 0;
8725 
8726     j = 0;
8727 
8728     while(ajListPop(idlist,(void**)&bid))
8729     {
8730 	cid = cbucket->Ids[j];
8731 	++j;
8732 
8733 	ajStrAssignS(&cid->id,bid->id);
8734 	cid->dbno = bid->dbno;
8735 	cid->dups = bid->dups;
8736 	cid->offset = bid->offset;
8737 
8738         if(cache->refcount)
8739         {
8740             for(iref=0; iref < cache->refcount; iref++)
8741                 cid->refoffsets[iref] = bid->refoffsets[iref];
8742 	}
8743 
8744 	++cbucket->Nentries;
8745 	ajBtreeIdDel(&bid);
8746     }
8747 
8748     if((iold < order) && parray[iold])
8749         newparray[i] = parray[iold++];
8750     else
8751         newparray[i] = cache->totsize;
8752     btreeWriteIdbucket(cache,cbucket,newparray[i]);
8753 
8754     nkeys = rbucketlimit;
8755 
8756     nodetype = BT_LEAF;
8757     v = nodetype;
8758     SBT_NODETYPE(rbuf,v);
8759     lv = prevsave;
8760     SBT_PREV(rbuf,lv);
8761     lv = overflow;
8762     SBT_OVERFLOW(rbuf,lv);
8763 
8764     btreeWriteNode(cache,rpage,newkarray,newparray,nkeys);
8765     rpage->dirty = BT_DIRTY;
8766 
8767     btreeIdbucketDel(&cbucket);
8768     ajListFree(&idlist);
8769 
8770 
8771 
8772     medianless = lblockno;
8773     mediangtr  = rblockno;
8774 
8775     btreeDeallocPriArray(cache,arrays1);
8776     btreeDeallocPriArray(cache,arrays2);
8777 
8778     if(rootnodetype == BT_ROOT)
8779     {
8780 	btreeWriteNodeSingle(cache,spage,mediankey,lblockno,rblockno);
8781 	spage->dirty = BT_LOCK;
8782         spage->lockfor = 1153;
8783 
8784 	ajStrDel(&mediankey);
8785 	++cache->plevel;
8786 
8787 	return spage;
8788     }
8789 
8790 
8791     page = btreePricacheRead(cache,prevsave);
8792     btreePriInsertKey(cache,page,mediankey,medianless,mediangtr);
8793     ajStrDel(&mediankey);
8794 
8795     page = btreePricacheRead(cache,prevsave);
8796 
8797     return page;
8798 }
8799 */
8800 #endif
8801 
8802 
8803 
8804 
8805 /* @funcstatic btreeKeyInsertShift ********************************************
8806 **
8807 ** Rebalance buckets on insertion of a key
8808 **
8809 ** @param [u] cache [AjPBtcache] cache
8810 ** @param [u] retpage [AjPBtpage*] page
8811 ** @param [r] key [const AjPStr] key
8812 **
8813 ** @return [ajulong] bucket block or 0UL if shift not possible
8814 **
8815 ** @release 6.5.0
8816 ** @@
8817 ******************************************************************************/
8818 
btreeKeyInsertShift(AjPBtcache cache,AjPBtpage * retpage,const AjPStr key)8819 static ajulong btreeKeyInsertShift(AjPBtcache cache, AjPBtpage *retpage,
8820                                    const AjPStr key)
8821 {
8822     unsigned char *tbuf = NULL;
8823     unsigned char *pbuf = NULL;
8824     unsigned char *sbuf = NULL;
8825 
8826     AjPBtpage ppage = NULL;
8827     AjPBtpage spage = NULL;
8828     AjPBtpage tpage = NULL;
8829 
8830     ajuint tkeys = 0U;
8831     ajuint pkeys = 0U;
8832     ajuint skeys = 0U;
8833     ajuint order = 0U;
8834 
8835     ajint ii;
8836     ajuint i;
8837     ajuint n;
8838 
8839     ajulong parent  = 0UL;
8840     ajulong blockno = 0UL;
8841 
8842     AjPBtMem arrays1 = NULL;
8843     AjPBtMem arrays2 = NULL;
8844     AjPBtMem arrays3 = NULL;
8845     AjPStr *kTarray = NULL;
8846     AjPStr *kParray = NULL;
8847     AjPStr *kSarray = NULL;
8848     ajulong *pTarray = NULL;
8849     ajulong *pParray = NULL;
8850     ajulong *pSarray = NULL;
8851 
8852     AjPStr *karray = NULL;
8853     ajulong *parray = NULL;
8854 
8855     ajuint ppos    = 0U;
8856     ajuint pkeypos = 0U;
8857     ajuint minsize = 0U;
8858 
8859     /* ajDebug("In btreeKeyInsertShift\n"); */
8860 
8861 
8862     tpage = *retpage;
8863 
8864     tbuf = tpage->buf;
8865 
8866     GBT_PREV(tbuf,&parent);
8867     GBT_NKEYS(tbuf,&tkeys);
8868 
8869     order = cache->porder;
8870     minsize = order / 2U;
8871 
8872     if(order % 2U)
8873 	++minsize;
8874 
8875     if(tkeys <= minsize)
8876 	return 0UL;
8877 
8878     ppage = btreePricacheRead(cache,parent);
8879 
8880     pbuf = ppage->buf;
8881     GBT_NKEYS(pbuf,&pkeys);
8882 
8883 
8884     arrays1 = btreeAllocPriArray(cache);
8885     kParray = arrays1->karray;
8886     pParray = arrays1->parray;
8887 
8888     arrays2 = btreeAllocPriArray(cache);
8889     kSarray = arrays2->karray;
8890     pSarray = arrays2->parray;
8891 
8892     arrays3 = btreeAllocPriArray(cache);
8893     kTarray = arrays3->karray;
8894     pTarray = arrays3->parray;
8895 
8896     btreeGetKeys(cache,pbuf,&kParray,&pParray);
8897 
8898     i=0;
8899 
8900     while(i!=pkeys && MAJSTRCMPS(kParray[i],key)<=0)
8901 	++i;
8902 
8903     pkeypos = i;
8904 
8905     if(i==pkeys)
8906     {
8907 	if(MAJSTRCMPS(kParray[i-1],key)>0)
8908 	    ppos = i-1;
8909 	else
8910 	    ppos = i;
8911     }
8912     else
8913 	ppos = i;
8914 
8915 
8916     if(ppos) /* There is another leaf to the left */
8917     {
8918 	spage = btreePricacheRead(cache,pParray[ppos-1]);
8919 	sbuf = spage->buf;
8920 	GBT_NKEYS(sbuf,&skeys);
8921     }
8922 
8923     if(i && skeys != order-1) /* There is space in the left leaf */
8924     {
8925 	/* ajDebug("Left shift\n"); */
8926 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
8927 	if(skeys)
8928 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
8929 
8930 	i = 0;
8931 
8932 	while(pParray[i] != tpage->pagepos)
8933 	    ++i;
8934 
8935 	--i;
8936 
8937 	pkeypos = i;
8938 
8939 	ajStrAssignS(&kSarray[skeys],kParray[pkeypos]);
8940 	pSarray[skeys+1] = pTarray[0];
8941 	++skeys;
8942 	--tkeys;
8943 	ajStrAssignS(&kParray[pkeypos],kTarray[0]);
8944 
8945 	for(i=0;i<tkeys;++i)
8946 	{
8947 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
8948 	    pTarray[i] = pTarray[i+1];
8949 	}
8950 
8951 	pTarray[i] = pTarray[i+1];
8952 	pTarray[i+1] = 0UL;
8953 
8954 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
8955 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
8956 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
8957 
8958 	if(!ppage->pagepos)
8959         {
8960 	    ppage->dirty = BT_LOCK;
8961             ppage->lockfor = 1161;
8962         }
8963 
8964 	i = 0;
8965 
8966 	while(i!=pkeys && MAJSTRCMPS(kParray[i],key)<=0)
8967 	    ++i;
8968 
8969 	if(i==pkeys)
8970 	{
8971 	    if(MAJSTRCMPS(kParray[i-1],key)>0)
8972 		blockno = pParray[i-1];
8973 	    else
8974 		blockno = pParray[i];
8975 	}
8976 	else
8977 	    blockno = pParray[i];
8978 
8979 	if(blockno == spage->pagepos)
8980 	{
8981 	    *retpage = spage;
8982 	    karray = kSarray;
8983 	    parray = pSarray;
8984 	    n = skeys;
8985 	}
8986 	else
8987 	{
8988 	    karray = kTarray;
8989 	    parray = pTarray;
8990 	    n = tkeys;
8991 	}
8992 
8993 
8994 	i = 0;
8995 
8996 	while(i!=n && MAJSTRCMPS(karray[i],key)<=0)
8997 	    ++i;
8998 
8999 	if(i==n)
9000 	{
9001 	    if(MAJSTRCMPS(karray[i-1],key)>0)
9002 		blockno = parray[i-1];
9003 	    else
9004 		blockno = parray[i];
9005 	}
9006 	else
9007 	    blockno = parray[i];
9008 
9009         btreeDeallocPriArray(cache,arrays1);
9010         btreeDeallocPriArray(cache,arrays2);
9011         btreeDeallocPriArray(cache,arrays3);
9012 
9013 	/* ajDebug("... returns blockno (a) %Lu\n",blockno); */
9014 
9015 	return blockno;
9016     }
9017 
9018 
9019     if(ppos != pkeys)	/* There is a right node */
9020     {
9021 	spage = btreePricacheRead(cache,pParray[ppos+1]);
9022 	sbuf = spage->buf;
9023 	GBT_NKEYS(sbuf,&skeys);
9024     }
9025 
9026 
9027     /* Space in the right leaf */
9028     if(ppos != pkeys && skeys != order-1)
9029     {
9030 	/* ajDebug("Right shift\n"); */
9031 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
9032 	btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
9033 
9034 	i = 0U;
9035 
9036 	while(pParray[i] != tpage->pagepos)
9037 	    ++i;
9038 
9039 	pkeypos = i;
9040 
9041 	pSarray[skeys+1] = pSarray[skeys];
9042 	for(ii=skeys-1;ii>-1;--ii)
9043 	{
9044 	    ajStrAssignS(&kSarray[ii+1],kSarray[ii]);
9045 	    pSarray[ii+1] = pSarray[ii];
9046 	}
9047 	ajStrAssignS(&kSarray[0],kParray[pkeypos]);
9048 	pSarray[0] = pTarray[tkeys];
9049 	ajStrAssignS(&kParray[pkeypos],kTarray[tkeys-1]);
9050 	++skeys;
9051 	--tkeys;
9052 	pTarray[tkeys+1] = 0UL;
9053 
9054 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
9055 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
9056 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
9057 	if(!ppage->pagepos)
9058         {
9059 	    ppage->dirty = BT_LOCK;
9060             ppage->lockfor = 1162;
9061         }
9062 
9063 	i = 0U;
9064 
9065 	while(i!=pkeys && MAJSTRCMPS(kParray[i],key)<=0)
9066 	    ++i;
9067 
9068 	if(i==pkeys)
9069 	{
9070 	    if(MAJSTRCMPS(kParray[i-1],key)>0)
9071 		blockno = pParray[i-1];
9072 	    else
9073 		blockno = pParray[i];
9074 	}
9075 	else
9076 	    blockno = pParray[i];
9077 
9078 	if(blockno == spage->pagepos)
9079 	{
9080 	    *retpage = spage;
9081 	    karray = kSarray;
9082 	    parray = pSarray;
9083 	    n = skeys;
9084 	}
9085 	else
9086 	{
9087 	    karray = kTarray;
9088 	    parray = pTarray;
9089 	    n = tkeys;
9090 	}
9091 
9092 	i = 0;
9093 
9094 	while(i!=n && MAJSTRCMPS(karray[i],key)<=0)
9095 	    ++i;
9096 
9097 	if(i==n)
9098 	{
9099 	    if(MAJSTRCMPS(karray[i-1],key)>0)
9100 		blockno = parray[i-1];
9101 	    else
9102 		blockno = parray[i];
9103 	}
9104 	else
9105 	    blockno = parray[i];
9106 
9107         btreeDeallocPriArray(cache,arrays1);
9108         btreeDeallocPriArray(cache,arrays2);
9109         btreeDeallocPriArray(cache,arrays3);
9110 
9111 	/* ajDebug("... returns blockno (b) %Lu\n",blockno); */
9112 
9113 	return blockno;
9114     }
9115 
9116 
9117     btreeDeallocPriArray(cache,arrays1);
9118     btreeDeallocPriArray(cache,arrays2);
9119     btreeDeallocPriArray(cache,arrays3);
9120 
9121     /* ajDebug("... returns 0UL\n"); */
9122 
9123     return 0UL;
9124 }
9125 
9126 
9127 
9128 
9129 /* @funcstatic btreePrimaryShift **********************************************
9130 **
9131 ** Rebalance primary nodes (identifiers or keywords) on insertion
9132 **
9133 ** @param [u] cache [AjPBtcache] cache
9134 ** @param [u] tpage [AjPBtpage] page
9135 **
9136 ** @return [void]
9137 **
9138 ** @release 6.5.0
9139 ** @@
9140 ******************************************************************************/
9141 
btreePrimaryShift(AjPBtcache cache,AjPBtpage tpage)9142 static void btreePrimaryShift(AjPBtcache cache, AjPBtpage tpage)
9143 {
9144     unsigned char *tbuf = NULL;
9145     unsigned char *pbuf = NULL;
9146     unsigned char *sbuf = NULL;
9147     unsigned char *buf  = NULL;
9148 
9149     AjPBtpage ppage = NULL;
9150     AjPBtpage spage = NULL;
9151     AjPBtpage page  = NULL;
9152 
9153     ajuint tkeys = 0U;
9154     ajuint pkeys = 0U;
9155     ajuint skeys = 0U;
9156     ajuint order = 0U;
9157 
9158     ajuint i;
9159     ajint  ii;
9160 
9161     ajulong parent  = 0UL;
9162 
9163     AjPBtMem Parrays = NULL;
9164     AjPBtMem Sarrays = NULL;
9165     AjPBtMem Tarrays = NULL;
9166     AjPStr *kTarray = NULL;
9167     AjPStr *kParray = NULL;
9168     AjPStr *kSarray = NULL;
9169     ajulong *pTarray = NULL;
9170     ajulong *pParray = NULL;
9171     ajulong *pSarray = NULL;
9172 
9173     ajuint pkeypos = 0U;
9174     ajuint minsize = 0U;
9175 
9176     ajulong lv = 0UL;
9177 
9178     /* ajDebug("In btreePrimaryShift\n"); */
9179 
9180     tbuf = tpage->buf;
9181 
9182     GBT_PREV(tbuf,&parent);
9183     GBT_NKEYS(tbuf,&tkeys);
9184 
9185     order = cache->porder;
9186     minsize = order / 2U;
9187 
9188     if(order % 2U)
9189 	++minsize;
9190 
9191     if(tkeys <= minsize)
9192 	return;
9193 
9194 
9195     ppage = btreePricacheRead(cache,parent);
9196     pbuf = ppage->buf;
9197     GBT_NKEYS(pbuf,&pkeys);
9198 
9199     Parrays = btreeAllocPriArray(cache);
9200     Sarrays = btreeAllocPriArray(cache);
9201     Tarrays = btreeAllocPriArray(cache);
9202     kParray = Parrays->karray;
9203     pParray = Parrays->parray;
9204     kSarray = Sarrays->karray;
9205     pSarray = Sarrays->parray;
9206     kTarray = Tarrays->karray;
9207     pTarray = Tarrays->parray;
9208 
9209     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
9210     GBT_NKEYS(tbuf,&tkeys);
9211 
9212     btreeGetKeys(cache,pbuf,&kParray,&pParray);
9213 
9214     i=0U;
9215 
9216     while(pParray[i] != tpage->pagepos)
9217 	++i;
9218 
9219     if(i) /* There is another leaf to the left */
9220     {
9221 	pkeypos = i-1U;
9222 	spage = btreePricacheRead(cache,pParray[pkeypos]);
9223 	sbuf = spage->buf;
9224 	GBT_NKEYS(sbuf,&skeys);
9225 
9226     }
9227 
9228     if(i && skeys != order-1U) /* There is space in the left leaf */
9229     {
9230 	if(skeys)
9231 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
9232 
9233 	ajStrAssignS(&kSarray[skeys],kParray[pkeypos]);
9234 	pSarray[skeys+1] = pTarray[0];
9235 	++skeys;
9236 	--tkeys;
9237 	ajStrAssignS(&kParray[pkeypos],kTarray[0]);
9238 
9239 	for(i=0;i<tkeys;++i)
9240 	{
9241 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
9242 	    pTarray[i] = pTarray[i+1];
9243 	}
9244 
9245 	pTarray[i] = pTarray[i+1];
9246 	pTarray[i+1] = 0UL;
9247 
9248 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
9249 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
9250 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
9251 	if(!ppage->pagepos)
9252         {
9253 	    ppage->dirty = BT_LOCK;
9254             ppage->lockfor = 1171;
9255         }
9256 
9257 	page = btreePricacheRead(cache,pSarray[skeys]);
9258 	buf = page->buf;
9259 	lv = spage->pagepos;
9260 	SBT_PREV(buf,lv);
9261 	page->dirty = BT_DIRTY;
9262 
9263         btreeDeallocPriArray(cache,Parrays);
9264         btreeDeallocPriArray(cache,Sarrays);
9265         btreeDeallocPriArray(cache,Tarrays);
9266 
9267 	return;
9268     }
9269 
9270 
9271 
9272     if(i != pkeys)	/* There is a right node */
9273     {
9274 	pkeypos = i;
9275 	spage = btreePricacheRead(cache,pParray[pkeypos+1]);
9276 	sbuf = spage->buf;
9277 	GBT_NKEYS(sbuf,&skeys);
9278     }
9279 
9280 
9281     if(i != pkeys && skeys != order-1) /* Space in the right node */
9282     {
9283 	if(skeys)
9284 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
9285 
9286 	pSarray[skeys+1] = pSarray[skeys];
9287 
9288 	for(ii=skeys-1;ii>-1;--ii)
9289 	{
9290 	    ajStrAssignS(&kSarray[ii+1],kSarray[ii]);
9291 	    pSarray[ii+1] = pSarray[ii];
9292 	}
9293 
9294 	ajStrAssignS(&kSarray[0],kParray[pkeypos]);
9295 	pSarray[0] = pTarray[tkeys];
9296 	ajStrAssignS(&kParray[pkeypos],kTarray[tkeys-1]);
9297 	++skeys;
9298 	--tkeys;
9299 	pTarray[tkeys+1] = 0UL;
9300 
9301 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
9302 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
9303 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
9304 	if(!ppage->pagepos)
9305         {
9306 	    ppage->dirty = BT_LOCK;
9307             ppage->lockfor = 1172;
9308         }
9309 
9310 	page = btreePricacheRead(cache,pSarray[0]);
9311 	buf = page->buf;
9312 	lv = spage->pagepos;
9313 	SBT_PREV(buf,lv);
9314 	page->dirty = BT_DIRTY;
9315 
9316         btreeDeallocPriArray(cache,Parrays);
9317         btreeDeallocPriArray(cache,Sarrays);
9318         btreeDeallocPriArray(cache,Tarrays);
9319 
9320 	return;
9321     }
9322 
9323     btreeDeallocPriArray(cache,Parrays);
9324     btreeDeallocPriArray(cache,Sarrays);
9325     btreeDeallocPriArray(cache,Tarrays);
9326 
9327     return;
9328 }
9329 
9330 
9331 
9332 
9333 #if 0
9334 /* @funcstatic btreeTraverseLeaves ********************************************
9335 **
9336 ** Find the next leaf by traversing the tree
9337 **
9338 ** @param [u] cache [AjPBtcache] cache
9339 ** @param [u] thys [AjPBtpage] current leaf page
9340 **
9341 ** @return [AjPBtpage] next leaf or NULL
9342 **
9343 ** @release 3.0.0
9344 ** @@
9345 ******************************************************************************/
9346 
9347 static AjPBtpage btreeTraverseLeaves(AjPBtcache cache, AjPBtpage thys)
9348 {
9349     AjPBtpage page = NULL;
9350 
9351     ajulong pagepos = 0UL;
9352     ajulong prev   = 0UL;
9353 
9354     AjPBtMem arrays = NULL;
9355     ajulong *parray = NULL;
9356 
9357     ajuint nodetype = 0U;
9358     ajuint nkeys    = 0U;
9359     ajuint apos     = 0U;
9360     ajuint order    = 0U;
9361     ajuint i;
9362 
9363     unsigned char *buf = NULL;
9364 
9365     if(!cache->plevel)
9366 	return NULL;
9367 
9368     order = cache->porder;
9369     arrays = btreeAllocPriArray(cache);
9370     parray = arrays->Parray;
9371 
9372     pagepos = thys->pagepos;
9373     buf = thys->buf;
9374     GBT_PREV(buf,&prev);
9375 
9376     page = btreePricacheRead(cache,prev);
9377     buf = page->buf;
9378     GBT_NKEYS(buf,&nkeys);
9379     GBT_NODETYPE(buf,&nodetype);
9380     btreeGetPointers(cache,buf,&parray);
9381 
9382     apos = 0U;
9383 
9384     while(parray[apos] != pagepos)
9385 	++apos;
9386 
9387     while(apos == nkeys)
9388     {
9389 	if(nodetype == BT_ROOT)
9390 	{
9391             btreeDeallocPriArray(cache,arrays);
9392 
9393 	    return NULL;
9394 	}
9395 
9396 	GBT_PREV(buf,&prev);
9397 	pagepos = page->pagepos;
9398 	page = btreePricacheRead(cache,prev);
9399 	buf = page->buf;
9400 	GBT_NKEYS(buf,&nkeys);
9401 	GBT_NODETYPE(buf,&nodetype);
9402 	btreeGetPointers(cache,buf,&parray);
9403 
9404 	apos = 0U;
9405 
9406 	while(parray[apos] != pagepos)
9407 	    ++apos;
9408     }
9409 
9410     page = btreePricacheRead(cache,parray[apos+1]);
9411     buf = page->buf;
9412     GBT_NODETYPE(buf,&nodetype);
9413     btreeGetPointers(cache,buf,&parray);
9414 
9415     while(nodetype != BT_LEAF)
9416     {
9417 	page = btreePricacheRead(cache,parray[0]);
9418 	buf = page->buf;
9419 	btreeGetPointers(cache,buf,&parray);
9420 	GBT_NODETYPE(buf,&nodetype);
9421     }
9422 
9423     btreeDeallocPriArray(cache,arrays);
9424 
9425     return page;
9426 }
9427 #endif
9428 
9429 
9430 
9431 
9432 #if 0
9433 /* @funcstatic btreeJoinLeaves ************************************************
9434 **
9435 ** Update all Left/Right Leaf Pointers
9436 **
9437 ** @param [u] cache [AjPBtcache] cache
9438 **
9439 ** @return [void] next leaf or NULL
9440 **
9441 ** @release 3.0.0
9442 ** @@
9443 ******************************************************************************/
9444 
9445 static void btreeJoinLeaves(AjPBtcache cache)
9446 {
9447     unsigned char *buf = NULL;
9448     AjPBtpage page     = NULL;
9449     AjPBtpage newpage  = NULL;
9450 
9451     ajuint nodetype = 0U;
9452     ajuint order    = 0U;
9453     ajuint i;
9454 
9455     AjPBtMem arrays = NULL;
9456     ajulong *parray = NULL;
9457 
9458     ajulong left    = 0UL;
9459     ajulong right   = 0UL;
9460 
9461     ajulong lv = 0UL;
9462 
9463     if(!cache->plevel)
9464 	return;
9465 
9466     order = cache->porder;
9467     arrays = btreeAllocPriArray(cache);
9468     parray = arrays->Parray;
9469 
9470     page = btreePricacheLocate(cache,0UL);
9471     buf = page->buf;
9472     btreeGetPointers(cache,buf,&parray);
9473     GBT_NODETYPE(buf,&nodetype);
9474 
9475     while(nodetype != BT_LEAF)
9476     {
9477 	page = btreePricacheRead(cache,parray[0]);
9478 	buf = page->buf;
9479 	btreeGetKeys(cache,buf,&karray,&parray);
9480 	GBT_NODETYPE(buf,&nodetype);
9481     }
9482 
9483     lv = left;
9484     SBT_LEFT(buf,lv);
9485 
9486     while((newpage = btreeTraverseLeaves(cache,page)))
9487     {
9488 	right = newpage->pagepos;
9489 	lv = right;
9490 	SBT_RIGHT(buf,lv);
9491 	page->dirty = BT_DIRTY;
9492 	left = page->pagepos;
9493 	buf = newpage->buf;
9494 	lv = left;
9495 	SBT_LEFT(buf,lv);
9496 	page = newpage;
9497     }
9498 
9499     right = 0UL;
9500     SBT_RIGHT(buf,right);
9501     page->dirty = BT_DIRTY;
9502 
9503     btreeDeallocPriArray(cache,arrays);
9504 
9505     return;
9506 }
9507 #endif
9508 
9509 
9510 
9511 
9512 /* @func ajBtreeIdwildNew *****************************************************
9513 **
9514 ** Construct a wildcard id search object
9515 **
9516 ** @param [u] cache [AjPBtcache] cache
9517 ** @param [r] wild [const AjPStr] wildcard id prefix (without asterisk)
9518 **
9519 ** @return [AjPBtIdwild] b+ tree wildcard object
9520 **
9521 ** @release 3.0.0
9522 ** @@
9523 ******************************************************************************/
9524 
ajBtreeIdwildNew(AjPBtcache cache,const AjPStr wild)9525 AjPBtIdwild ajBtreeIdwildNew(AjPBtcache cache, const AjPStr wild)
9526 {
9527     AjPBtIdwild thys = NULL;
9528 
9529     (void) cache;			/* make it used */
9530 
9531     AJNEW0(thys);
9532 
9533     thys->id   = ajStrNewS(wild);
9534     ajStrTrimC(&thys->id,"*"); /* Need to revisit this */
9535     thys->list = ajListNew();
9536 
9537     thys->first = ajTrue;
9538 
9539     return thys;
9540 }
9541 
9542 
9543 
9544 
9545 /* @func ajBtreeIdwildDel *****************************************************
9546 **
9547 ** Destroy a wildcard search object
9548 **
9549 ** @param [u] Pthis [AjPBtIdwild*] b+ tree identifier wildcard structure
9550 **
9551 ** @return [void]
9552 **
9553 ** @release 3.0.0
9554 ** @@
9555 ******************************************************************************/
9556 
ajBtreeIdwildDel(AjPBtIdwild * Pthis)9557 void ajBtreeIdwildDel(AjPBtIdwild *Pthis)
9558 {
9559     AjPBtIdwild thys = NULL;
9560     AjPBtId   id    = NULL;
9561 
9562     if(!Pthis || !*Pthis)
9563 	return;
9564 
9565     thys = *Pthis;
9566 
9567     ajStrDel(&thys->id);
9568 
9569     while(ajListPop(thys->list,(void **)&id))
9570 	ajBtreeIdDel(&id);
9571 
9572     ajListFree(&thys->list);
9573 
9574     AJFREE(thys);
9575     *Pthis = NULL;
9576 
9577     return;
9578 }
9579 
9580 
9581 
9582 
9583 /* @func ajBtreeKeywildNew ****************************************************
9584 **
9585 ** Construct a wildcard keyword search object
9586 **
9587 ** @param [u] cache [AjPBtcache] cache
9588 ** @param [r] wild [const AjPStr] wildcard keyword prefix (without asterisk)
9589 **
9590 ** @return [AjPBtKeywild] b+ tree keyword wildcard object
9591 **
9592 ** @release 3.0.0
9593 ** @@
9594 ******************************************************************************/
9595 
ajBtreeKeywildNew(AjPBtcache cache,const AjPStr wild)9596 AjPBtKeywild ajBtreeKeywildNew(AjPBtcache cache, const AjPStr wild)
9597 {
9598     AjPBtKeywild thys = NULL;
9599 
9600     char *p;
9601     char *cp;
9602 
9603     AJNEW0(thys);
9604 
9605     thys->keyword = ajStrNewS(wild);
9606     ajStrFmtQuery(&thys->keyword);
9607     if(MAJSTRGETLEN(thys->keyword) > cache->keylimit)
9608         ajStrTruncateLen(&thys->keyword, cache->keylimit);
9609 
9610     cp = MAJSTRGETPTR(thys->keyword);
9611     p = strpbrk(cp,"*?");
9612 
9613     if(p)
9614     {
9615 	if(p-cp)
9616 	    ajStrAssignSubS(&thys->prefix,thys->keyword,0,p-cp-1);
9617 	else
9618             ajStrAssignC(&thys->prefix, "");
9619     }
9620     else
9621 	ajStrAssignS(&thys->prefix,thys->keyword);
9622 
9623     thys->list   = ajListNew();
9624     thys->idlist = ajListNew();
9625 
9626     thys->first = ajTrue;
9627 
9628     return thys;
9629 }
9630 
9631 
9632 
9633 
9634 /* @func ajBtreeKeywildDel ****************************************************
9635 **
9636 ** Destroy a wildcard keyword search object
9637 **
9638 ** @param [u] Pthis [AjPBtKeywild*] b+ tree wildcard keyword structure
9639 **
9640 ** @return [void]
9641 **
9642 ** @release 3.0.0
9643 ** @@
9644 ******************************************************************************/
9645 
ajBtreeKeywildDel(AjPBtKeywild * Pthis)9646 void ajBtreeKeywildDel(AjPBtKeywild *Pthis)
9647 {
9648     AjPBtKeywild thys = NULL;
9649     AjPStr id = NULL;
9650     AjPBtPri pri = NULL;
9651 
9652     if(!Pthis || !*Pthis)
9653 	return;
9654 
9655     thys = *Pthis;
9656 
9657     ajStrDel(&thys->keyword);
9658     ajStrDel(&thys->prefix);
9659 
9660     while(ajListPop(thys->idlist,(void **)&id))
9661 	ajStrDel(&id);
9662 
9663     while(ajListPop(thys->list,(void **)&pri))
9664 	ajBtreePriDel(&pri);
9665 
9666     ajListFree(&thys->list);
9667 
9668     AJFREE(thys);
9669     *Pthis = NULL;
9670 
9671     return;
9672 }
9673 
9674 
9675 
9676 
9677 /* @funcstatic btreePrimaryFetchFindleafWild **********************************
9678 **
9679 ** Find the node that should contain a wildcard identifier or keyword
9680 **
9681 ** @param [u] cache [AjPBtcache] cache
9682 ** @param [r] key [const AjPStr] key to search for
9683 **
9684 ** @return [AjPBtpage] leaf node where item should be inserted
9685 **
9686 ** @release 6.5.0
9687 ** @@
9688 ******************************************************************************/
9689 
btreePrimaryFetchFindleafWild(AjPBtcache cache,const AjPStr key)9690 static AjPBtpage btreePrimaryFetchFindleafWild(AjPBtcache cache,
9691                                                const AjPStr key)
9692 {
9693     AjPBtpage root = NULL;
9694     AjPBtpage ret  = NULL;
9695 
9696     /* ajDebug("In btreePrimaryFetchFindleafWild\n"); */
9697 
9698     /* The root node should always be in the cache (BT_LOCKed) */
9699     root = btreePricacheLocate(cache,0UL);
9700 
9701     if(!cache->plevel)
9702 	return root;
9703 
9704     ret = btreePrimaryFindInodeWild(cache,root,key);
9705 
9706     return ret;
9707 }
9708 
9709 
9710 
9711 
9712 /* @funcstatic btreePrimaryFindInodeWild **************************************
9713 **
9714 ** Recursive search for node (wild)
9715 **
9716 ** @param [u] cache [AjPBtcache] cache
9717 ** @param [u] page [AjPBtpage] page
9718 ** @param [r] item [const AjPStr] key to search for
9719 **
9720 ** @return [AjPBtpage] leaf node where item should be inserted
9721 **
9722 ** @release 6.5.0
9723 ** @@
9724 ******************************************************************************/
9725 
btreePrimaryFindInodeWild(AjPBtcache cache,AjPBtpage page,const AjPStr item)9726 static AjPBtpage btreePrimaryFindInodeWild(AjPBtcache cache, AjPBtpage page,
9727                                            const AjPStr item)
9728 {
9729     AjPBtpage ret = NULL;
9730     AjPBtpage pg  = NULL;
9731 
9732     unsigned char *buf = NULL;
9733 
9734     ajuint status = 0;
9735     ajuint ival   = 0;
9736 
9737     /* ajDebug("In btreePrimaryFindInodeWild\n"); */
9738 
9739     ret = page;
9740     buf = page->buf;
9741     GBT_NODETYPE(buf,&ival);
9742 
9743     if(ival != BT_LEAF)
9744     {
9745 	status = ret->dirty;
9746 	ret->dirty = BT_LOCK;	/* Lock in case of lots of overflow pages */
9747         ret->lockfor = 1181;
9748 	pg = btreePrimaryPageDownWild(cache,buf,item);
9749 	ret->dirty = status;
9750 	ret = btreePrimaryFindInodeWild(cache,pg,item);
9751     }
9752 
9753     return ret;
9754 }
9755 
9756 
9757 
9758 
9759 /* @funcstatic btreePrimaryPageDownWild ***************************************
9760 **
9761 ** Return next lower index page given a wildcard key
9762 **
9763 ** @param [u] cache [AjPBtcache] cache
9764 ** @param [u] buf [unsigned char *] page buffer
9765 ** @param [r] key [const AjPStr] key to search for
9766 **
9767 ** @return [AjPBtpage] pointer to a page
9768 **
9769 ** @release 2.8.0
9770 ** @@
9771 ******************************************************************************/
9772 
btreePrimaryPageDownWild(AjPBtcache cache,unsigned char * buf,const AjPStr key)9773 static AjPBtpage btreePrimaryPageDownWild(AjPBtcache cache, unsigned char *buf,
9774                                           const AjPStr key)
9775 {
9776     ajulong blockno = 0UL;
9777     AjPBtpage page = NULL;
9778 
9779     unsigned char *rootbuf = NULL;
9780 
9781     /* ajDebug("In btreePrimaryPageDownWild\n"); */
9782 
9783     rootbuf = buf;
9784 
9785     blockno = btreeGetBlockS(cache,rootbuf,key);
9786     page =  btreePricacheRead(cache,blockno);
9787 
9788     return page;
9789 }
9790 
9791 
9792 
9793 
9794 /* @funcstatic btreeIdleafFetch ***********************************************
9795 **
9796 ** Read all leaf identifiers into a list
9797 **
9798 ** @param [u] cache [AjPBtcache] cache
9799 ** @param [u] page [AjPBtpage] page
9800 ** @param [w] list [AjPList] list of AjPBtIds
9801 **
9802 ** @return [void]
9803 **
9804 ** @release 6.5.0
9805 ** @@
9806 ******************************************************************************/
9807 
btreeIdleafFetch(AjPBtcache cache,AjPBtpage page,AjPList list)9808 static void btreeIdleafFetch(AjPBtcache cache, AjPBtpage page,
9809                                AjPList list)
9810 {
9811     unsigned char *buf = NULL;
9812     AjPBtMem arrays    = NULL;
9813     ajulong *parray     = NULL;
9814 
9815     ajuint keylimit = 0;
9816     ajuint nkeys    = 0;
9817 
9818     ajuint i;
9819 
9820     /* ajDebug("In btreeIdleafFetch\n"); */
9821 
9822     buf = page->buf;
9823 
9824     GBT_NKEYS(buf,&nkeys);
9825 
9826     if(!nkeys)
9827         return;
9828 
9829     arrays = btreeAllocPriArray(cache);
9830     parray = arrays->parray;
9831 
9832     btreeGetPointers(cache,buf,&parray);
9833 
9834     GBT_NKEYS(buf,&nkeys);
9835 
9836     keylimit = nkeys+1;
9837 
9838     for(i=0;i<keylimit;++i)
9839 	btreeIdbucketIdlistAll(cache,parray[i],list);
9840 
9841     /*
9842     ** sort the list so we can compare prefix until no more matches
9843     */
9844 
9845     ajListSort(list, &btreeIdCompare);
9846 
9847     btreeDeallocPriArray(cache,arrays);
9848 
9849     return;
9850 }
9851 
9852 
9853 
9854 
9855 /* @func ajBtreeIdwildQuery ***************************************************
9856 **
9857 ** Wildcard retrieval of entries by identifier
9858 **
9859 ** @param [u] cache [AjPBtcache] cache
9860 ** @param [u] wild [AjPBtIdwild] Wildcard
9861 **
9862 ** @return [AjPBtId] next matching Id or NULL
9863 **
9864 ** @release 6.5.0
9865 ** @@
9866 ******************************************************************************/
9867 
ajBtreeIdwildQuery(AjPBtcache cache,AjPBtIdwild wild)9868 AjPBtId ajBtreeIdwildQuery(AjPBtcache cache, AjPBtIdwild wild)
9869 {
9870 
9871     AjPBtId id     = NULL;
9872     AjPBtpage page = NULL;
9873     const AjPStr key = NULL;
9874     AjPList list   = NULL;
9875     AjBool found   = ajFalse;
9876 
9877     ajulong pagepos = 0UL;
9878 
9879     unsigned char *buf = NULL;
9880 
9881     key  = wild->id;
9882     list = wild->list;
9883 
9884     found = ajFalse;
9885 
9886     if(wild->first)
9887     {
9888 	page = btreePrimaryFetchFindleafWild(cache,key);
9889 	page->dirty = BT_LOCK;
9890         page->lockfor = 1191;
9891 	wild->pagepos = page->pagepos;
9892 
9893 	btreeIdleafFetch(cache,page,list);
9894 
9895 	page->dirty = BT_CLEAN;
9896 
9897 	if(!ajListGetLength(list))
9898 	    return NULL;
9899 
9900 	while(ajListPop(list,(void **)&id))
9901 	{
9902 	    if(ajStrPrefixS(id->id,key))
9903 	    {
9904 		found = ajTrue;
9905 		break;
9906 	    }
9907 
9908 	    ajBtreeIdDel(&id);
9909 	}
9910 
9911 	wild->first = ajFalse;
9912 
9913 
9914 	if(!found)	/* Check the next leaf just in case key==internal */
9915 	{
9916 	    buf = page->buf;
9917 	    GBT_RIGHT(buf,&pagepos);
9918 
9919 	    if(!pagepos)
9920 		return NULL;
9921 
9922 	    page = btreePricacheRead(cache,pagepos);
9923 	    wild->pagepos = pagepos;
9924 	    page->dirty = BT_LOCK;
9925             page->lockfor = 1192;
9926 
9927 	    btreeIdleafFetch(cache,page,list);
9928 
9929 	    page->dirty = BT_CLEAN;
9930 
9931 	    if(!ajListGetLength(list))
9932 		return NULL;
9933 
9934 	    found = ajFalse;
9935 
9936 	    while(ajListPop(list,(void **)&id))
9937 	    {
9938 		if(ajStrPrefixS(id->id,key))
9939 		{
9940 		    found = ajTrue;
9941 		    break;
9942 		}
9943 
9944 		ajBtreeIdDel(&id);
9945 	    }
9946 	}
9947 
9948 
9949 	if(!found)
9950 	    return NULL;
9951 
9952 	return id;
9953     }
9954 
9955 
9956     if(!ajListGetLength(list))
9957     {
9958 	page = btreePricacheRead(cache,wild->pagepos);
9959 	buf = page->buf;
9960 	GBT_RIGHT(buf,&pagepos);
9961 
9962 	if(!pagepos)
9963 	    return NULL;
9964 
9965 	page = btreePricacheRead(cache,pagepos);
9966 	wild->pagepos = pagepos;
9967 	page->dirty = BT_LOCK;
9968         page->lockfor = 1193;
9969 
9970 	btreeIdleafFetch(cache,page,list);
9971 
9972 	page->dirty = BT_CLEAN;
9973 
9974 	if(!ajListGetLength(list))
9975 	    return NULL;
9976     }
9977 
9978 
9979 
9980     while(ajListPop(list,(void **)&id))
9981     {
9982 	if(ajStrPrefixS(id->id,key))
9983 	{
9984 	    found = ajTrue;
9985 	    break;
9986 	}
9987 
9988 	ajBtreeIdDel(&id);
9989     }
9990 
9991 
9992     if(!found)
9993 	return NULL;
9994 
9995     return id;
9996 }
9997 
9998 
9999 
10000 
10001 /* @func ajBtreeIdentFetchwildId **********************************************
10002 **
10003 ** Wildcard retrieval of entries
10004 **
10005 ** @param [u] cache [AjPBtcache] cache
10006 ** @param [r] key [const AjPStr] Wildcard key
10007 ** @param [u] idlist [AjPList] list of matching AjPBtIds
10008 **
10009 ** @return [void]
10010 **
10011 ** @release 6.5.0
10012 ** @@
10013 ******************************************************************************/
10014 
ajBtreeIdentFetchwildId(AjPBtcache cache,const AjPStr key,AjPList idlist)10015 void ajBtreeIdentFetchwildId(AjPBtcache cache, const AjPStr key,
10016                              AjPList idlist)
10017 {
10018 
10019     AjPBtId id     = NULL;
10020     AjPBtpage page = NULL;
10021     AjPList list   = NULL;
10022     AjBool found   = ajFalse;
10023 
10024     ajulong pripagepos = 0UL;
10025     ajulong right = 0UL;
10026 
10027     unsigned char *buf = NULL;
10028     AjBool finished = ajFalse;
10029 
10030     AjPStr prefix = NULL;
10031     AjPStr keystr = NULL;
10032 
10033     char *p;
10034     char *cp;
10035 
10036     ajDebug("ajBtreeIdentFetchwildId '%S' list: %Lu\n",
10037             key, ajListGetLength(list));
10038 
10039     if(!cache->countunique && !cache->countall)
10040         return;
10041 
10042     keystr = ajStrNewS(key);
10043     ajStrFmtQuery(&keystr);
10044 
10045     if(MAJSTRGETLEN(keystr) > cache->keylimit)
10046         ajStrTruncateLen(&keystr, cache->keylimit);
10047 
10048     cp = MAJSTRGETPTR(keystr);
10049     p = strpbrk(cp,"*?");
10050 
10051     if(p)
10052     {
10053 	if(p-cp)
10054 	    ajStrAssignSubS(&prefix,keystr,0,p-cp-1);
10055 	else
10056 	{
10057 	    btreeKeyFullSearchId(cache,keystr,idlist);
10058             ajStrDel(&keystr);
10059 	    ajStrDel(&prefix);
10060 	    return;
10061 	}
10062     }
10063     else
10064 	ajStrAssignS(&prefix,keystr);
10065 
10066     ajStrFmtQuery(&prefix);
10067 
10068     if(MAJSTRGETLEN(prefix) > cache->keylimit)
10069         ajStrTruncateLen(&prefix, cache->keylimit);
10070 
10071     ajDebug("ajBtreeIdentFetchwild '%S' prefix: '%S'\n",
10072             keystr, prefix);
10073 
10074     list = ajListNew();
10075 
10076     found = ajFalse;
10077 
10078     page = btreePrimaryFetchFindleafWild(cache,prefix);
10079     page->dirty = BT_LOCK;
10080     page->lockfor = 1201;
10081     pripagepos = page->pagepos;
10082 
10083     btreeIdleafFetch(cache,page,list);
10084     page->dirty = BT_CLEAN;
10085 
10086 
10087     while(ajListPop(list,(void **)&id))
10088     {
10089 	if(ajStrPrefixS(id->id,prefix))
10090 	{
10091 	    found = ajTrue;
10092 	    break;
10093 	}
10094 	else
10095 	    ajBtreeIdDel(&id);
10096     }
10097 
10098     ajDebug("ajBtreeIdentFetchwild first leaf prefix found: %B\n", found);
10099 
10100     if(!found)	/* Check the next leaf just in case key==internal */
10101     {
10102 	buf = page->buf;
10103 	GBT_RIGHT(buf,&right);
10104 
10105 	if(!right)
10106 	{
10107             ajStrDel(&keystr);
10108 	    ajStrDel(&prefix);
10109 	    ajListFree(&list);
10110 
10111 	    return;
10112 	}
10113 
10114 	page = btreePricacheRead(cache,right);
10115 	pripagepos = right;
10116 	page->dirty = BT_LOCK;
10117         page->lockfor = 1202;
10118 
10119 	btreeIdleafFetch(cache,page,list);
10120 
10121 	page->dirty = BT_CLEAN;
10122 
10123 	if(!ajListGetLength(list))
10124 	{
10125             ajStrDel(&keystr);
10126 	    ajStrDel(&prefix);
10127 	    ajListFree(&list);
10128 
10129 	    return;
10130 	}
10131 
10132 
10133 	found = ajFalse;
10134 
10135 	while(ajListPop(list,(void **)&id))
10136 	{
10137 	    if(ajStrPrefixS(id->id,prefix))
10138 	    {
10139 		found = ajTrue;
10140 		break;
10141 	    }
10142 	    else
10143 		ajBtreeIdDel(&id);
10144 	}
10145         ajDebug("ajBtreeIdentFetchwild check next prefix found: %B\n", found);
10146     }
10147 
10148 
10149     if(!found)
10150     {
10151         ajStrDel(&keystr);
10152 	ajStrDel(&prefix);
10153 	ajListFree(&list);
10154 
10155 	return;
10156     }
10157 
10158 
10159     finished = ajFalse;
10160 
10161     /*
10162     ** check current ID against full wildcard query
10163     ** read next ID (new list when list is empty)
10164     ** stop when no more to read or prefix no longer matches
10165     */
10166 
10167     while(!finished)
10168     {
10169 	if(ajStrMatchWildS(id->id,keystr))
10170         {
10171 	    ajListPush(idlist,(void *)id);
10172             id = NULL;
10173         }
10174 	else
10175 	    ajBtreeIdDel(&id);
10176 
10177 	if(!ajListGetLength(list))
10178 	{
10179 	    page = btreePricacheRead(cache,pripagepos);
10180 	    buf = page->buf;
10181 	    GBT_RIGHT(buf,&right);
10182 
10183 	    if(!right)
10184 	    {
10185 		finished = ajTrue;
10186 		continue;
10187 	    }
10188 
10189 	    page = btreePricacheRead(cache,right);
10190 	    page->dirty = BT_LOCK;
10191             page->lockfor = 1203;
10192 	    buf = page->buf;
10193 	    pripagepos = right;
10194 
10195 	    btreeIdleafFetch(cache,page,list);
10196 
10197 	    page->dirty = BT_CLEAN;
10198 
10199 	    if(!ajListGetLength(list))
10200 	    {
10201 		finished = ajTrue;
10202 		continue;
10203 	    }
10204 	}
10205 
10206 	ajListPop(list,(void **)&id);
10207 
10208 	if(!ajStrPrefixS(id->id,prefix))
10209 	{
10210 	    finished = ajTrue;
10211 	    ajBtreeIdDel(&id);
10212 	}
10213     }
10214 
10215     /*
10216     ** clear remaining (non-matching) ids in the latest list
10217     */
10218 
10219     while(ajListPop(list,(void **)&id))
10220 	ajBtreeIdDel(&id);
10221     ajListFree(&list);
10222 
10223     ajListSortTwoUnique(idlist, &btreeIdDbnoCompare, &btreeIdOffsetCompare,
10224                         &btreeIdDelFromList);
10225 
10226     ajStrDel(&prefix);
10227     ajStrDel(&keystr);
10228 
10229     return;
10230 }
10231 
10232 
10233 
10234 
10235 /* @func ajBtreeIdentFetchwildHit *********************************************
10236 **
10237 ** Wildcard retrieval of entries
10238 **
10239 ** @param [u] cache [AjPBtcache] cache
10240 ** @param [r] key [const AjPStr] Wildcard key
10241 ** @param [u] hitlist [AjPList] list of matching AjPBtHits
10242 **
10243 ** @return [void]
10244 **
10245 ** @release 6.5.0
10246 ** @@
10247 ******************************************************************************/
10248 
ajBtreeIdentFetchwildHit(AjPBtcache cache,const AjPStr key,AjPList hitlist)10249 void ajBtreeIdentFetchwildHit(AjPBtcache cache, const AjPStr key,
10250                               AjPList hitlist)
10251 {
10252 
10253     AjPBtId id     = NULL;
10254     AjPBtHit hit   = NULL;
10255     AjPBtpage page = NULL;
10256     AjPList list   = NULL;
10257     AjBool found   = ajFalse;
10258 
10259     ajulong pripagepos = 0UL;
10260     ajulong right = 0UL;
10261 
10262     unsigned char *buf = NULL;
10263     AjBool finished = ajFalse;
10264 
10265     AjPStr prefix = NULL;
10266     AjPStr keystr = NULL;
10267 
10268     char *p;
10269     char *cp;
10270 
10271     ajDebug("ajBtreeIdentFetchwildHit '%S' list: %Lu\n",
10272             key, ajListGetLength(list));
10273 
10274     if(!cache->countunique && !cache->countall)
10275         return;
10276 
10277     keystr = ajStrNewS(key);
10278     ajStrFmtQuery(&keystr);
10279 
10280     if(MAJSTRGETLEN(keystr) > cache->keylimit)
10281         ajStrTruncateLen(&keystr, cache->keylimit);
10282 
10283     cp = MAJSTRGETPTR(keystr);
10284     p = strpbrk(cp,"*?");
10285 
10286     if(p)
10287     {
10288 	if(p-cp)
10289 	    ajStrAssignSubS(&prefix,keystr,0,p-cp-1);
10290 	else
10291 	{
10292 	    btreeKeyFullSearchHit(cache,keystr,hitlist);
10293             ajStrDel(&keystr);
10294 	    ajStrDel(&prefix);
10295 	    return;
10296 	}
10297     }
10298     else
10299 	ajStrAssignS(&prefix,keystr);
10300 
10301     ajStrFmtQuery(&prefix);
10302 
10303     if(MAJSTRGETLEN(prefix) > cache->keylimit)
10304         ajStrTruncateLen(&prefix, cache->keylimit);
10305 
10306     ajDebug("ajBtreeIdentFetchwildHit '%S' prefix: '%S'\n",
10307             keystr, prefix);
10308 
10309     list = ajListNew();
10310 
10311     found = ajFalse;
10312 
10313     page = btreePrimaryFetchFindleafWild(cache,prefix);
10314     page->dirty = BT_LOCK;
10315     page->lockfor = 1201;
10316     pripagepos = page->pagepos;
10317 
10318     btreeIdleafFetch(cache,page,list);
10319     page->dirty = BT_CLEAN;
10320 
10321 
10322     while(ajListPop(list,(void **)&id))
10323     {
10324 	if(ajStrPrefixS(id->id,prefix))
10325 	{
10326 	    found = ajTrue;
10327 	    break;
10328 	}
10329 	else
10330 	    ajBtreeIdDel(&id);
10331     }
10332 
10333     ajDebug("ajBtreeIdentFetchwildHit first leaf prefix found: %B\n", found);
10334 
10335     if(!found)	/* Check the next leaf just in case key==internal */
10336     {
10337 	buf = page->buf;
10338 	GBT_RIGHT(buf,&right);
10339 
10340 	if(!right)
10341 	{
10342             ajStrDel(&keystr);
10343 	    ajStrDel(&prefix);
10344 	    ajListFree(&list);
10345 
10346 	    return;
10347 	}
10348 
10349 	page = btreePricacheRead(cache,right);
10350 	pripagepos = right;
10351 	page->dirty = BT_LOCK;
10352         page->lockfor = 1202;
10353 
10354 	btreeIdleafFetch(cache,page,list);
10355 
10356 	page->dirty = BT_CLEAN;
10357 
10358 	if(!ajListGetLength(list))
10359 	{
10360             ajStrDel(&keystr);
10361 	    ajStrDel(&prefix);
10362 	    ajListFree(&list);
10363 
10364 	    return;
10365 	}
10366 
10367 
10368 	found = ajFalse;
10369 
10370 	while(ajListPop(list,(void **)&id))
10371 	{
10372 	    if(ajStrPrefixS(id->id,prefix))
10373 	    {
10374 		found = ajTrue;
10375 		break;
10376 	    }
10377 	    else
10378 		ajBtreeIdDel(&id);
10379 	}
10380         ajDebug("ajBtreeIdentFetchwildHit check next prefix found: %B\n",
10381                 found);
10382     }
10383 
10384 
10385     if(!found)
10386     {
10387         ajStrDel(&keystr);
10388 	ajStrDel(&prefix);
10389 	ajListFree(&list);
10390 
10391 	return;
10392     }
10393 
10394 
10395     finished = ajFalse;
10396 
10397     /*
10398     ** check current ID against full wildcard query
10399     ** read next ID (new list when list is empty)
10400     ** stop when no more to read or prefix no longer matches
10401     */
10402 
10403     while(!finished)
10404     {
10405 	if(ajStrMatchWildS(id->id,keystr))
10406         {
10407             hit = ajBtreeHitNewId(id);
10408 	    ajListPush(hitlist,(void *)hit);
10409             hit = NULL;
10410         }
10411         ajBtreeIdDel(&id);
10412 
10413 	if(!ajListGetLength(list))
10414 	{
10415 	    page = btreePricacheRead(cache,pripagepos);
10416 	    buf = page->buf;
10417 	    GBT_RIGHT(buf,&right);
10418 
10419 	    if(!right)
10420 	    {
10421 		finished = ajTrue;
10422 		continue;
10423 	    }
10424 
10425 	    page = btreePricacheRead(cache,right);
10426 	    page->dirty = BT_LOCK;
10427             page->lockfor = 1203;
10428 	    buf = page->buf;
10429 	    pripagepos = right;
10430 
10431 	    btreeIdleafFetch(cache,page,list);
10432 
10433 	    page->dirty = BT_CLEAN;
10434 
10435 	    if(!ajListGetLength(list))
10436 	    {
10437 		finished = ajTrue;
10438 		continue;
10439 	    }
10440 	}
10441 
10442 	ajListPop(list,(void **)&id);
10443 
10444 	if(!ajStrPrefixS(id->id,prefix))
10445 	{
10446 	    finished = ajTrue;
10447 	    ajBtreeIdDel(&id);
10448 	}
10449     }
10450 
10451     /*
10452     ** clear remaining (non-matching) ids in the latest list
10453     */
10454 
10455     while(ajListPop(list,(void **)&id))
10456 	ajBtreeIdDel(&id);
10457     ajListFree(&list);
10458 
10459     ajListSortTwoUnique(hitlist,
10460                         &btreeHitDbnoCompare, &btreeHitOffsetCompare,
10461                         &btreeHitDelFromList);
10462 
10463     ajStrDel(&prefix);
10464     ajStrDel(&keystr);
10465 
10466     return;
10467 }
10468 
10469 
10470 
10471 
10472 /* @func ajBtreeIdentFetchwildHitref ******************************************
10473 **
10474 ** Wildcard retrieval of entries
10475 **
10476 ** @param [u] cache [AjPBtcache] cache
10477 ** @param [r] key [const AjPStr] Wildcard key
10478 ** @param [u] hitlist [AjPList] list of matching AjPBtHitrefs
10479 **
10480 ** @return [void]
10481 **
10482 ** @release 6.5.0
10483 ** @@
10484 ******************************************************************************/
10485 
ajBtreeIdentFetchwildHitref(AjPBtcache cache,const AjPStr key,AjPList hitlist)10486 void ajBtreeIdentFetchwildHitref(AjPBtcache cache, const AjPStr key,
10487                                  AjPList hitlist)
10488 {
10489 
10490     AjPBtId id     = NULL;
10491     AjPBtHitref hitref = NULL;
10492     AjPBtpage page = NULL;
10493     AjPList list   = NULL;
10494     AjBool found   = ajFalse;
10495 
10496     ajulong pripagepos = 0UL;
10497     ajulong right = 0UL;
10498 
10499     unsigned char *buf = NULL;
10500     AjBool finished = ajFalse;
10501 
10502     AjPStr prefix = NULL;
10503     AjPStr keystr = NULL;
10504 
10505     char *p;
10506     char *cp;
10507 
10508     ajDebug("ajBtreeIdentFetchwildHitref '%S' list: %Lu\n",
10509             key, ajListGetLength(list));
10510 
10511     if(!cache->countunique && !cache->countall)
10512         return;
10513 
10514     keystr = ajStrNewS(key);
10515     ajStrFmtQuery(&keystr);
10516 
10517     if(MAJSTRGETLEN(keystr) > cache->keylimit)
10518         ajStrTruncateLen(&keystr, cache->keylimit);
10519 
10520     cp = MAJSTRGETPTR(keystr);
10521     p = strpbrk(cp,"*?");
10522 
10523     if(p)
10524     {
10525 	if(p-cp)
10526 	    ajStrAssignSubS(&prefix,keystr,0,p-cp-1);
10527 	else
10528 	{
10529 	    btreeKeyFullSearchHitref(cache,keystr,hitlist);
10530             ajStrDel(&keystr);
10531 	    ajStrDel(&prefix);
10532 	    return;
10533 	}
10534     }
10535     else
10536 	ajStrAssignS(&prefix,keystr);
10537 
10538     ajStrFmtQuery(&prefix);
10539 
10540     if(MAJSTRGETLEN(prefix) > cache->keylimit)
10541         ajStrTruncateLen(&prefix, cache->keylimit);
10542 
10543     ajDebug("ajBtreeIdentFetchwildHitref '%S' prefix: '%S'\n",
10544             keystr, prefix);
10545 
10546     list = ajListNew();
10547 
10548     found = ajFalse;
10549 
10550     page = btreePrimaryFetchFindleafWild(cache,prefix);
10551     page->dirty = BT_LOCK;
10552     page->lockfor = 1201;
10553     pripagepos = page->pagepos;
10554 
10555     btreeIdleafFetch(cache,page,list);
10556     page->dirty = BT_CLEAN;
10557 
10558 
10559     while(ajListPop(list,(void **)&id))
10560     {
10561 	if(ajStrPrefixS(id->id,prefix))
10562 	{
10563 	    found = ajTrue;
10564 	    break;
10565 	}
10566 	else
10567 	    ajBtreeIdDel(&id);
10568     }
10569 
10570     ajDebug("ajBtreeIdentFetchwildHitref first leaf prefix found: %B\n", found);
10571 
10572     if(!found)	/* Check the next leaf just in case key==internal */
10573     {
10574 	buf = page->buf;
10575 	GBT_RIGHT(buf,&right);
10576 
10577 	if(!right)
10578 	{
10579             ajStrDel(&keystr);
10580 	    ajStrDel(&prefix);
10581 	    ajListFree(&list);
10582 
10583 	    return;
10584 	}
10585 
10586 	page = btreePricacheRead(cache,right);
10587 	pripagepos = right;
10588 	page->dirty = BT_LOCK;
10589         page->lockfor = 1202;
10590 
10591 	btreeIdleafFetch(cache,page,list);
10592 
10593 	page->dirty = BT_CLEAN;
10594 
10595 	if(!ajListGetLength(list))
10596 	{
10597             ajStrDel(&keystr);
10598 	    ajStrDel(&prefix);
10599 	    ajListFree(&list);
10600 
10601 	    return;
10602 	}
10603 
10604 
10605 	found = ajFalse;
10606 
10607 	while(ajListPop(list,(void **)&id))
10608 	{
10609 	    if(ajStrPrefixS(id->id,prefix))
10610 	    {
10611 		found = ajTrue;
10612 		break;
10613 	    }
10614 	    else
10615 		ajBtreeIdDel(&id);
10616 	}
10617         ajDebug("ajBtreeIdentFetchwildHitref check next prefix found: %B\n",
10618                 found);
10619     }
10620 
10621 
10622     if(!found)
10623     {
10624         ajStrDel(&keystr);
10625 	ajStrDel(&prefix);
10626 	ajListFree(&list);
10627 
10628 	return;
10629     }
10630 
10631 
10632     finished = ajFalse;
10633 
10634     /*
10635     ** check current ID against full wildcard query
10636     ** read next ID (new list when list is empty)
10637     ** stop when no more to read or prefix no longer matches
10638     */
10639 
10640     while(!finished)
10641     {
10642 	if(ajStrMatchWildS(id->id,keystr))
10643         {
10644             hitref = ajBtreeHitrefNewId(id);
10645 	    ajListPush(hitlist,(void *)hitref);
10646             hitref = NULL;
10647         }
10648         ajBtreeIdDel(&id);
10649 
10650 	if(!ajListGetLength(list))
10651 	{
10652 	    page = btreePricacheRead(cache,pripagepos);
10653 	    buf = page->buf;
10654 	    GBT_RIGHT(buf,&right);
10655 
10656 	    if(!right)
10657 	    {
10658 		finished = ajTrue;
10659 		continue;
10660 	    }
10661 
10662 	    page = btreePricacheRead(cache,right);
10663 	    page->dirty = BT_LOCK;
10664             page->lockfor = 1203;
10665 	    buf = page->buf;
10666 	    pripagepos = right;
10667 
10668 	    btreeIdleafFetch(cache,page,list);
10669 
10670 	    page->dirty = BT_CLEAN;
10671 
10672 	    if(!ajListGetLength(list))
10673 	    {
10674 		finished = ajTrue;
10675 		continue;
10676 	    }
10677 	}
10678 
10679 	ajListPop(list,(void **)&id);
10680 
10681 	if(!ajStrPrefixS(id->id,prefix))
10682 	{
10683 	    finished = ajTrue;
10684 	    ajBtreeIdDel(&id);
10685 	}
10686     }
10687 
10688     /*
10689     ** clear remaining (non-matching) ids in the latest list
10690     */
10691 
10692     while(ajListPop(list,(void **)&id))
10693 	ajBtreeIdDel(&id);
10694     ajListFree(&list);
10695 
10696     ajListSortTwoUnique(hitlist,
10697                         &btreeHitrefDbnoCompare, &btreeHitrefOffsetCompare,
10698                         &btreeHitrefDelFromList);
10699 
10700     ajStrDel(&prefix);
10701     ajStrDel(&keystr);
10702 
10703     return;
10704 }
10705 
10706 
10707 
10708 
10709 /* @funcstatic btreeKeyFullSearchId *******************************************
10710 **
10711 ** Wildcard retrieval of id/acc/sv entries. Whole index scan. Only used for
10712 ** wildcard searches with keys beginning with '?' or '*'
10713 **
10714 ** @param [u] cache [AjPBtcache] cache
10715 ** @param [r] key [const AjPStr] Wildcard key
10716 ** @param [u] idlist [AjPList] list of matching AjPBtIds
10717 **
10718 ** @return [void]
10719 **
10720 ** @release 6.5.0
10721 ** @@
10722 ******************************************************************************/
10723 
btreeKeyFullSearchId(AjPBtcache cache,const AjPStr key,AjPList idlist)10724 static void btreeKeyFullSearchId(AjPBtcache cache, const AjPStr key,
10725                                  AjPList idlist)
10726 {
10727     AjPBtId id     = NULL;
10728     AjPBtpage root = NULL;
10729     AjPBtpage page = NULL;
10730     ajulong right   = 0UL;
10731     ajuint nodetype = 0U;
10732 
10733     AjPBtMem arrays = NULL;
10734     ajulong *parray = NULL;
10735 
10736     AjPList list   = NULL;
10737 
10738     unsigned char *buf = NULL;
10739 
10740     ajulong idlen = 0UL;
10741 
10742     ajDebug("btreeKeyFullSearchId key '%S'\n", key);
10743 
10744     list   = ajListNew();
10745 
10746     root = btreePricacheLocate(cache, 0UL);
10747     page = root;
10748 
10749     buf = root->buf;
10750     GBT_NODETYPE(buf,&nodetype);
10751 
10752     if(cache->plevel)
10753     {
10754         arrays = btreeAllocPriArray(cache);
10755         parray = arrays->parray;
10756 
10757         /* down to the leftmost leaf node */
10758 
10759 	while(nodetype != BT_LEAF)
10760 	{
10761 	    btreeGetPointers(cache,buf,&parray);
10762 	    page = btreePricacheRead(cache, parray[0]);
10763 	    buf = page->buf;
10764 	    GBT_NODETYPE(buf,&nodetype);
10765 	    page->dirty = BT_CLEAN;
10766 	}
10767 
10768         btreeDeallocPriArray(cache,arrays);
10769     }
10770 
10771     right = 99L;
10772 
10773     /* read left-to-right through all leaf nodes */
10774 
10775     while(right)
10776     {
10777 	btreeIdleafFetch(cache,page,list);
10778 
10779 	while(ajListPop(list,(void **)&id))
10780 	{
10781 	    if(ajStrMatchWildS(id->id,key))
10782 		ajListPushAppend(idlist,(void *)id);
10783 	    else
10784 		ajBtreeIdDel(&id);
10785 	}
10786 
10787         if(idlen != ajListGetLength(idlist))
10788         {
10789             ajDebug("Page %Lu found %Lu\n", page->pagepos,
10790                     ajListGetLength(idlist) - idlen);
10791             idlen = ajListGetLength(idlist);
10792         }
10793 
10794 	GBT_RIGHT(buf,&right);
10795 
10796 	if(right)
10797 	{
10798 	    page = btreePricacheRead(cache,right);
10799 	    buf = page->buf;
10800 	}
10801     }
10802 
10803 
10804     ajListSortTwoUnique(idlist, &btreeIdDbnoCompare, &btreeIdOffsetCompare,
10805                         &btreeIdDelFromList);
10806 
10807     ajListFree(&list);
10808 
10809     return;
10810 }
10811 
10812 
10813 
10814 
10815 /* @funcstatic btreeKeyFullSearchHit ******************************************
10816 **
10817 ** Wildcard retrieval of id/acc/sv entries. Whole index scan. Only used for
10818 ** wildcard searches with keys beginning with '?' or '*'
10819 **
10820 ** @param [u] cache [AjPBtcache] cache
10821 ** @param [r] key [const AjPStr] Wildcard key
10822 ** @param [u] hitlist [AjPList] list of matching AjPBtHits
10823 **
10824 ** @return [void]
10825 **
10826 ** @release 3.0.0
10827 ** @@
10828 ******************************************************************************/
10829 
btreeKeyFullSearchHit(AjPBtcache cache,const AjPStr key,AjPList hitlist)10830 static void btreeKeyFullSearchHit(AjPBtcache cache, const AjPStr key,
10831                                   AjPList hitlist)
10832 {
10833     AjPBtId  id    = NULL;
10834     AjPBtHit hit   = NULL;
10835     AjPBtpage root = NULL;
10836     AjPBtpage page = NULL;
10837     ajulong right   = 0UL;
10838     ajuint nodetype = 0U;
10839 
10840     AjPBtMem arrays = NULL;
10841     ajulong *parray = NULL;
10842 
10843     AjPList list   = NULL;
10844 
10845     unsigned char *buf = NULL;
10846 
10847     ajulong hitlen = 0UL;
10848 
10849     ajDebug("btreeKeyFullSearchId key '%S'\n", key);
10850 
10851     list   = ajListNew();
10852 
10853     root = btreePricacheLocate(cache, 0UL);
10854     page = root;
10855 
10856     buf = root->buf;
10857     GBT_NODETYPE(buf,&nodetype);
10858 
10859     if(cache->plevel)
10860     {
10861         arrays = btreeAllocPriArray(cache);
10862         parray = arrays->parray;
10863 
10864         /* down to the leftmost leaf node */
10865 
10866 	while(nodetype != BT_LEAF)
10867 	{
10868 	    btreeGetPointers(cache,buf,&parray);
10869 	    page = btreePricacheRead(cache, parray[0]);
10870 	    buf = page->buf;
10871 	    GBT_NODETYPE(buf,&nodetype);
10872 	    page->dirty = BT_CLEAN;
10873 	}
10874 
10875         btreeDeallocPriArray(cache,arrays);
10876     }
10877 
10878     right = 99L;
10879 
10880     /* read left-to-right through all leaf nodes */
10881 
10882     while(right)
10883     {
10884 	btreeIdleafFetch(cache,page,list);
10885 
10886 	while(ajListPop(list,(void **)&id))
10887 	{
10888 	    if(ajStrMatchWildS(id->id,key))
10889             {
10890                 hit = ajBtreeHitNewId(id);
10891 		ajListPushAppend(hitlist,(void *)hit);
10892                 hit = NULL;
10893             }
10894             ajBtreeIdDel(&id);
10895 	}
10896 
10897         if(hitlen != ajListGetLength(hitlist))
10898         {
10899             ajDebug("Page %Lu found %Lu\n", page->pagepos,
10900                     ajListGetLength(hitlist) - hitlen);
10901             hitlen = ajListGetLength(hitlist);
10902         }
10903 
10904 	GBT_RIGHT(buf,&right);
10905 
10906 	if(right)
10907 	{
10908 	    page = btreePricacheRead(cache,right);
10909 	    buf = page->buf;
10910 	}
10911     }
10912 
10913 
10914     ajListSortTwoUnique(hitlist, &btreeHitDbnoCompare, &btreeHitOffsetCompare,
10915                         &btreeHitDelFromList);
10916 
10917     ajListFree(&list);
10918 
10919     return;
10920 }
10921 
10922 
10923 
10924 
10925 /* @funcstatic btreeKeyFullSearchHitref ***************************************
10926 **
10927 ** Wildcard retrieval of id/acc/sv entries. Whole index scan. Only used for
10928 ** wildcard searches with keys beginning with '?' or '*'
10929 **
10930 ** @param [u] cache [AjPBtcache] cache
10931 ** @param [r] key [const AjPStr] Wildcard key
10932 ** @param [u] hitlist [AjPList] list of matching AjPBtHitrefs
10933 **
10934 ** @return [void]
10935 **
10936 ** @release 3.0.0
10937 ** @@
10938 ******************************************************************************/
10939 
btreeKeyFullSearchHitref(AjPBtcache cache,const AjPStr key,AjPList hitlist)10940 static void btreeKeyFullSearchHitref(AjPBtcache cache, const AjPStr key,
10941                                      AjPList hitlist)
10942 {
10943     AjPBtId  id    = NULL;
10944     AjPBtHitref hitref = NULL;
10945     AjPBtpage root = NULL;
10946     AjPBtpage page = NULL;
10947     ajulong right   = 0UL;
10948     ajuint nodetype = 0U;
10949 
10950     AjPBtMem arrays = NULL;
10951     ajulong *parray = NULL;
10952 
10953     AjPList list   = NULL;
10954 
10955     unsigned char *buf = NULL;
10956 
10957     ajulong hitlen = 0UL;
10958 
10959     ajDebug("btreeKeyFullSearchId key '%S'\n", key);
10960 
10961     list   = ajListNew();
10962 
10963     root = btreePricacheLocate(cache, 0UL);
10964     page = root;
10965 
10966     buf = root->buf;
10967     GBT_NODETYPE(buf,&nodetype);
10968 
10969     if(cache->plevel)
10970     {
10971         arrays = btreeAllocPriArray(cache);
10972         parray = arrays->parray;
10973 
10974         /* down to the leftmost leaf node */
10975 
10976 	while(nodetype != BT_LEAF)
10977 	{
10978 	    btreeGetPointers(cache,buf,&parray);
10979 	    page = btreePricacheRead(cache, parray[0]);
10980 	    buf = page->buf;
10981 	    GBT_NODETYPE(buf,&nodetype);
10982 	    page->dirty = BT_CLEAN;
10983 	}
10984 
10985         btreeDeallocPriArray(cache,arrays);
10986     }
10987 
10988     right = 99L;
10989 
10990     /* read left-to-right through all leaf nodes */
10991 
10992     while(right)
10993     {
10994 	btreeIdleafFetch(cache,page,list);
10995 
10996 	while(ajListPop(list,(void **)&id))
10997 	{
10998 	    if(ajStrMatchWildS(id->id,key))
10999             {
11000                 hitref = ajBtreeHitrefNewId(id);
11001 		ajListPushAppend(hitlist,(void *)hitref);
11002                 hitref = NULL;
11003             }
11004             ajBtreeIdDel(&id);
11005 	}
11006 
11007         if(hitlen != ajListGetLength(hitlist))
11008         {
11009             ajDebug("Page %Lu found %Lu\n", page->pagepos,
11010                     ajListGetLength(hitlist) - hitlen);
11011             hitlen = ajListGetLength(hitlist);
11012         }
11013 
11014 	GBT_RIGHT(buf,&right);
11015 
11016 	if(right)
11017 	{
11018 	    page = btreePricacheRead(cache,right);
11019 	    buf = page->buf;
11020 	}
11021     }
11022 
11023 
11024     ajListSortTwoUnique(hitlist,
11025                         &btreeHitrefDbnoCompare, &btreeHitrefOffsetCompare,
11026                         &btreeHitrefDelFromList);
11027 
11028     ajListFree(&list);
11029 
11030     return;
11031 }
11032 
11033 
11034 
11035 
11036 /* @func ajBtreeReplaceId *****************************************************
11037 **
11038 ** Replace an ID structure in a leaf node given a key
11039 **
11040 ** @param [u] cache [AjPBtcache] cache
11041 ** @param [r] rid [const AjPBtId] replacement id object
11042 **
11043 ** @return [AjBool] true if success
11044 **
11045 ** @release 3.0.0
11046 ** @@
11047 ******************************************************************************/
11048 
ajBtreeReplaceId(AjPBtcache cache,const AjPBtId rid)11049 AjBool ajBtreeReplaceId(AjPBtcache cache, const AjPBtId rid)
11050 {
11051     AjPBtpage page   = NULL;
11052     AjPIdbucket bucket = NULL;
11053     AjPBtId   id     = NULL;
11054     const AjPStr key = NULL;
11055 
11056     unsigned char *buf = NULL;
11057 
11058     ajuint nentries = 0U;
11059 
11060     ajuint i;
11061     ajuint iref;
11062 
11063     ajulong blockno = 0UL;
11064     AjBool found   = ajFalse;
11065 
11066 
11067     key = rid->id;
11068 
11069     page = btreeIdentFind(cache,key);
11070     buf = page->buf;
11071 
11072     blockno = btreeGetBlockS(cache,buf,key);
11073     bucket = btreeReadIdbucket(cache,blockno);
11074 
11075     nentries = bucket->Nentries;
11076 
11077     found = ajFalse;
11078 
11079     for(i=0;i<nentries;++i)
11080     {
11081 	if(ajStrMatchS(key,bucket->Ids[i]->id))
11082 	{
11083 	    found = ajTrue;
11084 	    break;
11085 	}
11086     }
11087 
11088     if(found)
11089     {
11090 	id->dbno = rid->dbno;
11091 	id->dups = rid->dups;
11092 	id->offset = rid->offset;
11093 
11094         if(cache->refcount)
11095         {
11096             for(iref=0; iref < cache->refcount; iref++)
11097                 id->refoffsets[iref] = rid->refoffsets[iref];
11098         }
11099 
11100 	btreeWriteIdbucket(cache,bucket,blockno);
11101     }
11102 
11103     btreeIdbucketDel(&bucket);
11104 
11105     return ajTrue;
11106 }
11107 
11108 
11109 
11110 
11111 /* @func ajBtreeReadEntriesC **************************************************
11112 **
11113 ** Read B+ tree entries from file
11114 **
11115 ** @param [r] filename [const char*] file name
11116 ** @param [r] indexdir [const char*] index file directory
11117 ** @param [r] directory [const char*] file directory
11118 ** @param [w] seqfiles [AjPStr**] sequence file names
11119 ** @param [w] reffiles [AjPStr***] reference file names (if any)
11120 ** @param [w] refcount [ajuint*] Number of reference file(s) per entry
11121 **
11122 ** @return [ajuint] number of entries
11123 **
11124 ** @release 6.5.0
11125 ** @@
11126 ******************************************************************************/
11127 
ajBtreeReadEntriesC(const char * filename,const char * indexdir,const char * directory,AjPStr ** seqfiles,AjPStr *** reffiles,ajuint * refcount)11128 ajuint ajBtreeReadEntriesC(const char *filename, const char *indexdir,
11129                            const char *directory,
11130                            AjPStr **seqfiles, AjPStr ***reffiles,
11131                            ajuint *refcount)
11132 {
11133     AjPStr line = NULL;
11134     AjPStr fn   = NULL;
11135 
11136     AjPList list;
11137     AjPList *reflist;
11138 
11139     AjPStr seqname = NULL;
11140     AjPStr refname = NULL;
11141     AjPStr tseqname = NULL;
11142     AjPStr trefname = NULL;
11143 
11144     AjPFile inf   = NULL;
11145     char p;
11146     ajulong entries = 0UL;
11147     ajuint iref;
11148 
11149     AjPStrTok handle = NULL;
11150 
11151     *refcount = 0;
11152 
11153     line    = ajStrNew();
11154     list    = ajListNew();
11155 
11156     tseqname = ajStrNew();
11157     trefname = ajStrNew();
11158 
11159     fn = ajStrNew();
11160 
11161     if(!*indexdir)
11162         ajFmtPrintS(&fn, "%s", filename);
11163     else
11164         ajFmtPrintS(&fn, "%s%s%s", indexdir, SLASH_STRING, filename);
11165 
11166     ajStrAppendC(&fn,".ent");
11167 
11168     inf = ajFileNewInNameS(fn);
11169 
11170     if(!inf)
11171 	ajFatal("Cannot open database entries file %S",fn);
11172 
11173     while(ajReadlineTrim(inf, &line))
11174     {
11175 	p = ajStrGetCharFirst(line);
11176 
11177 	if(p == '#' || !ajStrGetLen(line))
11178 	    continue;
11179 
11180 	if(ajStrPrefixC(line,"Dual"))
11181 	    *refcount = 1;
11182 
11183 	if(ajStrPrefixC(line,"Reference "))
11184         {
11185 	    ajFmtScanS(line,"%S%d",&trefname, refcount);
11186             --*refcount;
11187         }
11188 
11189 	break;
11190     }
11191 
11192 
11193     if(!*refcount)
11194     {
11195 	while(ajReadlineTrim(inf, &line))
11196 	{
11197 	    seqname = ajStrNew();
11198 	    ajFmtScanS(line,"%S",&tseqname);
11199 	    ajFmtPrintS(&seqname,"%s%s%S",directory,SLASH_STRING,tseqname);
11200 	    ajListPushAppend(list,(void *)seqname);
11201 	}
11202 
11203 	ajListToarray(list,(void ***)&(*seqfiles));
11204 	entries = ajListGetLength(list);
11205     }
11206     else
11207     {
11208         AJCNEW(reflist, *refcount);
11209         for(iref=0; iref < *refcount; iref++)
11210             reflist[iref] = ajListNew();
11211 
11212 	while(ajReadlineTrim(inf, &line))
11213 	{
11214 	    seqname = ajStrNew();
11215 	    refname = ajStrNew();
11216             handle = ajStrTokenNewC(line, " \t");
11217             ajStrTokenNextParse(handle, &tseqname);
11218 	    ajFmtPrintS(&seqname,"%s%s%S",directory,SLASH_STRING,tseqname);
11219 	    ajListPushAppend(list,(void *)seqname);
11220 
11221             for(iref=0; iref < *refcount; iref++)
11222             {
11223                 ajStrTokenNextParse(handle, &trefname);
11224                 ajFmtPrintS(&refname,"%s%s%S",directory,SLASH_STRING,trefname);
11225                 ajListPushAppend(reflist[iref],(void *)refname);
11226             }
11227         }
11228 
11229 	ajListToarray(list,(void ***)&(*seqfiles));
11230 
11231         for(iref=0; iref < *refcount; iref++)
11232             ajListToarray(reflist[iref],(void ***)&(*reffiles[iref]));
11233 
11234 	entries = ajListGetLength(list);
11235     }
11236 
11237 
11238     ajStrTokenDel(&handle);
11239     ajListFree(&list);
11240 
11241     if(*refcount)
11242     {
11243         for(iref=0; iref < *refcount; iref++)
11244             ajListFree(&reflist[iref]);
11245 
11246         AJFREE(reflist);
11247     }
11248 
11249     ajStrDel(&line);
11250     ajStrDel(&fn);
11251 
11252     ajStrDel(&tseqname);
11253     ajStrDel(&trefname);
11254 
11255 
11256     ajFileClose(&inf);
11257 
11258     return (ajuint) entries;
11259 }
11260 
11261 
11262 
11263 
11264 /* @func ajBtreeReadEntriesS **************************************************
11265 **
11266 ** Read B+ tree entries from file
11267 **
11268 ** @param [r] filename [const AjPStr] file name
11269 ** @param [r] indexdir [const AjPStr] index file directory
11270 ** @param [r] directory [const AjPStr] file directory
11271 ** @param [w] seqfiles [AjPStr**] sequence file names
11272 ** @param [w] reffiles [AjPStr***] reference file names (if any)
11273 ** @param [w] refcount [ajuint*] Number of reference file(s) per entry
11274 **
11275 ** @return [ajuint] number of entries
11276 **
11277 ** @release 6.4.0
11278 ** @@
11279 ******************************************************************************/
11280 
ajBtreeReadEntriesS(const AjPStr filename,const AjPStr indexdir,const AjPStr directory,AjPStr ** seqfiles,AjPStr *** reffiles,ajuint * refcount)11281 ajuint ajBtreeReadEntriesS(const AjPStr filename, const AjPStr indexdir,
11282                            const AjPStr directory,
11283                            AjPStr **seqfiles, AjPStr ***reffiles,
11284                            ajuint *refcount)
11285 {
11286     AjPStr line = NULL;
11287     AjPStr fn   = NULL;
11288 
11289     AjPList list;
11290     AjPList *reflist;
11291 
11292     AjPStr seqname = NULL;
11293     AjPStr refname = NULL;
11294     AjPStr tseqname = NULL;
11295     AjPStr trefname = NULL;
11296 
11297     AjPFile inf   = NULL;
11298     char p;
11299     ajulong entries = 0UL;
11300     ajuint iref;
11301 
11302     AjPStrTok handle = NULL;
11303 
11304     *refcount = 0;
11305 
11306     line    = ajStrNew();
11307     list    = ajListNew();
11308 
11309     tseqname = ajStrNew();
11310     trefname = ajStrNew();
11311 
11312     fn = ajStrNew();
11313 
11314     if(!ajStrGetLen(indexdir))
11315         ajFmtPrintS(&fn,"%S.ent",filename);
11316     else
11317         ajFmtPrintS(&fn,"%S%s%S.ent",indexdir,SLASH_STRING,filename);
11318 
11319     inf = ajFileNewInNameS(fn);
11320 
11321     if(!inf)
11322 	ajFatal("Cannot open database entries file %S", fn);
11323 
11324     while(ajReadlineTrim(inf, &line))
11325     {
11326 	p = ajStrGetCharFirst(line);
11327 
11328 	if(p == '#' || !ajStrGetLen(line))
11329 	    continue;
11330 
11331 	if(ajStrPrefixC(line,"Dual"))
11332 	    *refcount = 1;
11333 
11334 	if(ajStrPrefixC(line,"Reference "))
11335         {
11336 	    ajFmtScanS(line,"%S%d",&trefname, refcount);
11337             --*refcount;
11338         }
11339 
11340 	break;
11341     }
11342 
11343 
11344     if(!*refcount)
11345     {
11346 	while(ajReadlineTrim(inf, &line))
11347 	{
11348 	    seqname = ajStrNew();
11349 	    ajFmtScanS(line,"%S",&tseqname);
11350 	    ajFmtPrintS(&seqname,"%S%s%S",directory,SLASH_STRING,tseqname);
11351 	    ajListPushAppend(list,(void *)seqname);
11352 	}
11353 
11354 	ajListToarray(list,(void ***)&(*seqfiles));
11355 	entries = ajListGetLength(list);
11356     }
11357     else
11358     {
11359         AJCNEW(reflist, *refcount);
11360         for(iref=0; iref < *refcount; iref++)
11361             reflist[iref] = ajListNew();
11362 
11363 	while(ajReadlineTrim(inf, &line))
11364 	{
11365 	    seqname = ajStrNew();
11366 	    refname = ajStrNew();
11367             ajStrTokenAssignC(&handle, line, " \t");
11368             ajStrTokenNextParse(handle, &tseqname);
11369 	    ajFmtPrintS(&seqname,"%S%s%S",directory,SLASH_STRING,tseqname);
11370 	    ajListPushAppend(list,(void *)seqname);
11371 
11372             for(iref=0; iref < *refcount; iref++)
11373             {
11374                 ajStrTokenNextParse(handle, &trefname);
11375                 ajFmtPrintS(&refname,"%S%s%S",directory,SLASH_STRING,trefname);
11376                 ajListPushAppend(reflist[iref],(void *)refname);
11377             }
11378         }
11379 
11380 	ajListToarray(list,(void ***)&(*seqfiles));
11381 
11382         if(reffiles)
11383         {
11384             AJCNEW0(*reffiles, *refcount);
11385             for(iref=0; iref < *refcount; iref++)
11386                 ajListToarray(reflist[iref],(void ***)&(*reffiles[iref]));
11387         }
11388 
11389 	entries = ajListGetLength(list);
11390     }
11391 
11392     ajStrTokenDel(&handle);
11393     ajListFree(&list);
11394 
11395     if(*refcount)
11396     {
11397         for(iref=0; iref < *refcount; iref++)
11398             ajListFree(&reflist[iref]);
11399 
11400         AJFREE(reflist);
11401     }
11402 
11403     ajStrDel(&line);
11404     ajStrDel(&fn);
11405 
11406     ajStrDel(&tseqname);
11407     ajStrDel(&trefname);
11408 
11409 
11410     ajFileClose(&inf);
11411 
11412     return (ajuint) entries;
11413 }
11414 
11415 
11416 
11417 
11418 #if AJINDEX_STATIC
11419 /* #funcstatic btreeKeyListDuplicates *****************************************
11420 **
11421 ** Write B+ tree duplicate entries matching key to a list
11422 **
11423 ** #param [u] cache [AjPBtcache] cache
11424 ** #param [r] key [const AjPStr] key
11425 **
11426 ** #return [AjPList] list of matching AjPBtIds or NULL
11427 **
11428 ** #release 6.5.0
11429 ** ##
11430 ******************************************************************************/
11431 /*
11432 static AjPList btreeKeyListDuplicates(AjPBtcache cache, const AjPStr key)
11433 {
11434     AjPList list = NULL;
11435     AjPBtId id   = NULL;
11436     ajuint i;
11437     ajuint dups;
11438 
11439     AjPStr dupkey = NULL;
11440     AjPStr okey   = NULL;
11441 
11442     if(!(id = btreeIdentQueryId(cache,key)))
11443 	return NULL;
11444 
11445     dupkey = ajStrNew();
11446     okey   = ajStrNew();
11447     list   = ajListNew();
11448     ajListPush(list,(void *)id);
11449 
11450 
11451     if(id->dups)
11452     {
11453 	ajStrAssignS(&okey,id->id);
11454 	dups = id->dups;
11455 
11456 	for(i=0;i<dups;++i)
11457 	{
11458 	    ajFmtPrintS(&dupkey,"%S%c%u",okey,'\1',i+1);
11459 	    id = btreeIdentQueryId(cache,dupkey);
11460 
11461 	    if(!id)
11462 		ajFatal("DupFromKey: Id not found\n");
11463 
11464 	    ajListPushAppend(list,(void *)id);
11465 	}
11466     }
11467 
11468     ajStrDel(&okey);
11469     ajStrDel(&dupkey);
11470 
11471 
11472     return list;
11473 }
11474 */
11475 #endif
11476 
11477 
11478 
11479 
11480 /* @funcstatic btreePribucketNew **********************************************
11481 **
11482 ** Construct a primary keyword bucket object
11483 **
11484 ** @param [r] n [ajuint] Number of IDs
11485 **
11486 ** @return [AjPPribucket] initialised disc block cache structure
11487 **
11488 ** @release 3.0.0
11489 ** @@
11490 ******************************************************************************/
11491 
btreePribucketNew(ajuint n)11492 static AjPPribucket btreePribucketNew(ajuint n)
11493 {
11494     AjPPribucket bucket = NULL;
11495     ajuint i;
11496 
11497     /*ajDebug("In btreePribucketNew %u statsave: %u empty: %u\n",
11498       n, statSavePribucketNext, statSavePribucketEmptyNext);*/
11499 
11500 
11501     if(n)
11502     {
11503         if(statSavePribucketNext)
11504         {
11505             bucket = statSavePribucket[--statSavePribucketNext];
11506             for(i=0;i<bucket->Maxentries;++i)
11507             {
11508                 ajStrAssignClear(&bucket->codes[i]->id);
11509                 ajStrAssignClear(&bucket->codes[i]->keyword);
11510                 bucket->codes[i]->treeblock = 0UL;
11511             }
11512             if(n > bucket->Maxentries)
11513             {
11514                 AJCRESIZE0(bucket->keylen,bucket->Maxentries,n);
11515                 AJCRESIZE0(bucket->codes,bucket->Maxentries,n);
11516                 for(i=bucket->Maxentries;i<n;++i)
11517                     bucket->codes[i] = ajBtreePriNew();
11518                 bucket->Maxentries = n;
11519             }
11520         }
11521         else
11522         {
11523             AJNEW0(bucket);
11524             AJCNEW0(bucket->codes,n);
11525             AJCNEW0(bucket->keylen,n);
11526             for(i=0;i<n;++i)
11527                 bucket->codes[i] = ajBtreePriNew();
11528             bucket->Maxentries = n;
11529         }
11530 
11531     }
11532     else
11533     {
11534         if(statSavePribucketEmptyNext)
11535             bucket = statSavePribucketEmpty[--statSavePribucketEmptyNext];
11536         else
11537             AJNEW0(bucket);
11538     }
11539 
11540     bucket->NodeType = BT_PRIBUCKET;
11541     bucket->Nentries = n;
11542     bucket->Overflow = 0UL;
11543 
11544     return bucket;
11545 }
11546 
11547 
11548 
11549 
11550 /* @funcstatic btreePribucketDel **********************************************
11551 **
11552 ** Delete a keyword primary bucket object
11553 **
11554 ** @param [w] thys [AjPPribucket*] bucket
11555 **
11556 ** @return [void]
11557 **
11558 ** @release 3.0.0
11559 ** @@
11560 ******************************************************************************/
11561 
btreePribucketDel(AjPPribucket * thys)11562 static void btreePribucketDel(AjPPribucket *thys)
11563 {
11564     AjPPribucket pthis = NULL;
11565     ajuint newmax;
11566 
11567     if(!thys || !*thys)
11568 	return;
11569 
11570     pthis = *thys;
11571 
11572     /*ajDebug("In btreePribucketDel maxentries: %u savepribucket %u empty %u\n",
11573             pthis->Maxentries, statSavePribucketNext,
11574             statSavePribucketEmptyNext);*/
11575 
11576     if(!statSavePribucket)
11577     {
11578         statSavePribucketMax=2048;
11579         statSavePribucketNext=0;
11580         AJCNEW0(statSavePribucket,statSavePribucketMax);
11581     }
11582 
11583     if(!statSavePribucketEmpty)
11584     {
11585         statSavePribucketEmptyMax=2048;
11586         statSavePribucketEmptyNext=0;
11587         AJCNEW0(statSavePribucketEmpty,statSavePribucketEmptyMax);
11588     }
11589 
11590     if(pthis->Maxentries)
11591     {
11592         if(statSavePribucketNext >= statSavePribucketMax)
11593         {
11594             newmax = statSavePribucketMax + statSavePribucketMax;
11595             AJCRESIZE0(statSavePribucket,statSavePribucketMax,newmax);
11596             statSavePribucketMax = newmax;
11597         }
11598 
11599         statSavePribucket[statSavePribucketNext++] = pthis;
11600     }
11601     else
11602     {
11603         if(statSavePribucketEmptyNext >= statSavePribucketEmptyMax)
11604         {
11605             newmax = statSavePribucketEmptyMax + statSavePribucketEmptyMax;
11606             AJCRESIZE0(statSavePribucketEmpty,statSavePribucketEmptyMax,newmax);
11607             statSavePribucketEmptyMax = newmax;
11608         }
11609         statSavePribucketEmpty[statSavePribucketEmptyNext++] = pthis;
11610     }
11611 
11612     *thys = NULL;
11613 
11614     return;
11615 }
11616 
11617 
11618 
11619 
11620 /* @funcstatic btreePribucketFree *********************************************
11621 **
11622 ** Delete a keyword primary bucket object
11623 **
11624 ** @param [w] thys [AjPPribucket*] bucket
11625 **
11626 ** @return [void]
11627 **
11628 ** @release 6.4.0
11629 ** @@
11630 ******************************************************************************/
11631 
btreePribucketFree(AjPPribucket * thys)11632 static void btreePribucketFree(AjPPribucket *thys)
11633 {
11634     AjPPribucket pthis = NULL;
11635     ajuint n;
11636     ajuint i;
11637 
11638     if(!thys || !*thys)
11639 	return;
11640 
11641     pthis = *thys;
11642     n = pthis->Maxentries;
11643 
11644     /*ajDebug("In btreePribucketFree %u\n",pthis->Maxentries);*/
11645 
11646     for(i=0;i<n;++i)
11647 	btreePriFree(&pthis->codes[i]);
11648 
11649     AJFREE(pthis->keylen);
11650     AJFREE(pthis->codes);
11651 
11652     AJFREE(pthis);
11653 
11654     *thys = NULL;
11655 
11656     return;
11657 }
11658 
11659 
11660 
11661 
11662 /* @func ajBtreePriNew ********************************************************
11663 **
11664 ** Constructor for index bucket keyword information
11665 **
11666 **
11667 ** @return [AjPBtPri] Index ID object
11668 **
11669 ** @release 3.0.0
11670 ** @@
11671 ******************************************************************************/
11672 
ajBtreePriNew(void)11673 AjPBtPri ajBtreePriNew(void)
11674 {
11675     AjPBtPri pri = NULL;
11676 
11677     /* ajDebug("In ajBtreePriNew\n"); */
11678 
11679     if(statSaveBtreePriNext)
11680     {
11681         pri = statSaveBtreePri[--statSaveBtreePriNext];
11682         MAJSTRASSIGNCLEAR(&pri->id);
11683         MAJSTRASSIGNCLEAR(&pri->keyword);
11684         pri->treeblock = 0UL;
11685     }
11686     else
11687     {
11688         AJNEW0(pri);
11689         pri->keyword   = ajStrNew();
11690         pri->id        = ajStrNew();
11691         pri->treeblock = 0UL;
11692     }
11693 
11694     return pri;
11695 }
11696 
11697 
11698 
11699 
11700 /* @func ajBtreePriDel ********************************************************
11701 **
11702 ** Destructor for keyword index primary bucket information
11703 **
11704 ** @param [w] thys [AjPBtPri*] index keyword primary object
11705 **
11706 ** @return [void]
11707 **
11708 ** @release 3.0.0
11709 ** @@
11710 ******************************************************************************/
11711 
ajBtreePriDel(AjPBtPri * thys)11712 void ajBtreePriDel(AjPBtPri *thys)
11713 {
11714     ajuint newmax;
11715 
11716     /* ajDebug("In ajBtreePriDel\n"); */
11717 
11718     if(!statSaveBtreePri)
11719     {
11720         statSaveBtreePriMax = 2048;
11721         AJCNEW0(statSaveBtreePri, statSaveBtreePriMax);
11722         statSaveBtreePriNext = 0;
11723     }
11724 
11725     if(!thys || !*thys)
11726 	return;
11727 
11728     if(statSaveBtreePriNext >= statSaveBtreePriMax)
11729     {
11730         newmax = statSaveBtreePriMax + statSaveBtreePriMax;
11731         AJCRESIZE0(statSaveBtreePri,statSaveBtreePriMax,newmax);
11732         statSaveBtreePriMax = newmax;
11733     }
11734 
11735     statSaveBtreePri[statSaveBtreePriNext++] = *thys;
11736 
11737     *thys = NULL;
11738 
11739     return;
11740 }
11741 
11742 
11743 
11744 
11745 /* @funcstatic btreePriFree ***************************************************
11746 **
11747 ** Destructor for index primary bucket information
11748 **
11749 ** @param [w] thys [AjPBtPri*] index keyword primary object
11750 **
11751 ** @return [void]
11752 **
11753 ** @release 6.4.0
11754 ** @@
11755 ******************************************************************************/
11756 
btreePriFree(AjPBtPri * thys)11757 static void btreePriFree(AjPBtPri *thys)
11758 {
11759     AjPBtPri pri = NULL;
11760 
11761     if(!thys || !*thys)
11762 	return;
11763     pri = *thys;
11764 
11765     /*ajDebug("In btreePriFree id '%S' key '%S'\n",
11766       pri->id, pri->keyword);*/
11767 
11768     ajStrDel(&pri->id);
11769     ajStrDel(&pri->keyword);
11770     AJFREE(pri);
11771     *thys = NULL;
11772 
11773     return;
11774 }
11775 
11776 
11777 
11778 
11779 /* @funcstatic btreePribucketIdlist *******************************************
11780 **
11781 ** Copies all primary keys into a list
11782 **
11783 ** @param [u] cache [AjPBtcache] cache
11784 ** @param [r] pagepos [ajulong] page number
11785 ** @param [u] idlist [AjPList] list to hold keys
11786 **
11787 ** @return [ajulong] Overflow
11788 **
11789 ** @release 6.4.0
11790 ** @@
11791 ******************************************************************************/
11792 
btreePribucketIdlist(AjPBtcache cache,ajulong pagepos,AjPList idlist)11793 static ajulong btreePribucketIdlist(AjPBtcache cache, ajulong pagepos,
11794                                    AjPList idlist)
11795 {
11796     AjPBtpage page      = NULL;
11797     AjPBtpage lpage     = NULL;
11798     unsigned char *buf  = NULL;
11799     unsigned char *kptr = NULL;
11800     AjPBtPri pri        = NULL;
11801 
11802     unsigned char *codeptr = NULL;
11803 
11804     ajuint  nodetype  = 0U;
11805     ajuint  nentries  = 0U;
11806     ajulong overflow  = 0UL;
11807     ajulong pageoverflow = 0UL;
11808     ajuint  dirtysave = 0U;
11809 
11810     ajuint  i;
11811     ajuint  len  = 0U;
11812     ajuint idlen = 0U;
11813 
11814     /* ajDebug("In btreePribucketIdlist\n"); */
11815 
11816     if(!pagepos)
11817 	ajFatal("PribucketIdlist: cannot read bucket from root page cache %S",
11818                 cache->filename);
11819 
11820     page  = btreePricacheRead(cache,pagepos);
11821     lpage = page;
11822     dirtysave = lpage->dirty;
11823     lpage->dirty = BT_LOCK;
11824     lpage->lockfor = 1211;
11825 
11826     buf = lpage->buf;
11827 
11828     GBT_BUCKNODETYPE(buf,&nodetype);
11829     if(nodetype != BT_PRIBUCKET)
11830 	ajFatal("PribucketIdlist: NodeType mismatch. "
11831                 "Not primary bucket (%u) cache %S",
11832 		nodetype, cache->filename);
11833 
11834     GBT_BUCKNENTRIES(buf,&nentries);
11835 
11836 
11837     GBT_BUCKOVERFLOW(buf,&overflow);
11838     pageoverflow = overflow;
11839 
11840     kptr  = PBT_BUCKKEYLEN(buf);
11841     codeptr = kptr + (nentries * sizeof(ajuint));
11842 
11843     for(i=0;i<nentries;++i)
11844     {
11845 	BT_GETAJUINT(kptr,&len);
11846         idlen = len - sizeof(ajulong) - 1;
11847 
11848 /*
11849 //        if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
11850 //	{
11851 //	    /# ajDebug("PribucketRead: Overflow\n"); #/
11852 //	    page  = btreePricacheRead(cache,overflow);
11853 //	    buf = page->buf;
11854 //	    GBT_BUCKNODETYPE(buf,&nodetype);
11855 //	    if(nodetype != BT_PRIBUCKET)
11856 //		ajFatal("PribucketIdlist: NodeType mismatch. Not primary "
11857 //			"bucket (%u) cache %S",
11858 //                        nodetype, cache->filename);
11859 //	    GBT_BUCKOVERFLOW(buf,&overflow);
11860 //	    /# overflow bucket ids start at the keylen position #/
11861 //	    codeptr = PBT_BUCKKEYLEN(buf);
11862 //	}
11863 */
11864 
11865 	pri = ajBtreePriNew();
11866 
11867 	/* Fill ID objects */
11868 	ajStrAssignLenC(&pri->keyword,(const char *)codeptr,idlen);
11869 	codeptr += (idlen + 1);
11870 	BT_GETAJULONG(codeptr,&pri->treeblock);
11871 	codeptr += sizeof(ajulong);
11872 
11873 	kptr += sizeof(ajuint);
11874         ajListPushAppend(idlist, pri);
11875     }
11876 
11877     lpage->dirty = dirtysave;
11878 
11879     return pageoverflow;
11880 }
11881 
11882 
11883 
11884 
11885 /* @funcstatic btreePribucketRead *********************************************
11886 **
11887 ** Constructor for keyword index primary bucket given a disc page number
11888 ** Creates one empty key slot for possible addition
11889 **
11890 ** @param [u] cache [AjPBtcache] cache
11891 ** @param [r] pagepos [ajulong] page number
11892 **
11893 ** @return [AjPPribucket] bucket
11894 **
11895 ** @release 6.5.0
11896 ** @@
11897 ******************************************************************************/
11898 
btreePribucketRead(AjPBtcache cache,ajulong pagepos)11899 static AjPPribucket btreePribucketRead(AjPBtcache cache, ajulong pagepos)
11900 {
11901     AjPPribucket bucket = NULL;
11902     AjPBtpage page      = NULL;
11903     AjPBtpage lpage     = NULL;
11904     unsigned char *buf  = NULL;
11905     unsigned char *kptr = NULL;
11906     AjPBtPri pri        = NULL;
11907 
11908     unsigned char *codeptr = NULL;
11909 
11910     ajuint  nodetype  = 0U;
11911     ajuint  nentries  = 0U;
11912     ajulong overflow  = 0UL;
11913     ajuint  dirtysave = 0U;
11914 
11915     ajuint  i;
11916     ajuint  len  = 0U;
11917 
11918     /* ajDebug("In btreePribucketRead\n"); */
11919 
11920     if(!pagepos)
11921 	ajFatal("PribucketRead: cannot read bucket from root page cache %S",
11922                 cache->filename);
11923 
11924     page  = btreePricacheRead(cache,pagepos);
11925     lpage = page;
11926     dirtysave = lpage->dirty;
11927     lpage->dirty = BT_LOCK;
11928     lpage->lockfor = 1221;
11929 
11930     buf = lpage->buf;
11931 
11932     GBT_BUCKNODETYPE(buf,&nodetype);
11933 
11934     if(nodetype != BT_PRIBUCKET)
11935 	ajFatal("PriReadBucket: NodeType mismatch. "
11936                 "Not primary bucket (%u) page %Lu cache %S",
11937 		nodetype, pagepos, cache->filename);
11938 
11939     GBT_BUCKNENTRIES(buf,&nentries);
11940 
11941     if(nentries > cache->pnperbucket)
11942 	ajFatal("PriReadBucket: Bucket too full page: %Lu "
11943                 "entries: %u max: %u page %Lu cache %S",
11944                 pagepos, nentries, cache->pnperbucket,
11945                 pagepos, cache->filename);
11946 
11947 
11948     GBT_BUCKOVERFLOW(buf,&overflow);
11949 
11950     bucket = btreePribucketNew(cache->pnperbucket);
11951     bucket->Nentries = nentries;
11952 
11953     kptr  = PBT_BUCKKEYLEN(buf);
11954     codeptr = kptr + (nentries * sizeof(ajuint));
11955 
11956     for(i=0;i<nentries;++i)
11957     {
11958 	BT_GETAJUINT(kptr,&len);
11959 
11960 /*
11961 //      if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
11962 //	{
11963 //#if AJINDEX_DEBUG
11964 //	    ajDebug("PriReadBucket: Overflow\n");
11965 //#endif
11966 //	    page  = btreePricacheRead(cache,overflow);
11967 //	    buf = page->buf;
11968 //	    GBT_BUCKNODETYPE(buf,&nodetype);
11969 //
11970 //	    if(nodetype != BT_PRIBUCKET)
11971 //		ajFatal("PriReadBucket: NodeType mismatch. Not primary "
11972 //			"bucket (%u) page %Lu", nodetype, pagepos);
11973 //
11974 //	    GBT_BUCKOVERFLOW(buf,&overflow);
11975 //	    /# overflow bucket ids start at the keylen position #/
11976 //	    codeptr = PBT_BUCKKEYLEN(buf);
11977 //	}
11978 */
11979 
11980 	pri = bucket->codes[i];
11981 
11982 	/* Fill ID objects */
11983 	ajStrAssignC(&pri->keyword,(const char *)codeptr);
11984 	codeptr += (strlen((const char *)codeptr) + 1);
11985 	BT_GETAJULONG(codeptr,&pri->treeblock);
11986 	codeptr += sizeof(ajulong);
11987 
11988 	kptr += sizeof(ajuint);
11989     }
11990 
11991     lpage->dirty = dirtysave;
11992 
11993     return bucket;
11994 }
11995 
11996 
11997 
11998 
11999 /* @funcstatic btreePribucketSort *********************************************
12000 **
12001 ** Sorts IDs in a primary keyword bucket
12002 **
12003 ** @param [u] thys [AjPPribucket] cache
12004 **
12005 ** @return [void]
12006 ** @@
12007 ******************************************************************************/
12008 
btreePribucketSort(AjPPribucket thys)12009 static void btreePribucketSort(AjPPribucket thys)
12010 {
12011     qsort(thys->codes, thys->Nentries, sizeof(AjPBtPri), btreeKeywordCompare);
12012 
12013     return;
12014 }
12015 
12016 
12017 
12018 
12019 /* @funcstatic btreeWritePribucket ********************************************
12020 **
12021 ** Write primary keyword index bucket object to the cache given a disc page
12022 ** number
12023 **
12024 ** @param [u] cache [AjPBtcache] cache
12025 ** @param [r] bucket [const AjPPribucket] bucket
12026 ** @param [r] pagepos [ajulong] page number
12027 **
12028 ** @return [void]
12029 **
12030 ** @release 3.0.0
12031 ** @@
12032 ******************************************************************************/
12033 
btreeWritePribucket(AjPBtcache cache,const AjPPribucket bucket,ajulong pagepos)12034 static void btreeWritePribucket(AjPBtcache cache, const AjPPribucket bucket,
12035 				ajulong pagepos)
12036 {
12037     AjPBtpage page  = NULL;
12038     AjPBtpage lpage = NULL;
12039 
12040     unsigned char *buf  = NULL;
12041     unsigned char *lbuf = NULL;
12042 
12043     ajuint  v   = 0U;
12044     ajuint i   = 0U;
12045     ajuint len = 0U;
12046     ajulong lv  = 0UL;
12047 
12048     AjPBtPri pri    = NULL;
12049     ajuint nentries = 0U;
12050     ajulong overflow = 0UL;
12051 
12052     unsigned char *keyptr = NULL;
12053     unsigned char *lptr   = NULL;
12054 
12055 /*    ajulong pno = 0UL;*/
12056 
12057 
12058     /* ajDebug("In btreeWritePribucket\n"); */
12059 
12060     if(pagepos == cache->totsize)	/* Create a new page */
12061     {
12062 	/* pno = pagepos; */
12063 	page = btreePricacheBucketnew(cache);
12064 	buf = page->buf;
12065         overflow = 0UL;
12066     }
12067     else
12068     {
12069 	page = btreePricacheRead(cache,pagepos);
12070 	buf = page->buf;
12071 	GBT_BUCKOVERFLOW(buf,&overflow);
12072     }
12073 
12074     v = BT_PRIBUCKET;
12075     SBT_BUCKNODETYPE(buf,v);
12076 
12077     lbuf = buf;
12078     page->dirty = BT_LOCK;
12079     page->lockfor = 1231;
12080     lpage = page;
12081 
12082     nentries = bucket->Nentries;
12083     v = nentries;
12084     SBT_BUCKNENTRIES(buf,v);
12085 
12086     /* Write out key lengths */
12087     keyptr = PBT_BUCKKEYLEN(lbuf);
12088 
12089     for(i=0;i<nentries;++i)
12090     {
12091         /* drop check - tested before write - pagesize dependency */
12092 /*
12093 //	if((ajuint)((keyptr-lbuf+1)+sizeof(ajuint)) > cache->pagesize)
12094 //	    ajFatal("WritePribucket: Bucket cannot hold more than %u keys",
12095 //		    i-1);
12096 */
12097 
12098 	pri = bucket->codes[i];
12099 	/* Need to alter this if bucket primary keyword structure changes */
12100 	len = BT_BUCKPRILEN(pri->keyword);
12101         v = len;
12102 	BT_SETAJUINT(keyptr,v);
12103 	keyptr += sizeof(ajuint);
12104     }
12105 
12106 
12107     /* Write out IDs using overflow if necessary */
12108     lptr = keyptr;
12109     for(i=0;i<nentries;++i)
12110     {
12111 	pri = bucket->codes[i];
12112 	len = BT_BUCKPRILEN(pri->keyword);
12113 
12114 /*
12115 //	if((lptr-buf+1)+len > cache->pagesize) /# overflow #/
12116 //	{
12117 //    	    /#ajDebug("WritePribucket: Overflow\n");#/
12118 //
12119 //	    if(!overflow)		/# No overflow buckets yet #/
12120 //	    {
12121 //		pno = cache->totsize;
12122 //                lv = pno;
12123 //		SBT_BUCKOVERFLOW(buf,lv);
12124 //		page = btreePricacheBucketnew(cache);
12125 //                buf = page->buf;
12126 //		v = BT_PRIBUCKET;
12127 //		SBT_BUCKNODETYPE(buf,v);
12128 //	    }
12129 //	    else
12130 //	    {
12131 //		page = btreePricacheRead(cache,overflow);
12132 //		buf  = page->buf;
12133 //		GBT_BUCKOVERFLOW(buf,&overflow);
12134 //	    }
12135 //
12136 //	    page->dirty = BT_DIRTY;
12137 //
12138 //	    lptr = PBT_BUCKKEYLEN(buf);
12139 //	}
12140 */
12141 
12142 	sprintf((char *)lptr,"%s",ajStrGetPtr(pri->keyword));
12143 	lptr += (ajStrGetLen(pri->keyword) + 1);
12144         lv = pri->treeblock;
12145 	BT_SETAJULONG(lptr,lv);
12146 	lptr += sizeof(ajulong);
12147     }
12148 
12149     lv = 0UL;
12150     SBT_BUCKOVERFLOW(buf,lv);
12151 
12152     lpage->dirty = BT_DIRTY;
12153 
12154     return;
12155 }
12156 
12157 
12158 
12159 
12160 /* @funcstatic btreeWritePribucketEmpty ***************************************
12161 **
12162 ** Write empty primary keyword index bucket object to the cache given a
12163 ** disc page number
12164 **
12165 ** @param [u] cache [AjPBtcache] cache
12166 ** @param [r] pagepos [ajulong] page number
12167 **
12168 ** @return [void]
12169 **
12170 ** @release 6.4.0
12171 ** @@
12172 ******************************************************************************/
12173 
btreeWritePribucketEmpty(AjPBtcache cache,ajulong pagepos)12174 static void btreeWritePribucketEmpty(AjPBtcache cache, ajulong pagepos)
12175 {
12176     AjPBtpage page  = NULL;
12177     AjPBtpage lpage = NULL;
12178 
12179     unsigned char *buf  = NULL;
12180 
12181     ajuint  v   = 0U;
12182     ajulong lv  = 0UL;
12183 
12184     ajulong overflow = 0UL;
12185 
12186     /* ajDebug("In btreeWritePribucketEmpty\n"); */
12187 
12188     if(pagepos == cache->totsize)	/* Create a new page */
12189     {
12190 	page = btreePricacheBucketnew(cache);
12191 	buf = page->buf;
12192 	overflow = 0UL;
12193 	lv = overflow;
12194 	SBT_BUCKOVERFLOW(buf,lv);
12195     }
12196     else
12197     {
12198 	page = btreePricacheRead(cache,pagepos);
12199 	buf = page->buf;
12200 	GBT_BUCKOVERFLOW(buf,&overflow);
12201     }
12202 
12203     v = BT_PRIBUCKET;
12204     SBT_BUCKNODETYPE(buf,v);
12205 
12206     page->dirty = BT_LOCK;      /* cleared at end */
12207     page->lockfor = 1241;
12208     lpage = page;
12209 
12210     v = 0U;
12211     SBT_BUCKNENTRIES(buf,v);
12212 
12213     lv = 0UL;
12214     SBT_BUCKOVERFLOW(buf,lv);
12215 
12216     lpage->dirty = BT_DIRTY;    /* clear the lock */
12217 
12218     return;
12219 }
12220 
12221 
12222 
12223 
12224 /* @funcstatic btreePribucketAdd **********************************************
12225 **
12226 ** Add a keyword ID to a primary bucket
12227 ** Only called if there is room in the bucket
12228 **
12229 ** The primary bucket entry for the keyword stores the root page of
12230 ** the secondary tree of ids
12231 **
12232 ** @param [u] cache [AjPBtcache] cache
12233 ** @param [r] pagepos [ajulong] page number of bucket
12234 ** @param [r] keyword [const AjPStr] Keyword
12235 ** @param [r] id      [const AjPStr] Identifier
12236 **
12237 ** @return [void]
12238 **
12239 ** @release 6.5.0
12240 ** @@
12241 ******************************************************************************/
12242 
btreePribucketAdd(AjPBtcache cache,ajulong pagepos,const AjPStr keyword,const AjPStr id)12243 static void btreePribucketAdd(AjPBtcache cache, ajulong pagepos,
12244                               const AjPStr keyword, const AjPStr id)
12245 {
12246     unsigned char *buf  = NULL;
12247     unsigned char *kptr = NULL;
12248     unsigned char *src  = NULL;
12249     unsigned char *dest = NULL;
12250 
12251 /*    unsigned char *lastptr = NULL;*/
12252     unsigned char *endptr  = NULL;
12253 
12254     ajuint nentries = 0U;
12255     ajuint nodetype = 0U;
12256     ajuint idlen    = 0U;
12257 
12258     ajuint sum = 0U;
12259     ajuint len = 0U;
12260     ajuint i;
12261     ajuint v;
12262     ajulong lv;
12263 
12264     ajulong secrootpage = 0UL;
12265     ajulong treeblock  = 0UL;
12266 
12267     AjPBtpage page = NULL;
12268     static ajuint calls = 0U;
12269 /*    static ajuint overflowcalls=0U;*/
12270 
12271     calls++;
12272 
12273     treeblock = 0UL;
12274 
12275     /* secondary tree does not exist, can use secbucket */
12276 
12277     secrootpage = cache->totsize;
12278 
12279     btreeWriteSecbucketEmpty(cache,secrootpage);
12280     btreeSecbucketAdd(cache,secrootpage,id);
12281 
12282     treeblock = secrootpage;
12283     cache->secrootblock = secrootpage;
12284     cache->slevel = 0;
12285 
12286     /* add to the Pribucket page */
12287 
12288     page = btreePricacheRead(cache,pagepos);
12289     buf  = page->buf;
12290 
12291     GBT_BUCKNODETYPE(buf,&nodetype);
12292 
12293     if(nodetype != BT_PRIBUCKET)
12294         ajFatal("Wrong nodetype in PribucketAdd cache %S",
12295                 cache->filename);
12296 
12297     GBT_BUCKNENTRIES(buf,&nentries);
12298 
12299     if(nentries >= cache->pnperbucket)
12300         ajFatal("Bucket too full in PribucketAdd page: %Lu "
12301                 "entries: %u max: %u cache %S",
12302                 pagepos, nentries, cache->pnperbucket,
12303                 cache->filename);
12304 
12305     kptr = PBT_BUCKKEYLEN(buf);
12306     src  = kptr + (nentries * sizeof(ajuint));
12307 
12308     for(i=0;i<nentries;++i)
12309     {
12310         BT_GETAJUINT(kptr,&len);
12311         sum += len;
12312         kptr += sizeof(ajuint);
12313     }
12314 
12315     endptr  = src + sum - 1;
12316     idlen   = MAJSTRGETLEN(keyword);
12317 
12318 /*
12319 //    lastptr = endptr + sizeof(ajulong) + idlen + 1;
12320 //
12321 //    if((ajuint) (lastptr - buf) >= cache->pripagesize)
12322 //    {
12323 //        overflowcalls++;
12324 //
12325 //        ajWarn("\nOverflow in PribucketAdd nentries:%u fails %u/%u cache %S",
12326 //               nentries, overflowcalls, calls, cache->filename);
12327 //
12328 //        btreePribucketAddFull(cache,pagepos,keyword, id);
12329 //        return;
12330 //    }
12331 */
12332 
12333     dest = src + sizeof(ajuint);
12334     memmove((void *)dest, (void *)src, sum);
12335 
12336     v = BT_BUCKPRILEN(keyword);
12337     BT_SETAJUINT(src,v);
12338 
12339     endptr += sizeof(ajuint) + 1;
12340     strcpy((char *)endptr,MAJSTRGETPTR(keyword));
12341 
12342     endptr += idlen + 1;
12343     lv = treeblock;
12344     BT_SETAJULONG(endptr,lv);
12345     endptr += sizeof(ajulong);
12346 
12347     v = nentries + 1;
12348     SBT_BUCKNENTRIES(buf,v);
12349 
12350     page->dirty = BT_DIRTY;
12351 
12352     return;
12353 }
12354 
12355 
12356 
12357 
12358 #if 0
12359 /* #funcstatic btreePribucketAddFull ******************************************
12360 **
12361 ** Add a keyword ID to a full primary bucket
12362 ** by making space in a new bucket
12363 **
12364 ** #param [u] cache [AjPBtcache] cache
12365 ** #param [r] pagepos [ajulong] page number of bucket
12366 ** #param [r] keyword [const AjPStr] Keyword
12367 ** #param [r] id      [const AjPStr] Identifier
12368 **
12369 ** #return [void]
12370 **
12371 ** #release 6.5.0
12372 ** ##
12373 ******************************************************************************/
12374 /*
12375 //static void btreePribucketAddFull(AjPBtcache cache, ajulong pagepos,
12376 //                                  const AjPStr keyword, const AjPStr id)
12377 //{
12378 //    AjPPribucket bucket  = NULL;
12379 //    AjPBtPri     dest    = NULL;
12380 //    AjPBtpage    page    = NULL;
12381 //
12382 //    ajuint nentries;
12383 //    ajulong secrootpage = 0UL;
12384 //    ajulong right       = 0UL;
12385 //
12386 //    unsigned char *buf;
12387 //
12388 //    /# ajDebug("In btreePribucketAddFull\n"); #/
12389 //
12390 //    bucket   = btreePribucketRead(cache,pagepos);
12391 //    nentries = bucket->Nentries;
12392 //
12393 //
12394 //    /# Reading a bucket always gives one extra ID position #/
12395 //    bucket->codes[nentries] = ajBtreePriNew();
12396 //    dest = bucket->codes[nentries];
12397 //
12398 //    ajStrAssignS(&dest->keyword,keyword);
12399 //
12400 //    /# Need to add code here to access secondary tree #/
12401 //    dest->treeblock = 0UL;
12402 //
12403 //
12404 //    /# See if secondary tree exists, if not then create it #/
12405 //    if(!dest->treeblock)
12406 //    {
12407 //	secrootpage = cache->totsize;
12408 //
12409 //	btreeSecrootCreate(cache,secrootpage);
12410 //	cache->secrootblock = secrootpage;
12411 //	page = btreeSeccacheWrite(cache,secrootpage);
12412 //	page->dirty = BT_LOCK;
12413 //        page->lockfor = 1261;
12414 //        if(btreeDoRootSync)
12415 //            btreeCacheRootSync(cache,secrootpage);
12416 //	dest->treeblock = secrootpage;
12417 //	buf = page->buf;
12418 //	cache->slevel = 0;
12419 //
12420 //	/# ajDebug("Created 2ry tree at block %u\n",(ajuint)secrootpage); #/
12421 //    }
12422 //    else
12423 //    {
12424 //	cache->secrootblock = dest->treeblock;
12425 //	page = btreeSeccacheWrite(cache,cache->secrootblock);
12426 //	page->dirty = BT_LOCK;
12427 //        page->lockfor = 1262;
12428 //	buf = page->buf;
12429 //	GBT_RIGHT(buf, &right);
12430 //	cache->slevel = (ajuint) right;
12431 //    }
12432 //
12433 //    btreeKeyidInsert(cache, id);
12434 //
12435 //    right = (ajulong) cache->slevel;
12436 //
12437 //    SBT_RIGHT(buf, right);
12438 //    page->dirty = BT_DIRTY;
12439 //
12440 //    ++bucket->Nentries;
12441 //
12442 //    btreeWritePribucket(cache,bucket,pagepos);
12443 //
12444 //    btreePribucketDel(&bucket);
12445 //
12446 //    return;
12447 //}
12448 */
12449 #endif
12450 
12451 
12452 
12453 
12454 /* @funcstatic btreeNumInPribucket ********************************************
12455 **
12456 ** Return number of entries in a primary keyword bucket
12457 **
12458 ** @param [u] cache [AjPBtcache] cache
12459 ** @param [r] pagepos [ajulong] page number
12460 **
12461 ** @return [ajuint] Number of entries in bucket
12462 **
12463 ** @release 3.0.0
12464 ** @@
12465 ******************************************************************************/
12466 
btreeNumInPribucket(AjPBtcache cache,ajulong pagepos)12467 static ajuint btreeNumInPribucket(AjPBtcache cache, ajulong pagepos)
12468 {
12469     AjPBtpage page     = NULL;
12470     unsigned char *buf = NULL;
12471 
12472     ajuint  nodetype = 0;
12473     ajuint nentries = 0;
12474 
12475     /* ajDebug("In btreeNumInPribucket\n"); */
12476 
12477     if(!pagepos)
12478 	ajFatal("NumInPribucket: Attempt to read bucket from root page\n");
12479 
12480     page  = btreePricacheRead(cache,pagepos);
12481 
12482     buf = page->buf;
12483 
12484     GBT_BUCKNODETYPE(buf,&nodetype);
12485 
12486     if(nodetype != BT_PRIBUCKET)
12487 	ajFatal("PriReadBucket: NodeType mismatch. Not primary bucket (%u)",
12488 		nodetype);
12489 
12490     GBT_BUCKNENTRIES(buf,&nentries);
12491 
12492     return nentries;
12493 }
12494 
12495 
12496 
12497 
12498 /* @funcstatic btreeKeywordCompare ********************************************
12499 **
12500 ** Comparison function for ajListSort
12501 **
12502 ** @param [r] a [const void*] ID 1
12503 ** @param [r] b [const void*] ID 2
12504 **
12505 ** @return [ajint] 0 = bases match
12506 **
12507 ** @release 3.0.0
12508 ** @@
12509 ******************************************************************************/
12510 
btreeKeywordCompare(const void * a,const void * b)12511 static ajint btreeKeywordCompare(const void *a, const void *b)
12512 {
12513     return MAJSTRCMPS((*(AjPBtPri const *)a)->keyword,
12514                       (*(AjPBtPri const *)b)->keyword);
12515 }
12516 
12517 
12518 
12519 
12520 /* @funcstatic btreePribucketsReorder *****************************************
12521 **
12522 ** Re-order primary keyword leaf buckets
12523 ** Must only be called if one of the buckets is full
12524 **
12525 ** @param [u] cache [AjPBtcache] cache
12526 ** @param [u] leaf [AjPBtpage] leaf page
12527 **
12528 ** @return [AjBool] true if reorder was successful i.e. leaf not full
12529 **
12530 ** @release 6.5.0
12531 ** @@
12532 ******************************************************************************/
12533 
btreePribucketsReorder(AjPBtcache cache,AjPBtpage leaf)12534 static AjBool btreePribucketsReorder(AjPBtcache cache, AjPBtpage leaf)
12535 {
12536     ajuint nkeys = 0;
12537     unsigned char *lbuf   = NULL;
12538 
12539     ajulong *ptrs      = NULL;
12540     AjPStr *newkeys      = NULL;
12541     ajulong *newptrs      = NULL;
12542     ajulong *overflows = NULL;
12543     AjPBtMem arrays1 = NULL;
12544     AjPBtMem arrays2 = NULL;
12545 
12546     ajuint i = 0;
12547 
12548     ajuint order;
12549     ajuint totalkeys     = 0;
12550     ajuint maxnperbucket = 0;
12551     ajuint count         = 0;
12552     ajuint keylimit      = 0;
12553     ajuint bucketlimit   = 0;
12554     ajuint nodetype      = 0;
12555     ajuint dirtysave     = 0;
12556 
12557     AjPList idlist       = NULL;
12558     AjPBtPri bid         = NULL;
12559     AjPPribucket cbucket = NULL;
12560     AjPBtPri cid         = NULL;
12561 
12562     ajuint iold = 0;
12563 
12564     /* ajDebug("In btreePribucketsReorder\n"); */
12565 
12566     dirtysave = leaf->dirty;
12567 
12568     leaf->dirty = BT_LOCK;
12569     leaf->lockfor = 1271;
12570     lbuf = leaf->buf;
12571 
12572     GBT_NODETYPE(lbuf,&nodetype);
12573 
12574     order = cache->porder;
12575 
12576     /* Read keys/ptrs */
12577     arrays1 = btreeAllocPriArray(cache);
12578     ptrs = arrays1->parray;
12579     arrays2 = btreeAllocPriArray(cache);
12580     newkeys = arrays2->karray;
12581     newptrs = arrays2->parray;
12582     overflows = arrays2->overflows;
12583 
12584     btreeGetPointers(cache,lbuf,&ptrs);
12585 
12586     GBT_NKEYS(lbuf,&nkeys);
12587     keylimit = nkeys + 1;
12588 
12589     if(!nkeys)
12590 	ajFatal("PribucketsReorder: Attempt to reorder empty leaf");
12591 
12592     for(i=0;i<nkeys;++i)
12593 	totalkeys += btreeNumInPribucket(cache,ptrs[i]);
12594     totalkeys += btreeNumInPribucket(cache,ptrs[i]);
12595 
12596     btreeBucketCalc(totalkeys, keylimit, cache->pnperbucket,
12597                     &bucketlimit, &maxnperbucket);
12598 
12599     if(bucketlimit >= order)
12600     {
12601         btreeDeallocPriArray(cache,arrays1);
12602         btreeDeallocPriArray(cache,arrays2);
12603 	leaf->dirty = dirtysave;
12604 
12605 	return ajFalse;
12606     }
12607 
12608 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
12609     ajDebug("btreePribucketsReorder '%S' %Lu id '%S' key '%S'\n",
12610             cache->basename, leaf->pagepos, indexId, indexKeyword);
12611 #endif
12612     ++statCallPribucketsReorder;
12613 
12614     /* Read IDs from all buckets and push to list and sort (increasing kw) */
12615     idlist  = ajListNew();
12616 
12617     for(i=0;i<keylimit;++i)
12618         overflows[i] = btreePribucketIdlist(cache,ptrs[i],idlist);
12619 
12620     ajListSort(idlist, &btreeKeywordCompare);
12621 
12622     cbucket = btreePribucketNew(cache->pnperbucket);
12623 
12624     iold = 0;
12625     for(i=0;i<bucketlimit;++i)
12626     {
12627 	cbucket->Overflow = overflows[i];
12628 	cbucket->Nentries = 0;
12629 
12630         count = 0;
12631 	while(count!=maxnperbucket)
12632 	{
12633 	    ajListPop(idlist,(void **)&bid);
12634 
12635 	    cid = cbucket->codes[count];
12636 	    ajStrAssignS(&cid->keyword,bid->keyword);
12637 	    cid->treeblock = bid->treeblock;
12638 
12639 	    cbucket->keylen[count] = BT_BUCKPRILEN(bid->keyword);
12640 	    ++cbucket->Nentries;
12641 	    ++count;
12642 	    ajBtreePriDel(&bid);
12643 	}
12644 
12645 
12646 	ajListPeek(idlist,(void **)&bid);
12647 	ajStrAssignS(&newkeys[i],bid->keyword);
12648 
12649 	if((iold < order) && ptrs[iold])
12650             newptrs[i] = ptrs[iold++];
12651         else
12652 	    newptrs[i] = cache->totsize;
12653 	btreeWritePribucket(cache,cbucket,newptrs[i]);
12654     }
12655 
12656 
12657     /* Deal with greater-than bucket */
12658 
12659     cbucket->Overflow = overflows[i];
12660     cbucket->Nentries = 0;
12661 
12662     count = 0;
12663 
12664     while(ajListPop(idlist,(void **)&bid))
12665     {
12666 	cid = cbucket->codes[count];
12667 	ajStrAssignS(&cid->keyword,bid->keyword);
12668 	cid->treeblock = bid->treeblock;
12669 
12670 	++cbucket->Nentries;
12671 	++count;
12672 	ajBtreePriDel(&bid);
12673     }
12674 
12675 
12676     if((iold < order) && ptrs[iold])
12677         newptrs[i] = ptrs[iold++];
12678     else
12679         newptrs[i] = cache->totsize;
12680     btreeWritePribucket(cache,cbucket,newptrs[i]);
12681 
12682     btreePribucketDel(&cbucket);
12683 
12684 #if AJINDEX_DEBUG
12685     if(bucketlimit <= keylimit)
12686         ajDebug("btreePribucketsReorder '%S' %u -> %u",
12687                 cache->filename, keylimit, bucketlimit);
12688 #endif
12689 
12690     for(i = bucketlimit + 1; i <= keylimit; i++)
12691     {
12692         btreePripageSetfree(cache, ptrs[i]);
12693     }
12694 
12695     /* Now write out a modified leaf with new keys/ptrs */
12696 
12697     nkeys = bucketlimit;
12698     btreeWriteNode(cache,leaf,newkeys,newptrs,nkeys);
12699     leaf->dirty = BT_DIRTY;
12700 
12701     if(nodetype == BT_ROOT)
12702     {
12703 	leaf->dirty = BT_LOCK;
12704         leaf->lockfor = 1272;
12705     }
12706 
12707     btreeDeallocPriArray(cache,arrays1);
12708     btreeDeallocPriArray(cache,arrays2);
12709 
12710     ajListFree(&idlist);
12711 
12712     return ajTrue;
12713 }
12714 
12715 
12716 
12717 
12718 /* @func ajBtreeKeyIndex ******************************************************
12719 **
12720 ** Insert a keyword structure into the tree.
12721 **
12722 ** This is the primary function for adding a keyword and identifier to
12723 ** a secondary keyword index
12724 **
12725 ** @param [u] cache   [AjPBtcache] cache
12726 ** @param [r] keyword [const AjPStr] keyword
12727 ** @param [r] id      [const AjPStr] entry identifier
12728 **
12729 ** @return [AjBool] True if keyword and ID combination was inserted
12730 **                  False if keyword exists already for ID
12731 **
12732 ** @release 6.5.0
12733 ** @@
12734 ******************************************************************************/
12735 
ajBtreeKeyIndex(AjPBtcache cache,const AjPStr keyword,const AjPStr id)12736 AjBool ajBtreeKeyIndex(AjPBtcache cache, const AjPStr keyword, const AjPStr id)
12737 {
12738     AjPBtpage spage   = NULL;
12739     AjPBtpage page    = NULL;
12740     ajulong lblockno = 0UL;
12741     ajulong rblockno = 0UL;
12742     ajulong blockno  = 0UL;
12743     ajulong shift    = 0UL;
12744 
12745     ajulong nkeys = 0U;
12746 
12747     ajuint nodetype = 0U;
12748     ajulong right   = 0UL;
12749 
12750     ajuint n;
12751     ajuint savedirty;
12752     unsigned char *buf   = NULL;
12753 
12754     ajulong treeblock = 0UL;
12755     AjBool newid = ajTrue;
12756 
12757 #if AJINDEX_DEBUG
12758     /*ajDebug("ajBtreeKeyIndex '%S' '%S'\n", pri->keyword, pri->id);*/
12759 #endif
12760 
12761     if(!MAJSTRGETLEN(keyword))
12762 	return ajFalse;
12763 
12764     ajStrAssignS(&indexKeyword, keyword);
12765     ajStrAssignS(&indexId, id);
12766     ajStrFmtQuery(&indexId);
12767     ajStrFmtQuery(&indexKeyword);
12768 
12769     if(MAJSTRGETLEN(indexKeyword) > cache->keylimit)
12770         ajStrTruncateLen(&indexKeyword, cache->keylimit);
12771 
12772     if(MAJSTRGETLEN(indexId) > cache->idlimit)
12773         ajStrTruncateLen(&indexId, cache->idlimit);
12774 
12775     /*
12776     ** Only insert a primary key if that key doesn't exist
12777     **
12778     ** We always have a secondary identifier if there is a match
12779     ** (for Identifiers there may be no secondary data so their code differs)
12780     */
12781 
12782     if(btreeKeyFind(cache, indexKeyword, &treeblock))
12783     {
12784         /* we have the keyword in the index */
12785 
12786 	cache->secrootblock = treeblock;
12787 	page = btreeSeccacheWrite(cache,cache->secrootblock);
12788         savedirty = page->dirty;
12789 	page->dirty = BT_LOCK;
12790         page->lockfor = 1281;
12791 	buf = page->buf;
12792 	GBT_RIGHT(buf,&right);
12793 	cache->slevel = (ajuint) right;
12794 
12795 	if(btreeKeyidInsert(cache,indexId))
12796         {
12797             ++cache->countall;
12798             GBT_NODETYPE(buf,&nodetype);
12799             if(nodetype != BT_SECBUCKET)
12800             {
12801                 right = (ajulong) cache->slevel;
12802                 SBT_RIGHT(buf,right);
12803                 page->dirty = BT_DIRTY;
12804             }
12805         }
12806         else
12807         {
12808             newid = ajFalse;
12809             page->dirty = savedirty;
12810         }
12811 
12812 	return newid;
12813     }
12814 
12815     /* this is a new key */
12816 
12817     spage = btreeIdentFind(cache,indexKeyword);
12818     buf = spage->buf;
12819 
12820     GBT_NKEYS(buf,&nkeys);
12821     GBT_NODETYPE(buf,&nodetype);
12822 
12823     if(!nkeys)
12824     {
12825         /* new primary leaf node with no keys yet */
12826 
12827         /* create two buckets, add to the right-hand one */
12828 
12829 	lblockno = cache->totsize;
12830 	btreeWritePribucketEmpty(cache,lblockno);
12831 
12832 	rblockno = cache->totsize;
12833 	btreeWritePribucketEmpty(cache,rblockno);
12834 
12835 	btreeWriteNodeSingle(cache,spage,indexKeyword,lblockno,rblockno);
12836 
12837 	GBT_BLOCKNUMBER(buf,&blockno);
12838 
12839 	if(!blockno)
12840         {
12841 	    spage->dirty = BT_LOCK; /* root page */
12842             spage->lockfor = 1282;
12843         }
12844 
12845 	btreePribucketAdd(cache,rblockno,indexKeyword, indexId);
12846 	++cache->countunique;
12847 	++cache->countall;
12848 
12849 	return ajTrue;
12850     }
12851 
12852     blockno = btreeGetBlockS(cache,buf,indexKeyword);
12853     if(nodetype != BT_ROOT)
12854 	if((shift = btreeKeyInsertShift(cache,&spage,indexKeyword)))
12855 	    blockno = shift;
12856 
12857     buf = spage->buf;
12858 
12859     n = btreeNumInPribucket(cache,blockno);
12860 
12861     if(n == cache->pnperbucket)
12862     {
12863 	if(btreePribucketsReorder(cache,spage))
12864 	{
12865             blockno = btreeGetBlockS(cache,buf,indexKeyword);
12866 	}
12867 	else
12868 	{
12869 	    btreeKeySplitleaf(cache,spage);
12870 	    spage = btreeIdentFind(cache,indexKeyword);
12871 	    buf = spage->buf;
12872 
12873             blockno = btreeGetBlockS(cache,buf,indexKeyword);
12874 	}
12875     }
12876 
12877     btreePribucketAdd(cache, blockno, indexKeyword, indexId);
12878     ++cache->countunique;
12879     ++cache->countall;
12880 
12881     return ajTrue;
12882 }
12883 
12884 
12885 
12886 
12887 /* @funcstatic btreeKeyFind ***************************************************
12888 **
12889 ** Get secondary root block matching a keyword
12890 **
12891 ** @param [u] cache [AjPBtcache] cache
12892 ** @param [r] key [const AjPStr] key
12893 ** @param [w] treeblock [ajulong*] Tree block number for id
12894 **
12895 ** @return [AjBool] ajTrue if found
12896 **
12897 ** @release 6.4.0
12898 ** @@
12899 ******************************************************************************/
12900 
btreeKeyFind(AjPBtcache cache,const AjPStr key,ajulong * treeblock)12901 static AjBool btreeKeyFind(AjPBtcache cache, const AjPStr key,
12902                            ajulong* treeblock)
12903 {
12904     AjPBtpage page      = NULL;
12905 
12906     unsigned char *buf = NULL;
12907 
12908     ajulong blockno = 0UL;
12909 
12910     /*ajDebug("btreeKeyFind '%s'\n", ckey);*/
12911 
12912     if(!cache->countunique)
12913 	return ajFalse;
12914 
12915     page = btreeIdentFind(cache,key);
12916 
12917     buf = page->buf;
12918 
12919     blockno = btreeGetBlockS(cache, buf, key);
12920 
12921     return btreePribucketFindId(cache,blockno,key,treeblock);
12922 }
12923 
12924 
12925 
12926 
12927 /* @func ajBtreeKeyFindLen ****************************************************
12928 **
12929 ** Get keyword structure matching a keyword truncated to maximum indexed length
12930 **
12931 ** @param [u] cache [AjPBtcache] cache
12932 ** @param [r] key [const AjPStr] key
12933 ** @param [w] treeblock [ajulong*] Tree block number for id
12934 **
12935 ** @return [AjBool] ajTrue if found
12936 **
12937 ** @release 6.4.0
12938 ** @@
12939 ******************************************************************************/
12940 
ajBtreeKeyFindLen(AjPBtcache cache,const AjPStr key,ajulong * treeblock)12941 AjBool ajBtreeKeyFindLen(AjPBtcache cache, const AjPStr key,
12942                          ajulong* treeblock)
12943 {
12944     AjBool ret = ajFalse;
12945 
12946     AjPBtpage page      = NULL;
12947     unsigned char *buf = NULL;
12948     ajulong blockno = 0UL;
12949 
12950     AjPStr keystr = NULL;
12951     const AjPStr keytest = key;
12952 
12953 #if AJINDEX_DEBUG
12954     ajDebug("ajBtreeKeyFindLen '%S'\n", key);
12955 #endif
12956 
12957     if(!cache->countunique)
12958 	return ajFalse;
12959 
12960     keystr = ajStrNewS(key);
12961     ajStrFmtQuery(&keystr);
12962     if(MAJSTRGETLEN(keystr) > cache->keylimit)
12963         ajStrTruncateLen(&keystr, cache->keylimit);
12964     keytest = keystr;
12965 
12966     ajDebug("ajBtreeKeyFindLen '%S'\n", keystr);
12967 
12968     page = btreeIdentFind(cache,keytest);
12969 
12970     buf = page->buf;
12971 
12972     blockno = btreeGetBlockS(cache, buf, keytest);
12973 
12974     ret = btreePribucketFindId(cache,blockno,keytest,treeblock);
12975 
12976     ajStrDel(&keystr);
12977 
12978     return ret;
12979 }
12980 
12981 
12982 
12983 
12984 /* @funcstatic btreePribucketFindId *******************************************
12985 **
12986 ** Tests for an ID in a primary bucket
12987 **
12988 ** @param [u] cache [AjPBtcache] cache
12989 ** @param [r] pagepos [ajulong] page number
12990 ** @param [r] id [const AjPStr] id to search for
12991 ** @param [w] treeblock [ajulong*] Tree block number for id
12992 **
12993 ** @return [AjBool] ajTrue if found
12994 **
12995 ** @release 6.4.0
12996 ** @@
12997 ******************************************************************************/
12998 
btreePribucketFindId(AjPBtcache cache,ajulong pagepos,const AjPStr id,ajulong * treeblock)12999 static AjBool btreePribucketFindId(AjPBtcache cache, ajulong pagepos,
13000                                    const AjPStr id, ajulong* treeblock)
13001 {
13002     AjPBtpage page      = NULL;
13003     AjPBtpage lpage     = NULL;
13004     unsigned char *buf  = NULL;
13005     unsigned char *kptr = NULL;
13006 
13007     unsigned char *codeptr = NULL;
13008 
13009     ajuint  nodetype  = 0U;
13010     ajuint  nentries  = 0U;
13011     ajulong overflow  = 0UL;
13012     ajuint  dirtysave = 0U;
13013 
13014     ajuint  i;
13015     ajuint  len  = 0U;
13016     ajuint idlen = 0U;
13017 
13018     /* ajDebug("In btreePribucketFindId\n"); */
13019 
13020     if(!pagepos)
13021 	ajFatal("PribucketFindId: cannot read bucket from root page cache %S",
13022                 cache->filename);
13023 
13024     page  = btreePricacheRead(cache,pagepos);
13025     lpage = page;
13026     dirtysave = lpage->dirty;
13027     lpage->dirty = BT_LOCK;
13028     lpage->lockfor = 1291;
13029 
13030     buf = lpage->buf;
13031 
13032     GBT_BUCKNODETYPE(buf,&nodetype);
13033     if(nodetype != BT_PRIBUCKET)
13034 	ajFatal("PribucketFindId: NodeType mismatch. "
13035                 "Not primary bucket (%u) cache %S",
13036 		nodetype, cache->filename);
13037 
13038     GBT_BUCKNENTRIES(buf,&nentries);
13039     if(nentries > cache->pnperbucket)
13040 	ajFatal("PribucketFindId: Bucket too full  page: %Lu "
13041                 "entries: %u max: %u cache %S",
13042                 pagepos, nentries, cache->pnperbucket,
13043                 cache->filename);
13044 
13045 
13046     GBT_BUCKOVERFLOW(buf,&overflow);
13047 
13048     kptr  = PBT_BUCKKEYLEN(buf);
13049     codeptr = kptr + (nentries * sizeof(ajuint));
13050 
13051     for(i=0;i<nentries;++i)
13052     {
13053 	BT_GETAJUINT(kptr,&len);
13054         idlen = len - sizeof(ajulong) - 1;
13055 
13056 /*
13057 //        if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
13058 //	{
13059 //	    /# ajDebug("PribucketFindId: Overflow\n"); #/
13060 //	    page  = btreePricacheRead(cache,overflow);
13061 //	    buf = page->buf;
13062 //	    GBT_BUCKNODETYPE(buf,&nodetype);
13063 //	    if(nodetype != BT_PRIBUCKET)
13064 //		ajFatal("PribucketFindId: NodeType mismatch. "
13065 //                        "Not primary bucket (%u) cache %S",
13066 //                        nodetype, cache->filename);
13067 //	    GBT_BUCKOVERFLOW(buf,&overflow);
13068 //	    /# overflow bucket ids start at the keylen position #/
13069 //	    codeptr = PBT_BUCKKEYLEN(buf);
13070 //	}
13071 */
13072 
13073 	/* Fill ID objects */
13074 	if(ajStrMatchC(id, (const char *)codeptr))
13075         {
13076             codeptr += (idlen + 1);
13077             BT_GETAJULONG(codeptr,treeblock);
13078             lpage->dirty = dirtysave;
13079             return ajTrue;
13080         }
13081 
13082 	codeptr += (idlen + 1);
13083 	codeptr += sizeof(ajulong);
13084 
13085 	kptr += sizeof(ajuint);
13086     }
13087 
13088     lpage->dirty = dirtysave;
13089 
13090     return ajFalse;
13091 }
13092 
13093 
13094 
13095 
13096 /* @funcstatic btreeKeySplitleaf **********************************************
13097 **
13098 ** Split a keyword leaf and propagate up if necessary
13099 **
13100 ** @param [u] cache [AjPBtcache] cache
13101 ** @param [u] spage [AjPBtpage] page
13102 **
13103 ** @return [AjPBtpage] pointer to a parent page
13104 **
13105 ** @release 6.5.0
13106 ** @@
13107 ******************************************************************************/
13108 
btreeKeySplitleaf(AjPBtcache cache,AjPBtpage spage)13109 static AjPBtpage btreeKeySplitleaf(AjPBtcache cache, AjPBtpage spage)
13110 {
13111     ajuint nkeys     = 0U;
13112     ajuint order     = 0U;
13113     ajuint totalkeys = 0U;
13114     ajuint keylimit  = 0U;
13115     ajuint nodetype  = 0U;
13116 
13117     ajuint rootnodetype = 0U;
13118 
13119     ajuint i;
13120     ajuint j;
13121 
13122     AjPBtpage lpage = NULL;
13123     AjPBtpage rpage = NULL;
13124     AjPBtpage page  = NULL;
13125 
13126     AjPStr mediankey  = NULL;
13127     ajulong mediangtr  = 0UL;
13128     ajulong medianless = 0UL;
13129 
13130 
13131     AjPBtPri bid       = NULL;
13132     AjPBtPri cid       = NULL;
13133     unsigned char *buf  = NULL;
13134     unsigned char *lbuf = NULL;
13135     unsigned char *rbuf = NULL;
13136 
13137     AjPList idlist = NULL;
13138 
13139     AjPPribucket cbucket  = NULL;
13140 
13141     ajulong *parray = NULL;
13142     AjPStr *newkarray = NULL;
13143     ajulong *newparray = NULL;
13144     AjPBtMem arrays = NULL;
13145     AjPBtMem newarrays = NULL;
13146 
13147     ajuint lno    = 0U;
13148     ajuint rno    = 0U;
13149 
13150     ajuint lbucketlimit   = 0U;
13151     ajuint rbucketlimit   = 0U;
13152     ajuint lmaxnperbucket = 0U;
13153     ajuint rmaxnperbucket = 0U;
13154     ajuint count         = 0U;
13155 
13156     ajulong lblockno = 0UL;
13157     ajulong rblockno = 0UL;
13158     ajulong prev     = 0UL;
13159     ajulong overflow = 0UL;
13160     ajulong prevsave = 0UL;
13161 
13162     ajulong zero = 0UL;
13163     ajulong join = 0UL;
13164 
13165     ajulong lv = 0UL;
13166     ajuint  v  = 0U;
13167 
13168     ajuint iold = 0U;
13169 
13170 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
13171     ajDebug("btreeKeySplitleaf '%S' %Lu id '%S' key '%S'\n",
13172             cache->basename, spage->pagepos, indexId, indexKeyword);
13173 #endif
13174     ++statCallKeySplitleaf;
13175 
13176     /* ajDebug("In btreeKeySplitleaf\n"); */
13177 
13178     order = cache->porder;
13179 
13180     mediankey = ajStrNew();
13181     arrays = btreeAllocPriArray(cache);
13182     parray = arrays->parray;
13183 
13184     newarrays = btreeAllocPriArray(cache);
13185     newkarray = newarrays->karray;
13186     newparray = newarrays->parray;
13187 
13188     buf = spage->buf;
13189     lbuf = buf;
13190 
13191     GBT_NKEYS(buf,&nkeys);
13192     GBT_NODETYPE(buf,&rootnodetype);
13193 
13194     if(rootnodetype == BT_ROOT)
13195     {
13196 	lblockno = cache->totsize;
13197 	lpage = btreePricacheNodenew(cache);
13198 	lbuf = lpage->buf;
13199 	lv = prev;
13200 	SBT_PREV(lbuf,lv);
13201     }
13202     else
13203     {
13204 	lblockno = spage->pagepos;
13205 	lpage = spage;
13206     }
13207 
13208     lpage->dirty = BT_LOCK;
13209     lpage->lockfor = 1301;
13210 
13211     rblockno = cache->totsize;
13212     rpage = btreePricacheNodenew(cache);
13213     rbuf = rpage->buf;
13214     rpage->dirty = BT_LOCK;
13215     rpage->lockfor = 1302;
13216 
13217     if(rootnodetype == BT_ROOT)
13218     {
13219 	lv = zero;
13220 	SBT_RIGHT(rbuf,lv);
13221 	lv = zero;
13222 	SBT_LEFT(lbuf,lv);
13223     }
13224     else
13225     {
13226 	GBT_RIGHT(lbuf,&join);
13227 	lv = join;
13228 	SBT_RIGHT(rbuf,lv);
13229     }
13230 
13231     lv = lblockno;
13232     SBT_LEFT(rbuf,lv);
13233     lv = rblockno;
13234     SBT_RIGHT(lbuf,lv);
13235 
13236 
13237     btreeGetPointers(cache,buf,&parray);
13238 
13239 
13240     keylimit = nkeys+1;
13241     idlist = ajListNew();
13242 
13243     for(i=0;i<keylimit;++i)
13244 	btreePribucketIdlist(cache,parray[i],idlist);
13245 
13246     ajListSort(idlist, &btreeKeywordCompare);
13247 
13248     totalkeys = (ajuint) ajListGetLength(idlist);
13249 
13250     btreeBucketSplitCalc(totalkeys, keylimit, cache->pnperbucket,
13251                          &lbucketlimit,&lmaxnperbucket,&lno,
13252                          &rbucketlimit,&rmaxnperbucket,&rno);
13253 
13254     cbucket = btreePribucketNew(cache->pnperbucket);
13255 
13256     count = 0;
13257     iold = 0;
13258 
13259     for(i=0;i<lbucketlimit;++i)
13260     {
13261 	cbucket->Nentries = 0;
13262 
13263 	for(j=0;j<lmaxnperbucket;++j)
13264 	{
13265 	    ajListPop(idlist,(void **)&bid);
13266 
13267 	    cid = cbucket->codes[j];
13268 	    ajStrAssignS(&cid->keyword,bid->keyword);
13269 	    cid->treeblock = bid->treeblock;
13270 
13271 	    cbucket->keylen[j] = BT_BUCKPRILEN(bid->keyword);
13272 	    ++count;
13273 	    ++cbucket->Nentries;
13274 	    ajBtreePriDel(&bid);
13275 	}
13276 
13277 	ajListPeek(idlist,(void **)&bid);
13278 
13279 	ajStrAssignS(&newkarray[i],bid->keyword);
13280 
13281 	if((iold < order) && parray[iold])
13282             newparray[i] = parray[iold++];
13283         else
13284 	    newparray[i] = cache->totsize;
13285 	btreeWritePribucket(cache,cbucket,newparray[i]);
13286     }
13287 
13288     cbucket->Nentries = 0;
13289 
13290     j = 0;
13291 
13292     while(count != lno)
13293     {
13294 	ajListPop(idlist,(void **)&bid);
13295 	cid = cbucket->codes[j];
13296 	++j;
13297 	++count;
13298 
13299 	ajStrAssignS(&cid->keyword,bid->keyword);
13300 	cid->treeblock = bid->treeblock;
13301 	++cbucket->Nentries;
13302 	ajBtreePriDel(&bid);
13303     }
13304 
13305     if((iold < order) && parray[iold])
13306         newparray[i] = parray[iold++];
13307     else
13308         newparray[i] = cache->totsize;
13309     btreeWritePribucket(cache,cbucket,newparray[i]);
13310 
13311     nkeys = lbucketlimit;
13312     nodetype = BT_LEAF;
13313     v = nodetype;
13314     SBT_NODETYPE(lbuf,v);
13315 
13316     GBT_PREV(lbuf,&prevsave);
13317     lpage->dirty = BT_DIRTY;
13318 
13319     btreeWriteNode(cache,lpage,newkarray,newparray,nkeys);
13320 
13321     ajListPeek(idlist,(void **)&bid);
13322     ajStrAssignS(&mediankey,bid->keyword);
13323 
13324     for(i=0;i<rbucketlimit;++i)
13325     {
13326 	cbucket->Nentries = 0;
13327 
13328 	for(j=0;j<rmaxnperbucket;++j)
13329 	{
13330 	    ajListPop(idlist,(void **)&bid);
13331 
13332 	    cid = cbucket->codes[j];
13333 	    ajStrAssignS(&cid->keyword,bid->keyword);
13334 	    cid->treeblock = bid->treeblock;
13335 
13336 	    cbucket->keylen[j] = BT_BUCKPRILEN(bid->keyword);
13337 	    ++cbucket->Nentries;
13338 	    ajBtreePriDel(&bid);
13339 	}
13340 
13341 	ajListPeek(idlist,(void **)&bid);
13342 	ajStrAssignS(&newkarray[i],bid->keyword);
13343 
13344 	if((iold < order) && parray[iold])
13345             newparray[i] = parray[iold++];
13346         else
13347 	    newparray[i] = cache->totsize;
13348 	btreeWritePribucket(cache,cbucket,newparray[i]);
13349     }
13350 
13351     cbucket->Nentries = 0;
13352 
13353     j = 0;
13354 
13355     while(ajListPop(idlist,(void**)&bid))
13356     {
13357 	cid = cbucket->codes[j];
13358 	++j;
13359 
13360 	ajStrAssignS(&cid->keyword,bid->keyword);
13361 	cid->treeblock = bid->treeblock;
13362 	++cbucket->Nentries;
13363 	ajBtreePriDel(&bid);
13364     }
13365 
13366     if((iold < order) && parray[iold])
13367         newparray[i] = parray[iold++];
13368     else
13369         newparray[i] = cache->totsize;
13370     btreeWritePribucket(cache,cbucket,newparray[i]);
13371 
13372     nkeys = rbucketlimit;
13373 
13374     v = nkeys;
13375     nodetype = BT_LEAF;
13376     v = nodetype;
13377     SBT_NODETYPE(rbuf,v);
13378 
13379     lv = prevsave;
13380     SBT_PREV(rbuf,lv);
13381     lv = overflow;
13382     SBT_OVERFLOW(rbuf,lv);
13383 
13384     btreeWriteNode(cache,rpage,newkarray,newparray,nkeys);
13385     rpage->dirty = BT_DIRTY;
13386 
13387     btreePribucketDel(&cbucket);
13388     ajListFree(&idlist);
13389 
13390 
13391 
13392     medianless = lblockno;
13393     mediangtr  = rblockno;
13394 
13395     btreeDeallocPriArray(cache,arrays);
13396     btreeDeallocPriArray(cache,newarrays);
13397 
13398     if(rootnodetype == BT_ROOT)
13399     {
13400 	btreeWriteNodeSingle(cache,spage,mediankey,lblockno,rblockno);
13401 	spage->dirty = BT_LOCK;
13402         spage->lockfor = 1303;
13403 	ajStrDel(&mediankey);
13404 	++cache->plevel;
13405 
13406 	return spage;
13407     }
13408 
13409     page = btreePricacheRead(cache,prevsave);
13410     btreePriInsertKey(cache,page,mediankey,medianless,mediangtr);
13411     ajStrDel(&mediankey);
13412 
13413     page = btreePricacheRead(cache,prevsave);
13414 
13415     return page;
13416 }
13417 
13418 
13419 
13420 
13421 /* @funcstatic btreeSecbucketFindId *******************************************
13422 **
13423 ** Tests for an ID in a secondary bucket
13424 **
13425 ** @param [u] cache [AjPBtcache] cache
13426 ** @param [r] pagepos [ajulong] page number
13427 ** @param [r] id [const AjPStr] id to search for
13428 **
13429 ** @return [AjBool] ajTrue if found
13430 **
13431 ** @release 6.4.0
13432 ** @@
13433 ******************************************************************************/
13434 
btreeSecbucketFindId(AjPBtcache cache,ajulong pagepos,const AjPStr id)13435 static AjBool btreeSecbucketFindId(AjPBtcache cache, ajulong pagepos,
13436                                    const AjPStr id)
13437 {
13438     AjPBtpage page  = NULL;
13439     AjPBtpage lpage = NULL;
13440 
13441     unsigned char *buf  = NULL;
13442     unsigned char *kptr = NULL;
13443 
13444     unsigned char *codeptr = NULL;
13445 
13446     ajuint  nodetype = 0U;
13447     ajuint  nentries = 0U;
13448     ajulong overflow = 0UL;
13449     ajuint  dirtysave = 0U;
13450 
13451     ajuint  i;
13452     ajuint  len  = 0U;
13453 
13454     /* ajDebug("In btreeSecbucketFindId\n"); */
13455 
13456     /* Put in test here for secondary root page read
13457        instead of !pagepos - done */
13458 
13459 /*
13460   if(pagepos == cache->secrootblock)
13461 	ajFatal("SecbucketFindId: cannot read bucket from root page cache %S",
13462                 cache->filename);
13463 */
13464 
13465     page  = btreeSeccacheRead(cache,pagepos);
13466     lpage = page;
13467     dirtysave = lpage->dirty;
13468     lpage->dirty = BT_LOCK;
13469     lpage->lockfor = 1311;
13470 
13471     buf = lpage->buf;
13472 
13473     GBT_BUCKNODETYPE(buf,&nodetype);
13474     if(nodetype != BT_SECBUCKET)
13475 	ajFatal("SecbucketFindId: NodeType mismatch. "
13476                 "Not secondary bucket (%u) page %Lu cache %S",
13477 		nodetype, pagepos, cache->filename);
13478 
13479     GBT_BUCKNENTRIES(buf,&nentries);
13480     if(nentries > cache->snperbucket)
13481 	ajFatal("SecbucketFindId: Bucket too full page: %Lu "
13482                 "entries: %u max: %u cache %S",
13483                 pagepos, nentries, cache->snperbucket,
13484                 cache->filename);
13485 
13486 
13487     GBT_BUCKOVERFLOW(buf,&overflow);
13488 
13489     kptr  = PBT_BUCKKEYLEN(buf);
13490     codeptr = kptr + (nentries * sizeof(ajuint));
13491 
13492     for(i=0;i<nentries;++i)
13493     {
13494 	BT_GETAJUINT(kptr,&len);
13495 
13496 /*
13497 //	if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
13498 //	{
13499 //	    /# ajDebug("SecbucketFindId: Overflow\n"); #/
13500 //	    page  = btreeSeccacheRead(cache,overflow);
13501 //	    buf = page->buf;
13502 //	    GBT_BUCKNODETYPE(buf,&nodetype);
13503 //	    if(nodetype != BT_SECBUCKET)
13504 //		ajFatal("SecbucketFindId: NodeType mismatch. "
13505 //                        "Not secondary bucket (%u) page: %Lu cache %S",
13506 //                        nodetype, pagepos, cache->filename);
13507 //	    GBT_BUCKOVERFLOW(buf,&overflow);
13508 //	    /# overflow bucket ids start at the keylen position #/
13509 //	    codeptr = PBT_BUCKKEYLEN(buf);
13510 //	}
13511 */
13512 
13513 	/* Fill ID objects */
13514         if(ajStrMatchC(id, (const char *)codeptr))
13515         {
13516             lpage->dirty = dirtysave;
13517             return ajTrue;
13518         }
13519 
13520 	codeptr += len;
13521 
13522 	kptr += sizeof(ajuint);
13523     }
13524 
13525     lpage->dirty = dirtysave;
13526 
13527     return ajFalse;
13528 }
13529 
13530 
13531 
13532 
13533 /* @funcstatic btreeSecbucketIdcount ******************************************
13534 **
13535 ** Counts secondary IDs in a bucket
13536 **
13537 ** @param [u] cache [AjPBtcache] cache
13538 ** @param [r] pagepos [ajulong] page number
13539 **
13540 ** @return [ajulong] Number of IDs
13541 **
13542 ** @release 6.5.0
13543 ** @@
13544 ******************************************************************************/
13545 
btreeSecbucketIdcount(AjPBtcache cache,ajulong pagepos)13546 static ajulong btreeSecbucketIdcount(AjPBtcache cache, ajulong pagepos)
13547 {
13548     AjPBtpage page  = NULL;
13549 
13550     unsigned char *buf  = NULL;
13551 
13552     ajuint  nodetype = 0U;
13553     ajuint  nentries = 0U;
13554 
13555     ajulong lcount = 0UL;
13556 
13557 /*
13558     if(pagepos == cache->secrootblock)
13559 	ajFatal("SecbucketIdcount: cannot read bucket from root page cache %S",
13560                cache->filename);
13561 */
13562 
13563     page  = btreeSeccacheRead(cache,pagepos);
13564 
13565     buf = page->buf;
13566 
13567     GBT_BUCKNODETYPE(buf,&nodetype);
13568     if(nodetype != BT_SECBUCKET)
13569 	ajFatal("SecbucketIdcount: NodeType mismatch. "
13570                 "Not secondary bucket (%u) cache %S",
13571 		nodetype, cache->filename);
13572 
13573     GBT_BUCKNENTRIES(buf,&nentries);
13574     if(nentries > cache->snperbucket)
13575 	ajFatal("SecbucketIdcount: Bucket too full page: %Lu "
13576                 "entries: %u max: %u full cache %S",
13577                 pagepos, nentries, cache->snperbucket, cache->filename);
13578 
13579     lcount = nentries;
13580 
13581     return lcount;
13582 }
13583 
13584 
13585 
13586 
13587 /* @funcstatic btreeSecbucketIdlist *******************************************
13588 **
13589 ** Copies all secondary IDs into a list
13590 **
13591 ** @param [u] cache [AjPBtcache] cache
13592 ** @param [r] pagepos [ajulong] page number
13593 ** @param [u] idlist [AjPList] list to hold secondary ID strings
13594 **
13595 ** @return [ajulong] Overflow
13596 **
13597 ** @release 6.4.0
13598 ** @@
13599 ******************************************************************************/
13600 
btreeSecbucketIdlist(AjPBtcache cache,ajulong pagepos,AjPList idlist)13601 static ajulong btreeSecbucketIdlist(AjPBtcache cache, ajulong pagepos,
13602                                     AjPList idlist)
13603 {
13604     AjPStr tmpstr = NULL;
13605 
13606     AjPBtpage page  = NULL;
13607     AjPBtpage lpage = NULL;
13608 
13609     unsigned char *buf  = NULL;
13610     unsigned char *kptr = NULL;
13611 
13612     unsigned char *codeptr = NULL;
13613 
13614     ajuint  nodetype = 0U;
13615     ajuint  nentries = 0U;
13616     ajulong overflow = 0UL;
13617     ajulong pageoverflow = 0UL;
13618     ajuint  dirtysave = 0U;
13619 
13620     ajuint  i;
13621     ajuint  len  = 0U;
13622 
13623     /* Put in test here for secondary root page read
13624        instead of !pagepos - done*/
13625 
13626 /*
13627     if(pagepos == cache->secrootblock)
13628 	ajFatal("SecbucketIdlist: cannot read bucket from root page cache %S",
13629                cache->filename);
13630 */
13631 
13632     page  = btreeSeccacheRead(cache,pagepos);
13633     lpage = page;
13634     dirtysave = lpage->dirty;
13635     lpage->dirty = BT_LOCK;
13636     lpage->lockfor = 1321;
13637 
13638     buf = lpage->buf;
13639 
13640     GBT_BUCKNODETYPE(buf,&nodetype);
13641     if(nodetype != BT_SECBUCKET)
13642 	ajFatal("SecbucketIdlist: NodeType mismatch. "
13643                 "Not secondary bucket (%u) cache %S",
13644 		nodetype, cache->filename);
13645 
13646     GBT_BUCKNENTRIES(buf,&nentries);
13647     if(nentries > cache->snperbucket)
13648 	ajFatal("SecbucketIdlist: Bucket too full page: %Lu "
13649                 "entries: %u max: %u full cache %S",
13650                 pagepos, nentries, cache->snperbucket, cache->filename);
13651 
13652 
13653     GBT_BUCKOVERFLOW(buf,&overflow);
13654     pageoverflow = overflow;
13655 
13656     kptr  = PBT_BUCKKEYLEN(buf);
13657     codeptr = kptr + (nentries * sizeof(ajuint));
13658 
13659     for(i=0;i<nentries;++i)
13660     {
13661 	BT_GETAJUINT(kptr,&len);
13662 
13663 /*
13664 //        if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
13665 //	{
13666 //	    page  = btreeSeccacheRead(cache,overflow);
13667 //	    buf = page->buf;
13668 //	    GBT_BUCKNODETYPE(buf,&nodetype);
13669 //	    if(nodetype != BT_SECBUCKET)
13670 //		ajFatal("SecbucketIdlist: NodeType mismatch. "
13671 //                        "Not secondary bucket (%u) cache %S",
13672 //                        nodetype, cache->filename);
13673 //	    GBT_BUCKOVERFLOW(buf,&overflow);
13674 //	    /# overflow bucket ids start at the keylen position #/
13675 //	    codeptr = PBT_BUCKKEYLEN(buf);
13676 //	}
13677 */
13678 
13679 	/* Fill ID objects */
13680         if(statSaveSecIdNext)
13681             tmpstr = statSaveSecId[--statSaveSecIdNext];
13682 	ajStrAssignLenC(&tmpstr,(const char *)codeptr, len-1);
13683         /*ajStrFmtLower(&tmpstr);*/ /* already lowercase */
13684         ajListPushAppend(idlist, tmpstr);
13685         tmpstr = NULL;
13686 	codeptr += len;
13687 
13688 	kptr += sizeof(ajuint);
13689     }
13690 
13691     lpage->dirty = dirtysave;
13692 
13693     return pageoverflow;
13694 }
13695 
13696 
13697 
13698 
13699 /* @funcstatic btreeReadSecbucket *********************************************
13700 **
13701 ** Constructor for keyword index secondary bucket given a disc page number
13702 ** Creates one empty key slot for possible addition
13703 **
13704 ** @param [u] cache [AjPBtcache] cache
13705 ** @param [r] pagepos [ajulong] page number
13706 **
13707 ** @return [AjPSecbucket] bucket
13708 **
13709 ** @release 3.0.0
13710 ** @@
13711 ******************************************************************************/
13712 
btreeReadSecbucket(AjPBtcache cache,ajulong pagepos)13713 static AjPSecbucket btreeReadSecbucket(AjPBtcache cache, ajulong pagepos)
13714 {
13715     AjPSecbucket bucket = NULL;
13716 
13717     AjPBtpage page  = NULL;
13718     AjPBtpage lpage = NULL;
13719 
13720     unsigned char *buf  = NULL;
13721     unsigned char *kptr = NULL;
13722 
13723     unsigned char *codeptr = NULL;
13724 
13725     ajuint  nodetype = 0U;
13726     ajuint nentries = 0U;
13727     ajulong overflow = 0UL;
13728     ajuint  dirtysave = 0U;
13729 
13730     ajuint  i;
13731     ajuint  len  = 0U;
13732 
13733     /* ajDebug("In btreeReadSecbucket\n"); */
13734 
13735     if(pagepos == cache->secrootblock)
13736 	ajFatal("btreeReadSecbucket: cannot read bucket from "
13737                 "root page cache %S",
13738                 cache->filename);
13739 
13740     page  = btreeSeccacheRead(cache,pagepos);
13741     lpage = page;
13742     dirtysave = lpage->dirty;
13743     lpage->dirty = BT_LOCK;
13744     lpage->lockfor = 1331;
13745 
13746     buf = lpage->buf;
13747 
13748     GBT_BUCKNODETYPE(buf,&nodetype);
13749 
13750     if(nodetype != BT_SECBUCKET)
13751 	ajFatal("SecReadBucket: NodeType mismatch. "
13752                 "Not secondary bucket (%u) cache %S",
13753 		nodetype, cache->filename);
13754 
13755     GBT_BUCKNENTRIES(buf,&nentries);
13756 
13757     if(nentries > cache->snperbucket)
13758 	ajFatal("SecReadBucket: Bucket too full page: %Lu "
13759                 "entries: %u max: %u secondary: %B cache %S\n",
13760                 pagepos, nentries, cache->snperbucket,
13761                 cache->secondary, cache->filename);
13762 
13763 
13764     GBT_BUCKOVERFLOW(buf,&overflow);
13765 
13766     bucket = btreeSecbucketNew(cache->snperbucket,cache->idlimit);
13767     bucket->Nentries = nentries;
13768 
13769     kptr  = PBT_BUCKKEYLEN(buf);
13770     codeptr = kptr + (nentries * sizeof(ajuint));
13771 
13772     for(i=0;i<nentries;++i)
13773     {
13774 	BT_GETAJUINT(kptr,&len);
13775 
13776 /*
13777 //	if((codeptr-buf+1) + len > cache->pagesize)	/# overflow #/
13778 //	{
13779 //#if AJINDEX_DEBUG
13780 //	    ajDebug("SecReadBucket: Overflow\n");
13781 //#endif
13782 //	    page  = btreeSeccacheRead(cache,overflow);
13783 //	    buf = page->buf;
13784 //	    GBT_BUCKNODETYPE(buf,&nodetype);
13785 //
13786 //	    if(nodetype != BT_SECBUCKET)
13787 //		ajFatal("SecReadBucket: NodeType mismatch. Not secondary "
13788 //			"bucket (%u)",nodetype);
13789 //
13790 //	    GBT_BUCKOVERFLOW(buf,&overflow);
13791 //	    /# overflow bucket ids start at the keylen position #/
13792 //	    codeptr = PBT_BUCKKEYLEN(buf);
13793 //	}
13794 */
13795 
13796 	/* Fill ID objects */
13797 	ajStrAssignLenC(&bucket->SecIds[i],(const char *)codeptr,len-1);
13798 	codeptr += len;
13799 
13800 	kptr += sizeof(ajuint);
13801     }
13802 
13803     lpage->dirty = dirtysave;
13804 
13805     return bucket;
13806 }
13807 
13808 
13809 
13810 
13811 /* @funcstatic btreeWriteSecbucket ********************************************
13812 **
13813 ** Write primary keyword index bucket object to the cache given a disc page
13814 ** number
13815 **
13816 ** @param [u] cache [AjPBtcache] cache
13817 ** @param [r] bucket [const AjPSecbucket] bucket
13818 ** @param [r] pagepos [ajulong] page number
13819 **
13820 ** @return [void]
13821 **
13822 ** @release 3.0.0
13823 ** @@
13824 ******************************************************************************/
13825 
btreeWriteSecbucket(AjPBtcache cache,const AjPSecbucket bucket,ajulong pagepos)13826 static void btreeWriteSecbucket(AjPBtcache cache, const AjPSecbucket bucket,
13827 				ajulong pagepos)
13828 {
13829     AjPBtpage page  = NULL;
13830     AjPBtpage lpage = NULL;
13831 
13832     unsigned char *buf  = NULL;
13833     unsigned char *lbuf = NULL;
13834 
13835     ajuint  v   = 0U;
13836     ajuint i   = 0U;
13837     ajuint len = 0U;
13838     ajulong lv  = 0UL;
13839 
13840     AjPStr sec = NULL;
13841     ajuint nentries = 0U;
13842     ajulong overflow = 0UL;
13843     unsigned char *keyptr = NULL;
13844     unsigned char *lptr   = NULL;
13845 
13846 /*  ajulong   pno = 0UL;*/
13847 
13848     /* ajDebug("In btreeWriteSecbucket\n"); */
13849 
13850     if(pagepos == cache->totsize)	/* Create a new page */
13851     {
13852 	/* pno = pagepos; */
13853 	page = btreeSeccacheBucketnew(cache);
13854 	buf = page->buf;
13855 	overflow = 0UL;
13856     }
13857     else
13858     {
13859 	page = btreeSeccacheRead(cache,pagepos);
13860 	buf = page->buf;
13861 	GBT_BUCKOVERFLOW(buf,&overflow);
13862     }
13863 
13864     v = BT_SECBUCKET;
13865     SBT_BUCKNODETYPE(buf,v);
13866 
13867     lbuf = buf;
13868     page->dirty = BT_LOCK;
13869     page->lockfor = 1341;
13870     lpage = page;
13871 
13872     nentries = bucket->Nentries;
13873     v = nentries;
13874     SBT_BUCKNENTRIES(buf,v);
13875 
13876     /* Write out key lengths */
13877     keyptr = PBT_BUCKKEYLEN(lbuf);
13878 
13879     for(i=0;i<nentries;++i)
13880     {
13881 /* these checks removed - pagesize dependency and already checked before call */
13882 /*
13883 //	if((ajuint)((keyptr-lbuf+1)+sizeof(ajuint)) > cache->pagesize)
13884 //	    ajFatal("btreeWriteSecbucket: Bucket cannot hold more than %u keys",
13885 //		    i-1);
13886 */
13887 
13888 	sec = bucket->SecIds[i];
13889 	/* Need to alter this if bucket primary keyword structure changes */
13890 	len = BT_BUCKSECLEN(sec);
13891         v = len;
13892 	BT_SETAJUINT(keyptr,v);
13893 	keyptr += sizeof(ajuint);
13894     }
13895 
13896 
13897     /* Write out IDs using overflow if necessary */
13898     lptr = keyptr;
13899 
13900     for(i=0;i<nentries;++i)
13901     {
13902 	sec = bucket->SecIds[i];
13903 	len = BT_BUCKSECLEN(sec);
13904 
13905 /*
13906 //	if((lptr-buf+1)+len > cache->pagesize) /# overflow #/
13907 //	{
13908 //#if AJINDEX_DEBUG
13909 //    	    ajDebug("btreeWriteSecbucket: Overflow\n");
13910 //#endif
13911 //
13912 //	    if(!overflow)		/# No overflow buckets yet #/
13913 //	    {
13914 //		pno = cache->totsize;
13915 //                lv = pno;
13916 //		SBT_BUCKOVERFLOW(buf,lv);
13917 //		page = btreeSeccacheBucketnew(cache);
13918 //		buf = page->buf;
13919 //		v = BT_SECBUCKET;
13920 //		SBT_BUCKNODETYPE(buf,v);
13921 //	    }
13922 //	    else
13923 //	    {
13924 //		page = btreeSeccacheRead(cache,overflow);
13925 //		buf  = page->buf;
13926 //		GBT_BUCKOVERFLOW(buf,&overflow);
13927 //	    }
13928 //
13929 //	    page->dirty = BT_DIRTY;
13930 //
13931 //	    lptr = PBT_BUCKKEYLEN(buf);
13932 //	}
13933 */
13934 
13935 	sprintf((char *)lptr,"%s",MAJSTRGETPTR(sec));
13936 	lptr += len;
13937     }
13938 
13939     lv = 0UL;
13940     SBT_BUCKOVERFLOW(buf,lv);
13941 
13942     lpage->dirty = BT_DIRTY;
13943 
13944     return;
13945 }
13946 
13947 
13948 
13949 
13950 /* @funcstatic btreeWriteSecbucketEmpty ***************************************
13951 **
13952 ** Write empty secondary keyword index bucket object to the cache given a
13953 ** disc page number
13954 **
13955 ** @param [u] cache [AjPBtcache] cache
13956 ** @param [r] pagepos [ajulong] page number
13957 **
13958 ** @return [void]
13959 **
13960 ** @release 6.4.0
13961 ** @@
13962 ******************************************************************************/
13963 
btreeWriteSecbucketEmpty(AjPBtcache cache,ajulong pagepos)13964 static void btreeWriteSecbucketEmpty(AjPBtcache cache, ajulong pagepos)
13965 {
13966     AjPBtpage page  = NULL;
13967     AjPBtpage lpage = NULL;
13968 
13969     unsigned char *buf  = NULL;
13970 
13971     ajuint  v   = 0U;
13972     ajulong lv  = 0UL;
13973 
13974     ajulong overflow = 0UL;
13975 
13976     if(pagepos == cache->totsize)	/* Create a new page */
13977     {
13978 	page = btreeSeccacheBucketnew(cache);
13979 	buf = page->buf;
13980 	overflow = 0UL;
13981 	lv = overflow;
13982 	SBT_BUCKOVERFLOW(buf,lv);
13983     }
13984     else
13985     {
13986 	page = btreeSeccacheRead(cache,pagepos);
13987 	buf = page->buf;
13988 	GBT_BUCKOVERFLOW(buf,&overflow);
13989     }
13990 
13991     v = BT_SECBUCKET;
13992     SBT_BUCKNODETYPE(buf,v);
13993 
13994     page->dirty = BT_LOCK;      /* cleared at end */
13995     page->lockfor = 1351;
13996     lpage = page;
13997 
13998     v = 0U;
13999     SBT_BUCKNENTRIES(buf,v);
14000 
14001     lv = 0UL;
14002     SBT_BUCKOVERFLOW(buf,lv);
14003 
14004     lpage->dirty = BT_DIRTY;    /* clear the lock */
14005 
14006     return;
14007 }
14008 
14009 
14010 
14011 
14012 /* @func ajBtreeSeccacheNewC **************************************************
14013 **
14014 ** Open a b+tree index file and initialise a cache object for keyword index
14015 **
14016 ** @param [r] filetxt [const char *] name of file
14017 ** @param [r] exttxt [const char *] extension of file
14018 ** @param [r] idirtxt [const char *] index file directory
14019 ** @param [r] mode [const char *] opening mode
14020 ** @param [r] compressed [AjBool] Compressed index flag
14021 ** @param [r] kwlimit [ajuint] Max key size
14022 ** @param [r] idlimit [ajuint] Max secondary id size
14023 ** @param [r] pripagesize [ajuint] pagesize
14024 ** @param [r] secpagesize [ajuint] secondary pagesize
14025 ** @param [r] pricachesize [ajuint] size of cache
14026 ** @param [r] seccachesize [ajuint] size of secondary cache
14027 ** @param [r] pripagecount [ajulong] page count
14028 ** @param [r] secpagecount [ajulong] page count
14029 ** @param [r] order [ajuint] Tree order
14030 ** @param [r] fill [ajuint] Number of entries per bucket
14031 ** @param [r] level [ajuint] level of tree
14032 ** @param [r] sorder [ajuint] order of secondary tree
14033 ** @param [r] sfill [ajuint] Number of entries per secondary bucket
14034 ** @param [r] count [ajulong] Number of entries in the index
14035 ** @param [r] countall [ajulong] Number of total entries in the index
14036 **
14037 ** @return [AjPBtcache] initialised disc block cache structure
14038 **
14039 ** @release 3.0.0
14040 ** @@
14041 ******************************************************************************/
14042 
ajBtreeSeccacheNewC(const char * filetxt,const char * exttxt,const char * idirtxt,const char * mode,AjBool compressed,ajuint kwlimit,ajuint idlimit,ajuint pripagesize,ajuint secpagesize,ajuint pricachesize,ajuint seccachesize,ajulong pripagecount,ajulong secpagecount,ajuint order,ajuint fill,ajuint level,ajuint sorder,ajuint sfill,ajulong count,ajulong countall)14043 AjPBtcache ajBtreeSeccacheNewC(const char *filetxt, const char *exttxt,
14044 			       const char *idirtxt, const char *mode,
14045                                AjBool compressed,
14046                                ajuint kwlimit, ajuint idlimit,
14047                                ajuint pripagesize, ajuint secpagesize,
14048                                ajuint pricachesize, ajuint seccachesize,
14049                                ajulong pripagecount, ajulong secpagecount,
14050                                ajuint order, ajuint fill, ajuint level,
14051 			       ajuint sorder, ajuint sfill,
14052 			       ajulong count, ajulong countall)
14053 {
14054     AjPBtcache cache = NULL;
14055     AjPBtpage  page = NULL;
14056 #if defined (usestat64)
14057     struct stat64 buf;
14058 #else
14059     struct stat buf;
14060 #endif
14061     ajulong filelen = 0UL;
14062     AjBool douncompress = ajFalse;
14063     AjBool writemode = ajFalse;
14064     AjBool okcache = ajTrue;
14065 
14066     AJNEW0(cache);
14067 
14068     cache->prilistLength = 0U;
14069     cache->seclistLength = 0U;
14070 
14071     cache->plru   = NULL;
14072     cache->pmru   = NULL;
14073     cache->slru   = NULL;
14074     cache->smru   = NULL;
14075 
14076     cache->replace    = ajStrNew();
14077     cache->numreplace = 0UL;
14078 
14079     if(pripagesize>0)
14080 	cache->pripagesize = pripagesize;
14081     else
14082 	cache->pripagesize = BT_PAGESIZE;
14083 
14084     if(secpagesize>0)
14085 	cache->secpagesize = secpagesize;
14086     else
14087 	cache->secpagesize = cache->pripagesize;
14088 
14089     cache->plevel       = level;
14090     cache->porder       = order;
14091     cache->pnperbucket  = fill;
14092 
14093     if(pricachesize > 0)
14094         cache->pricachesize = pricachesize;
14095     else
14096         cache->pricachesize = BT_CACHESIZE;
14097 
14098     cache->pripagecount = pripagecount;
14099     cache->secpagecount = secpagecount;
14100 
14101     cache->slevel = 0;
14102     cache->sorder = sorder;
14103     cache->snperbucket = sfill;
14104     if(seccachesize > 0)
14105         cache->seccachesize = seccachesize;
14106     else
14107         cache->seccachesize = cache->pricachesize;
14108 
14109     cache->countunique = count;
14110     cache->countall = countall;
14111     cache->keylimit = kwlimit;
14112     cache->idlimit = idlimit;
14113     cache->compressed = compressed;
14114 
14115     cache->bmem = NULL;
14116     cache->tmem = NULL;
14117 
14118     cache->bsmem = NULL;
14119     cache->tsmem = NULL;
14120 
14121     cache->secondary = ajTrue;
14122 
14123     cache->pripagetable = ajTablelongNewConst(cache->pricachesize);
14124     cache->secpagetable = ajTablelongNewConst(cache->seccachesize);
14125 
14126     cache->basename = ajStrNewC(exttxt);
14127 
14128     cache->filename = ajStrNew();
14129     if(!*idirtxt)
14130         ajFmtPrintS(&cache->filename,"%s.%s",filetxt,exttxt);
14131     else if(idirtxt[strlen(idirtxt)-1] == SLASH_CHAR)
14132         ajFmtPrintS(&cache->filename,"%s%s.%s",idirtxt,filetxt,exttxt);
14133     else
14134         ajFmtPrintS(&cache->filename,"%s%s%s.%s",
14135 		    idirtxt,SLASH_STRING,filetxt,exttxt);
14136 
14137     if(cache->porder < 4)
14138     {
14139         ajErr("cache '%S' pagesize %u order %u too small, increase pagesize",
14140               cache->filename, cache->pripagesize, cache->porder);
14141         okcache = ajFalse;
14142     }
14143 
14144     if(cache->pnperbucket < 4)
14145     {
14146         ajErr("cache '%S' pagesize %u fill %u too small, increase pagesize",
14147               cache->filename, cache->pripagesize, cache->pnperbucket);
14148         okcache = ajFalse;
14149     }
14150 
14151     if(cache->sorder < 4)
14152     {
14153         ajErr("cache '%S' pagesize %u sorder %u too small, increase pagesize",
14154               cache->filename, cache->pripagesize, cache->sorder);
14155         okcache = ajFalse;
14156     }
14157 
14158     if(cache->snperbucket < 4)
14159     {
14160         ajErr("cache '%S' pagesize %u sfill %u too small, increase pagesize",
14161               cache->filename, cache->secpagesize, cache->snperbucket);
14162         okcache = ajFalse;
14163     }
14164 
14165     if(!okcache)
14166         return NULL;
14167 
14168     cache->fp = fopen(MAJSTRGETPTR(cache->filename),mode);
14169     if(!cache->fp)
14170 	return NULL;
14171 
14172     /* Commented out pending database updating */
14173     if(ajCharMatchC(mode, "rb"))
14174     {
14175 #if defined (usestat64)
14176 	if(!stat64(MAJSTRGETPTR(cache->filename), &buf))
14177 #else
14178         if(!stat(MAJSTRGETPTR(cache->filename), &buf))
14179 #endif
14180             filelen = buf.st_size;
14181 
14182         cache->readonly = ajTrue;
14183     }
14184     else if(ajCharMatchC(mode, "rb+"))
14185     {
14186 #if defined (usestat64)
14187 	if(!stat64(MAJSTRGETPTR(cache->filename), &buf))
14188 #else
14189         if(!stat(MAJSTRGETPTR(cache->filename), &buf))
14190 #endif
14191             filelen = buf.st_size;
14192         if(compressed)
14193             douncompress = ajTrue;
14194     }
14195     else if(ajCharMatchC(mode, "wb+")) /* create */
14196     {
14197         writemode = ajTrue;
14198     }
14199     else
14200     {
14201         ajWarn("ajBtreeSecCacheNewC unknown mode '%s'", mode);
14202     }
14203 
14204     cache->totsize    = filelen;
14205     cache->filesize  = filelen;
14206     cache->maxsize = filelen;
14207 
14208     if(writemode)
14209     {
14210         if(cache->maxsize)
14211             cache->maxsize += cache->maxsize/2;
14212         else
14213             cache->maxsize = cache->pricachesize*cache->pripagesize +
14214                 cache->seccachesize * cache->secpagesize;
14215         if(ftruncate(fileno(cache->fp), cache->maxsize) == -1)
14216         {
14217         }
14218     }
14219 
14220     if(douncompress)
14221         btreeCacheUncompress(cache);
14222 
14223     /* create or lock the root page */
14224 
14225     if(writemode)
14226         btreePrirootCreate(cache);
14227     else
14228     {
14229         page = btreePricacheRead(cache,0UL);
14230         page->dirty = BT_LOCK;
14231     }
14232 
14233     return cache;
14234 }
14235 
14236 
14237 
14238 
14239 /* @func ajBtreeSeccacheNewS **************************************************
14240 **
14241 ** Open a b+tree index file and initialise a cache object for keyword index
14242 **
14243 ** @param [r] file [const AjPStr] name of file
14244 ** @param [r] ext [const AjPStr] extension of file
14245 ** @param [r] idir [const AjPStr] index file directory
14246 ** @param [r] mode [const char *] opening mode
14247 ** @param [r] compressed [AjBool] Compressed index flag
14248 ** @param [r] kwlimit [ajuint] Max key size
14249 ** @param [r] idlimit [ajuint] Max secondary id size
14250 ** @param [r] pripagesize [ajuint] Primary pagesize
14251 ** @param [r] secpagesize [ajuint] Secondary pagesize
14252 ** @param [r] pricachesize [ajuint] size of primary cache
14253 ** @param [r] seccachesize [ajuint] size of secondary cache
14254 ** @param [r] pripagecount [ajulong] Primary page count
14255 ** @param [r] secpagecount [ajulong] Secondary page count
14256 ** @param [r] order [ajuint] Tree order
14257 ** @param [r] fill [ajuint] Number of entries per bucket
14258 ** @param [r] level [ajuint] level of tree
14259 ** @param [r] sorder [ajuint] order of secondary tree
14260 ** @param [r] sfill [ajuint] Number of entries per secondary bucket
14261 ** @param [r] count [ajulong] Number of entries in the index
14262 ** @param [r] countall [ajulong] Number of total entries in the index
14263 **
14264 ** @return [AjPBtcache] initialised disc block cache structure
14265 **
14266 ** @release 6.4.0
14267 ** @@
14268 ******************************************************************************/
14269 
ajBtreeSeccacheNewS(const AjPStr file,const AjPStr ext,const AjPStr idir,const char * mode,AjBool compressed,ajuint kwlimit,ajuint idlimit,ajuint pripagesize,ajuint secpagesize,ajuint pricachesize,ajuint seccachesize,ajulong pripagecount,ajulong secpagecount,ajuint order,ajuint fill,ajuint level,ajuint sorder,ajuint sfill,ajulong count,ajulong countall)14270 AjPBtcache ajBtreeSeccacheNewS(const AjPStr file, const AjPStr ext,
14271 			       const AjPStr idir, const char *mode,
14272                                AjBool compressed,
14273                                ajuint kwlimit, ajuint idlimit,
14274                                ajuint pripagesize, ajuint secpagesize,
14275                                ajuint pricachesize, ajuint seccachesize,
14276                                ajulong pripagecount, ajulong secpagecount,
14277                                ajuint order, ajuint fill, ajuint level,
14278 			       ajuint sorder, ajuint sfill,
14279 			       ajulong count, ajulong countall)
14280 {
14281     return ajBtreeSeccacheNewC(MAJSTRGETPTR(file), MAJSTRGETPTR(ext),
14282                                MAJSTRGETPTR(idir), mode,
14283                                compressed, kwlimit, idlimit,
14284                                pripagesize, secpagesize,
14285                                pricachesize, seccachesize,
14286                                pripagecount, secpagecount,
14287                                order, fill, level,
14288                                sorder, sfill,
14289                                count, countall);
14290 }
14291 
14292 
14293 
14294 
14295 /* @funcstatic btreeSecSplitleaf **********************************************
14296 **
14297 ** Split a leaf and propagate up if necessary
14298 **
14299 ** @param [u] cache [AjPBtcache] cache
14300 ** @param [u] spage [AjPBtpage] page
14301 **
14302 ** @return [AjPBtpage] pointer to a parent page
14303 **
14304 ** @release 3.0.0
14305 ** @@
14306 ******************************************************************************/
14307 
btreeSecSplitleaf(AjPBtcache cache,AjPBtpage spage)14308 static AjPBtpage btreeSecSplitleaf(AjPBtcache cache, AjPBtpage spage)
14309 {
14310     ajuint nkeys     = 0U;
14311     ajuint order     = 0U;
14312     ajuint totalkeys = 0U;
14313     ajuint keylimit  = 0U;
14314     ajuint nodetype  = 0U;
14315 
14316     ajuint rootnodetype = 0U;
14317 
14318     ajuint i;
14319     ajuint j;
14320 
14321     AjPBtpage lpage = NULL;
14322     AjPBtpage rpage = NULL;
14323     AjPBtpage page  = NULL;
14324 
14325     AjPStr mediankey  = NULL;
14326     ajulong mediangtr  = 0UL;
14327     ajulong medianless = 0UL;
14328 
14329     AjPStr bid = NULL;
14330 
14331     unsigned char *buf  = NULL;
14332     unsigned char *lbuf = NULL;
14333     unsigned char *rbuf = NULL;
14334 
14335     AjPList idlist = NULL;
14336 
14337     AjPSecbucket cbucket  = NULL;
14338 
14339     ajulong *parray = NULL;
14340     AjPBtMem arrays = NULL;
14341     AjPBtMem newarrays = NULL;
14342     AjPStr *newkarray = NULL;
14343     ajulong *newparray = NULL;
14344 
14345     ajuint lno    = 0U;
14346     ajuint rno    = 0U;
14347 
14348     ajuint lbucketlimit   = 0U;
14349     ajuint rbucketlimit   = 0U;
14350     ajuint lmaxnperbucket = 0U;
14351     ajuint rmaxnperbucket = 0U;
14352     ajuint count         = 0U;
14353 
14354     ajulong lblockno = 0UL;
14355     ajulong rblockno = 0UL;
14356     ajulong overflow = 0UL;
14357     ajulong prevsave = 0UL;
14358 
14359     ajulong zero = 0UL;
14360     ajulong join = 0UL;
14361 
14362     ajulong lv = 0UL;
14363     ajuint  v  = 0U;
14364     ajuint newmax;
14365     ajuint iold = 0U;
14366 
14367 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
14368     ajDebug("btreeSecSplitleaf '%S' %Lu id '%S' key '%S'\n",
14369             cache->basename, spage->pagepos, indexId, indexKeyword);
14370 #endif
14371     ++statCallSecSplitleaf;
14372 
14373     if(!statSaveSecId)
14374     {
14375         statSaveSecIdMax = 2048;
14376         AJCNEW0(statSaveSecId, statSaveSecIdMax);
14377         statSaveSecIdNext = 0;
14378     }
14379 
14380     order = cache->sorder;
14381 
14382     mediankey = ajStrNew();
14383 
14384     arrays = btreeAllocSecArray(cache);
14385     parray = arrays->parray;
14386     newarrays = btreeAllocSecArray(cache);
14387     newkarray = newarrays->karray;
14388     newparray = newarrays->parray;
14389 
14390     buf = spage->buf;
14391     lbuf = buf;
14392 
14393     GBT_NKEYS(buf,&nkeys);
14394     GBT_NODETYPE(buf,&rootnodetype);
14395 
14396     if(rootnodetype == BT_SECROOT)
14397     {
14398 	lblockno = cache->totsize;
14399 	lpage = btreeSeccacheNodenew(cache);
14400 	lbuf = lpage->buf;
14401 	lv = cache->secrootblock;
14402 	SBT_PREV(lbuf,lv);
14403     }
14404     else
14405     {
14406 	lblockno = spage->pagepos;
14407 	lpage = spage;
14408     }
14409 
14410     lpage->dirty = BT_LOCK;
14411     lpage->lockfor = 1361;
14412 
14413     rblockno = cache->totsize;
14414     rpage = btreeSeccacheNodenew(cache);
14415     rbuf = rpage->buf;
14416     rpage->dirty = BT_LOCK;
14417     rpage->lockfor = 1362;
14418 
14419     if(rootnodetype == BT_SECROOT)
14420     {
14421 	lv = zero;
14422 	SBT_RIGHT(rbuf,lv);
14423 	lv = zero;
14424 	SBT_LEFT(lbuf,lv);
14425     }
14426     else
14427     {
14428 	GBT_RIGHT(lbuf,&join);
14429 	lv = join;
14430 	SBT_RIGHT(rbuf,lv);
14431     }
14432 
14433     lv = lblockno;
14434     SBT_LEFT(rbuf,lv);
14435     lv = rblockno;
14436     SBT_RIGHT(lbuf,lv);
14437 
14438 
14439     btreeGetPointers(cache,buf,&parray);
14440 
14441 
14442     keylimit = nkeys+1;
14443 
14444     idlist = ajListNew();
14445 
14446     for(i=0;i<keylimit;++i)
14447         btreeSecbucketIdlist(cache, parray[i], idlist);
14448 
14449     ajListSort(idlist, &btreeKeywordIdCompare);
14450 
14451 
14452     totalkeys = (ajuint) ajListGetLength(idlist);
14453 
14454     btreeBucketSplitCalc(totalkeys, keylimit, cache->snperbucket,
14455                          &lbucketlimit,&lmaxnperbucket,&lno,
14456                          &rbucketlimit,&rmaxnperbucket,&rno);
14457 
14458     cbucket = btreeSecbucketNew(cache->snperbucket,cache->idlimit);
14459 
14460     count = 0;
14461     iold=0;
14462 
14463     for(i=0;i<lbucketlimit;++i)
14464     {
14465 	cbucket->Nentries = 0;
14466 
14467 	for(j=0;j<lmaxnperbucket;++j)
14468 	{
14469 	    ajListPop(idlist,(void **)&bid);
14470 
14471 
14472 	    ajStrAssignS(&cbucket->SecIds[j],bid);
14473 
14474 	    cbucket->keylen[j] = BT_BUCKSECLEN(bid);
14475 	    ++count;
14476 	    ++cbucket->Nentries;
14477             if(statSaveSecIdNext >= statSaveSecIdMax)
14478             {
14479                 newmax = statSaveSecIdMax + statSaveSecIdMax;
14480                 AJCRESIZE0(statSaveSecId,statSaveSecIdMax,newmax);
14481                 statSaveSecIdMax = newmax;
14482             }
14483             statSaveSecId[statSaveSecIdNext++] = bid;
14484             bid = NULL;
14485 	}
14486 
14487 	ajListPeek(idlist,(void **)&bid);
14488 
14489 	ajStrAssignS(&newkarray[i],bid);
14490 
14491 	if((iold < order) && parray[iold])
14492             newparray[i] = parray[iold++];
14493         else
14494 	    newparray[i] = cache->totsize;
14495 	btreeWriteSecbucket(cache,cbucket,newparray[i]);
14496     }
14497 
14498     cbucket->Nentries = 0;
14499 
14500     j = 0;
14501 
14502     while(count != lno)
14503     {
14504 	ajListPop(idlist,(void **)&bid);
14505 
14506 	ajStrAssignS(&cbucket->SecIds[j],bid);
14507 	++j;
14508 	++count;
14509 
14510 
14511 	++cbucket->Nentries;
14512 	ajStrDel(&bid);
14513     }
14514 
14515     if((iold < order) && parray[iold])
14516         newparray[i] = parray[iold++];
14517     else
14518         newparray[i] = cache->totsize;
14519 
14520     btreeWriteSecbucket(cache,cbucket,newparray[i]);
14521 
14522     nkeys = lbucketlimit;
14523 
14524     nodetype = BT_SECLEAF;
14525     v = nodetype;
14526     SBT_NODETYPE(lbuf,v);
14527 
14528     GBT_PREV(lbuf,&prevsave);
14529     lpage->dirty = BT_DIRTY;
14530 
14531     btreeWriteNode(cache,lpage,newkarray,newparray,nkeys);
14532 
14533     ajListPeek(idlist,(void **)&bid);
14534     ajStrAssignS(&mediankey,bid);
14535 
14536     for(i=0;i<rbucketlimit;++i)
14537     {
14538 	cbucket->Nentries = 0;
14539 
14540 	for(j=0;j<rmaxnperbucket;++j)
14541 	{
14542 	    ajListPop(idlist,(void **)&bid);
14543 
14544 
14545 	    ajStrAssignS(&cbucket->SecIds[j],bid);
14546 
14547 	    cbucket->keylen[j] = BT_BUCKSECLEN(bid);
14548 	    ++cbucket->Nentries;
14549 	    ajStrDel(&bid);
14550 	}
14551 
14552 	ajListPeek(idlist,(void **)&bid);
14553 	ajStrAssignS(&newkarray[i],bid);
14554 
14555 	if((iold < order) && parray[iold])
14556             newparray[i] = parray[iold++];
14557         else
14558             newparray[i] = cache->totsize;
14559 
14560 	btreeWriteSecbucket(cache,cbucket,newparray[i]);
14561     }
14562 
14563     cbucket->Nentries = 0;
14564 
14565     j = 0;
14566 
14567     while(ajListPop(idlist,(void**)&bid))
14568     {
14569 	ajStrAssignS(&cbucket->SecIds[j],bid);
14570 	++j;
14571 
14572 
14573 	++cbucket->Nentries;
14574 	ajStrDel(&bid);
14575     }
14576 
14577     if((iold < order) && parray[iold])
14578         newparray[i] = parray[iold++];
14579     else
14580         newparray[i] = cache->totsize;
14581     btreeWriteSecbucket(cache,cbucket,newparray[i]);
14582 
14583     nkeys = rbucketlimit;
14584 
14585     nodetype = BT_SECLEAF;
14586     v = nodetype;
14587     SBT_NODETYPE(rbuf,v);
14588 
14589     lv = prevsave;
14590     SBT_PREV(rbuf,lv);
14591     lv = overflow;
14592     SBT_OVERFLOW(rbuf,lv);
14593 
14594     btreeWriteNode(cache,rpage,newkarray,newparray,nkeys);
14595     rpage->dirty = BT_DIRTY;
14596 
14597     btreeSecbucketDel(&cbucket);
14598     ajListFree(&idlist);
14599 
14600 
14601 
14602     medianless = lblockno;
14603     mediangtr  = rblockno;
14604 
14605     btreeDeallocSecArray(cache,arrays);
14606     btreeDeallocSecArray(cache,newarrays);
14607 
14608     if(rootnodetype == BT_SECROOT)
14609     {
14610 	spage->dirty = BT_DIRTY;
14611 
14612 	btreeWriteNodeSingle(cache,spage,mediankey,lblockno,rblockno);
14613 
14614 	++cache->slevel;
14615 	lv = cache->slevel;
14616 	SBT_RIGHT(buf,lv);
14617 	spage->dirty = BT_LOCK;
14618         spage->lockfor = 1363;
14619 
14620 	ajStrDel(&mediankey);
14621 
14622 	return spage;
14623     }
14624 
14625 
14626     page = btreeSeccacheRead(cache,prevsave);
14627     btreeInsertKeySec(cache,page,mediankey,medianless,mediangtr);
14628     ajStrDel(&mediankey);
14629 
14630     page = btreeSeccacheRead(cache,prevsave);
14631 
14632     return page;
14633 }
14634 
14635 
14636 
14637 
14638 /* @funcstatic btreeKeywordIdCompare ******************************************
14639 **
14640 ** Comparison function for ajListSort
14641 **
14642 ** @param [r] a [const void*] ID 1
14643 ** @param [r] b [const void*] ID 2
14644 **
14645 ** @return [ajint] 0 = bases match
14646 **
14647 ** @release 3.0.0
14648 ** @@
14649 ******************************************************************************/
14650 
btreeKeywordIdCompare(const void * a,const void * b)14651 static ajint btreeKeywordIdCompare(const void *a, const void *b)
14652 {
14653     return MAJSTRCMPS((*(AjPStr const *)a),
14654                       (*(AjPStr const *)b));
14655 }
14656 
14657 
14658 
14659 
14660 /* @funcstatic btreeSecbucketNew **********************************************
14661 **
14662 ** Construct a primary keyword secondary bucket object
14663 **
14664 ** @param [r] n [ajuint] Number of IDs
14665 ** @param [r] idlen [ajuint] Maximum size of IDs
14666 **
14667 ** @return [AjPSecbucket] initialised disc block cache structure
14668 **
14669 ** @release 3.0.0
14670 ** @@
14671 ******************************************************************************/
14672 
btreeSecbucketNew(ajuint n,ajuint idlen)14673 static AjPSecbucket btreeSecbucketNew(ajuint n, ajuint idlen)
14674 {
14675     AjPSecbucket bucket = NULL;
14676     ajuint i;
14677 
14678     /* ajDebug("In btreeSecbucketNew\n"); */
14679 
14680     if(n)
14681     {
14682         if(statSaveSecbucketNext)
14683         {
14684             bucket = statSaveSecbucket[--statSaveSecbucketNext];
14685             for(i=0;i<bucket->Maxentries;++i)
14686                 MAJSTRASSIGNCLEAR(&bucket->SecIds[i]);
14687             if(n > bucket->Maxentries)
14688             {
14689                 AJCRESIZE0(bucket->keylen,bucket->Maxentries,n);
14690                 AJCRESIZE0(bucket->SecIds,bucket->Maxentries,n);
14691                 for(i=bucket->Maxentries;i<n;++i)
14692                     bucket->SecIds[i] = ajStrNewRes(idlen+1);
14693                 bucket->Maxentries = n;
14694             }
14695 
14696         }
14697         else
14698         {
14699             AJNEW0(bucket);
14700 
14701             AJCNEW0(bucket->SecIds,n);
14702             AJCNEW0(bucket->keylen,n);
14703             for(i=0;i<n;++i)
14704                 bucket->SecIds[i] = ajStrNewRes(idlen+1);
14705             bucket->Maxentries = n;
14706         }
14707     }
14708     else
14709     {
14710         if(statSaveSecbucketEmptyNext)
14711             bucket = statSaveSecbucketEmpty[--statSaveSecbucketEmptyNext];
14712         else
14713             AJNEW0(bucket);
14714     }
14715 
14716     bucket->NodeType = BT_SECBUCKET;
14717     bucket->Nentries = n;
14718     bucket->Overflow = 0UL;
14719 
14720     return bucket;
14721 }
14722 
14723 
14724 
14725 
14726 /* @funcstatic btreeSecbucketDel **********************************************
14727 **
14728 ** Delete a keyword primary bucket object
14729 **
14730 ** @param [w] thys [AjPSecbucket*] bucket
14731 **
14732 ** @return [void]
14733 **
14734 ** @release 3.0.0
14735 ** @@
14736 ******************************************************************************/
14737 
btreeSecbucketDel(AjPSecbucket * thys)14738 static void btreeSecbucketDel(AjPSecbucket *thys)
14739 {
14740     AjPSecbucket pthis = NULL;
14741     ajuint newmax;
14742 
14743     /* ajDebug("In btreeSecbucketDel\n"); */
14744 
14745     if(!thys || !*thys)
14746 	return;
14747 
14748     pthis = *thys;
14749 
14750     if(!statSaveSecbucket)
14751     {
14752         statSaveSecbucketMax=2048;
14753         statSaveSecbucketNext=0;
14754         AJCNEW0(statSaveSecbucket,statSaveSecbucketMax);
14755     }
14756 
14757     if(!statSaveSecbucketEmpty)
14758     {
14759         statSaveSecbucketEmptyMax=2048;
14760         statSaveSecbucketEmptyNext=0;
14761         AJCNEW0(statSaveSecbucketEmpty,statSaveSecbucketEmptyMax);
14762     }
14763 
14764 
14765     /*
14766     statCountSecbucketDel++;
14767     statCountSecbucket--;
14768     if(pthis->Nentries > statMaxSecbucket)
14769       statMaxSecbucket = pthis->Nentries;
14770     */
14771 
14772     if(pthis->Maxentries)
14773     {
14774         /*statReusedSecbucket++;
14775           statUsedSecbucket--;*/
14776         if(statSaveSecbucketNext >= statSaveSecbucketMax)
14777         {
14778             newmax = statSaveSecbucketMax + statSaveSecbucketMax;
14779             AJCRESIZE0(statSaveSecbucket,statSaveSecbucketMax,newmax);
14780             statSaveSecbucketMax = newmax;
14781         }
14782 
14783         statSaveSecbucket[statSaveSecbucketNext++] = pthis;
14784         /*if(ajListGetLength(statListSecbucket) > statMaxFreeSecbucket)
14785           statMaxFreeSecbucket = ajListGetLength(statListSecbucket);*/
14786     }
14787     else
14788     {
14789         if(statSaveSecbucketEmptyNext >= statSaveSecbucketEmptyMax)
14790         {
14791             newmax = statSaveSecbucketEmptyMax + statSaveSecbucketEmptyMax;
14792             AJCRESIZE0(statSaveSecbucketEmpty,statSaveSecbucketEmptyMax,newmax);
14793             statSaveSecbucketEmptyMax = newmax;
14794         }
14795         statSaveSecbucketEmpty[statSaveSecbucketEmptyNext++] = pthis;
14796         /*statReusedSecbucketEmpty++;
14797           statUsedSecbucketEmpty--;*/
14798         /*if(ajListGetLength(statListSecbucketEmpty) > statMaxFreeSecbucketEmpty)
14799           statMaxFreeSecbucketEmpty = ajListGetLength(statListSecbucketEmpty);*/
14800     }
14801 
14802     pthis = NULL;
14803 
14804     *thys = NULL;
14805 
14806     return;
14807 }
14808 
14809 
14810 
14811 
14812 /* @funcstatic btreeSecbucketFree *********************************************
14813 **
14814 ** Delete a keyword primary bucket object
14815 **
14816 ** @param [w] thys [AjPSecbucket*] bucket
14817 **
14818 ** @return [void]
14819 **
14820 ** @release 6.4.0
14821 ** @@
14822 ******************************************************************************/
14823 
btreeSecbucketFree(AjPSecbucket * thys)14824 static void btreeSecbucketFree(AjPSecbucket *thys)
14825 {
14826     AjPSecbucket pthis = NULL;
14827     ajuint n;
14828     ajuint i;
14829 
14830     /* ajDebug("In btreeSecbucketFree\n"); */
14831 
14832     if(!thys || !*thys)
14833 	return;
14834 
14835     pthis = *thys;
14836     n = pthis->Maxentries;
14837 
14838     for(i=0;i<n;++i)
14839 	ajStrDel(&pthis->SecIds[i]);
14840 
14841     AJFREE(pthis->keylen);
14842     AJFREE(pthis->SecIds);
14843 
14844     AJFREE(pthis);
14845 
14846     *thys = NULL;
14847 
14848     return;
14849 }
14850 
14851 
14852 
14853 
14854 /* @funcstatic btreeKeyidInsert ***********************************************
14855 **
14856 ** Insert an ID into the secondary keyword tree
14857 **
14858 ** Depends on secrootblock being set before the function is called
14859 **
14860 ** @param [u] cache [AjPBtcache] cache
14861 ** @param [r] id [const AjPStr] Id
14862 **
14863 ** @return [AjBool] True if ID was inserted
14864 **                  False if ID already exists
14865 **
14866 ** @release 6.5.0
14867 ** @@
14868 ******************************************************************************/
14869 
btreeKeyidInsert(AjPBtcache cache,const AjPStr id)14870 static AjBool btreeKeyidInsert(AjPBtcache cache, const AjPStr id)
14871 {
14872     AjPBtpage spage  = NULL;
14873 
14874     ajulong lblockno = 0UL;
14875     ajulong rblockno = 0UL;
14876     ajulong blockno  = 0UL;
14877     ajulong shift    = 0UL;
14878 
14879     ajuint nkeys = 0U;
14880 
14881     ajuint nodetype = 0U;
14882 
14883     unsigned char *buf = NULL;
14884 
14885     ajuint n;
14886 
14887     AjBool exists  = ajFalse;
14888 
14889     /* ajDebug("In btreeKeyidInsert\n"); */
14890 
14891 
14892     if(!MAJSTRGETLEN(id))
14893         return ajFalse;
14894 
14895     /* Only insert an ID if it doesn't exist */
14896     exists = btreeKeyidExists(cache,id);
14897 
14898     if(exists)
14899 	return ajFalse;
14900 
14901     spage = btreeKeyidFind(cache,id);
14902     buf = spage->buf;
14903 
14904     GBT_NODETYPE(buf,&nodetype);
14905     if(nodetype == BT_SECBUCKET)
14906     {
14907         blockno = spage->pagepos;
14908         n = btreeNumInSecbucket(cache,blockno);
14909         if(n < cache->snperbucket)
14910         {
14911             btreeSecbucketAdd(cache,blockno,id);
14912             return ajTrue;
14913         }
14914 
14915         /*
14916         ** make a list of the IDs in the bucket
14917         ** make a new root
14918         ** split the IDs into the two buckets
14919         */
14920 
14921         btreeKeyidMakeroot(cache, spage);
14922     }
14923 
14924     GBT_NKEYS(buf,&nkeys);
14925     if(!nkeys)
14926     {
14927         lblockno = cache->totsize;
14928         btreeWriteSecbucketEmpty(cache,lblockno);
14929 
14930         rblockno = cache->totsize;
14931         btreeWriteSecbucketEmpty(cache,rblockno);
14932 
14933         btreeWriteNodeSingle(cache,spage,id,lblockno,rblockno);
14934 
14935         btreeSecbucketAdd(cache,rblockno,id);
14936 
14937         return ajTrue;
14938     }
14939 
14940     blockno = btreeGetBlockS(cache,buf,id);
14941 
14942     if(nodetype != BT_SECROOT)
14943         if((shift = btreeKeyidInsertShift(cache,&spage,id)))
14944             blockno = shift;
14945 
14946     buf = spage->buf;
14947 
14948     n = btreeNumInSecbucket(cache,blockno);
14949 
14950     if(n == cache->snperbucket)
14951     {
14952 	if(btreeSecbucketsReorder(cache,spage))
14953 	{
14954             blockno = btreeGetBlockS(cache,buf,id);
14955 	}
14956 	else
14957 	{
14958 	    btreeSecSplitleaf(cache,spage);
14959 
14960 	    spage = btreeKeyidFind(cache,id);
14961 	    buf = spage->buf;
14962 
14963             blockno = btreeGetBlockS(cache,buf,id);
14964 	}
14965     }
14966 
14967 
14968     btreeSecbucketAdd(cache,blockno,id);
14969 
14970     return ajTrue;
14971 }
14972 
14973 
14974 
14975 
14976 /* @funcstatic btreeKeyidExists ***********************************************
14977 **
14978 ** See whether ID already exists in the tree
14979 **
14980 ** @param [u] cache [AjPBtcache] cache
14981 ** @param [r] key [const AjPStr] key
14982 **
14983 ** @return [AjBool] true if ID already added
14984 **
14985 ** @release 6.4.0
14986 ** @@
14987 ******************************************************************************/
14988 
btreeKeyidExists(AjPBtcache cache,const AjPStr key)14989 static AjBool btreeKeyidExists(AjPBtcache cache, const AjPStr key)
14990 {
14991     AjPBtpage page      = NULL;
14992     unsigned char *buf = NULL;
14993 
14994     ajuint nkeys    = 0U;
14995     ajuint nodetype;
14996 
14997     ajulong blockno = 0UL;
14998     AjBool found   = ajFalse;
14999 
15000     /* ajDebug("In btreeKeyidExists %u\n",cache->count); */
15001 
15002     if(!cache->countunique)
15003 	return ajFalse;
15004 
15005     page = btreeKeyidFind(cache,key);
15006     buf = page->buf;
15007     GBT_NODETYPE(buf, &nodetype);
15008 
15009     if(nodetype == BT_SECBUCKET)
15010     {
15011         blockno = page->pagepos;
15012     }
15013     else
15014     {
15015         GBT_NKEYS(buf,&nkeys);
15016 
15017         if(!nkeys)
15018             return ajFalse;
15019 
15020         blockno = btreeGetBlockS(cache,buf,key);
15021     }
15022 
15023     found = btreeSecbucketFindId(cache, blockno, key);
15024 
15025     return found;
15026 }
15027 
15028 
15029 
15030 
15031 /* @func btreeKeyidFind *************************************************
15032 **
15033 ** Find the node that should contain a new key for insertion
15034 **
15035 ** @param [u] cache [AjPBtcache] cache
15036 ** @param [r] key [const AjPStr] key to search for
15037 **
15038 ** @return [AjPBtpage] leaf node where item should be inserted
15039 **
15040 ** @release 3.0.0
15041 ** @@
15042 ******************************************************************************/
15043 
btreeKeyidFind(AjPBtcache cache,const AjPStr key)15044 AjPBtpage btreeKeyidFind(AjPBtcache cache, const AjPStr key)
15045 {
15046     AjPBtpage root = NULL;
15047     AjPBtpage ret  = NULL;
15048     ajuint nodetype;
15049 
15050     /* ajDebug("In btreeKeyidFind\n"); */
15051 
15052     /* The root node should always be in the cache (BT_LOCKed) */
15053     root = btreeSeccacheLocate(cache,cache->secrootblock);
15054 
15055     /* ajDebug("cache->slevel = %u root=%u\n",cache->slevel,(ajuint)root); */
15056 
15057     GBT_NODETYPE(root->buf, &nodetype);
15058 
15059     if(nodetype == BT_SECBUCKET)
15060         return root;
15061 
15062     if(!cache->slevel)
15063 	return root;
15064 
15065     ret = btreeKeyidFindINode(cache,root,key);
15066 
15067     return ret;
15068 }
15069 
15070 
15071 
15072 
15073 /* @funcstatic btreeNumInSecbucket ********************************************
15074 **
15075 ** Return number of entries in a secondary id bucket
15076 **
15077 ** @param [u] cache [AjPBtcache] cache
15078 ** @param [r] pagepos [ajulong] page number
15079 **
15080 ** @return [ajuint] Number of entries in bucket
15081 **
15082 ** @release 3.0.0
15083 ** @@
15084 ******************************************************************************/
15085 
btreeNumInSecbucket(AjPBtcache cache,ajulong pagepos)15086 static ajuint btreeNumInSecbucket(AjPBtcache cache, ajulong pagepos)
15087 {
15088     unsigned char *buf = NULL;
15089 
15090     AjPBtpage page = NULL;
15091     ajuint  nodetype = 0;
15092     ajuint nentries = 0;
15093 
15094     /* ajDebug("In btreeNumInPribucket\n"); */
15095 
15096 /*
15097     if(pagepos == cache->secrootblock)
15098 	ajFatal("NumInSecbucket: Attempt to read bucket from root page\n");
15099 */
15100 
15101     page  = btreeSeccacheRead(cache,pagepos);
15102 
15103     buf = page->buf;
15104 
15105     GBT_BUCKNODETYPE(buf,&nodetype);
15106 
15107     if(nodetype != BT_SECBUCKET)
15108 	ajFatal("SecReadBucket: NodeType mismatch. Not secondary bucket (%u)",
15109 		nodetype);
15110 
15111     GBT_BUCKNENTRIES(buf,&nentries);
15112 
15113     return nentries;
15114 }
15115 
15116 
15117 
15118 
15119 /* @funcstatic btreeSecbucketAdd **********************************************
15120 **
15121 ** Add a keyword ID to a secondary bucket
15122 ** Only called if there is room in the bucket
15123 **
15124 ** @param [u] cache [AjPBtcache] cache
15125 ** @param [r] pagepos [ajulong] page number of bucket
15126 ** @param [r] id [const AjPStr] ID
15127 **
15128 ** @return [void]
15129 **
15130 ** @release 3.0.0
15131 ** @@
15132 ******************************************************************************/
15133 
btreeSecbucketAdd(AjPBtcache cache,ajulong pagepos,const AjPStr id)15134 static void btreeSecbucketAdd(AjPBtcache cache, ajulong pagepos,
15135                               const AjPStr id)
15136 {
15137     unsigned char *buf  = NULL;
15138     unsigned char *kptr = NULL;
15139     unsigned char *src  = NULL;
15140     unsigned char *dest = NULL;
15141 
15142 /*    unsigned char *lastptr = NULL;*/
15143     unsigned char *endptr  = NULL;
15144 
15145     ajuint nentries = 0;
15146     ajuint nodetype = 0;
15147     ajuint idlen    = 0;
15148 
15149     ajuint sum = 0;
15150     ajuint len = 0;
15151     ajuint i;
15152 
15153     ajuint v;
15154 
15155     AjPBtpage page = NULL;
15156     static ajuint calls = 0;
15157 /*    static ajuint overflowcalls=0;*/
15158 
15159     calls++;
15160     page = btreeSeccacheRead(cache,pagepos);
15161     buf  = page->buf;
15162 
15163     GBT_BUCKNODETYPE(buf,&nodetype);
15164     if(nodetype != BT_SECBUCKET)
15165         ajFatal("Wrong nodetype in SecbucketAdd cache %S",
15166                 cache->filename);
15167 
15168     GBT_BUCKNENTRIES(buf,&nentries);
15169     if(nentries >= cache->snperbucket)
15170         ajFatal("Bucket too full in SecbucketAdd page %Lu %u %u cache %S",
15171                 pagepos, nentries, cache->snperbucket, cache->filename);
15172 
15173     kptr = PBT_BUCKKEYLEN(buf);
15174     src  = kptr + (nentries * sizeof(ajuint));
15175 
15176     sum = 0;
15177     for(i=0;i<nentries;++i)
15178     {
15179         BT_GETAJUINT(kptr,&len);
15180         sum += len;
15181         kptr += sizeof(ajuint);
15182     }
15183     /*sum += nentries;*/
15184 
15185     endptr  = src + sum;
15186     idlen   = MAJSTRGETLEN(id);
15187 
15188 /*
15189 //    lastptr = endptr + sizeof(ajuint) + idlen;
15190 //    if((ajuint) (lastptr - buf) >= cache->pagesize)
15191 //    {
15192 //        overflowcalls++;
15193 //        ajWarn("\nOverflow in SecbucketAdd nentries:%u fails %u/%u '%S' "
15194 //               "cache %S",
15195 //               nentries, overflowcalls,calls, id, cache->filename);
15196 //        btreeSecbucketAddFull(cache,pagepos,id);
15197 //        return;
15198 //    }
15199 */
15200 
15201     dest = src + sizeof(ajuint);
15202     memmove((void *)dest, (void *)src, sum);
15203 
15204     v = idlen+1;
15205     BT_SETAJUINT(src,v);
15206 
15207     endptr += sizeof(ajuint);
15208     strcpy((char *)endptr,MAJSTRGETPTR(id));
15209 
15210     v = nentries + 1;
15211     SBT_BUCKNENTRIES(buf,v);
15212 
15213     page->dirty = BT_DIRTY;
15214 
15215     return;
15216 }
15217 
15218 
15219 
15220 
15221 #if 0
15222 /* #funcstatic btreeSecbucketAddFull ******************************************
15223 **
15224 ** Add a keyword ID to a secondary bucket
15225 ** Only called if there is room in the bucket
15226 **
15227 ** #param [u] cache [AjPBtcache] cache
15228 ** #param [r] pagepos [ajulong] page number of bucket
15229 ** #param [r] id [const AjPStr] ID
15230 **
15231 ** #return [void]
15232 **
15233 ** #release 6.4.0
15234 ** ##
15235 ******************************************************************************/
15236 /*
15237 //static void btreeSecbucketAddFull(AjPBtcache cache, ajulong pagepos,
15238 //                                  const AjPStr id)
15239 //{
15240 //    AjPSecbucket bucket = NULL;
15241 //    ajuint nentries;
15242 //
15243 //    /# ajDebug("In btreeSecbucketAddFull\n"); #/
15244 //
15245 //    bucket   = btreeReadSecbucket(cache,pagepos);
15246 //    nentries = bucket->Nentries;
15247 //
15248 //
15249 //    /# Reading a bucket always gives one extra ID position #/
15250 //
15251 //    ajStrAssignS(&bucket->SecIds[nentries],id);
15252 //
15253 //    ++bucket->Nentries;
15254 //
15255 //    btreeWriteSecbucket(cache,bucket,pagepos);
15256 //
15257 //    btreeSecbucketDel(&bucket);
15258 //
15259 //    return;
15260 //}
15261 */
15262 #endif
15263 
15264 
15265 
15266 
15267 /* @funcstatic btreeKeyidMakeroot *********************************************
15268 **
15269 ** Re-write root secondary id buckets as a root node with two buckets
15270 ** Must only be called if the buckets is full
15271 **
15272 ** @param [u] cache [AjPBtcache] cache
15273 ** @param [u] bucket [AjPBtpage] secbucket page
15274 **
15275 ** @return [AjBool] true if reorder was successful i.e. leaf not full
15276 **
15277 ** @release 6.5.0
15278 ** @@
15279 ******************************************************************************/
15280 
btreeKeyidMakeroot(AjPBtcache cache,AjPBtpage bucket)15281 static AjBool btreeKeyidMakeroot(AjPBtcache cache, AjPBtpage bucket)
15282 {
15283     AjPList idlist;
15284     ajulong pagepos;
15285     ajulong lblockno;
15286     ajulong rblockno;
15287     AjPBtpage page;
15288     AjPSecbucket cbucket = NULL;
15289     AjPStr bid      = NULL;
15290     ajuint count;
15291     ajuint maxleft;
15292     ajuint newmax;
15293 
15294     pagepos = bucket->pagepos;
15295 
15296     if(!statSaveSecId)
15297     {
15298         statSaveSecIdMax = 2048;
15299         AJCNEW0(statSaveSecId, statSaveSecIdMax);
15300         statSaveSecIdNext = 0;
15301     }
15302 
15303     /* save the ids in the bucket ... a simple list of strings */
15304 
15305     idlist = ajListNew();
15306     btreeSecbucketIdlist(cache, bucket->pagepos, idlist);
15307     ajListSort(idlist, &btreeKeywordIdCompare);
15308 
15309     /* create a pair of buckets linked to the root */
15310 
15311     lblockno = cache->totsize;
15312     btreeWriteSecbucketEmpty(cache,lblockno);
15313 
15314     rblockno = cache->totsize;
15315     btreeWriteSecbucketEmpty(cache,rblockno);
15316 
15317     /* save half the ids to the left, half to the right */
15318 
15319     cbucket = btreeSecbucketNew(cache->snperbucket,cache->idlimit);
15320     cbucket->Overflow = 0UL;
15321     cbucket->Nentries = 0U;
15322     count = 0;
15323     maxleft = (ajuint) ajListGetLength(idlist)/2;
15324 
15325     while(count < maxleft)
15326     {
15327 	    ajListPop(idlist,(void **)&bid);
15328 	    ajStrAssignS(&cbucket->SecIds[count],bid);
15329 	    cbucket->keylen[count] = BT_BUCKSECLEN(bid);
15330 	    ++cbucket->Nentries;
15331 	    ++count;
15332 
15333             if(statSaveSecIdNext >= statSaveSecIdMax)
15334             {
15335                 newmax = statSaveSecIdMax + statSaveSecIdMax;
15336                 AJCRESIZE0(statSaveSecId,statSaveSecIdMax,newmax);
15337                 statSaveSecIdMax = newmax;
15338             }
15339             statSaveSecId[statSaveSecIdNext++] = bid;
15340             bid = NULL;
15341     }
15342 
15343     btreeWriteSecbucket(cache,cbucket,lblockno);
15344 
15345     /* make the bucket into the new secondary root */
15346 
15347     cache->secrootblock = pagepos;
15348     btreeSecrootCreate(cache, pagepos);
15349 
15350     page = btreeSeccacheWrite(cache,pagepos);
15351     page->dirty = BT_LOCK;
15352     page->lockfor = 1251;
15353 
15354     if(btreeDoRootSync)
15355         btreeCacheRootSync(cache,pagepos);
15356 
15357     cache->slevel = 0U;
15358 
15359     ajListPeek(idlist,(void **)&bid); /* mid-range key to save in root node */
15360     btreeWriteNodeSingle(cache,page,bid,lblockno,rblockno);
15361 
15362     cbucket->Overflow = 0UL;
15363     cbucket->Nentries = 0U;
15364 
15365     count = 0;
15366 
15367     while(ajListGetLength(idlist))
15368     {
15369 	    ajListPop(idlist,(void **)&bid);
15370 	    ajStrAssignS(&cbucket->SecIds[count],bid);
15371 	    cbucket->keylen[count] = BT_BUCKSECLEN(bid);
15372 	    ++cbucket->Nentries;
15373 	    ++count;
15374 
15375             if(statSaveSecIdNext >= statSaveSecIdMax)
15376             {
15377                 newmax = statSaveSecIdMax + statSaveSecIdMax;
15378                 AJCRESIZE0(statSaveSecId,statSaveSecIdMax,newmax);
15379                 statSaveSecIdMax = newmax;
15380             }
15381             statSaveSecId[statSaveSecIdNext++] = bid;
15382             bid = NULL;
15383     }
15384 
15385     btreeWriteSecbucket(cache,cbucket,rblockno);
15386 
15387     btreeSecbucketDel(&cbucket);
15388     ajListFree(&idlist);
15389 
15390     return ajTrue;
15391 }
15392 
15393 
15394 
15395 
15396 /* @funcstatic btreeSecbucketsReorder *****************************************
15397 **
15398 ** Re-order secondary id leaf buckets
15399 ** Must only be called if one of the buckets is full
15400 **
15401 ** @param [u] cache [AjPBtcache] cache
15402 ** @param [u] leaf [AjPBtpage] leaf page
15403 **
15404 ** @return [AjBool] true if reorder was successful i.e. leaf not full
15405 **
15406 ** @release 3.0.0
15407 ** @@
15408 ******************************************************************************/
15409 
btreeSecbucketsReorder(AjPBtcache cache,AjPBtpage leaf)15410 static AjBool btreeSecbucketsReorder(AjPBtcache cache, AjPBtpage leaf)
15411 {
15412     ajuint nkeys = 0;
15413     AjPSecbucket cbucket  = NULL;
15414     unsigned char *lbuf   = NULL;
15415 
15416     AjPBtMem arrays1 = NULL;
15417     AjPBtMem arrays2 = NULL;
15418     AjPStr *newkeys = NULL;
15419     ajulong *newptrs = NULL;
15420     ajulong *ptrs = NULL;
15421     ajulong *overflows = NULL;
15422 
15423     ajuint i = 0;
15424 
15425     ajuint order;
15426     ajuint totalkeys     = 0;
15427     ajuint maxnperbucket = 0;
15428     ajuint count         = 0;
15429     ajuint keylimit      = 0;
15430     ajuint bucketlimit   = 0;
15431     ajuint nodetype      = 0;
15432 
15433     AjPList idlist  = NULL;
15434     ajuint dirtysave = 0;
15435     AjPStr bid      = NULL;
15436 
15437 
15438     ajuint newmax;
15439     ajuint iold = 0;
15440 
15441     /* ajDebug("In btreeSecbucketsReorder\n"); */
15442 
15443     if(!statSaveSecId)
15444     {
15445         statSaveSecIdMax = 2048;
15446         AJCNEW0(statSaveSecId, statSaveSecIdMax);
15447         statSaveSecIdNext = 0;
15448     }
15449 
15450     dirtysave = leaf->dirty;
15451 
15452     leaf->dirty = BT_LOCK;
15453     leaf->lockfor = 1371;
15454     lbuf = leaf->buf;
15455 
15456     GBT_NODETYPE(lbuf,&nodetype);
15457 
15458     order = cache->sorder;
15459 
15460     /* Read keys/ptrs */
15461     arrays1 = btreeAllocSecArray(cache);
15462     ptrs = arrays1->parray;
15463     arrays2 = btreeAllocSecArray(cache);
15464     newkeys = arrays2->karray;
15465     newptrs = arrays2->parray;
15466     overflows = arrays2->overflows;
15467     btreeGetPointers(cache,lbuf,&ptrs);
15468 
15469 
15470     GBT_NKEYS(lbuf,&nkeys);
15471 
15472 
15473     if(!nkeys)
15474 	ajFatal("SecbucketsReorder: Attempt to reorder empty lea cache %Sf",
15475                 cache->filename);
15476 
15477     if(nkeys > order)
15478     {
15479         ajErr("SecbucketsReorder: nkeys %u > order %u cache %S",
15480               nkeys, order, cache->filename);
15481     }
15482 
15483     for(i=0;i<nkeys;++i)
15484 	totalkeys += btreeNumInSecbucket(cache,ptrs[i]);
15485 
15486     totalkeys += btreeNumInSecbucket(cache,ptrs[i]);
15487 
15488     keylimit = nkeys + 1;
15489 
15490     btreeBucketCalc(totalkeys, keylimit, cache->snperbucket,
15491                     &bucketlimit, &maxnperbucket);
15492 
15493     if(bucketlimit >= order)
15494     {
15495         btreeDeallocSecArray(cache,arrays1);
15496         btreeDeallocSecArray(cache,arrays2);
15497 	leaf->dirty = dirtysave;
15498 
15499 	return ajFalse;
15500     }
15501 
15502 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
15503     ajDebug("btreeSecbucketsReorder '%S' %Lu id '%S' key '%S'\n",
15504             cache->basename, leaf->pagepos, indexId, indexKeyword);
15505 #endif
15506     ++statCallSecbucketsReorder;
15507 
15508     /* Read IDs from all buckets and push to list and sort (increasing kw) */
15509     idlist  = ajListNew();
15510 
15511     for(i=0;i<keylimit;++i)
15512 	overflows[i] = btreeSecbucketIdlist(cache,ptrs[i],idlist);
15513 
15514 
15515     ajListSort(idlist, &btreeKeywordIdCompare);
15516 
15517     cbucket = btreeSecbucketNew(cache->snperbucket,cache->idlimit);
15518     iold = 0;
15519 
15520     for(i=0;i<bucketlimit;++i)
15521     {
15522 	cbucket->Overflow = overflows[i];
15523 	cbucket->Nentries = 0;
15524 
15525 	count = 0;
15526 	while(count!=maxnperbucket)
15527 	{
15528 	    ajListPop(idlist,(void **)&bid);
15529 
15530 	    ajStrAssignS(&cbucket->SecIds[count],bid);
15531 
15532 	    cbucket->keylen[count] = BT_BUCKSECLEN(bid);
15533 	    ++cbucket->Nentries;
15534 	    ++count;
15535             if(statSaveSecIdNext >= statSaveSecIdMax)
15536             {
15537                 newmax = statSaveSecIdMax + statSaveSecIdMax;
15538                 AJCRESIZE0(statSaveSecId,statSaveSecIdMax,newmax);
15539                 statSaveSecIdMax = newmax;
15540             }
15541             statSaveSecId[statSaveSecIdNext++] = bid;
15542             bid = NULL;
15543 	}
15544 
15545 
15546 	ajListPeek(idlist,(void **)&bid);
15547 	ajStrAssignS(&newkeys[i],bid);
15548 
15549 	if((iold < order) && ptrs[iold])
15550             newptrs[i]=ptrs[iold++];
15551         else
15552 	    newptrs[i] = cache->totsize;
15553 	btreeWriteSecbucket(cache,cbucket,newptrs[i]);
15554     }
15555 
15556 
15557     /* Deal with greater-than bucket */
15558 
15559     cbucket->Overflow = overflows[i];
15560     cbucket->Nentries = 0;
15561 
15562     count = 0;
15563 
15564     while(ajListPop(idlist,(void **)&bid))
15565     {
15566 	ajStrAssignS(&cbucket->SecIds[count],bid);
15567 
15568 	++cbucket->Nentries;
15569 	++count;
15570 	ajStrDel(&bid);
15571     }
15572 
15573     if((iold < order) && ptrs[iold])
15574         newptrs[i]=ptrs[iold++];
15575     else
15576         newptrs[i] = cache->totsize;
15577     btreeWriteSecbucket(cache,cbucket,newptrs[i]);
15578 
15579     btreeSecbucketDel(&cbucket);
15580 
15581 #if AJINDEX_DEBUG
15582     if(bucketlimit <= keylimit)
15583         ajDebug("btreeSecbucketsReorder '%S' %u -> %u",
15584                 cache->filename, keylimit, bucketlimit);
15585 #endif
15586 
15587     for(i = bucketlimit + 1; i <= keylimit; i++)
15588     {
15589         btreeSecpageSetfree(cache, ptrs[i]);
15590     }
15591 
15592     /* Now write out a modified leaf with new keys/ptrs */
15593 
15594     nkeys = bucketlimit;
15595     btreeWriteNode(cache,leaf,newkeys,newptrs,nkeys);
15596     leaf->dirty = BT_DIRTY;
15597 
15598     if(nodetype == BT_SECROOT)
15599     {
15600 	leaf->dirty = BT_LOCK;
15601         leaf->lockfor = 1372;
15602     }
15603 
15604     btreeDeallocSecArray(cache,arrays1);
15605     btreeDeallocSecArray(cache,arrays2);
15606 
15607     ajListFree(&idlist);
15608 
15609     return ajTrue;
15610 }
15611 
15612 
15613 
15614 
15615 #if 0
15616 /* @funcstatic btreeInsertIdOnly **********************************************
15617 **
15618 ** Add only a secondary ID: the primary keyword already exists
15619 **
15620 ** @param [u] cache [AjPBtcache] cache
15621 ** @param [r] pri [const AjPBtPri] keyword/id
15622 **
15623 ** @return [void]
15624 **
15625 ** @release 3.0.0
15626 ** @@
15627 ******************************************************************************/
15628 
15629 static void btreeInsertIdOnly(AjPBtcache cache, const AjPBtPri pri)
15630 {
15631     unsigned char *buf;
15632     AjPBtpage page = NULL;
15633     ajuint nodetype;
15634 
15635     ajulong right = 0UL;
15636 
15637     /* ajDebug("In btreeInsertIdOnly\n"); */
15638 
15639     if(!pri->treeblock)
15640     {
15641 	fprintf(stderr,"btreeInsertIdOnly: root page doesn't exist\n");
15642 	exit(-1);
15643     }
15644 
15645     cache->secrootblock = pri->treeblock;
15646     page = btreeSeccacheWrite(cache,cache->secrootblock);
15647     page->dirty = BT_LOCK;
15648     page->lockfor = 1381;
15649     buf = page->buf;
15650     GBT_RIGHT(buf, &right);
15651     cache->slevel = (ajuint) right;
15652 
15653     btreeKeyidInsert(cache, pri->id);
15654 
15655     GBT_NODETYPE(buf, &nodetype);
15656     if(nodetype != BT_SECBUCKET)
15657     {
15658         right = (ajulong) cache->slevel;
15659         SBT_RIGHT(buf, right);
15660         page->dirty = BT_DIRTY;
15661     }
15662 
15663     return;
15664 }
15665 #endif
15666 
15667 
15668 
15669 
15670 /* @funcstatic btreeSecSplitroot **********************************************
15671 **
15672 ** Split a secondary root node
15673 **
15674 ** @param [u] cache [AjPBtcache] cache
15675 **
15676 ** @return [void]
15677 **
15678 ** @release 3.0.0
15679 ** @@
15680 ******************************************************************************/
15681 
btreeSecSplitroot(AjPBtcache cache)15682 static void btreeSecSplitroot(AjPBtcache cache)
15683 {
15684     AjPBtpage rootpage = NULL;
15685     AjPBtpage rpage    = NULL;
15686     AjPBtpage lpage    = NULL;
15687     AjPBtpage tpage    = NULL;
15688 
15689     AjPBtMem arrays = NULL;
15690     AjPBtMem tarrays = NULL;
15691     AjPStr *karray  = NULL;
15692     AjPStr *tkarray = NULL;
15693     ajulong *parray  = NULL;
15694     ajulong *tparray = NULL;
15695 
15696     ajuint order  = 0U;
15697     ajuint nkeys  = 0U;
15698     ajuint keypos = 0U;
15699 
15700     ajulong rblockno = 0UL;
15701     ajulong lblockno = 0UL;
15702 
15703     ajulong right;
15704 
15705     AjPStr key = NULL;
15706     ajuint i;
15707 
15708     unsigned char *rootbuf = NULL;
15709     unsigned char *rbuf    = NULL;
15710     unsigned char *lbuf    = NULL;
15711     unsigned char *tbuf    = NULL;
15712 
15713     ajuint nodetype  = 0U;
15714     ajulong overflow = 0UL;
15715     ajulong zero     = 0UL;
15716     ajuint totlen    = 0U;
15717     ajuint rkeyno    = 0U;
15718     ajuint n         = 0U;
15719 
15720     ajulong lv = 0UL;
15721     ajuint  v  = 0U;
15722 
15723 
15724 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
15725     ajDebug("btreeSecSplitroot '%S' %Lu id '%S' key '%S'\n",
15726             cache->basename, cache->secrootblock, indexId, indexKeyword);
15727 #endif
15728     ++statCallSecSplitroot;
15729 
15730     /* ajDebug("In btreeSecSplitroot\n"); */
15731 
15732     order = cache->sorder;
15733 
15734     arrays = btreeAllocSecArray(cache);
15735     tarrays = btreeAllocSecArray(cache);
15736     karray = arrays->karray;
15737     parray = arrays->parray;
15738     tkarray = tarrays->karray;
15739     tparray = tarrays->parray;
15740 
15741     rootpage = btreeSeccacheLocate(cache,cache->secrootblock);
15742     rootbuf = rootpage->buf;
15743 
15744     nkeys = order - 1;
15745 
15746     keypos = nkeys / 2;
15747 
15748     if(!(nkeys % 2))
15749 	--keypos;
15750 
15751 
15752     rblockno = cache->totsize;
15753     rpage = btreeSeccacheNodenew(cache);
15754     rpage->dirty = BT_LOCK;
15755     rpage->lockfor = 1391;
15756 
15757     lblockno = cache->totsize;
15758     lpage = btreeSeccacheNodenew(cache);
15759     lpage->dirty = BT_DIRTY;
15760     lpage->lockfor = 1392;
15761 
15762     /* Comment this next block out after the beta test */
15763     if(!cache->slevel)
15764     {
15765 	fprintf(stderr,"btreeSecSplitroot Shouldn't get here\n");
15766 	exit(0);
15767 	lv = zero;
15768 	SBT_LEFT(lpage->buf,lv);
15769 	lv = rblockno;
15770 	SBT_RIGHT(lpage->buf,lv);
15771 	lv = lblockno;
15772 	SBT_LEFT(rpage->buf,lv);
15773 	lv = zero;
15774 	SBT_RIGHT(rpage->buf,lv);
15775     }
15776 
15777     btreeGetKeys(cache,rootbuf,&karray,&parray);
15778 
15779     /* Get key for root node and write new root node */
15780     ajStrAssignS(&tkarray[0],karray[keypos]);
15781     tparray[0] = lblockno;
15782     tparray[1] = rblockno;
15783 
15784 
15785     n = 1;
15786     v = n;
15787     SBT_NKEYS(rootbuf,v);
15788     btreeWriteNode(cache,rootpage,tkarray,tparray,1);
15789     right = (ajulong)(cache->slevel + 1);
15790     SBT_RIGHT(rootbuf,right);
15791 
15792     rootpage->dirty = BT_LOCK;
15793     rootpage->lockfor = 1393;
15794 
15795     rbuf = rpage->buf;
15796     lbuf = lpage->buf;
15797 
15798     if(cache->slevel)
15799 	nodetype = BT_SECINTERNAL;
15800     else
15801 	nodetype = BT_SECLEAF;
15802 
15803     v = nodetype;
15804     SBT_NODETYPE(rbuf,v);
15805     v = nodetype;
15806     SBT_NODETYPE(lbuf,v);
15807     lv = overflow;
15808     SBT_OVERFLOW(rbuf,lv);
15809     lv = cache->secrootblock;
15810     SBT_PREV(rbuf,lv);
15811     lv = overflow;
15812     SBT_OVERFLOW(lbuf,lv);
15813     lv = cache->secrootblock;
15814     SBT_PREV(lbuf,lv);
15815 
15816     totlen = 0;
15817 
15818     for(i=0;i<keypos;++i)
15819     {
15820 	ajStrAssignS(&tkarray[i],karray[i]);
15821 	totlen += ajStrGetLen(karray[i]);
15822 	tparray[i] = parray[i];
15823     }
15824 
15825     tparray[i] = parray[i];
15826     n = i;
15827     btreeWriteNode(cache,lpage,tkarray,tparray,n);
15828 
15829     for(i=0;i<=n;++i)
15830     {
15831 	tpage = btreeSeccacheRead(cache,tparray[i]);
15832 	tbuf = tpage->buf;
15833 	lv = lblockno;
15834 	SBT_PREV(tbuf,lv);
15835 	tpage->dirty = BT_DIRTY;
15836     }
15837 
15838     totlen = 0;
15839 
15840     for(i=keypos+1;i<nkeys;++i)
15841     {
15842 	ajStrAssignS(&tkarray[i-(keypos+1)],karray[i]);
15843 	totlen += ajStrGetLen(karray[i]);
15844 	tparray[i-(keypos+1)] = parray[i];
15845     }
15846 
15847     tparray[i-(keypos+1)] = parray[i];
15848 
15849     rkeyno = (nkeys-keypos) - 1;
15850     rpage->dirty = BT_DIRTY;
15851 
15852     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
15853 
15854     for(i=0;i<rkeyno+1;++i)
15855     {
15856 	tpage = btreeSeccacheRead(cache,tparray[i]);
15857 	tbuf = tpage->buf;
15858 	lv = rblockno;
15859 	SBT_PREV(tbuf,lv);
15860 	tpage->dirty = BT_DIRTY;
15861     }
15862 
15863 
15864     ++cache->slevel;
15865 
15866     btreeDeallocSecArray(cache,arrays);
15867     btreeDeallocSecArray(cache,tarrays);
15868 
15869     ajStrDel(&key);
15870 
15871     return;
15872 }
15873 
15874 
15875 
15876 
15877 /* @funcstatic btreeInsertKeySec **********************************************
15878 **
15879 ** Insert a secondary key into a potentially full node
15880 **
15881 ** @param [u] cache [AjPBtcache] cache
15882 ** @param [u] page [AjPBtpage] original page
15883 ** @param [r] key [const AjPStr] key to insert
15884 ** @param [r] less [ajulong] less-than pointer
15885 ** @param [r] greater [ajulong] greater-than pointer
15886 **
15887 ** @return [void]
15888 **
15889 ** @release 3.0.0
15890 ** @@
15891 ******************************************************************************/
15892 
btreeInsertKeySec(AjPBtcache cache,AjPBtpage page,const AjPStr key,ajulong less,ajulong greater)15893 static void btreeInsertKeySec(AjPBtcache cache, AjPBtpage page,
15894 			      const AjPStr key, ajulong less, ajulong greater)
15895 {
15896     unsigned char *lbuf = NULL;
15897     unsigned char *rbuf = NULL;
15898     unsigned char *tbuf = NULL;
15899 
15900     AjPStr *karray  = NULL;
15901     ajulong *parray  = NULL;
15902     AjPStr *tkarray = NULL;
15903     ajulong *tparray = NULL;
15904     AjPBtMem savekeyarrays = NULL;
15905     AjPBtMem tarrays = NULL;
15906 
15907     ajuint nkeys  = 0U;
15908     ajuint order  = 0U;
15909     ajuint keypos = 0U;
15910     ajuint rkeyno = 0U;
15911 
15912     ajuint i = 0U;
15913     ajuint n = 0U;
15914 
15915     ajuint nodetype = 0U;
15916     AjPBtpage ipage = NULL;
15917     AjPBtpage lpage = NULL;
15918     AjPBtpage rpage = NULL;
15919     AjPBtpage tpage = NULL;
15920 
15921     ajulong blockno  = 0UL;
15922     ajulong rblockno = 0UL;
15923     ajulong lblockno = 0UL;
15924     ajulong ibn      = 0UL;
15925 
15926     AjPStr mediankey  = NULL;
15927     ajulong medianless = 0UL;
15928     ajulong mediangtr  = 0UL;
15929     ajulong overflow   = 0UL;
15930     ajulong prev       = 0UL;
15931     ajuint  totlen     = 0U;
15932 
15933     ajulong lv = 0UL;
15934     ajuint  v  = 0U;
15935 
15936     /* ajDebug("In btreeInsertKeySec\n"); */
15937 
15938     if(!btreeNodeIsFullSec(cache,page))
15939     {
15940 	btreeInsertNonfullSec(cache,page,key,less,greater);
15941 
15942 	return;
15943     }
15944 
15945     order = cache->sorder;
15946     lbuf = page->buf;
15947     GBT_NODETYPE(lbuf,&nodetype);
15948     page->dirty = BT_LOCK;
15949     page->lockfor = 1401;
15950 
15951     if(nodetype == BT_SECROOT)
15952     {
15953 	btreeSecSplitroot(cache);
15954 	page->dirty = BT_DIRTY;
15955 
15956 	blockno = btreeGetBlockFirstS(cache,lbuf,key);
15957 
15958 	ipage = btreeSeccacheRead(cache,blockno);
15959 	btreeInsertNonfullSec(cache,ipage,key,less,greater);
15960 
15961 	return;
15962     }
15963 
15964     savekeyarrays = btreeAllocSecArray(cache);
15965     tarrays = btreeAllocSecArray(cache);
15966     karray = savekeyarrays->karray;
15967     parray = savekeyarrays->parray;
15968     tkarray = tarrays->karray;
15969     tparray = tarrays->parray;
15970 
15971     lpage = page;
15972     lbuf = lpage->buf;
15973 
15974     btreeGetKeys(cache,lbuf,&karray,&parray);
15975 
15976     GBT_BLOCKNUMBER(lbuf,&lblockno);
15977 
15978     rblockno = cache->totsize;
15979     rpage = btreeSeccacheNodenew(cache);
15980     rbuf = rpage->buf;
15981     rpage->dirty = BT_LOCK;
15982     rpage->lockfor = 1402;
15983 
15984     GBT_PREV(lbuf,&prev);
15985     lv = prev;
15986     SBT_PREV(rbuf,lv);
15987 
15988     nkeys = order - 1;
15989     keypos = nkeys / 2;
15990 
15991     if(!(nkeys % 2))
15992 	--keypos;
15993 
15994     mediankey = ajStrNewS(karray[keypos]);
15995     medianless = lblockno;
15996     mediangtr  = rblockno;
15997 
15998 
15999     GBT_NODETYPE(lbuf,&nodetype);
16000     v = nodetype;
16001     SBT_NODETYPE(rbuf,v);
16002     lv = overflow;
16003     SBT_OVERFLOW(rbuf,lv);
16004 
16005 
16006     totlen = 0;
16007 
16008     for(i=0;i<keypos;++i)
16009     {
16010 	ajStrAssignS(&tkarray[i],karray[i]);
16011 	totlen += ajStrGetLen(karray[i]);
16012 	tparray[i] = parray[i];
16013     }
16014 
16015     tparray[i] = parray[i];
16016     n = i;
16017     btreeWriteNode(cache,lpage,tkarray,tparray,n);
16018 
16019 
16020 
16021     for(i=0;i<n+1;++i)
16022     {
16023 	tpage = btreeSeccacheRead(cache,tparray[i]);
16024 	tbuf = tpage->buf;
16025 	lv = lblockno;
16026 	SBT_PREV(tbuf,lv);
16027 	tpage->dirty = BT_DIRTY;
16028     }
16029 
16030 
16031     totlen = 0;
16032 
16033     for(i=keypos+1;i<nkeys;++i)
16034     {
16035 	ajStrAssignS(&tkarray[i-(keypos+1)],karray[i]);
16036 	totlen += ajStrGetLen(karray[i]);
16037 	tparray[i-(keypos+1)] = parray[i];
16038     }
16039 
16040     tparray[i-(keypos+1)] = parray[i];
16041     rkeyno = (nkeys-keypos) - 1;
16042     rpage->dirty = BT_DIRTY;
16043 
16044     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
16045 
16046 
16047     for(i=0;i<rkeyno+1;++i)
16048     {
16049 	tpage = btreeSeccacheRead(cache,tparray[i]);
16050 	tbuf = tpage->buf;
16051 	lv = rblockno;
16052 	SBT_PREV(tbuf,lv);
16053 	tpage->dirty = BT_DIRTY;
16054     }
16055 
16056 
16057     ibn = rblockno;
16058 
16059     if(MAJSTRCMPS(key,mediankey)<0)
16060 	ibn = lblockno;
16061 
16062     ipage = btreeSeccacheRead(cache,ibn);
16063 
16064     btreeInsertNonfullSec(cache,ipage,key,less,greater);
16065 
16066 
16067 
16068     ipage = btreeSeccacheRead(cache,prev);
16069 
16070     btreeInsertKeySec(cache,ipage,mediankey,medianless,mediangtr);
16071     btreeDeallocSecArray(cache,savekeyarrays); /* mediankey points here */
16072     btreeDeallocSecArray(cache,tarrays);
16073 
16074     ajStrDel(&mediankey);
16075 
16076     return;
16077 }
16078 
16079 
16080 
16081 
16082 /* @funcstatic btreeKeyidInsertShift ******************************************
16083 **
16084 ** Rebalance buckets on insertion of a secondary identifier
16085 **
16086 ** @param [u] cache [AjPBtcache] cache
16087 ** @param [u] retpage [AjPBtpage*] page
16088 ** @param [r] key [const AjPStr] key
16089 **
16090 ** @return [ajulong] bucket block or 0UL if shift not possible
16091 **
16092 ** @release 6.5.0
16093 ** @@
16094 ******************************************************************************/
16095 
btreeKeyidInsertShift(AjPBtcache cache,AjPBtpage * retpage,const AjPStr key)16096 static ajulong btreeKeyidInsertShift(AjPBtcache cache, AjPBtpage *retpage,
16097                                      const AjPStr key)
16098 {
16099     unsigned char *tbuf = NULL;
16100     unsigned char *pbuf = NULL;
16101     unsigned char *sbuf = NULL;
16102 
16103     AjPBtpage ppage = NULL;
16104     AjPBtpage spage = NULL;
16105     AjPBtpage tpage = NULL;
16106 
16107     ajuint tkeys = 0U;
16108     ajuint pkeys = 0U;
16109     ajuint skeys = 0U;
16110     ajuint order = 0U;
16111 
16112     ajuint i;
16113     ajuint n;
16114     ajint ii;
16115 
16116     ajulong parent  = 0UL;
16117     ajulong blockno = 0UL;
16118 
16119     AjPBtMem arrays1 = NULL;
16120     AjPBtMem arrays2 = NULL;
16121     AjPBtMem arrays3 = NULL;
16122 
16123     AjPStr *kTarray = NULL;
16124     AjPStr *kParray = NULL;
16125     AjPStr *kSarray = NULL;
16126     ajulong *pTarray = NULL;
16127     ajulong *pParray = NULL;
16128     ajulong *pSarray = NULL;
16129 
16130     AjPStr *karray = NULL;
16131     ajulong *parray = NULL;
16132 
16133     ajuint ppos    = 0U;
16134     ajuint pkeypos = 0U;
16135     ajuint minsize = 0U;
16136 
16137     /* ajDebug("In btreeKeyidInsertShift\n"); */
16138 
16139 
16140     tpage = *retpage;
16141 
16142     tbuf = tpage->buf;
16143 
16144     GBT_PREV(tbuf,&parent);
16145     GBT_NKEYS(tbuf,&tkeys);
16146 
16147     order = cache->sorder;
16148     minsize = order / 2U;
16149 
16150     if(order % 2U)
16151 	++minsize;
16152 
16153     if(tkeys <= minsize)
16154 	return 0UL;
16155 
16156 
16157     ppage = btreeSeccacheRead(cache,parent);
16158     pbuf = ppage->buf;
16159     GBT_NKEYS(pbuf,&pkeys);
16160 
16161     arrays1 = btreeAllocSecArray(cache);
16162     kParray = arrays1->karray;
16163     pParray = arrays1->parray;
16164 
16165     arrays2 = btreeAllocSecArray(cache);
16166     kSarray = arrays2->karray;
16167     pSarray = arrays2->parray;
16168 
16169     arrays3 = btreeAllocSecArray(cache);
16170     kTarray = arrays3->karray;
16171     pTarray = arrays3->parray;
16172 
16173     btreeGetKeys(cache,pbuf,&kParray,&pParray);
16174 
16175     i=0;
16176 
16177     while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
16178 	++i;
16179 
16180     pkeypos = i;
16181 
16182     if(i==pkeys)
16183     {
16184 	if(MAJSTRCMPS(key,kParray[i-1])<0)
16185 	    ppos = i-1;
16186 	else
16187 	    ppos = i;
16188     }
16189     else
16190 	ppos = i;
16191 
16192 
16193     if(ppos) /* There is another leaf to the left */
16194     {
16195 	spage = btreeSeccacheRead(cache,pParray[ppos-1]);
16196 	sbuf = spage->buf;
16197 	GBT_NKEYS(sbuf,&skeys);
16198     }
16199 
16200     if(i && skeys != order-1) /* There is space in the left leaf */
16201     {
16202 	/* ajDebug("Left shift\n"); */
16203 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
16204 	if(skeys)
16205 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
16206 
16207 	i = 0;
16208 
16209 	while(pParray[i] != tpage->pagepos)
16210 	    ++i;
16211 
16212 	--i;
16213 
16214 	pkeypos = i;
16215 
16216 	ajStrAssignS(&kSarray[skeys],kParray[pkeypos]);
16217 	pSarray[skeys+1] = pTarray[0];
16218 	++skeys;
16219 	--tkeys;
16220 	ajStrAssignS(&kParray[pkeypos],kTarray[0]);
16221 
16222 	for(i=0;i<tkeys;++i)
16223 	{
16224 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
16225 	    pTarray[i] = pTarray[i+1];
16226 	}
16227 
16228 	pTarray[i] = pTarray[i+1];
16229 	pTarray[i+1] = 0UL;
16230 
16231 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
16232 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
16233 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
16234 
16235 	if(ppage->pagepos == cache->secrootblock)
16236         {
16237 	    ppage->dirty = BT_LOCK;
16238             ppage->lockfor = 1411;
16239         }
16240 
16241 	i = 0;
16242 
16243 	while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
16244 	    ++i;
16245 
16246 	if(i==pkeys)
16247 	{
16248 	    if(MAJSTRCMPS(key,kParray[i-1])<0)
16249 		blockno = pParray[i-1];
16250 	    else
16251 		blockno = pParray[i];
16252 	}
16253 	else
16254 	    blockno = pParray[i];
16255 
16256 	if(blockno == spage->pagepos)
16257 	{
16258 	    *retpage = spage;
16259 	    karray = kSarray;
16260 	    parray = pSarray;
16261 	    n = skeys;
16262 	}
16263 	else
16264 	{
16265 	    karray = kTarray;
16266 	    parray = pTarray;
16267 	    n = tkeys;
16268 	}
16269 
16270 
16271 	i = 0;
16272 
16273 	while(i!=n && MAJSTRCMPS(key,karray[i])>=0)
16274 	    ++i;
16275 
16276 	if(i==n)
16277 	{
16278 	    if(MAJSTRCMPS(key,karray[i-1])<0)
16279 		blockno = parray[i-1];
16280 	    else
16281 		blockno = parray[i];
16282 	}
16283 	else
16284 	    blockno = parray[i];
16285 
16286         btreeDeallocSecArray(cache,arrays1);
16287         btreeDeallocSecArray(cache,arrays2);
16288         btreeDeallocSecArray(cache,arrays3);
16289 
16290 	return blockno;
16291     }
16292 
16293 
16294     if(ppos != pkeys)	/* There is a right node */
16295     {
16296 	spage = btreeSeccacheRead(cache,pParray[ppos+1]);
16297 	sbuf = spage->buf;
16298 	GBT_NKEYS(sbuf,&skeys);
16299     }
16300 
16301 
16302     /* Space in the right leaf */
16303     if(ppos != pkeys && skeys != order-1)
16304     {
16305 	/* ajDebug("Right shift\n");*/
16306 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
16307 	btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
16308 
16309 	i = 0;
16310 
16311 	while(pParray[i] != tpage->pagepos)
16312 	    ++i;
16313 
16314 	pkeypos = i;
16315 
16316 	pSarray[skeys+1] = pSarray[skeys];
16317 
16318 	for(ii=skeys-1;ii>-1;--ii)
16319 	{
16320 	    ajStrAssignS(&kSarray[ii+1],kSarray[ii]);
16321 	    pSarray[ii+1] = pSarray[ii];
16322 	}
16323 
16324 	ajStrAssignS(&kSarray[0],kParray[pkeypos]);
16325 	pSarray[0] = pTarray[tkeys];
16326 	ajStrAssignS(&kParray[pkeypos],kTarray[tkeys-1]);
16327 	++skeys;
16328 	--tkeys;
16329 	pTarray[tkeys+1] = 0UL;
16330 
16331 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
16332 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
16333 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
16334 
16335 	if(ppage->pagepos == cache->secrootblock)
16336         {
16337 	    ppage->dirty = BT_LOCK;
16338             ppage->lockfor = 1412;
16339         }
16340 
16341 	i = 0;
16342 
16343 	while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
16344 	    ++i;
16345 
16346 	if(i==pkeys)
16347 	{
16348 	    if(MAJSTRCMPS(key,kParray[i-1])<0)
16349 		blockno = pParray[i-1];
16350 	    else
16351 		blockno = pParray[i];
16352 	}
16353 	else
16354 	    blockno = pParray[i];
16355 
16356 	if(blockno == spage->pagepos)
16357 	{
16358 	    *retpage = spage;
16359 	    karray = kSarray;
16360 	    parray = pSarray;
16361 	    n = skeys;
16362 	}
16363 	else
16364 	{
16365 	    karray = kTarray;
16366 	    parray = pTarray;
16367 	    n = tkeys;
16368 	}
16369 
16370 	i = 0;
16371 
16372 	while(i!=n && MAJSTRCMPS(key,karray[i])>=0)
16373 	    ++i;
16374 
16375 	if(i==n)
16376 	{
16377 	    if(MAJSTRCMPS(key,karray[i-1])<0)
16378 		blockno = parray[i-1];
16379 	    else
16380 		blockno = parray[i];
16381 	}
16382 	else
16383 	    blockno = parray[i];
16384 
16385         btreeDeallocSecArray(cache,arrays1);
16386         btreeDeallocSecArray(cache,arrays2);
16387         btreeDeallocSecArray(cache,arrays3);
16388 
16389 	return blockno;
16390     }
16391 
16392     btreeDeallocSecArray(cache,arrays1);
16393     btreeDeallocSecArray(cache,arrays2);
16394     btreeDeallocSecArray(cache,arrays3);
16395 
16396     return 0UL;
16397 }
16398 
16399 
16400 
16401 
16402 /* @funcstatic btreeInsertNonfullSec ******************************************
16403 **
16404 ** Insert a key into a non-full node
16405 **
16406 ** @param [u] cache [AjPBtcache] cache
16407 ** @param [u] page [AjPBtpage] original page
16408 ** @param [r] key [const AjPStr] key to insert
16409 ** @param [r] less [ajulong] less-than pointer
16410 ** @param [r] greater [ajulong] greater-than pointer
16411 **
16412 ** @return [void]
16413 **
16414 ** @release 3.0.0
16415 ** @@
16416 ******************************************************************************/
16417 
btreeInsertNonfullSec(AjPBtcache cache,AjPBtpage page,const AjPStr key,ajulong less,ajulong greater)16418 static void btreeInsertNonfullSec(AjPBtcache cache, AjPBtpage page,
16419 				  const AjPStr key, ajulong less,
16420 				  ajulong greater)
16421 {
16422     unsigned char *buf = NULL;
16423 
16424     AjPBtMem arrays = NULL;
16425     AjPStr *karray = NULL;
16426     ajulong *parray = NULL;
16427 
16428     ajuint nkeys = 0U;
16429     ajuint ipos  = 0U;
16430     ajuint i;
16431     ajuint count = 0U;
16432 
16433     ajulong lv = 0UL;
16434     ajuint  v  = 0U;
16435 
16436 
16437     AjPBtpage ppage = NULL;
16438     ajulong pagepos   = 0UL;
16439 
16440     ajuint nodetype = 0U;
16441 
16442     /* ajDebug("In btreeInsertNonfullSec\n"); */
16443 
16444     arrays = btreeAllocSecArray(cache);
16445     karray = arrays->karray;
16446     parray = arrays->parray;
16447 
16448     buf = page->buf;
16449     GBT_NKEYS(buf,&nkeys);
16450     GBT_NODETYPE(buf,&nodetype);
16451 
16452     btreeGetKeys(cache,buf,&karray,&parray);
16453 
16454     i = 0;
16455 
16456     while(i!=nkeys && MAJSTRCMPS(key,karray[i]) >= 0)
16457 	++i;
16458 
16459     ipos = i;
16460 
16461     count = nkeys - ipos;
16462 
16463 
16464     if(ipos == nkeys)
16465     {
16466 	ajStrAssignS(&karray[ipos],key);
16467 	parray[ipos+1] = greater;
16468 	parray[ipos]   = less;
16469     }
16470     else
16471     {
16472 	parray[nkeys+1] = parray[nkeys];
16473 
16474 	for(i=nkeys-1; count>0; --count, --i)
16475 	{
16476 	    ajStrAssignS(&karray[i+1],karray[i]);
16477 	    parray[i+1] = parray[i];
16478 	}
16479 
16480 	ajStrAssignS(&karray[ipos],key);
16481 	parray[ipos] = less;
16482 	parray[ipos+1] = greater;
16483     }
16484 
16485     ++nkeys;
16486     v = nkeys;
16487     SBT_NKEYS(buf,v);
16488 
16489     btreeWriteNode(cache,page,karray,parray,nkeys);
16490 
16491     if(nodetype == BT_SECROOT)
16492     {
16493 	page->dirty = BT_LOCK;
16494         page->lockfor = 1421;
16495     }
16496 
16497     pagepos = page->pagepos;
16498     ppage = btreeSeccacheRead(cache,less);
16499     lv = pagepos;
16500     SBT_PREV(ppage->buf,lv);
16501     ppage->dirty = BT_DIRTY;
16502 
16503     ppage = btreeSeccacheRead(cache,greater);
16504     lv = pagepos;
16505     SBT_PREV(ppage->buf,lv);
16506     ppage->dirty = BT_DIRTY;
16507 
16508     btreeDeallocSecArray(cache,arrays);
16509 
16510     if(nodetype != BT_SECROOT)
16511 	btreeKeyShiftSec(cache,page);
16512 
16513     return;
16514 }
16515 
16516 
16517 
16518 
16519 /* @funcstatic btreeKeyShiftSec ***********************************************
16520 **
16521 ** Rebalance secondary Nodes on insertion
16522 **
16523 ** @param [u] cache [AjPBtcache] cache
16524 ** @param [u] tpage [AjPBtpage] page
16525 **
16526 ** @return [void]
16527 **
16528 ** @release 3.0.0
16529 ** @@
16530 ******************************************************************************/
16531 
btreeKeyShiftSec(AjPBtcache cache,AjPBtpage tpage)16532 static void btreeKeyShiftSec(AjPBtcache cache, AjPBtpage tpage)
16533 {
16534     unsigned char *tbuf = NULL;
16535     unsigned char *pbuf = NULL;
16536     unsigned char *sbuf = NULL;
16537     unsigned char *buf  = NULL;
16538 
16539     AjPBtpage ppage = NULL;
16540     AjPBtpage spage = NULL;
16541     AjPBtpage page  = NULL;
16542 
16543     ajuint tkeys = 0U;
16544     ajuint pkeys = 0U;
16545     ajuint skeys = 0U;
16546     ajuint order = 0U;
16547 
16548     ajuint i;
16549     ajint ii;
16550 
16551     ajulong parent = 0UL;
16552 
16553     AjPBtMem Parrays = NULL;
16554     AjPBtMem Sarrays = NULL;
16555     AjPBtMem Tarrays = NULL;
16556     AjPStr *kTarray = NULL;
16557     AjPStr *kParray = NULL;
16558     AjPStr *kSarray = NULL;
16559     ajulong *pTarray = NULL;
16560     ajulong *pParray = NULL;
16561     ajulong *pSarray = NULL;
16562 
16563     ajuint pkeypos = 0U;
16564     ajuint minsize = 0U;
16565 
16566     ajulong lv = 0UL;
16567 
16568     /* ajDebug("In btreeKeyShiftSec\n"); */
16569 
16570     tbuf = tpage->buf;
16571 
16572     GBT_PREV(tbuf,&parent);
16573     GBT_NKEYS(tbuf,&tkeys);
16574 
16575     order = cache->sorder;
16576     minsize = order / 2U;
16577 
16578     if(order % 2U)
16579 	++minsize;
16580 
16581     if(tkeys <= minsize)
16582 	return;
16583 
16584 
16585     ppage = btreeSeccacheRead(cache,parent);
16586     pbuf = ppage->buf;
16587     GBT_NKEYS(pbuf,&pkeys);
16588 
16589     Parrays = btreeAllocSecArray(cache);
16590     Sarrays = btreeAllocSecArray(cache);
16591     Tarrays = btreeAllocSecArray(cache);
16592     kParray = Parrays->karray;
16593     pParray = Parrays->parray;
16594     kSarray = Sarrays->karray;
16595     pSarray = Sarrays->parray;
16596     kTarray = Tarrays->karray;
16597     pTarray = Tarrays->parray;
16598 
16599     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
16600     GBT_NKEYS(tbuf,&tkeys);
16601 
16602 
16603     btreeGetKeys(cache,pbuf,&kParray,&pParray);
16604 
16605     i=0;
16606 
16607     while(pParray[i] != tpage->pagepos)
16608 	++i;
16609 
16610     if(i) /* There is another leaf to the left */
16611     {
16612 	pkeypos = i-1;
16613 	spage = btreeSeccacheRead(cache,pParray[pkeypos]);
16614 	sbuf = spage->buf;
16615 	GBT_NKEYS(sbuf,&skeys);
16616 
16617     }
16618 
16619     if(i && skeys != order-1) /* There is space in the left leaf */
16620     {
16621 	if(skeys)
16622 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
16623 
16624 	ajStrAssignS(&kSarray[skeys],kParray[pkeypos]);
16625 	pSarray[skeys+1] = pTarray[0];
16626 	++skeys;
16627 	--tkeys;
16628 	ajStrAssignS(&kParray[pkeypos],kTarray[0]);
16629 
16630 	for(i=0;i<tkeys;++i)
16631 	{
16632 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
16633 	    pTarray[i] = pTarray[i+1];
16634 	}
16635 
16636 	pTarray[i] = pTarray[i+1];
16637 	pTarray[i+1] = 0UL;
16638 
16639 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
16640 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
16641 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
16642 
16643 	if(ppage->pagepos == cache->secrootblock)
16644         {
16645 	    ppage->dirty = BT_LOCK;
16646             ppage->lockfor = 1431;
16647         }
16648 
16649 	page = btreeSeccacheRead(cache,pSarray[skeys]);
16650 	buf = page->buf;
16651 	lv = spage->pagepos;
16652 	SBT_PREV(buf,lv);
16653 	page->dirty = BT_DIRTY;
16654 
16655 
16656         btreeDeallocSecArray(cache,Parrays);
16657         btreeDeallocSecArray(cache,Sarrays);
16658         btreeDeallocSecArray(cache,Tarrays);
16659 
16660 	return;
16661     }
16662 
16663 
16664 
16665     if(i != pkeys)	/* There is a right node */
16666     {
16667 	pkeypos = i;
16668 	spage = btreeSeccacheRead(cache,pParray[pkeypos+1]);
16669 	sbuf = spage->buf;
16670 	GBT_NKEYS(sbuf,&skeys);
16671     }
16672 
16673 
16674     if(i != pkeys && skeys != order-1) /* Space in the right node */
16675     {
16676 	if(skeys)
16677 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
16678 
16679 	pSarray[skeys+1] = pSarray[skeys];
16680 
16681 	for(ii=skeys-1;ii>-1;--ii)
16682 	{
16683 	    ajStrAssignS(&kSarray[ii+1],kSarray[ii]);
16684 	    pSarray[ii+1] = pSarray[ii];
16685 	}
16686 
16687 	ajStrAssignS(&kSarray[0],kParray[pkeypos]);
16688 	pSarray[0] = pTarray[tkeys];
16689 	ajStrAssignS(&kParray[pkeypos],kTarray[tkeys-1]);
16690 	++skeys;
16691 	--tkeys;
16692 	pTarray[tkeys+1] = 0UL;
16693 
16694 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
16695 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
16696 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
16697 
16698 	if(ppage->pagepos == cache->secrootblock)
16699         {
16700 	    ppage->dirty = BT_LOCK;
16701             ppage->lockfor = 1432;
16702         }
16703 
16704 	page = btreeSeccacheRead(cache,pSarray[0]);
16705 	buf = page->buf;
16706 	lv = spage->pagepos;
16707 	SBT_PREV(buf,lv);
16708 	page->dirty = BT_DIRTY;
16709 
16710 
16711         btreeDeallocSecArray(cache,Parrays);
16712         btreeDeallocSecArray(cache,Sarrays);
16713         btreeDeallocSecArray(cache,Tarrays);
16714 
16715 	return;
16716     }
16717 
16718     btreeDeallocSecArray(cache,Parrays);
16719     btreeDeallocSecArray(cache,Sarrays);
16720     btreeDeallocSecArray(cache,Tarrays);
16721 
16722     return;
16723 }
16724 
16725 
16726 
16727 
16728 #if AJINDEX_STATIC
16729 /* #func ajBtreeLockTest ******************************************************
16730 **
16731 ** Test function: show if a primary tree root block is unlocked
16732 **
16733 ** #param [u] cache [AjPBtcache] cache
16734 **
16735 ** #return [void]
16736 **
16737 ** #release 3.0.0
16738 ** ##
16739 ******************************************************************************/
16740 /*
16741 static void btreeLockTest(AjPBtcache cache)
16742 {
16743     AjPBtpage page = NULL;
16744 
16745     page = btreePricacheLocate(cache,0UL);
16746 
16747     if(page->dirty != BT_LOCK)
16748 	ajFatal("Root page unlocked\n");
16749 
16750     return;
16751 }
16752 */
16753 #endif
16754 
16755 
16756 
16757 
16758 /* @funcstatic btreeSecTreeCount **********************************************
16759 **
16760 ** Count the IDs in a secondary tree of identifiers for primary keywords
16761 **
16762 ** @param [u] cache [AjPBtcache] cache
16763 ** @param [r] rootblock [ajulong] root page of secondary tree
16764 **
16765 ** @return [ajulong] Number of IDs in secondary tree
16766 **
16767 ** @release 6.5.0
16768 ** @@
16769 ******************************************************************************/
16770 
btreeSecTreeCount(AjPBtcache cache,ajulong rootblock)16771 static ajulong btreeSecTreeCount(AjPBtcache cache, ajulong rootblock)
16772 {
16773     AjPBtMem arrays = NULL;
16774     ajulong *parray;
16775 
16776     AjPBtpage page;
16777     unsigned char *buf;
16778     ajuint nodetype;
16779     ajuint j;
16780     ajulong level = 0UL;
16781 
16782     ajuint nkeys;
16783     ajulong right;
16784 
16785     ajulong lcount = 0UL;
16786 
16787     page = btreeSeccacheRead(cache,rootblock);
16788     buf = page->buf;
16789     GBT_NODETYPE(buf,&nodetype);
16790 
16791     if(nodetype == BT_SECBUCKET)
16792     {
16793         return btreeSecbucketIdcount(cache, page->pagepos);
16794     }
16795 
16796     GBT_RIGHT(buf,&level);
16797     cache->slevel = (ajuint)level;
16798 
16799     arrays = btreeAllocSecArray(cache);
16800     parray = arrays->parray;
16801 
16802     btreeGetPointers(cache,buf,&parray);
16803 
16804     while(nodetype != BT_SECLEAF && cache->slevel!=0)
16805     {
16806 	page = btreeSeccacheRead(cache,parray[0]);
16807 	buf = page->buf;
16808 	btreeGetPointers(cache,buf,&parray);
16809 	GBT_NODETYPE(buf,&nodetype);
16810     }
16811 
16812     do
16813     {
16814 	GBT_NKEYS(buf,&nkeys);
16815 
16816         page->dirty = BT_LOCK;
16817         page->lockfor = 9901;
16818 
16819 	for(j=0;j<=nkeys;++j)
16820 	    lcount += btreeNumInSecbucket(cache, parray[j]);
16821 
16822         page->dirty = BT_CLEAN;
16823 	right = 0UL;
16824 
16825 	if(cache->slevel)
16826 	{
16827 	    GBT_RIGHT(buf,&right);
16828 
16829 	    if(right)
16830 	    {
16831 		page = btreeSeccacheRead(cache,right);
16832 		buf = page->buf;
16833 		btreeGetPointers(cache,buf,&parray);
16834 	    }
16835 	}
16836 
16837     } while(right);
16838 
16839     btreeDeallocSecArray(cache,arrays);
16840 
16841     return lcount;
16842 }
16843 
16844 
16845 
16846 
16847 /* @funcstatic btreeSecTreeList ***********************************************
16848 **
16849 ** Read the leaves of a secondary tree, storing the entry secondary
16850 ** identifiers in a list
16851 **
16852 ** @param [u] cache [AjPBtcache] cache
16853 ** @param [r] rootblock [ajulong] root page of secondary tree
16854 **
16855 ** @return [AjPList] List of IDs matching a keyword
16856 **
16857 ** @release 6.5.0
16858 ** @@
16859 ******************************************************************************/
16860 
btreeSecTreeList(AjPBtcache cache,ajulong rootblock)16861 static AjPList btreeSecTreeList(AjPBtcache cache, ajulong rootblock)
16862 {
16863     AjPList list;
16864 
16865     AjPBtMem arrays = NULL;
16866     ajulong *parray;
16867 
16868     AjPBtpage page;
16869     unsigned char *buf;
16870     ajuint nodetype;
16871     ajuint j;
16872     ajuint k;
16873     ajulong level = 0UL;
16874 
16875     AjPSecbucket bucket;
16876     ajuint nkeys;
16877     AjPStr id = NULL;
16878     ajulong right;
16879 
16880     list = ajListNew();
16881 
16882     page = btreeSeccacheRead(cache,rootblock);
16883     buf = page->buf;
16884     GBT_NODETYPE(buf,&nodetype);
16885 
16886     if(nodetype == BT_SECBUCKET)
16887     {
16888         btreeSecbucketIdlist(cache, page->pagepos, list);
16889         return list;
16890     }
16891 
16892     GBT_RIGHT(buf,&level);
16893     cache->slevel = (ajuint)level;
16894 
16895     arrays = btreeAllocSecArray(cache);
16896     parray = arrays->parray;
16897 
16898     btreeGetPointers(cache,buf,&parray);
16899 
16900     while(nodetype != BT_SECLEAF && cache->slevel!=0)
16901     {
16902 	page = btreeSeccacheRead(cache,parray[0]);
16903 	buf = page->buf;
16904 	btreeGetPointers(cache,buf,&parray);
16905 	GBT_NODETYPE(buf,&nodetype);
16906     }
16907 
16908     do
16909     {
16910 	GBT_NKEYS(buf,&nkeys);
16911 
16912         page->dirty = BT_LOCK;
16913         page->lockfor = 9901;
16914 
16915 	for(j=0;j<=nkeys;++j)
16916 	{
16917 	    bucket = btreeReadSecbucket(cache, parray[j]);
16918 
16919 	    for(k=0;k<bucket->Nentries;++k)
16920 	    {
16921 		id = ajStrNewS(bucket->SecIds[k]);
16922 		ajListPushAppend(list, (void *)id);
16923 	    }
16924 
16925 	    btreeSecbucketDel(&bucket);
16926 	}
16927 
16928         page->dirty = BT_CLEAN;
16929 	right = 0UL;
16930 
16931 	if(cache->slevel)
16932 	{
16933 	    GBT_RIGHT(buf,&right);
16934 
16935 	    if(right)
16936 	    {
16937 		page = btreeSeccacheRead(cache,right);
16938 		buf = page->buf;
16939 		btreeGetPointers(cache,buf,&parray);
16940 	    }
16941 	}
16942 
16943     } while(right);
16944 
16945     btreeDeallocSecArray(cache,arrays);
16946 
16947     return list;
16948 }
16949 
16950 
16951 
16952 
16953 #if AJINDEX_STATIC
16954 /* #funcstatic btreeKeyidVerify ***********************************************
16955 **
16956 ** Test routine: test for ID within a secondary tree
16957 **
16958 ** #param [u] cache [AjPBtcache] cache
16959 ** #param [r] rootblock [ajulong] root page of secondary tree
16960 ** #param [r] id [const AjPStr] test ID
16961 **
16962 ** #return [AjBool] true if ID found
16963 **
16964 ** #release 6.5.0
16965 ** ##
16966 ******************************************************************************/
16967 /*
16968 static AjBool btreeKeyidVerify(AjPBtcache cache, ajulong rootblock,
16969                                const AjPStr id)
16970 {
16971     AjPBtpage page;
16972     AjPBtpage spage;
16973     unsigned char *buf;
16974     ajulong blockno;
16975     AjBool found;
16976     ajulong right;
16977     ajuint dirtysave;
16978 
16979     cache->secrootblock = rootblock;
16980     page = btreeSeccacheRead(cache,rootblock);
16981     dirtysave = page->dirty;
16982     page->dirty = BT_LOCK;
16983     page->lockfor = 1441;
16984     buf = page->buf;
16985     GBT_RIGHT(buf,&right);
16986     cache->slevel = (ajuint)right;
16987 
16988     spage = btreeKeyidFind(cache,id);
16989 
16990     if(!spage)
16991 	return ajFalse;
16992 
16993     buf = spage->buf;
16994 
16995     blockno = btreeGetBlockS(cache,buf,id);
16996 
16997     found = btreeSecbucketFindId(cache, blockno, id);
16998 
16999     cache->secrootblock = 0UL;
17000     page->dirty = dirtysave;
17001 
17002     return found;
17003 }
17004 */
17005 #endif
17006 
17007 
17008 
17009 
17010 /* @func ajBtreeKeywildQuery **************************************************
17011 **
17012 ** Wildcard retrieval of keyword index entries
17013 **
17014 ** @param [u] cache [AjPBtcache] cache
17015 ** @param [u] wild [AjPBtKeywild] Wildcard
17016 ** @param [u] idcache [AjPBtcache] id cache
17017 **
17018 ** @return [AjPBtId] next matching Id or NULL
17019 **
17020 ** @release 3.0.0
17021 ** @@
17022 ******************************************************************************/
17023 
ajBtreeKeywildQuery(AjPBtcache cache,AjPBtKeywild wild,AjPBtcache idcache)17024 AjPBtId ajBtreeKeywildQuery(AjPBtcache cache, AjPBtKeywild wild,
17025                             AjPBtcache idcache)
17026 {
17027 
17028     AjPBtPri pri   = NULL;
17029     AjPBtpage page = NULL;
17030     AjPStr key      = NULL;
17031     AjPList list   = NULL;
17032     AjBool found   = ajFalse;
17033     AjPBtId btid   = NULL;
17034     AjPStr  id     = NULL;
17035 
17036     ajulong pagepos = 0UL;
17037 
17038     unsigned char *buf = NULL;
17039 
17040     key  = ajStrNewS(wild->prefix);
17041 
17042     list = wild->list;
17043 
17044     found = ajFalse;
17045 
17046     /*
17047     ** note prefix may be empty if query started with a wildcard
17048     */
17049 
17050     if(wild->first)
17051     {
17052 	page = btreePrimaryFetchFindleafWild(cache,key);
17053 	page->dirty = BT_LOCK;
17054         page->lockfor = 1451;
17055 	wild->pagepos = page->pagepos;
17056 
17057 	btreePrileafFetch(cache,page,list);
17058 
17059 	page->dirty = BT_CLEAN;
17060 
17061 	if(!ajListGetLength(list))
17062         {
17063             ajStrDel(&key);
17064 	    return NULL;
17065         }
17066 
17067 	while(ajListPop(list,(void **)&pri))
17068 	{
17069 	    if(ajStrPrefixS(pri->keyword,key))
17070 	    {
17071 		found = ajTrue;
17072 		break;
17073 	    }
17074 	    else
17075 		ajBtreePriDel(&pri);
17076 	}
17077 
17078 
17079 	wild->first = ajFalse;
17080 
17081 	if(found)
17082 	{
17083 	    cache->secrootblock = pri->treeblock;
17084 	    btreeSecLeftLeaf(cache, wild);
17085 	}
17086 	else /* Check the next leaf just in case key==internal */
17087 	{
17088 	    buf = page->buf;
17089 	    GBT_RIGHT(buf,&pagepos);
17090 
17091 	    if(!pagepos)
17092             {
17093                 ajStrDel(&key);
17094 		return NULL;
17095             }
17096 
17097 	    page = btreePricacheRead(cache,pagepos);
17098 	    wild->pagepos = pagepos;
17099 	    page->dirty = BT_LOCK;
17100             page->lockfor = 1452;
17101 
17102 	    btreePrileafFetch(cache,page,list);
17103 
17104 	    page->dirty = BT_CLEAN;
17105 
17106 	    if(!ajListGetLength(list))
17107             {
17108                 ajStrDel(&key);
17109 		return NULL;
17110 	    }
17111 
17112 	    found = ajFalse;
17113 
17114 	    while(ajListPop(list,(void **)&pri))
17115 	    {
17116 		if(ajStrPrefixS(pri->keyword,key))
17117 		{
17118 		    found = ajTrue;
17119 		    break;
17120 		}
17121 		else
17122 		    ajBtreePriDel(&pri);
17123 	    }
17124 
17125 	    if(!found)
17126             {
17127                 ajStrDel(&key);
17128 		return NULL;
17129             }
17130 
17131 	    cache->secrootblock = pri->treeblock;
17132 	    btreeSecLeftLeaf(cache, wild);
17133 	}
17134 
17135 
17136     }
17137 
17138 
17139     if(ajListGetLength(wild->idlist))
17140     {
17141 	ajListPop(wild->idlist,(void **)&id);
17142 	btid = btreeIdentQueryId(idcache,id);
17143 	ajStrDel(&id);
17144         ajStrDel(&key);
17145 
17146 	return btid;
17147     }
17148     else if((btreeKeywildNextList(cache,wild)))
17149     {
17150 	if(ajListGetLength(wild->idlist))
17151 	{
17152 	    ajListPop(wild->idlist,(void **)&id);
17153 	    btid = btreeIdentQueryId(idcache,id);
17154 	    ajStrDel(&id);
17155             ajStrDel(&key);
17156 
17157 	    return btid;
17158 	}
17159 
17160         ajStrDel(&key);
17161 	return NULL;
17162     }
17163 
17164 
17165     /* Done for the current keyword so get the next one */
17166 
17167     if(!ajListGetLength(list))
17168     {
17169 	page = btreePricacheRead(cache,wild->pagepos);
17170 	buf = page->buf;
17171 	GBT_RIGHT(buf,&pagepos);
17172 
17173 	if(!pagepos)
17174         {
17175             ajStrDel(&key);
17176             return NULL;
17177         }
17178 
17179 	page = btreePricacheRead(cache,pagepos);
17180 	wild->pagepos = pagepos;
17181 	page->dirty = BT_LOCK;
17182         page->lockfor = 1453;
17183 
17184 	btreePrileafFetch(cache,page,list);
17185 
17186 	page->dirty = BT_CLEAN;
17187 
17188 	if(!ajListGetLength(list))
17189         {
17190             ajStrDel(&key);
17191 	    return NULL;
17192         }
17193     }
17194 
17195     while(ajListPop(list,(void **)&pri))
17196     {
17197 	if(ajStrPrefixS(pri->keyword,key))
17198 	{
17199 	    found = ajTrue;
17200 	    break;
17201 	}
17202 
17203 	ajBtreePriDel(&pri);
17204     }
17205 
17206     ajStrDel(&key);
17207 
17208     if(!found)
17209 	return NULL;
17210 
17211     cache->secrootblock = pri->treeblock;
17212     btreeSecLeftLeaf(cache, wild);
17213 
17214     if(ajListGetLength(wild->idlist))
17215     {
17216 	ajListPop(wild->idlist,(void **)&id);
17217 	btid = btreeIdentQueryId(idcache,id);
17218 	ajStrDel(&id);
17219     }
17220     else
17221 	return NULL;
17222 
17223     return btid;
17224 }
17225 
17226 
17227 
17228 
17229 /* @func ajBtreeKeyFetchId ****************************************************
17230 **
17231 ** Retrieval of keyword index entries with a perfect match to the keyword
17232 **
17233 ** @param [u] cache [AjPBtcache] cache
17234 ** @param [u] idcache [AjPBtcache] id cache
17235 ** @param [r] key [const AjPStr] key
17236 ** @param [u] btidlist [AjPList] List of matching AjPBtId entries
17237 **
17238 ** @return [void]
17239 **
17240 ** @release 6.5.0
17241 ** @@
17242 ******************************************************************************/
17243 
ajBtreeKeyFetchId(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList btidlist)17244 void ajBtreeKeyFetchId(AjPBtcache cache, AjPBtcache idcache,
17245                        const AjPStr key, AjPList btidlist)
17246 {
17247     AjPBtId   id   = NULL;
17248     AjPList tlist = NULL;
17249     AjPStr   kwid  = NULL;
17250     ajulong treeblock = 0UL;
17251 
17252     if(ajBtreeKeyFindLen(cache, key,
17253                          &treeblock))
17254     {
17255         tlist = btreeSecTreeList(cache, treeblock);
17256 
17257         while(ajListPop(tlist,(void **)&kwid))
17258         {
17259             id = btreeIdentQueryId(idcache, kwid);
17260             ajDebug("id '%S' entry: %p\n", kwid, id);
17261 
17262             if(id)
17263             {
17264                 ajDebug("entry id: '%S' dups: %u offset: %Ld\n",
17265                         id->id, id->dups, id->offset);
17266                 if(!id->dups)
17267                 {
17268                     ajListPushAppend(btidlist,(void *)id);
17269                     id = NULL;
17270                 }
17271                 else
17272                 {
17273                     btreeIdentFetchMulti(idcache, id->id, id->offset,
17274                                            btidlist);
17275                     ajBtreeIdDel(&id);
17276                 }
17277             }
17278 
17279             ajStrDel(&kwid);
17280         }
17281 
17282         ajListFree(&tlist);
17283     }
17284 }
17285 
17286 
17287 
17288 
17289 /* @func ajBtreeKeyFetchHit ***************************************************
17290 **
17291 ** Retrieval of keyword hits with a perfect match to the keyword
17292 **
17293 ** @param [u] cache [AjPBtcache] cache
17294 ** @param [u] idcache [AjPBtcache] id cache
17295 ** @param [r] key [const AjPStr] key
17296 ** @param [u] hitlist [AjPList] List of matching AjPBtHit entries
17297 **
17298 ** @return [void]
17299 **
17300 ** @release 6.5.0
17301 ** @@
17302 ******************************************************************************/
17303 
ajBtreeKeyFetchHit(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList hitlist)17304 void ajBtreeKeyFetchHit(AjPBtcache cache, AjPBtcache idcache,
17305                         const AjPStr key, AjPList hitlist)
17306 {
17307     AjPBtId id  = NULL;
17308     AjPBtHit hit  = NULL;
17309     AjPList tlist = NULL;
17310     AjPStr   kwid  = NULL;
17311     ajulong treeblock = 0UL;
17312 
17313     ajDebug("ajBtreeKeyFetchHit '%S'\n", key);
17314 
17315     if(ajBtreeKeyFindLen(cache, key,
17316                          &treeblock))
17317     {
17318         tlist = btreeSecTreeList(cache, treeblock);
17319 
17320         while(ajListPop(tlist,(void **)&kwid))
17321         {
17322             id = btreeIdentQueryId(idcache, kwid);
17323             ajDebug("id '%S' entry: %p\n", kwid, id);
17324 
17325             if(id)
17326             {
17327                 ajDebug("entry id: '%S' dups: %u offset: %Ld\n",
17328                         id->id, id->dups, id->offset);
17329                 if(!id->dups)
17330                 {
17331                     hit = ajBtreeHitNewId(id);
17332                     ajListPushAppend(hitlist,(void *)hit);
17333                     hit = NULL;
17334                     ajBtreeIdDel(&id);
17335                 }
17336                 else
17337                 {
17338                     btreeIdentFetchMultiHit(idcache, id->offset,
17339                                             hitlist);
17340                     ajBtreeIdDel(&id);
17341                 }
17342             }
17343 
17344             ajStrDel(&kwid);
17345         }
17346 
17347         ajListFree(&tlist);
17348     }
17349 }
17350 
17351 
17352 
17353 
17354 /* @func ajBtreeKeyFetchHitref ************************************************
17355 **
17356 ** Retrieval of keyword hits with a perfect match to the keyword
17357 **
17358 ** @param [u] cache [AjPBtcache] cache
17359 ** @param [u] idcache [AjPBtcache] id cache
17360 ** @param [r] key [const AjPStr] key
17361 ** @param [u] hitlist [AjPList] List of matching AjPBtHitref entries
17362 **
17363 ** @return [void]
17364 **
17365 ** @release 6.5.0
17366 ** @@
17367 ******************************************************************************/
17368 
ajBtreeKeyFetchHitref(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList hitlist)17369 void ajBtreeKeyFetchHitref(AjPBtcache cache, AjPBtcache idcache,
17370                            const AjPStr key, AjPList hitlist)
17371 {
17372     AjPBtId id  = NULL;
17373     AjPBtHitref hitref = NULL;
17374     AjPList tlist = NULL;
17375     AjPStr   kwid  = NULL;
17376     ajulong treeblock = 0UL;
17377 
17378     ajDebug("ajBtreeKeyFetchHitref '%S'\n", key);
17379 
17380     if(ajBtreeKeyFindLen(cache, key,
17381                          &treeblock))
17382     {
17383         tlist = btreeSecTreeList(cache, treeblock);
17384 
17385         while(ajListPop(tlist,(void **)&kwid))
17386         {
17387             id = btreeIdentQueryId(idcache, kwid);
17388             ajDebug("id '%S' entry: %p\n", kwid, id);
17389 
17390             if(id)
17391             {
17392                 ajDebug("entry id: '%S' dups: %u offset: %Ld\n",
17393                         id->id, id->dups, id->offset);
17394                 if(!id->dups)
17395                 {
17396                     hitref = ajBtreeHitrefNewId(id);
17397                     ajListPushAppend(hitlist,(void *)hitref);
17398                     hitref = NULL;
17399                     ajBtreeIdDel(&id);
17400                 }
17401                 else
17402                 {
17403                     btreeIdentFetchMultiHitref(idcache, id->offset,
17404                                             hitlist);
17405                     ajBtreeIdDel(&id);
17406                 }
17407             }
17408 
17409             ajStrDel(&kwid);
17410         }
17411 
17412         ajListFree(&tlist);
17413     }
17414 }
17415 
17416 
17417 
17418 
17419 /* @func ajBtreeKeyFetchwildId ************************************************
17420 **
17421 ** Wildcard retrieval of keyword index entries
17422 **
17423 ** @param [u] cache [AjPBtcache] cache
17424 ** @param [u] idcache [AjPBtcache] id cache
17425 ** @param [r] key [const AjPStr] key
17426 ** @param [u] btidlist [AjPList] List of matching AjPBtId entries
17427 **
17428 ** @return [void]
17429 **
17430 ** @release 6.5.0
17431 ** @@
17432 ******************************************************************************/
17433 
ajBtreeKeyFetchwildId(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList btidlist)17434 void ajBtreeKeyFetchwildId(AjPBtcache cache, AjPBtcache idcache,
17435                            const AjPStr key, AjPList btidlist)
17436 {
17437     AjPBtPri pri     = NULL;
17438     AjPBtpage page   = NULL;
17439     AjPList prilist  = NULL;
17440     AjPList strlist  = NULL;
17441 
17442     AjPBtId btid = NULL;
17443     AjPStr id    = NULL;
17444 
17445     AjBool found     = ajFalse;
17446     AjBool finished  = ajFalse;
17447 
17448     ajulong pripagepossave = 0UL;
17449     ajulong pagepos        = 0UL;
17450     ajulong right         = 0UL;
17451 
17452     unsigned char *buf = NULL;
17453 
17454     AjPStr prefix = NULL;
17455 
17456     char *p;
17457     char *cp;
17458 
17459     ajDebug("ajBtreeKeyFetchwildId '%S' list: %Lu\n",
17460             key, ajListGetLength(btidlist));
17461 
17462     prefix = ajStrNew();
17463     cp = MAJSTRGETPTR(key);
17464     p = strpbrk(cp,"*?");
17465 
17466     if(p)
17467     {
17468 	if(p-cp)
17469 	    ajStrAssignSubS(&prefix,key,0,p-cp-1);
17470 	else
17471 	{
17472 	    ajStrDel(&prefix);
17473 	    btreeKeywordFullSearchId(cache,key,idcache,btidlist);
17474 
17475 	    return;
17476 	}
17477     }
17478     else
17479 	ajStrAssignS(&prefix,key);
17480 
17481     ajDebug("ajBtreeKeyFetchwildId prefix '%S' list: %Lu\n",
17482             prefix, ajListGetLength(btidlist));
17483 
17484     prilist  = ajListNew();
17485 
17486     found   = ajFalse;
17487 
17488     page = btreePrimaryFetchFindleafWild(cache,prefix);
17489     page->dirty = BT_LOCK;
17490     page->lockfor = 1461;
17491     pripagepossave = page->pagepos;
17492     btreePrileafFetch(cache,page,prilist);
17493     page->dirty = BT_CLEAN;
17494 
17495     if(!ajListGetLength(prilist))
17496     {
17497 	ajStrDel(&prefix);
17498 	ajListFree(&prilist);
17499 
17500 	return;
17501     }
17502 
17503 
17504     while(ajListPop(prilist,(void **)&pri))
17505     {
17506 	if(ajStrPrefixS(pri->keyword,prefix))
17507 	{
17508 	    found = ajTrue;
17509 	    break;
17510 	}
17511 	else
17512 	    ajBtreePriDel(&pri);
17513     }
17514 
17515     if(!found)	/* check next leaf in case key == internal */
17516     {
17517 	buf = page->buf;
17518 	GBT_RIGHT(buf,&pagepos);
17519 
17520 	if(!pagepos)
17521 	{
17522 	    ajStrDel(&prefix);
17523 	    ajListFree(&prilist);
17524 
17525 	    return;
17526 	}
17527 	page = btreePricacheRead(cache,pagepos);
17528 	page->dirty = BT_LOCK;
17529         page->lockfor = 1462;
17530 	pripagepossave = pagepos;
17531 	btreePrileafFetch(cache,page,prilist);
17532 	page->dirty = BT_CLEAN;
17533 
17534 	if(!ajListGetLength(prilist))
17535 	{
17536 	    ajStrDel(&prefix);
17537 	    ajListFree(&prilist);
17538 
17539 	    return;
17540 	}
17541 
17542 	while(ajListPop(prilist,(void **)&pri))
17543 	{
17544 	    if(ajStrPrefixS(pri->keyword,prefix))
17545 	    {
17546 		found = ajTrue;
17547 		break;
17548 	    }
17549 	    else
17550 		ajBtreePriDel(&pri);
17551 	}
17552     }
17553 
17554     if(!found)
17555     {
17556 	ajStrDel(&prefix);
17557 	ajListFree(&prilist);
17558 	return;
17559     }
17560 
17561     finished = ajFalse;
17562 
17563     strlist  = ajListNew();
17564 
17565     while(!finished)
17566     {
17567 	if(ajStrMatchWildS(pri->keyword,key))
17568 	{
17569 	    cache->secrootblock = pri->treeblock;
17570 	    btreeReadAllSecLeaves(cache,strlist);
17571 	}
17572 
17573 	ajBtreePriDel(&pri);
17574 
17575 	if(!ajListGetLength(prilist))
17576 	{
17577 	    page = btreePricacheRead(cache,pripagepossave);
17578 
17579 	    buf = page->buf;
17580 	    GBT_RIGHT(buf,&right);
17581 
17582 	    if(!right)
17583 	    {
17584 		finished = ajTrue;
17585 		continue;
17586 	    }
17587 
17588 	    page->dirty = BT_LOCK;
17589             page->lockfor = 1463;
17590 	    btreePrileafFetch(cache,page,prilist);
17591 	    page->dirty = BT_CLEAN;
17592 	    pripagepossave = right;
17593 
17594 	    if(!ajListGetLength(prilist))
17595 	    {
17596 		finished = ajTrue;
17597 		continue;
17598 	    }
17599 	}
17600 
17601 
17602 	ajListPop(prilist,(void **)&pri);
17603 
17604 	if(!ajStrPrefixS(pri->keyword,prefix))
17605 	{
17606 	    ajBtreePriDel(&pri);
17607 	    finished = ajTrue;
17608 	}
17609     }
17610 
17611 
17612     while(ajListPop(prilist,(void **)&pri))
17613 	ajBtreePriDel(&pri);
17614 
17615     ajListFree(&prilist);
17616 
17617 
17618     if(ajListGetLength(strlist))
17619     {
17620 	ajListSortUnique(strlist, &ajStrVcmp, &btreeStrDel);
17621 
17622 	while(ajListPop(strlist,(void **)&id))
17623 	{
17624 	    btid = btreeIdentQueryId(idcache,id);
17625             if(btid)
17626             {
17627                 ajDebug("ajBtreeKeyFetchwildId id: %S btid: '%S'\n",
17628                         id, btid->id);
17629                 ajListPushAppend(btidlist,(void *)btid);
17630             }
17631             else
17632             {
17633                 ajDebug("ajBtreeKeyFetchwildId id: %S not found\n",
17634                         id);
17635             }
17636 	    ajStrDel(&id);
17637 	}
17638     }
17639 
17640     ajListFree(&strlist);
17641 
17642     ajStrDel(&prefix);
17643 
17644     return;
17645 }
17646 
17647 
17648 
17649 
17650 /* @func ajBtreeKeyFetchwildHit ***********************************************
17651 **
17652 ** Wildcard retrieval of keyword hit entries
17653 **
17654 ** @param [u] cache [AjPBtcache] cache
17655 ** @param [u] idcache [AjPBtcache] id cache
17656 ** @param [r] key [const AjPStr] key
17657 ** @param [u] hitlist [AjPList] List of matching AjPBtHit entries
17658 **
17659 ** @return [void]
17660 **
17661 ** @release 6.5.0
17662 ** @@
17663 ******************************************************************************/
17664 
ajBtreeKeyFetchwildHit(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList hitlist)17665 void ajBtreeKeyFetchwildHit(AjPBtcache cache, AjPBtcache idcache,
17666                             const AjPStr key, AjPList hitlist)
17667 {
17668     AjPBtPri pri     = NULL;
17669     AjPBtpage page   = NULL;
17670     AjPList prilist  = NULL;
17671     AjPList strlist  = NULL;
17672 
17673     AjPBtId btid = NULL;
17674     AjPBtHit hit = NULL;
17675     AjPStr id    = NULL;
17676 
17677     AjBool found     = ajFalse;
17678     AjBool finished  = ajFalse;
17679 
17680     ajulong pripagepossave = 0UL;
17681     ajulong pagepos        = 0UL;
17682     ajulong right         = 0UL;
17683 
17684     unsigned char *buf = NULL;
17685 
17686     AjPStr prefix = NULL;
17687 
17688     char *p;
17689     char *cp;
17690 
17691     ajDebug("ajBtreeKeyFetchwildHit '%S' list: %Lu\n",
17692             key, ajListGetLength(hitlist));
17693 
17694     prefix = ajStrNew();
17695     cp = MAJSTRGETPTR(key);
17696     p = strpbrk(cp,"*?");
17697 
17698     if(p)
17699     {
17700 	if(p-cp)
17701 	    ajStrAssignSubS(&prefix,key,0,p-cp-1);
17702 	else
17703 	{
17704 	    ajStrDel(&prefix);
17705 	    btreeKeywordFullSearchHit(cache,key,idcache,hitlist);
17706 
17707 	    return;
17708 	}
17709     }
17710     else
17711 	ajStrAssignS(&prefix,key);
17712 
17713     ajDebug("ajBtreeKeyFetchwildHit prefix '%S' list: %Lu\n",
17714             prefix, ajListGetLength(hitlist));
17715 
17716     prilist  = ajListNew();
17717 
17718     found   = ajFalse;
17719 
17720     page = btreePrimaryFetchFindleafWild(cache,prefix);
17721     page->dirty = BT_LOCK;
17722     page->lockfor = 1461;
17723     pripagepossave = page->pagepos;
17724     btreePrileafFetch(cache,page,prilist);
17725     page->dirty = BT_CLEAN;
17726 
17727     if(!ajListGetLength(prilist))
17728     {
17729 	ajStrDel(&prefix);
17730 	ajListFree(&prilist);
17731 
17732 	return;
17733     }
17734 
17735 
17736     while(ajListPop(prilist,(void **)&pri))
17737     {
17738 	if(ajStrPrefixS(pri->keyword,prefix))
17739 	{
17740 	    found = ajTrue;
17741 	    break;
17742 	}
17743 	else
17744 	    ajBtreePriDel(&pri);
17745     }
17746 
17747     if(!found)	/* check next leaf in case key == internal */
17748     {
17749 	buf = page->buf;
17750 	GBT_RIGHT(buf,&pagepos);
17751 
17752 	if(!pagepos)
17753 	{
17754 	    ajStrDel(&prefix);
17755 	    ajListFree(&prilist);
17756 
17757 	    return;
17758 	}
17759 	page = btreePricacheRead(cache,pagepos);
17760 	page->dirty = BT_LOCK;
17761         page->lockfor = 1462;
17762 	pripagepossave = pagepos;
17763 	btreePrileafFetch(cache,page,prilist);
17764 	page->dirty = BT_CLEAN;
17765 
17766 	if(!ajListGetLength(prilist))
17767 	{
17768 	    ajStrDel(&prefix);
17769 	    ajListFree(&prilist);
17770 
17771 	    return;
17772 	}
17773 
17774 	while(ajListPop(prilist,(void **)&pri))
17775 	{
17776 	    if(ajStrPrefixS(pri->keyword,prefix))
17777 	    {
17778 		found = ajTrue;
17779 		break;
17780 	    }
17781 	    else
17782 		ajBtreePriDel(&pri);
17783 	}
17784     }
17785 
17786     if(!found)
17787     {
17788 	ajStrDel(&prefix);
17789 	ajListFree(&prilist);
17790 	return;
17791     }
17792 
17793     finished = ajFalse;
17794 
17795     strlist  = ajListNew();
17796 
17797     while(!finished)
17798     {
17799 	if(ajStrMatchWildS(pri->keyword,key))
17800 	{
17801 	    cache->secrootblock = pri->treeblock;
17802 	    btreeReadAllSecLeaves(cache,strlist);
17803 	}
17804 
17805 	ajBtreePriDel(&pri);
17806 
17807 	if(!ajListGetLength(prilist))
17808 	{
17809 	    page = btreePricacheRead(cache,pripagepossave);
17810 
17811 	    buf = page->buf;
17812 	    GBT_RIGHT(buf,&right);
17813 
17814 	    if(!right)
17815 	    {
17816 		finished = ajTrue;
17817 		continue;
17818 	    }
17819 
17820 	    page->dirty = BT_LOCK;
17821             page->lockfor = 1463;
17822 	    btreePrileafFetch(cache,page,prilist);
17823 	    page->dirty = BT_CLEAN;
17824 	    pripagepossave = right;
17825 
17826 	    if(!ajListGetLength(prilist))
17827 	    {
17828 		finished = ajTrue;
17829 		continue;
17830 	    }
17831 	}
17832 
17833 
17834 	ajListPop(prilist,(void **)&pri);
17835 
17836 	if(!ajStrPrefixS(pri->keyword,prefix))
17837 	{
17838 	    ajBtreePriDel(&pri);
17839 	    finished = ajTrue;
17840 	}
17841     }
17842 
17843 
17844     while(ajListPop(prilist,(void **)&pri))
17845 	ajBtreePriDel(&pri);
17846 
17847     ajListFree(&prilist);
17848 
17849 
17850     if(ajListGetLength(strlist))
17851     {
17852 	ajListSortUnique(strlist, &ajStrVcmp, &btreeStrDel);
17853 
17854 	while(ajListPop(strlist,(void **)&id))
17855 	{
17856 	    btid = btreeIdentQueryId(idcache,id);
17857             if(btid)
17858             {
17859                 ajDebug("ajBtreeKeyFetchwildHit id: %S btid: '%S'\n",
17860                         id, btid->id);
17861                 hit = ajBtreeHitNewId(btid);
17862                 ajListPushAppend(hitlist,(void *)hit);
17863                 hit = NULL;
17864                 ajBtreeIdDel(&btid);
17865             }
17866             else
17867             {
17868                 ajDebug("ajBtreeKeyFetchwildHit id: %S not found\n",
17869                         id);
17870             }
17871 	    ajStrDel(&id);
17872 	}
17873     }
17874 
17875     ajListFree(&strlist);
17876 
17877     ajStrDel(&prefix);
17878 
17879     return;
17880 }
17881 
17882 
17883 
17884 
17885 /* @func ajBtreeKeyFetchwildHitref ********************************************
17886 **
17887 ** Wildcard retrieval of keyword reference hit entries
17888 **
17889 ** @param [u] cache [AjPBtcache] cache
17890 ** @param [u] idcache [AjPBtcache] id cache
17891 ** @param [r] key [const AjPStr] key
17892 ** @param [u] hitlist [AjPList] List of matching AjPBtHitref entries
17893 **
17894 ** @return [void]
17895 **
17896 ** @release 6.5.0
17897 ** @@
17898 ******************************************************************************/
17899 
ajBtreeKeyFetchwildHitref(AjPBtcache cache,AjPBtcache idcache,const AjPStr key,AjPList hitlist)17900 void ajBtreeKeyFetchwildHitref(AjPBtcache cache, AjPBtcache idcache,
17901                                const AjPStr key, AjPList hitlist)
17902 {
17903     AjPBtPri pri     = NULL;
17904     AjPBtpage page   = NULL;
17905     AjPList prilist  = NULL;
17906     AjPList strlist  = NULL;
17907 
17908     AjPBtId btid = NULL;
17909     AjPBtHitref hitref = NULL;
17910     AjPStr id    = NULL;
17911 
17912     AjBool found     = ajFalse;
17913     AjBool finished  = ajFalse;
17914 
17915     ajulong pripagepossave = 0UL;
17916     ajulong pagepos        = 0UL;
17917     ajulong right         = 0UL;
17918 
17919     unsigned char *buf = NULL;
17920 
17921     AjPStr prefix = NULL;
17922 
17923     char *p;
17924     char *cp;
17925 
17926     ajDebug("ajBtreeKeyFetchwildHitref '%S' list: %Lu\n",
17927             key, ajListGetLength(hitlist));
17928 
17929     prefix = ajStrNew();
17930     cp = MAJSTRGETPTR(key);
17931     p = strpbrk(cp,"*?");
17932 
17933     if(p)
17934     {
17935 	if(p-cp)
17936 	    ajStrAssignSubS(&prefix,key,0,p-cp-1);
17937 	else
17938 	{
17939 	    ajStrDel(&prefix);
17940 	    btreeKeywordFullSearchHitref(cache,key,idcache,hitlist);
17941 
17942 	    return;
17943 	}
17944     }
17945     else
17946 	ajStrAssignS(&prefix,key);
17947 
17948     ajDebug("ajBtreeKeyFetchwildHitref prefix '%S' list: %Lu\n",
17949             prefix, ajListGetLength(hitlist));
17950 
17951     prilist  = ajListNew();
17952 
17953     found   = ajFalse;
17954 
17955     page = btreePrimaryFetchFindleafWild(cache,prefix);
17956     page->dirty = BT_LOCK;
17957     page->lockfor = 1461;
17958     pripagepossave = page->pagepos;
17959     btreePrileafFetch(cache,page,prilist);
17960     page->dirty = BT_CLEAN;
17961 
17962     if(!ajListGetLength(prilist))
17963     {
17964 	ajStrDel(&prefix);
17965 	ajListFree(&prilist);
17966 
17967 	return;
17968     }
17969 
17970 
17971     while(ajListPop(prilist,(void **)&pri))
17972     {
17973 	if(ajStrPrefixS(pri->keyword,prefix))
17974 	{
17975 	    found = ajTrue;
17976 	    break;
17977 	}
17978 	else
17979 	    ajBtreePriDel(&pri);
17980     }
17981 
17982     if(!found)	/* check next leaf in case key == internal */
17983     {
17984 	buf = page->buf;
17985 	GBT_RIGHT(buf,&pagepos);
17986 
17987 	if(!pagepos)
17988 	{
17989 	    ajStrDel(&prefix);
17990 	    ajListFree(&prilist);
17991 
17992 	    return;
17993 	}
17994 	page = btreePricacheRead(cache,pagepos);
17995 	page->dirty = BT_LOCK;
17996         page->lockfor = 1462;
17997 	pripagepossave = pagepos;
17998 	btreePrileafFetch(cache,page,prilist);
17999 	page->dirty = BT_CLEAN;
18000 
18001 	if(!ajListGetLength(prilist))
18002 	{
18003 	    ajStrDel(&prefix);
18004 	    ajListFree(&prilist);
18005 
18006 	    return;
18007 	}
18008 
18009 	while(ajListPop(prilist,(void **)&pri))
18010 	{
18011 	    if(ajStrPrefixS(pri->keyword,prefix))
18012 	    {
18013 		found = ajTrue;
18014 		break;
18015 	    }
18016 	    else
18017 		ajBtreePriDel(&pri);
18018 	}
18019     }
18020 
18021     if(!found)
18022     {
18023 	ajStrDel(&prefix);
18024 	ajListFree(&prilist);
18025 	return;
18026     }
18027 
18028     finished = ajFalse;
18029 
18030     strlist  = ajListNew();
18031 
18032     while(!finished)
18033     {
18034 	if(ajStrMatchWildS(pri->keyword,key))
18035 	{
18036 	    cache->secrootblock = pri->treeblock;
18037 	    btreeReadAllSecLeaves(cache,strlist);
18038 	}
18039 
18040 	ajBtreePriDel(&pri);
18041 
18042 	if(!ajListGetLength(prilist))
18043 	{
18044 	    page = btreePricacheRead(cache,pripagepossave);
18045 
18046 	    buf = page->buf;
18047 	    GBT_RIGHT(buf,&right);
18048 
18049 	    if(!right)
18050 	    {
18051 		finished = ajTrue;
18052 		continue;
18053 	    }
18054 
18055 	    page->dirty = BT_LOCK;
18056             page->lockfor = 1463;
18057 	    btreePrileafFetch(cache,page,prilist);
18058 	    page->dirty = BT_CLEAN;
18059 	    pripagepossave = right;
18060 
18061 	    if(!ajListGetLength(prilist))
18062 	    {
18063 		finished = ajTrue;
18064 		continue;
18065 	    }
18066 	}
18067 
18068 
18069 	ajListPop(prilist,(void **)&pri);
18070 
18071 	if(!ajStrPrefixS(pri->keyword,prefix))
18072 	{
18073 	    ajBtreePriDel(&pri);
18074 	    finished = ajTrue;
18075 	}
18076     }
18077 
18078 
18079     while(ajListPop(prilist,(void **)&pri))
18080 	ajBtreePriDel(&pri);
18081 
18082     ajListFree(&prilist);
18083 
18084 
18085     if(ajListGetLength(strlist))
18086     {
18087 	ajListSortUnique(strlist, &ajStrVcmp, &btreeStrDel);
18088 
18089 	while(ajListPop(strlist,(void **)&id))
18090 	{
18091 	    btid = btreeIdentQueryId(idcache,id);
18092             if(btid)
18093             {
18094                 ajDebug("ajBtreeKeyFetchwildHitref id: %S btid: '%S'\n",
18095                         id, btid->id);
18096                 hitref = ajBtreeHitrefNewId(btid);
18097                 ajListPushAppend(hitlist,(void *)hitref);
18098                 hitref = NULL;
18099                 ajBtreeIdDel(&btid);
18100             }
18101             else
18102             {
18103                 ajDebug("ajBtreeKeyFetchwildHitref id: %S not found\n",
18104                         id);
18105             }
18106 	    ajStrDel(&id);
18107 	}
18108     }
18109 
18110     ajListFree(&strlist);
18111 
18112     ajStrDel(&prefix);
18113 
18114     return;
18115 }
18116 
18117 
18118 
18119 
18120 /* @funcstatic btreeKeywordFullSearchId ***************************************
18121 **
18122 ** Wildcard retrieval of key/des/org entries. Whole index scan. Only used for
18123 ** wildcard searches with keys beginning with '?' or '*'
18124 **
18125 ** @param [u] cache [AjPBtcache] cache
18126 ** @param [r] key [const AjPStr] Wildcard key
18127 ** @param [u] idcache [AjPBtcache] id index cache
18128 ** @param [u] idlist [AjPList] list of matching AjPBtIds
18129 **
18130 ** @return [void]
18131 **
18132 ** @release 3.0.0
18133 ** @@
18134 ******************************************************************************/
18135 
btreeKeywordFullSearchId(AjPBtcache cache,const AjPStr key,AjPBtcache idcache,AjPList idlist)18136 static void btreeKeywordFullSearchId(AjPBtcache cache, const AjPStr key,
18137                                      AjPBtcache idcache, AjPList idlist)
18138 {
18139     AjPBtPri pri   = NULL;
18140     AjPBtpage root = NULL;
18141     AjPBtpage page = NULL;
18142     AjPBtId btid   = NULL;
18143     ajulong right   = 0UL;
18144     ajuint nodetype = 0U;
18145 
18146     AjPBtMem arrays = NULL;
18147     ajulong *parray = NULL;
18148 
18149     AjPList list   = NULL;
18150     AjPList strlist = NULL;
18151     AjPStr id = NULL;
18152 
18153     unsigned char *buf = NULL;
18154 
18155     ajulong nids = 0U;
18156     ajulong i;
18157 
18158     AjPTable strtable = NULL;
18159     AjPStr *idarray = NULL;
18160 
18161     list    = ajListNew();
18162     strlist = ajListNew();
18163     strtable = ajTablestrNew(idcache->countunique);
18164 
18165     root = btreePricacheLocate(cache, 0UL);
18166     page = root;
18167 
18168     buf = root->buf;
18169     GBT_NODETYPE(buf,&nodetype);
18170 
18171     if(cache->plevel)
18172     {
18173         arrays = btreeAllocPriArray(cache);
18174         parray = arrays->parray;
18175 	while(nodetype != BT_LEAF)
18176 	{
18177 	    btreeGetPointers(cache,buf,&parray);
18178 	    page = btreePricacheRead(cache, parray[0]);
18179 	    buf = page->buf;
18180 	    GBT_NODETYPE(buf,&nodetype);
18181 	    page->dirty = BT_CLEAN;
18182 	}
18183         btreeDeallocPriArray(cache,arrays);
18184     }
18185 
18186     right = 99L;
18187 
18188     while(right)
18189     {
18190 	btreePrileafFetch(cache,page,list);
18191 
18192 	while(ajListPop(list,(void **)&pri))
18193 	{
18194 	    if(ajStrMatchWildS(pri->keyword,key))
18195 	    {
18196 		cache->secrootblock = pri->treeblock;
18197 		btreeReadAllSecLeaves(cache,strlist);
18198 
18199                 while(ajListPop(strlist,(void **)&id))
18200                     ajTablePut(strtable, id, NULL);
18201 	    }
18202 
18203 	    ajBtreePriDel(&pri);
18204 	}
18205 
18206 	GBT_RIGHT(buf,&right);
18207 
18208 	if(right)
18209 	{
18210 	    page = btreePricacheRead(cache,right);
18211 	    buf = page->buf;
18212 	}
18213     }
18214 
18215     if(ajTableGetLength(strtable))
18216     {
18217         nids = ajTableToarrayKeys(strtable, (void***)&idarray);
18218 
18219         for(i=0; i < nids; i++)
18220 	{
18221 	    btid = btreeIdentQueryId(idcache,idarray[i]);
18222             if(btid)
18223                 ajListPushAppend(idlist,(void *)btid);
18224 	}
18225     }
18226 
18227     AJFREE(idarray);
18228     ajListFree(&list);
18229     ajListFree(&strlist);
18230     ajTableDel(&strtable);
18231 
18232     return;
18233 }
18234 
18235 
18236 
18237 
18238 /* @funcstatic btreeKeywordFullSearchHit **************************************
18239 **
18240 ** Wildcard retrieval of key/des/org entries. Whole index scan. Only used for
18241 ** wildcard searches with keys beginning with '?' or '*'
18242 **
18243 ** @param [u] cache [AjPBtcache] cache
18244 ** @param [r] key [const AjPStr] Wildcard key
18245 ** @param [u] idcache [AjPBtcache] id index cache
18246 ** @param [u] hitlist [AjPList] list of matching AjPBtHits
18247 **
18248 ** @return [void]
18249 **
18250 ** @release 6.5.0
18251 ** @@
18252 ******************************************************************************/
18253 
btreeKeywordFullSearchHit(AjPBtcache cache,const AjPStr key,AjPBtcache idcache,AjPList hitlist)18254 static void btreeKeywordFullSearchHit(AjPBtcache cache, const AjPStr key,
18255                                       AjPBtcache idcache, AjPList hitlist)
18256 {
18257     AjPBtPri pri   = NULL;
18258     AjPBtpage root = NULL;
18259     AjPBtpage page = NULL;
18260     AjPBtHit  hit  = NULL;
18261     ajulong right   = 0UL;
18262     ajuint nodetype = 0U;
18263 
18264     AjPBtMem arrays = NULL;
18265     ajulong *parray = NULL;
18266 
18267     AjPList list   = NULL;
18268     AjPList strlist = NULL;
18269     AjPStr id = NULL;
18270 
18271     unsigned char *buf = NULL;
18272 
18273     ajulong nids = 0U;
18274     ajulong i;
18275 
18276     AjPTable strtable = NULL;
18277     AjPStr *idarray = NULL;
18278 
18279     list    = ajListNew();
18280     strlist = ajListNew();
18281     strtable = ajTablestrNew(idcache->countunique);
18282 
18283     root = btreePricacheLocate(cache, 0UL);
18284     page = root;
18285 
18286     buf = root->buf;
18287     GBT_NODETYPE(buf,&nodetype);
18288 
18289     if(cache->plevel)
18290     {
18291         arrays = btreeAllocPriArray(cache);
18292         parray = arrays->parray;
18293 	while(nodetype != BT_LEAF)
18294 	{
18295 	    btreeGetPointers(cache,buf,&parray);
18296 	    page = btreePricacheRead(cache, parray[0]);
18297 	    buf = page->buf;
18298 	    GBT_NODETYPE(buf,&nodetype);
18299 	    page->dirty = BT_CLEAN;
18300 	}
18301         btreeDeallocPriArray(cache,arrays);
18302     }
18303 
18304     right = 99L;
18305 
18306     while(right)
18307     {
18308 	btreePrileafFetch(cache,page,list);
18309 
18310 	while(ajListPop(list,(void **)&pri))
18311 	{
18312 	    if(ajStrMatchWildS(pri->keyword,key))
18313 	    {
18314 		cache->secrootblock = pri->treeblock;
18315 		btreeReadAllSecLeaves(cache,strlist);
18316 
18317                 while(ajListPop(strlist,(void **)&id))
18318                     ajTablePut(strtable, id, NULL);
18319             }
18320 
18321 	    ajBtreePriDel(&pri);
18322 	}
18323 
18324 	GBT_RIGHT(buf,&right);
18325 
18326 	if(right)
18327 	{
18328 	    page = btreePricacheRead(cache,right);
18329 	    buf = page->buf;
18330 	}
18331     }
18332 
18333     if(ajTableGetLength(strtable))
18334     {
18335         nids = ajTableToarrayKeys(strtable, (void***)&idarray);
18336         for(i=0; i < nids; i++)
18337 	{
18338 	    hit = btreeIdentQueryHit(idcache,idarray[i]);
18339 	    if(hit)
18340                 ajListPushAppend(hitlist,(void *)hit);
18341 	}
18342     }
18343 
18344     AJFREE(idarray);
18345     ajListFree(&list);
18346     ajListFree(&strlist);
18347     ajTableDel(&strtable);
18348 
18349     return;
18350 }
18351 
18352 
18353 
18354 
18355 /* @funcstatic btreeKeywordFullSearchHitref ***********************************
18356 **
18357 ** Wildcard retrieval of key/des/org entries. Whole index scan. Only used for
18358 ** wildcard searches with keys beginning with '?' or '*'
18359 **
18360 ** @param [u] cache [AjPBtcache] cache
18361 ** @param [r] key [const AjPStr] Wildcard key
18362 ** @param [u] idcache [AjPBtcache] id index cache
18363 ** @param [u] hitlist [AjPList] list of matching AjPBtHitrefs
18364 **
18365 ** @return [void]
18366 **
18367 ** @release 6.5.0
18368 ** @@
18369 ******************************************************************************/
18370 
btreeKeywordFullSearchHitref(AjPBtcache cache,const AjPStr key,AjPBtcache idcache,AjPList hitlist)18371 static void btreeKeywordFullSearchHitref(AjPBtcache cache, const AjPStr key,
18372                                          AjPBtcache idcache, AjPList hitlist)
18373 {
18374     AjPBtPri pri   = NULL;
18375     AjPBtpage root = NULL;
18376     AjPBtpage page = NULL;
18377     AjPBtHitref hitref = NULL;
18378     ajulong right   = 0UL;
18379     ajuint nodetype = 0U;
18380 
18381     AjPBtMem arrays = NULL;
18382     ajulong *parray = NULL;
18383 
18384     AjPList list   = NULL;
18385     AjPList strlist = NULL;
18386     AjPStr id = NULL;
18387 
18388     unsigned char *buf = NULL;
18389 
18390     ajulong nids = 0U;
18391     ajulong i;
18392 
18393     AjPTable strtable = NULL;
18394     AjPStr *idarray = NULL;
18395 
18396     list    = ajListNew();
18397     strlist = ajListNew();
18398     strtable = ajTablestrNew(idcache->countunique);
18399 
18400     root = btreePricacheLocate(cache, 0UL);
18401     page = root;
18402 
18403     buf = root->buf;
18404     GBT_NODETYPE(buf,&nodetype);
18405 
18406     if(cache->plevel)
18407     {
18408         arrays = btreeAllocPriArray(cache);
18409         parray = arrays->parray;
18410 	while(nodetype != BT_LEAF)
18411 	{
18412 	    btreeGetPointers(cache,buf,&parray);
18413 	    page = btreePricacheRead(cache, parray[0]);
18414 	    buf = page->buf;
18415 	    GBT_NODETYPE(buf,&nodetype);
18416 	    page->dirty = BT_CLEAN;
18417 	}
18418         btreeDeallocPriArray(cache,arrays);
18419     }
18420 
18421     right = 99L;
18422 
18423     while(right)
18424     {
18425 	btreePrileafFetch(cache,page,list);
18426 
18427 	while(ajListPop(list,(void **)&pri))
18428 	{
18429 	    if(ajStrMatchWildS(pri->keyword,key))
18430 	    {
18431 		cache->secrootblock = pri->treeblock;
18432 		btreeReadAllSecLeaves(cache,strlist);
18433 
18434                 while(ajListPop(strlist,(void **)&id))
18435                     ajTablePut(strtable, id, NULL);
18436 	    }
18437 
18438 	    ajBtreePriDel(&pri);
18439 	}
18440 
18441 	GBT_RIGHT(buf,&right);
18442 
18443 	if(right)
18444 	{
18445 	    page = btreePricacheRead(cache,right);
18446 	    buf = page->buf;
18447 	}
18448     }
18449 
18450 
18451     if(ajTableGetLength(strtable))
18452     {
18453         nids = ajTableToarrayKeys(strtable, (void***)&idarray);
18454 
18455         for(i=0; i < nids; i++)
18456 	{
18457 	    hitref = btreeIdentQueryHitref(idcache,idarray[i]);
18458             if(hitref)
18459                 ajListPushAppend(hitlist,(void *)hitref);
18460 	}
18461     }
18462 
18463     AJFREE(idarray);
18464     ajListFree(&list);
18465     ajListFree(&strlist);
18466     ajTableDel(&strtable);
18467 
18468     return;
18469 }
18470 
18471 
18472 
18473 
18474 /* @funcstatic btreePrileafFetch **********************************************
18475 **
18476 ** Read all primary index leaf keywords into a list
18477 **
18478 ** @param [u] cache [AjPBtcache] cache
18479 ** @param [u] page [AjPBtpage] page
18480 ** @param [w] list [AjPList] list of AjPBtPri objects
18481 **
18482 ** @return [void]
18483 **
18484 ** @release 6.5.0
18485 ** @@
18486 ******************************************************************************/
18487 
btreePrileafFetch(AjPBtcache cache,AjPBtpage page,AjPList list)18488 static void btreePrileafFetch(AjPBtcache cache, AjPBtpage page, AjPList list)
18489 {
18490     unsigned char *buf = NULL;
18491     AjPBtMem arrays    = NULL;
18492     ajulong *parray     = NULL;
18493 
18494     ajuint keylimit = 0;
18495     ajuint nkeys    = 0;
18496 
18497     ajuint i;
18498 
18499     /* ajDebug("In PrileafFetch\n"); */
18500 
18501     buf = page->buf;
18502 
18503     arrays = btreeAllocPriArray(cache);
18504     parray = arrays->parray;
18505 
18506     btreeGetPointers(cache,buf,&parray);
18507 
18508     GBT_NKEYS(buf,&nkeys);
18509 
18510     keylimit = nkeys+1;
18511 
18512     for(i=0;i<keylimit;++i)
18513 	btreePribucketIdlist(cache,parray[i],list);
18514 
18515     ajListSort(list, &btreeKeywordCompare);
18516 
18517     btreeDeallocPriArray(cache,arrays);
18518 
18519     return;
18520 }
18521 
18522 
18523 
18524 
18525 /* @funcstatic btreeSecLeftLeaf ***********************************************
18526 **
18527 ** Read all secondary index leaf IDs into a list from the lefthand-most
18528 ** leaf or the root node if the level is 0.
18529 **
18530 ** @param [u] cache [AjPBtcache] cache
18531 ** @param [u] wild [AjPBtKeywild] wildcard keyword object
18532 **
18533 ** @return [void]
18534 **
18535 ** @release 3.0.0
18536 ** @@
18537 ******************************************************************************/
18538 
btreeSecLeftLeaf(AjPBtcache cache,AjPBtKeywild wild)18539 static void btreeSecLeftLeaf(AjPBtcache cache, AjPBtKeywild wild)
18540 {
18541     AjPBtpage root = NULL;
18542     AjPBtpage page = NULL;
18543     ajulong right = 0UL;
18544     ajuint nodetype = 0U;
18545     ajuint nkeys = 0U;
18546     ajuint keylimit = 0U;
18547 
18548     ajuint i;
18549 
18550     AjPBtMem arrays = NULL;
18551     ajulong *parray = NULL;
18552 
18553     unsigned char *buf;
18554 
18555 
18556     root = btreeSeccacheRead(cache, cache->secrootblock);
18557     root->dirty = BT_LOCK;
18558     root->lockfor = 1471;
18559 
18560     buf = root->buf;
18561     GBT_RIGHT(buf,&right);
18562     cache->slevel = (ajuint) right;
18563     GBT_NODETYPE(buf,&nodetype);
18564 
18565     arrays = btreeAllocSecArray(cache);
18566     parray = arrays->parray;
18567 
18568     btreeGetPointers(cache,buf,&parray);
18569 
18570     GBT_NODETYPE(buf,&nodetype);
18571 
18572     if(cache->slevel)
18573     {
18574 	while(nodetype != BT_SECLEAF)
18575 	{
18576 	    btreeGetPointers(cache,buf,&parray);
18577 	    page = btreeSeccacheRead(cache, parray[0]);
18578 	    buf = page->buf;
18579 	    GBT_NODETYPE(buf,&nodetype);
18580 	    page->dirty = BT_CLEAN;
18581 	}
18582 
18583 	wild->secpagepos = parray[0];
18584 	btreeGetPointers(cache,buf,&parray);
18585     }
18586     else
18587 	wild->secpagepos = cache->secrootblock;
18588 
18589 
18590 
18591     GBT_NKEYS(buf,&nkeys);
18592 
18593 
18594     keylimit = nkeys+1;
18595     for(i=0;i<keylimit;++i)
18596 	btreeSecbucketIdlist(cache,parray[i],wild->idlist);
18597 
18598     ajListSort(wild->idlist, &ajStrVcmp);
18599 
18600     root->dirty = BT_CLEAN;
18601 
18602     btreeDeallocSecArray(cache,arrays);
18603 
18604     cache->secrootblock = 0UL;
18605 
18606     return;
18607 }
18608 
18609 
18610 
18611 
18612 /* @funcstatic btreeKeywildNextList *******************************************
18613 **
18614 ** Get next wadge of secondary index leaf IDs into a list after a successful
18615 ** wildcard keyword search
18616 **
18617 ** @param [u] cache [AjPBtcache] cache
18618 ** @param [u] wild [AjPBtKeywild] wildcard keyword object
18619 **
18620 ** @return [AjBool] true if a wadge was successfully read
18621 **
18622 ** @release 6.5.0
18623 ** @@
18624 ******************************************************************************/
18625 
btreeKeywildNextList(AjPBtcache cache,AjPBtKeywild wild)18626 static AjBool btreeKeywildNextList(AjPBtcache cache, AjPBtKeywild wild)
18627 {
18628     AjPBtpage page = NULL;
18629     unsigned char *buf;
18630     ajulong right = 0UL;
18631     ajuint nkeys = 0U;
18632     ajuint keylimit = 0U;
18633     ajuint i;
18634 
18635     AjPBtMem arrays = NULL;
18636     ajulong *parray = NULL;
18637 
18638     if(cache->secrootblock == wild->secpagepos)
18639 	return ajFalse;
18640 
18641 
18642     page = btreeSeccacheRead(cache,wild->secpagepos);
18643     buf = page->buf;
18644     GBT_RIGHT(buf,&right);
18645     page->dirty = BT_CLEAN;
18646 
18647     if(!right)
18648 	return ajFalse;
18649 
18650     page = btreeSeccacheRead(cache,right);
18651     wild->secpagepos = right;
18652 
18653     arrays = btreeAllocSecArray(cache);
18654     parray = arrays->parray;
18655 
18656     btreeGetPointers(cache,buf,&parray);
18657     GBT_NKEYS(buf,&nkeys);
18658 
18659     keylimit = nkeys + 1;
18660 
18661     for(i=0;i<keylimit;++i)
18662 	btreeSecbucketIdlist(cache,parray[i],wild->idlist);
18663 
18664     ajListSort(wild->idlist, &ajStrVcmp);
18665 
18666     btreeDeallocSecArray(cache,arrays);
18667 
18668     return ajTrue;
18669 }
18670 
18671 
18672 
18673 
18674 /* @funcstatic btreeReadAllSecLeaves ******************************************
18675 **
18676 ** Read all the IDs from a secondary tree
18677 **
18678 ** @param [u] cache [AjPBtcache] cache
18679 ** @param [u] list [AjPList] list of IDs to return
18680 **
18681 ** @return [void]
18682 **
18683 ** @release 3.0.0
18684 ** @@
18685 ******************************************************************************/
18686 
btreeReadAllSecLeaves(AjPBtcache cache,AjPList list)18687 static void btreeReadAllSecLeaves(AjPBtcache cache, AjPList list)
18688 {
18689     AjPBtpage root = NULL;
18690     AjPBtpage page = NULL;
18691     ajulong right = 0UL;
18692     ajuint nodetype = 0U;
18693     ajuint nkeys = 0U;
18694     ajuint keylimit = 0U;
18695 
18696     ajuint i;
18697 
18698     AjPBtMem arrays = NULL;
18699     ajulong *parray = NULL;
18700 
18701     unsigned char *buf;
18702 
18703     ajulong secpagepos = 0UL;
18704 
18705     root = btreeSeccacheRead(cache,cache->secrootblock);
18706     buf = root->buf;
18707     GBT_NODETYPE(buf,&nodetype);
18708 
18709     if(nodetype == BT_SECBUCKET)
18710     {
18711         btreeSecbucketIdlist(cache, root->pagepos, list);
18712         return;
18713     }
18714 
18715     root->dirty = BT_LOCK;
18716     root->lockfor = 1481;
18717     GBT_RIGHT(buf,&right);
18718     cache->slevel = (ajuint) right;
18719 
18720     arrays = btreeAllocSecArray(cache);
18721     parray = arrays->parray;
18722 
18723     if(cache->slevel)
18724     {
18725 	while(nodetype != BT_SECLEAF)
18726 	{
18727 	    btreeGetPointers(cache,buf,&parray);
18728 	    page = btreeSeccacheRead(cache,parray[0]);
18729 	    buf = page->buf;
18730 	    GBT_NODETYPE(buf,&nodetype);
18731 	    page->dirty = BT_CLEAN;
18732 	}
18733 
18734 	secpagepos = parray[0];
18735     }
18736     else
18737 	secpagepos = cache->secrootblock;
18738 
18739 
18740     btreeGetPointers(cache,buf,&parray);
18741 
18742     GBT_NKEYS(buf,&nkeys);
18743     keylimit = nkeys + 1;
18744 
18745     for(i=0;i<keylimit;++i)
18746 	btreeSecbucketIdlist(cache,parray[i],list);
18747 
18748     if(cache->sorder)
18749     {
18750 	page = btreeSeccacheRead(cache,secpagepos);
18751 	buf  = page->buf;
18752 	GBT_RIGHT(buf,&right);
18753 	page->dirty = BT_CLEAN;
18754     }
18755 
18756 
18757     while(right && secpagepos != cache->secrootblock)
18758     {
18759 	secpagepos = right;
18760 	page = btreeSeccacheRead(cache,secpagepos);
18761 	page->dirty = BT_LOCK;
18762         page->lockfor = 1482;
18763 	buf = page->buf;
18764 
18765 	btreeGetPointers(cache,buf,&parray);
18766 	GBT_NKEYS(buf,&nkeys);
18767 	keylimit = nkeys + 1;
18768 	for(i=0; i < keylimit;++i)
18769             btreeSecbucketIdlist(cache,parray[i],list);
18770 
18771 	GBT_RIGHT(buf,&right);
18772 	page->dirty = BT_CLEAN;
18773     }
18774 
18775 
18776     root->dirty = BT_CLEAN;
18777 
18778     btreeDeallocSecArray(cache,arrays);
18779 
18780     return;
18781 }
18782 
18783 
18784 
18785 
18786 /* @funcstatic btreeStrDel ****************************************************
18787 **
18788 ** Deletes an AjPStr entry from a list
18789 **
18790 ** @param [r] Pstr [void**] Address of an AjPStr object
18791 ** @param [r] cl [void*] Standard unused argument, usually NULL.
18792 ** @return [void]
18793 **
18794 ** @release 3.0.0
18795 ** @@
18796 ******************************************************************************/
18797 
btreeStrDel(void ** Pstr,void * cl)18798 static void btreeStrDel(void** Pstr, void* cl)
18799 {
18800     AjPStr str = NULL;
18801 
18802     (void) cl;				/* make it used */
18803 
18804     str = *((AjPStr *)Pstr);
18805     ajStrDel(&str);
18806 
18807     return;
18808 }
18809 
18810 
18811 
18812 
18813 /* @funcstatic btreeIdDelFromList *********************************************
18814 **
18815 ** Deletes an AjPBtId entry from a list
18816 **
18817 ** @param [r] pentry [void**] Address of an AjPBtId object
18818 ** @param [r] cl [void*] Standard unused argument, usually NULL.
18819 ** @return [void]
18820 **
18821 ** @release 3.0.0
18822 ** @@
18823 ******************************************************************************/
18824 
btreeIdDelFromList(void ** pentry,void * cl)18825 static void btreeIdDelFromList(void** pentry, void* cl)
18826 {
18827     AjPBtId id = NULL;
18828 
18829     (void) cl;				/* make it used */
18830 
18831     id = *((AjPBtId *)pentry);
18832 
18833     ajDebug("btreeIdDelFromList '%S'\n", id->id);
18834     ajBtreeIdDel(&id);
18835 
18836     return;
18837 }
18838 
18839 
18840 
18841 
18842 /* @funcstatic btreeHitDelFromList ********************************************
18843 **
18844 ** Deletes an AjPBtHit entry from a list
18845 **
18846 ** @param [r] pentry [void**] Address of an AjPBtHit object
18847 ** @param [r] cl [void*] Standard unused argument, usually NULL.
18848 ** @return [void]
18849 **
18850 ** @release 6.5.0
18851 ** @@
18852 ******************************************************************************/
18853 
btreeHitDelFromList(void ** pentry,void * cl)18854 static void btreeHitDelFromList(void** pentry, void* cl)
18855 {
18856     AjPBtHit hit = NULL;
18857 
18858     (void) cl;				/* make it used */
18859 
18860     hit = *((AjPBtHit *)pentry);
18861 
18862     ajBtreeHitDel(&hit);
18863 
18864     return;
18865 }
18866 
18867 
18868 
18869 
18870 /* @funcstatic btreeHitrefDelFromList *****************************************
18871 **
18872 ** Deletes an AjPBtHitref entry from a list
18873 **
18874 ** @param [r] pentry [void**] Address of an AjPBtHitref object
18875 ** @param [r] cl [void*] Standard unused argument, usually NULL.
18876 ** @return [void]
18877 **
18878 ** @release 6.5.0
18879 ** @@
18880 ******************************************************************************/
18881 
btreeHitrefDelFromList(void ** pentry,void * cl)18882 static void btreeHitrefDelFromList(void** pentry, void* cl)
18883 {
18884     AjPBtHitref hitref = NULL;
18885 
18886     (void) cl;				/* make it used */
18887 
18888     hitref = *((AjPBtHitref *)pentry);
18889 
18890     ajBtreeHitrefDel(&hitref);
18891 
18892     return;
18893 }
18894 
18895 
18896 
18897 
18898 /* @funcstatic btreeIdOffsetCompare *******************************************
18899 **
18900 ** Comparison function for ajListUnique2
18901 **
18902 ** @param [r] a [const void*] ID 1
18903 ** @param [r] b [const void*] ID 2
18904 **
18905 ** @return [ajint] 0 = bases match
18906 **
18907 ** @release 6.5.0
18908 ** @@
18909 ******************************************************************************/
18910 
btreeIdOffsetCompare(const void * a,const void * b)18911 static ajint btreeIdOffsetCompare(const void *a, const void *b)
18912 {
18913     ajlong val;
18914 
18915     val = (ajlong) ((*(AjPBtId const *)a)->offset -
18916                     (*(AjPBtId const *)b)->offset);
18917 
18918     if(!val)
18919       return 0;
18920 
18921     return (val < 0L) ? -1 : 1;
18922 }
18923 
18924 
18925 
18926 
18927 /* @funcstatic btreeHitOffsetCompare ******************************************
18928 **
18929 ** Comparison function for ajListUnique2
18930 **
18931 ** @param [r] a [const void*] ID 1
18932 ** @param [r] b [const void*] ID 2
18933 **
18934 ** @return [ajint] 0 = offsets match
18935 **
18936 ** @release 6.5.0
18937 ** @@
18938 ******************************************************************************/
18939 
btreeHitOffsetCompare(const void * a,const void * b)18940 static ajint btreeHitOffsetCompare(const void *a, const void *b)
18941 {
18942     ajlong val;
18943 
18944     val = (ajlong) ((*(AjPBtHit const *)a)->offset -
18945                     (*(AjPBtHit const *)b)->offset);
18946 
18947     if(!val)
18948       return 0;
18949 
18950     return (val < 0L) ? -1 : 1;
18951 }
18952 
18953 
18954 
18955 
18956 /* @funcstatic btreeHitrefOffsetCompare ***************************************
18957 **
18958 ** Comparison function for ajListUnique2
18959 **
18960 ** @param [r] a [const void*] ID 1
18961 ** @param [r] b [const void*] ID 2
18962 **
18963 ** @return [ajint] 0 = offsets match
18964 **
18965 ** @release 6.5.0
18966 ** @@
18967 ******************************************************************************/
18968 
btreeHitrefOffsetCompare(const void * a,const void * b)18969 static ajint btreeHitrefOffsetCompare(const void *a, const void *b)
18970 {
18971     ajlong val;
18972 
18973     val = (ajlong) ((*(AjPBtHitref const *)a)->offset -
18974                     (*(AjPBtHitref const *)b)->offset);
18975 
18976     if(!val)
18977       return 0;
18978 
18979     return (val < 0L) ? -1 : 1;
18980 }
18981 
18982 
18983 
18984 
18985 /* @funcstatic btreeHitDbnoCompare ********************************************
18986 **
18987 ** Second comparison function for ajListUnique2
18988 **
18989 ** @param [r] a [const void*] ID 1
18990 ** @param [r] b [const void*] ID 2
18991 **
18992 ** @return [ajint] 0 = values match
18993 **
18994 ** @release 6.5.0
18995 ** @@
18996 ******************************************************************************/
18997 
btreeHitDbnoCompare(const void * a,const void * b)18998 static ajint btreeHitDbnoCompare(const void *a, const void *b)
18999 {
19000     return (ajuint) (*(AjPBtHit const *)a)->dbno -
19001            (ajuint) (*(AjPBtHit const *)b)->dbno;
19002 }
19003 
19004 
19005 
19006 
19007 /* @funcstatic btreeHitrefDbnoCompare *****************************************
19008 **
19009 ** Second comparison function for ajListUnique2
19010 **
19011 ** @param [r] a [const void*] ID 1
19012 ** @param [r] b [const void*] ID 2
19013 **
19014 ** @return [ajint] 0 = values match
19015 **
19016 ** @release 6.5.0
19017 ** @@
19018 ******************************************************************************/
19019 
btreeHitrefDbnoCompare(const void * a,const void * b)19020 static ajint btreeHitrefDbnoCompare(const void *a, const void *b)
19021 {
19022     return (ajuint) (*(AjPBtHitref const *)a)->dbno -
19023            (ajuint) (*(AjPBtHitref const *)b)->dbno;
19024 }
19025 
19026 
19027 
19028 
19029 /* @funcstatic btreeIdDbnoCompare *********************************************
19030 **
19031 ** Second comparison function for ajListUnique2
19032 **
19033 ** @param [r] a [const void*] ID 1
19034 ** @param [r] b [const void*] ID 2
19035 **
19036 ** @return [ajint] 0 = values match
19037 **
19038 ** @release 6.5.0
19039 ** @@
19040 ******************************************************************************/
19041 
btreeIdDbnoCompare(const void * a,const void * b)19042 static ajint btreeIdDbnoCompare(const void *a, const void *b)
19043 {
19044     return (ajuint) (*(AjPBtId const *)a)->dbno -
19045            (ajuint) (*(AjPBtId const *)b)->dbno;
19046 }
19047 
19048 
19049 
19050 
19051 #if 0
19052 /* #func ajBtreeHybNew ********************************************************
19053 **
19054 ** Constructor for index bucket ID information
19055 **
19056 ** #param [r] refcount [ajuint] Number of reference file(s) per entry
19057 ** #return [AjPBtHybrid] Index ID object
19058 **
19059 ** #release 4.0.0
19060 ** ##
19061 ******************************************************************************/
19062 /*
19063 AjPBtHybrid ajBtreeHybNew(ajuint refcount)
19064 {
19065     AjPBtHybrid Id = NULL;
19066 
19067     /# ajDebug("In ajBtreeHybNew\n"); #/
19068 
19069     AJNEW0(Id);
19070     Id->key1 = ajStrNew();
19071     Id->dbno = 0U;
19072     Id->dups = 0U;
19073     Id->offset = 0UL;
19074     Id->refcount = refcount;
19075 
19076     if(refcount)
19077         AJCNEW(Id->refoffsets, refcount);
19078 
19079     Id->treeblock = 0UL;
19080 
19081     return Id;
19082 }
19083 */
19084 #endif
19085 
19086 
19087 
19088 
19089 #if 0
19090 /* #func ajBtreeHybDel ********************************************************
19091 **
19092 ** Destructor for index bucket ID information
19093 **
19094 ** #param [w] Pthis [AjPBtHybrid*] index ID object
19095 **
19096 ** #return [void]
19097 **
19098 ** #release 4.0.0
19099 ** ##
19100 ******************************************************************************/
19101 /*
19102 void ajBtreeHybDel(AjPBtHybrid *Pthis)
19103 {
19104     AjPBtHybrid Id = NULL;
19105 
19106     /# ajDebug("In ajBtreeHybDel\n"); #/
19107 
19108     if(!Pthis || !*Pthis)
19109 	return;
19110 
19111     Id = *Pthis;
19112 
19113     ajStrDel(&Id->key1);
19114 
19115     if(Id->refcount)
19116         AJFREE(Id->refoffsets);
19117 
19118     AJFREE(Id);
19119 
19120     *Pthis = NULL;
19121 
19122     return;
19123 }
19124 */
19125 #endif
19126 
19127 
19128 
19129 
19130 /* @funcstatic btreeAllocPriArray *********************************************
19131 **
19132 ** Allocate karray and parray arrays for a primary key
19133 **
19134 ** @param [u] cache [AjPBtcache] cache
19135 **
19136 ** @return [AjPBtMem] memory node
19137 **
19138 ** @release 4.0.0
19139 ** @@
19140 ******************************************************************************/
19141 
btreeAllocPriArray(AjPBtcache cache)19142 static AjPBtMem btreeAllocPriArray(AjPBtcache cache)
19143 {
19144     AjPBtMem node = NULL;
19145     ajuint i;
19146     ajuint limit;
19147     AjPBtMem p = NULL;
19148     ajuint klen;
19149 
19150     limit = cache->porder;
19151 
19152     if(!cache->bmem)
19153     {
19154         statCountAllocPriArrayNew++;
19155 
19156         AJNEW0(node);
19157         cache->bmem = node;
19158         cache->tmem = node;
19159         node->prev = NULL;
19160         node->next = NULL;
19161         node->used = ajTrue;
19162         AJCNEW0(node->karray,limit);
19163         AJCNEW0(node->parray,limit);
19164         AJCNEW0(node->overflows,limit);
19165 
19166         klen = cache->keylimit+1;
19167         for(i=0;i<limit;++i)
19168             node->karray[i] = ajStrNewRes(klen);
19169 
19170         return node;
19171     }
19172 
19173     if(!cache->bmem->used)
19174     {
19175         statCountAllocPriArrayReuse++;
19176 
19177         cache->bmem->used = ajTrue;
19178 
19179         if(cache->bmem->next)
19180         {
19181             p = cache->bmem->next;
19182 
19183             cache->tmem->next = cache->bmem;
19184             cache->bmem->next = NULL;
19185 
19186             cache->bmem->prev = cache->tmem;
19187 
19188             cache->tmem = cache->bmem;
19189 
19190             cache->bmem = p;
19191             cache->bmem->prev = NULL;
19192 
19193 	    memset(cache->tmem->parray,0,sizeof(ajulong)*limit);
19194 
19195 	    return cache->tmem;
19196         }
19197 
19198 	memset(cache->bmem->parray,0,sizeof(ajulong)*limit);
19199 
19200         return cache->bmem;
19201     }
19202 
19203 
19204     statCountAllocPriArrayNew++;
19205 
19206     AJNEW0(node);
19207     node->used = ajTrue;
19208     node->next = NULL;
19209     node->prev = cache->tmem;
19210     cache->tmem->next = node;
19211     cache->tmem = node;
19212 
19213     AJCNEW0(node->karray,limit);
19214     AJCNEW0(node->parray,limit);
19215     AJCNEW0(node->overflows,limit);
19216 
19217     klen = cache->keylimit+1;
19218     for(i=0;i<limit;++i)
19219         node->karray[i] = ajStrNewRes(klen);
19220 
19221     return node;
19222 }
19223 
19224 
19225 
19226 
19227 /* @funcstatic btreeDeallocPriArray *******************************************
19228 **
19229 ** Deallocate karray and parray arrays for a primary key
19230 **
19231 ** @param [u] cache [AjPBtcache] cache
19232 ** @param [u] node [AjPBtMem] node
19233 **
19234 ** @return [void]
19235 **
19236 ** @release 4.0.0
19237 ** @@
19238 ******************************************************************************/
19239 
btreeDeallocPriArray(AjPBtcache cache,AjPBtMem node)19240 static void btreeDeallocPriArray(AjPBtcache cache, AjPBtMem node)
19241 {
19242     statCountAllocPriArrayDel++;
19243 
19244     node->used = ajFalse;
19245 
19246     if(!node->prev)
19247         return;
19248 
19249     node->prev->next = node->next;
19250 
19251     if(node->next)
19252         node->next->prev = node->prev;
19253     else
19254         cache->tmem = node->prev;
19255 
19256     node->next = cache->bmem;
19257     cache->bmem->prev = node;
19258     cache->bmem = node;
19259     node->prev = NULL;
19260 
19261     return;
19262 }
19263 
19264 
19265 
19266 
19267 /* @funcstatic btreeAllocSecArray *********************************************
19268 **
19269 ** Allocate karray and parray arrays for a primary key
19270 **
19271 ** @param [u] cache [AjPBtcache] cache
19272 **
19273 ** @return [AjPBtMem] memory node
19274 **
19275 ** @release 4.0.0
19276 ** @@
19277 ******************************************************************************/
19278 
btreeAllocSecArray(AjPBtcache cache)19279 static AjPBtMem btreeAllocSecArray(AjPBtcache cache)
19280 {
19281     AjPBtMem node = NULL;
19282     ajuint i;
19283     ajuint limit;
19284     AjPBtMem p = NULL;
19285     ajuint klen;
19286 
19287     limit = cache->sorder;
19288 
19289     if(!cache->bsmem)
19290     {
19291         statCountAllocSecArrayNew++;
19292 
19293         AJNEW0(node);
19294         cache->bsmem = node;
19295         cache->tsmem = node;
19296         node->prev = NULL;
19297         node->next = NULL;
19298         node->used = ajTrue;
19299         AJCNEW0(node->karray,limit);
19300         AJCNEW0(node->parray,limit);
19301         AJCNEW0(node->overflows,limit);
19302 
19303         klen = cache->idlimit + 1;
19304         for(i=0;i<limit;++i)
19305             node->karray[i] = ajStrNewRes(klen);
19306 
19307         return node;
19308     }
19309 
19310     if(!cache->bsmem->used)
19311     {
19312         statCountAllocSecArrayReuse++;
19313 
19314         cache->bsmem->used = ajTrue;
19315 
19316         if(cache->bsmem->next)
19317         {
19318             p = cache->bsmem->next;
19319 
19320             cache->tsmem->next = cache->bsmem;
19321             cache->bsmem->next = NULL;
19322 
19323             cache->bsmem->prev = cache->tsmem;
19324 
19325             cache->tsmem = cache->bsmem;
19326 
19327             cache->bsmem = p;
19328             cache->bsmem->prev = NULL;
19329 
19330 	    memset(cache->tsmem->parray,0,sizeof(ajulong)*limit);
19331 
19332 	    return cache->tsmem;
19333         }
19334 
19335 	memset(cache->bsmem->parray,0,sizeof(ajulong)*limit);
19336 
19337         return cache->bsmem;
19338     }
19339 
19340     statCountAllocSecArrayNew++;
19341 
19342     AJNEW0(node);
19343     node->used = ajTrue;
19344     node->next = NULL;
19345     node->prev = cache->tsmem;
19346     cache->tsmem->next = node;
19347     cache->tsmem = node;
19348 
19349     AJCNEW0(node->karray,limit);
19350     AJCNEW0(node->parray,limit);
19351     AJCNEW0(node->overflows,limit);
19352 
19353     klen = cache->idlimit + 1;
19354     for(i=0;i<limit;++i)
19355         node->karray[i] = ajStrNewRes(klen);
19356 
19357     return node;
19358 }
19359 
19360 
19361 
19362 
19363 /* @funcstatic btreeDeallocSecArray *******************************************
19364 **
19365 ** Deallocate karray and parray arrays for a primary key
19366 **
19367 ** @param [u] cache [AjPBtcache] cache
19368 ** @param [u] node [AjPBtMem] node
19369 **
19370 ** @return [void]
19371 **
19372 ** @release 4.0.0
19373 ** @@
19374 ******************************************************************************/
19375 
btreeDeallocSecArray(AjPBtcache cache,AjPBtMem node)19376 static void btreeDeallocSecArray(AjPBtcache cache, AjPBtMem node)
19377 {
19378     node->used = ajFalse;
19379 
19380     if(!node->prev)
19381         return;
19382 
19383     node->prev->next = node->next;
19384 
19385     if(node->next)
19386         node->next->prev = node->prev;
19387     else
19388         cache->tsmem = node->prev;
19389 
19390     node->next = cache->bsmem;
19391     cache->bsmem->prev = node;
19392     cache->bsmem = node;
19393     node->prev = NULL;
19394 
19395     return;
19396 }
19397 
19398 
19399 
19400 
19401 #if 0
19402 /* #funcstatic btreeHybbucketAdd **********************************************
19403 **
19404 ** Add a hybrid ID to a bucket
19405 ** Only called if there is room in the bucket
19406 **
19407 ** Use btreeIdbucketAdd using an ID object which is the same with no
19408 ** (unused) treeblock.
19409 **
19410 ** #param [u] cache [AjPBtcache] cache
19411 ** #param [r] pagepos [ajulong] page number of bucket
19412 ** #param [r] hyb [const AjPBtHybrid] ID info
19413 **
19414 ** #return [void]
19415 **
19416 ** #release 4.0.0
19417 ** ##
19418 ******************************************************************************/
19419 /*
19420 static void btreeHybbucketAdd(AjPBtcache cache, ajulong pagepos,
19421                               const AjPBtHybrid hyb)
19422 {
19423     unsigned char *buf  = NULL;
19424     unsigned char *kptr = NULL;
19425     unsigned char *src  = NULL;
19426     unsigned char *dest = NULL;
19427 
19428 /#    unsigned char *lastptr = NULL;#/
19429     unsigned char *endptr  = NULL;
19430 
19431     ajuint nentries = 0;
19432     ajuint nodetype = 0;
19433 
19434     ajuint sum = 0;
19435     ajuint len = 0;
19436     ajuint i;
19437     ajuint v;
19438     ajuint uv;
19439     ajulong lv;
19440     ajuint iref = 0;
19441 
19442     AjPBtpage page = NULL;
19443     static ajuint calls = 0;
19444     ajuint refskip = cache->refcount*BT_EXTRA;
19445 
19446 /#    ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1;#/
19447 /#    ajuint idlen    = 0;#/
19448 /#    static ajuint overflowcalls=0;#/
19449 
19450     calls++;
19451 
19452     page = btreePricacheRead(cache,pagepos);
19453     buf  = page->buf;
19454 
19455     GBT_BUCKNODETYPE(buf,&nodetype);
19456     if(nodetype != BT_IDBUCKET)
19457         ajFatal("Wrong nodetype in HybbucketAdd cache %S", cache->filename);
19458 
19459     GBT_BUCKNENTRIES(buf,&nentries);
19460     if(nentries == cache->pnperbucket)
19461         ajFatal("Bucket too full in HybbucketAdd page: %Lu "
19462                 "entries: %u max: %u cache %S",
19463                 pagepos, nentries, cache->pnperbucket, cache->filename);
19464 
19465     kptr = PBT_BUCKKEYLEN(buf);
19466     src  = kptr + (nentries * sizeof(ajuint));
19467 
19468     sum = 0;
19469     for(i=0;i<nentries;++i)
19470     {
19471         BT_GETAJUINT(kptr,&len);
19472         sum += len;
19473         kptr += sizeof(ajuint);
19474     }
19475     /#sum += nentries;#/
19476 
19477     endptr  = src + sum - 1;
19478 
19479 /#
19480 //    idlen   = MAJSTRGETLEN(hyb->key1);
19481 //    lastptr = endptr + sizeof(ajuint) + idlen + keyskip;
19482 //
19483 //    if((ajuint) (lastptr - buf) >= cache->pagesize)
19484 //    {
19485 //        overflowcalls++;
19486 //        ajWarn("\nOverflow in HybbucketAdd nentries:%u fails %u/%u '%S' "
19487 //               "cache %S",
19488 //               nentries, overflowcalls,calls, hyb->key1, cache->filename);
19489 //        btreeHybbucketAddFull(cache,pagepos,hyb);
19490 //        return;
19491 //    }
19492 #/
19493 
19494     dest = src + sizeof(ajuint);
19495     memmove((void *)dest, (void *)src, sum);
19496 
19497     v = BT_BUCKIDLEN(hyb->key1) + refskip;
19498     BT_SETAJUINT(src,v);
19499 
19500     endptr += sizeof(ajuint) + 1;
19501     strcpy((char *)endptr,MAJSTRGETPTR(hyb->key1));
19502     endptr += (MAJSTRGETLEN(hyb->key1) + 1);
19503     uv = hyb->dbno;
19504     BT_SETAJUINT(endptr,uv);
19505     endptr += sizeof(ajuint);
19506     uv = hyb->dups;
19507     BT_SETAJUINT(endptr,uv);
19508     endptr += sizeof(ajuint);
19509     lv = hyb->offset;
19510     BT_SETAJULONG(endptr,lv);
19511     endptr += sizeof(ajulong);
19512 
19513     if(cache->refcount)
19514     {
19515         for(iref=0; iref < cache->refcount; iref++)
19516         {
19517             lv = hyb->refoffsets[iref];
19518             BT_SETAJULONG(endptr,lv);
19519             endptr += sizeof(ajulong);
19520         }
19521     }
19522 
19523     v = nentries + 1;
19524     SBT_BUCKNENTRIES(buf,v);
19525 
19526     page->dirty = BT_DIRTY;
19527 
19528     return;
19529 }
19530 */
19531 #endif
19532 
19533 
19534 
19535 
19536 #if 0
19537 /* #funcstatic btreeHybbucketAddFull ******************************************
19538 **
19539 ** Add an ID to a bucket
19540 ** Only called if there is room in the bucket
19541 **
19542 ** #param [u] cache [AjPBtcache] cache
19543 ** #param [r] pagepos [ajulong] page number of bucket
19544 ** #param [r] hyb [const AjPBtHybrid] ID info
19545 **
19546 ** #return [void]
19547 **
19548 ** #release 6.4.0
19549 ** ##
19550 ******************************************************************************/
19551 /*
19552 //static void btreeHybbucketAddFull(AjPBtcache cache, ajulong pagepos,
19553 //                                  const AjPBtHybrid hyb)
19554 //{
19555 //    AjPIdbucket bucket = NULL;
19556 //    AjPBtId   destid = NULL;
19557 //
19558 //    ajuint nentries;
19559 //    ajuint iref;
19560 //
19561 //    /# ajDebug("In btreeHybbucketAddFull\n"); #/
19562 //
19563 //    bucket   = btreeReadIdbucket(cache,pagepos);
19564 //    nentries = bucket->Nentries;
19565 //
19566 //
19567 //    /# Reading a bucket always gives one extra ID position #/
19568 //    destid = bucket->Ids[nentries];
19569 //
19570 //    ajStrAssignS(&destid->id,hyb->key1);
19571 //    destid->dbno      = hyb->dbno;
19572 //    destid->dups      = hyb->dups;
19573 //    destid->offset    = hyb->offset;
19574 //
19575 //    if(cache->refcount)
19576 //    {
19577 //        for(iref=0; iref < cache->refcount; iref++)
19578 //            destid->refoffsets[iref] = hyb->refoffsets[iref];
19579 //    }
19580 //
19581 //    ++bucket->Nentries;
19582 //
19583 //    btreeWriteIdbucket(cache,bucket,pagepos);
19584 //
19585 //    btreeIdbucketDel(&bucket);
19586 //
19587 //    return;
19588 //}
19589 */
19590 #endif
19591 
19592 
19593 
19594 
19595 /* @funcstatic btreeIdentFind *************************************************
19596 **
19597 ** Find the node that should contain a new key for insertion
19598 **
19599 ** @param [u] cache [AjPBtcache] cache
19600 ** @param [r] key [const AjPStr] key to search for
19601 **
19602 ** @return [AjPBtpage] leaf node where item should be inserted
19603 **
19604 ** @release 4.0.0
19605 ** @@
19606 ******************************************************************************/
19607 
btreeIdentFind(AjPBtcache cache,const AjPStr key)19608 static AjPBtpage btreeIdentFind(AjPBtcache cache, const AjPStr key)
19609 {
19610     AjPBtpage root = NULL;
19611     AjPBtpage ret  = NULL;
19612 
19613     /* ajDebug("In btreeIdentFind\n"); */
19614 
19615     /* The root node should always be in the cache (BT_LOCKed) */
19616     root = btreePricacheLocate(cache,0UL);
19617 
19618     if(!root)
19619 	ajFatal("The master root cache page has been unlocked\n");
19620 
19621     if(!cache->plevel)
19622 	return root;
19623 
19624     ret = btreePrimaryFindInode(cache,root,key);
19625 
19626     return ret;
19627 }
19628 
19629 
19630 
19631 
19632 /* @funcstatic btreeIdentInsertShift ******************************************
19633 **
19634 ** Rebalance buckets on insertion of an identifier
19635 **
19636 ** @param [u] cache [AjPBtcache] cache
19637 ** @param [u] retpage [AjPBtpage*] page
19638 ** @param [r] key [const AjPStr] key
19639 **
19640 ** @return [ajulong] bucket block or 0UL if shift not possible
19641 **
19642 ** @release 6.5.0
19643 ** @@
19644 ******************************************************************************/
19645 
btreeIdentInsertShift(AjPBtcache cache,AjPBtpage * retpage,const AjPStr key)19646 static ajulong btreeIdentInsertShift(AjPBtcache cache, AjPBtpage *retpage,
19647                                      const AjPStr key)
19648 {
19649     unsigned char *tbuf = NULL;
19650     unsigned char *pbuf = NULL;
19651     unsigned char *sbuf = NULL;
19652 
19653     AjPBtpage ppage = NULL;
19654     AjPBtpage spage = NULL;
19655     AjPBtpage tpage = NULL;
19656 
19657     ajuint tkeys = 0U;
19658     ajuint pkeys = 0U;
19659     ajuint skeys = 0U;
19660     ajuint order = 0U;
19661 
19662     ajuint i;
19663     ajuint n;
19664     ajint ii;
19665 
19666     ajulong parent  = 0UL;
19667     ajulong blockno = 0UL;
19668 
19669     AjPStr *kTarray = NULL;
19670     AjPStr *kParray = NULL;
19671     AjPStr *kSarray = NULL;
19672     ajulong *pTarray = NULL;
19673     ajulong *pParray = NULL;
19674     ajulong *pSarray = NULL;
19675 
19676     AjPStr *karray = NULL;
19677     ajulong *parray = NULL;
19678 
19679     ajuint ppos    = 0U;
19680     ajuint pkeypos = 0U;
19681     ajuint minsize = 0U;
19682 
19683     AjPBtMem arrays1 = NULL;
19684     AjPBtMem arrays2 = NULL;
19685     AjPBtMem arrays3 = NULL;
19686 
19687     tpage = *retpage;
19688 
19689     tbuf = tpage->buf;
19690 
19691     GBT_PREV(tbuf,&parent);
19692     GBT_NKEYS(tbuf,&tkeys);
19693 
19694 #if AJINDEX_DEBUG
19695     ajDebug("btreeIdentInsertShift %Lu parent: %Lu '%S'\n",
19696             (*retpage)->pagepos, parent, key);
19697 #endif
19698 
19699     order = cache->porder;
19700     minsize = order / 2U;
19701 
19702     if(order % 2U)
19703 	++minsize;
19704 
19705     if(tkeys <= minsize)
19706 	return 0UL;
19707 
19708     ppage = btreePricacheRead(cache,parent);
19709 
19710     pbuf = ppage->buf;
19711     GBT_NKEYS(pbuf,&pkeys);
19712 
19713 
19714     arrays1 = btreeAllocPriArray(cache);
19715     kParray = arrays1->karray;
19716     pParray = arrays1->parray;
19717 
19718     arrays2 = btreeAllocPriArray(cache);
19719     kSarray = arrays2->karray;
19720     pSarray = arrays2->parray;
19721 
19722     arrays3 = btreeAllocPriArray(cache);
19723     kTarray = arrays3->karray;
19724     pTarray = arrays3->parray;
19725 
19726 
19727     btreeGetKeys(cache,pbuf,&kParray,&pParray);
19728 
19729     i=0;
19730 
19731     while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
19732 	++i;
19733 
19734     pkeypos = i;
19735 
19736     if(i==pkeys)
19737     {
19738 	if(MAJSTRCMPS(key,kParray[i-1])<0)
19739 	    ppos = i-1;
19740 	else
19741 	    ppos = i;
19742     }
19743     else
19744 	ppos = i;
19745 
19746 
19747 #if AJINDEX_DEBUG
19748     ajDebug("    InsertShift tpage: %Lu ppage: %Lu ppos: %u i: %u "
19749             "skeys: %u order: %u\n",
19750             tpage->pagepos, ppage->pagepos, ppos, i, skeys, order);
19751 #endif
19752 
19753     if(ppos) /* There is another leaf to the left */
19754     {
19755 	spage = btreePricacheRead(cache,pParray[ppos-1]);
19756 	sbuf = spage->buf;
19757 	GBT_NKEYS(sbuf,&skeys);
19758     }
19759 
19760     if(i && skeys != order-1) /* There is space in the left leaf */
19761     {
19762 
19763 #if AJINDEX_DEBUG
19764     ajDebug("    InsertShift left leaf\n");
19765 #endif
19766 
19767 	/* ajDebug("Left shift\n"); */
19768 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
19769 	if(skeys)
19770 	    btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
19771 
19772 	i = 0;
19773 
19774 	while(pParray[i] != tpage->pagepos)
19775 	    ++i;
19776 	--i;
19777 
19778 	pkeypos = i;
19779 
19780 	ajStrAssignS(&kSarray[skeys],kParray[pkeypos]);
19781 	pSarray[skeys+1] = pTarray[0];
19782 	++skeys;
19783 	--tkeys;
19784 	ajStrAssignS(&kParray[pkeypos],kTarray[0]);
19785 	for(i=0;i<tkeys;++i)
19786 	{
19787 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
19788 	    pTarray[i] = pTarray[i+1];
19789 	}
19790 	pTarray[i] = pTarray[i+1];
19791 	pTarray[i+1] = 0UL;
19792 
19793 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
19794 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
19795 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
19796 
19797 	if(!ppage->pagepos)
19798         {
19799 	    ppage->dirty = BT_LOCK;
19800             ppage->lockfor = 1501;
19801         }
19802 
19803 	i = 0;
19804 
19805 	while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
19806 	    ++i;
19807 
19808 	if(i==pkeys)
19809 	{
19810 	    if(MAJSTRCMPS(key,kParray[i-1])<0)
19811 		blockno = pParray[i-1];
19812 	    else
19813 		blockno = pParray[i];
19814 	}
19815 	else
19816 	    blockno = pParray[i];
19817 
19818 	if(blockno == spage->pagepos)
19819 	{
19820 	    *retpage = spage;
19821 	    karray = kSarray;
19822 	    parray = pSarray;
19823 	    n = skeys;
19824 	}
19825 	else
19826 	{
19827 	    karray = kTarray;
19828 	    parray = pTarray;
19829 	    n = tkeys;
19830 	}
19831 
19832 
19833 	i = 0;
19834 
19835 	while(i!=n && MAJSTRCMPS(key,karray[i])>=0)
19836 	    ++i;
19837 
19838 	if(i==n)
19839 	{
19840 	    if(MAJSTRCMPS(key,karray[i-1])<0)
19841 		blockno = parray[i-1];
19842 	    else
19843 		blockno = parray[i];
19844 	}
19845 	else
19846 	    blockno = parray[i];
19847 
19848 	btreeDeallocPriArray(cache,arrays1);
19849 	btreeDeallocPriArray(cache,arrays2);
19850 	btreeDeallocPriArray(cache,arrays3);
19851 
19852 	/* ajDebug("... returns blockno (a) %Lu\n",blockno); */
19853 
19854 	return blockno;
19855     }
19856 
19857 
19858     if(ppos != pkeys)	/* There is a right node */
19859     {
19860 	spage = btreePricacheRead(cache,pParray[ppos+1]);
19861 	sbuf = spage->buf;
19862 	GBT_NKEYS(sbuf,&skeys);
19863 
19864 #if AJINDEX_DEBUG
19865         ajDebug("    InsertShift right, new spage: %Lu skeys: %u\n",
19866                 spage->pagepos, skeys);
19867 #endif
19868 
19869     }
19870 
19871 
19872 #if AJINDEX_DEBUG
19873     ajDebug("    InsertShift test again tpage: %Lu ppage: %Lu spage: %Lu "
19874             "ppos: %u i: %u skeys: %u order: %u\n",
19875             tpage->pagepos, ppage->pagepos, spage->pagepos,
19876             ppos, i, skeys, order);
19877 #endif
19878 
19879     /* Space in the right leaf */
19880     if(ppos != pkeys && skeys != order-1)
19881     {
19882 
19883 #if AJINDEX_DEBUG
19884         ajDebug("    InsertShift right leaf tpage: %Lu\n", tpage->pagepos);
19885 #endif
19886 
19887 	/* ajDebug("Right shift\n"); */
19888 	btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
19889 	btreeGetKeys(cache,sbuf,&kSarray,&pSarray);
19890 
19891 	i = 0;
19892 
19893 	while(pParray[i] != tpage->pagepos) /* fails here */
19894         {
19895 
19896 #if AJINDEX_DEBUG
19897             ajDebug("    InsertShift right pParray[%u] %Lu\n",
19898                     i, pParray[i]);
19899 #endif
19900 	    ++i;
19901         }
19902 
19903 	pkeypos = i;
19904 
19905 	pSarray[skeys+1] = pSarray[skeys];
19906 
19907 	for(ii=skeys-1;ii>-1;--ii)
19908 	{
19909 	    ajStrAssignS(&kSarray[ii+1],kSarray[ii]);
19910 	    pSarray[ii+1] = pSarray[ii];
19911 	}
19912 
19913 	ajStrAssignS(&kSarray[0],kParray[pkeypos]);
19914 	pSarray[0] = pTarray[tkeys];
19915 	ajStrAssignS(&kParray[pkeypos],kTarray[tkeys-1]);
19916 	++skeys;
19917 	--tkeys;
19918 	pTarray[tkeys+1] = 0UL;
19919 
19920 	btreeWriteNode(cache,spage,kSarray,pSarray,skeys);
19921 	btreeWriteNode(cache,tpage,kTarray,pTarray,tkeys);
19922 	btreeWriteNode(cache,ppage,kParray,pParray,pkeys);
19923 
19924 	if(!ppage->pagepos)
19925 	{
19926             ppage->dirty = BT_LOCK;
19927             ppage->lockfor = 1502;
19928         }
19929 
19930 	i = 0U;
19931 
19932 	while(i!=pkeys && MAJSTRCMPS(key,kParray[i])>=0)
19933 	    ++i;
19934 
19935 	if(i==pkeys)
19936 	{
19937 	    if(MAJSTRCMPS(key,kParray[i-1])<0)
19938 		blockno = pParray[i-1];
19939 	    else
19940 		blockno = pParray[i];
19941 	}
19942 	else
19943 	    blockno = pParray[i];
19944 
19945 	if(blockno == spage->pagepos)
19946 	{
19947 	    *retpage = spage;
19948 	    karray = kSarray;
19949 	    parray = pSarray;
19950 	    n = skeys;
19951 	}
19952 	else
19953 	{
19954 	    karray = kTarray;
19955 	    parray = pTarray;
19956 	    n = tkeys;
19957 	}
19958 
19959 	i = 0;
19960 
19961 	while(i!=n && MAJSTRCMPS(key,karray[i])>=0)
19962 	    ++i;
19963 
19964 	if(i==n)
19965 	{
19966 	    if(MAJSTRCMPS(key,karray[i-1])<0)
19967 		blockno = parray[i-1];
19968 	    else
19969 		blockno = parray[i];
19970 	}
19971 	else
19972 	    blockno = parray[i];
19973 
19974 	btreeDeallocPriArray(cache,arrays1);
19975 	btreeDeallocPriArray(cache,arrays2);
19976 	btreeDeallocPriArray(cache,arrays3);
19977 
19978 	/* ajDebug("... returns blockno (b) %Lu\n",blockno); */
19979 
19980 	return blockno;
19981     }
19982 
19983 
19984     btreeDeallocPriArray(cache,arrays1);
19985     btreeDeallocPriArray(cache,arrays2);
19986     btreeDeallocPriArray(cache,arrays3);
19987 
19988 
19989 #if AJINDEX_DEBUG
19990     ajDebug("    InsertShift returning zero ...\n");
19991 #endif
19992 
19993     /* ajDebug("... returns 0UL\n"); */
19994 
19995     return 0UL;
19996 }
19997 
19998 
19999 
20000 
20001 #if 0
20002 /* #funcstatic btreeHybbucketsReorder *****************************************
20003 **
20004 ** Re-order leaf buckets
20005 ** Must only be called if one of the buckets is full
20006 **
20007 ** #param [u] cache [AjPBtcache] cache
20008 ** #param [u] leaf [AjPBtpage] leaf page
20009 **
20010 ** #return [AjBool] true if reorder was successful i.e. leaf not full
20011 **
20012 ** #release 4.0.0
20013 ** ##
20014 ******************************************************************************/
20015 /*
20016 static AjBool btreeHybbucketsReorder(AjPBtcache cache, AjPBtpage leaf)
20017 {
20018     ajuint nkeys = 0;
20019     unsigned char *lbuf = NULL;
20020 
20021     ajulong *ptrs        = NULL;
20022     ajulong *overflows   = NULL;
20023     AjPStr *newkeys     = NULL;
20024     ajulong *newptrs     = NULL;
20025     AjPBtMem arrays     = NULL;
20026     AjPBtMem newarrays  = NULL;
20027 
20028     ajuint i = 0;
20029     ajuint iref = 0;
20030 
20031     ajuint order;
20032     ajuint totalkeys     = 0;
20033     ajuint maxnperbucket = 0;
20034     ajuint count         = 0;
20035     ajuint keylimit      = 0;
20036     ajuint bucketlimit   = 0;
20037     ajuint nodetype      = 0;
20038 
20039     AjPList idlist    = NULL;
20040     ajuint  dirtysave = 0;
20041     AjPBtId bid       = NULL;
20042     AjPIdbucket cbucket = NULL;
20043     AjPBtId cid       = NULL;
20044 
20045     ajuint iold = 0;
20046     ajuint refskip = cache->refcount*BT_EXTRA;
20047 
20048     /# ajDebug("In btreeHybbucketsReorder\n"); #/
20049 
20050     dirtysave = leaf->dirty;
20051 
20052     leaf->dirty = BT_LOCK;
20053     leaf->lockfor = 1511;
20054     lbuf = leaf->buf;
20055 
20056     GBT_NODETYPE(lbuf,&nodetype);
20057 
20058     order = cache->porder;
20059 
20060 
20061     /# Read keys/ptrs #/
20062     arrays    = btreeAllocPriArray(cache);
20063     ptrs      = arrays->parray;
20064     newarrays    = btreeAllocPriArray(cache);
20065     newkeys      = newarrays->karray;
20066     newptrs      = newarrays->parray;
20067     overflows = newarrays->overflows;
20068 
20069     btreeGetPointers(cache,lbuf,&ptrs);
20070 
20071     GBT_NKEYS(lbuf,&nkeys);
20072 
20073 
20074     if(!nkeys)
20075 	ajFatal("HybbucketsReorder: Attempt to reorder empty leaf");
20076 
20077     for(i=0;i<=nkeys;++i)
20078 	totalkeys += btreeIdbucketCount(cache,ptrs[i]);
20079 
20080     keylimit = nkeys + 1;
20081 
20082     btreeBucketCalc(totalkeys, keylimit, cache->pnperbucket,
20083                     &bucketlimit, &maxnperbucket);
20084 
20085 
20086     if(bucketlimit >= order)
20087     {
20088 	btreeDeallocPriArray(cache,arrays);
20089 	btreeDeallocPriArray(cache,newarrays);
20090 
20091 	leaf->dirty = dirtysave;
20092 	return ajFalse;
20093     }
20094 
20095 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
20096     ajDebug("btreeHybucketsReorder '%S' %Lu id '%S' key '%S'\n",
20097             cache->basename, leaf->pagepos, indexid, indexKeyword);
20098 #endif
20099     ++statCallHybbucketsReorder;
20100 
20101     /# Read buckets #/
20102 
20103     idlist  = ajListNew();
20104 
20105     for(i=0;i<keylimit;++i)
20106 	overflows[i] = btreeHybbucketIdlist(cache,ptrs[i],idlist);
20107 
20108     ajListSort(idlist, &btreeIdCompare);
20109 
20110     cbucket = btreeIdbucketNew(cache->pnperbucket, cache->refcount);
20111 
20112     iold = 0;
20113     for(i=0;i<bucketlimit;++i)
20114     {
20115 	cbucket->Overflow = overflows[i];
20116 	cbucket->Nentries = 0;
20117 
20118 	count = 0;
20119 
20120 	while(count!=maxnperbucket)
20121 	{
20122 	    ajListPop(idlist,(void **)&bid);
20123 
20124 	    cid = cbucket->Ids[count];
20125 	    ajStrAssignS(&cid->id,bid->id);
20126 	    cid->dbno = bid->dbno;
20127 	    cid->dups = bid->dups;
20128 	    cid->offset = bid->offset;
20129 
20130             if(cache->refcount)
20131             {
20132                 for(iref=0; iref< cache->refcount; iref++)
20133                     cid->refoffsets[iref] = bid->refoffsets[iref];
20134 	    }
20135 
20136 	    cbucket->keylen[count] = BT_BUCKIDLEN(bid->id) + refskip;
20137 	    ++cbucket->Nentries;
20138 	    ++count;
20139 	    ajBtreeIdDel(&bid);
20140 	}
20141 
20142 
20143 	ajListPeek(idlist,(void **)&bid);
20144 	ajStrAssignS(&newkeys[i],bid->id);
20145 
20146 	if((iold < order) && ptrs[iold])
20147             newptrs[i] = ptrs[iold++];
20148         else
20149 	    newptrs[i] = cache->totsize;
20150 	btreeWriteIdbucket(cache,cbucket,newptrs[i]);
20151     }
20152 
20153 
20154     /# Deal with greater-than bucket #/
20155 
20156     cbucket->Overflow = overflows[i];
20157     cbucket->Nentries = 0;
20158 
20159     count = 0;
20160 
20161     while(ajListPop(idlist,(void **)&bid))
20162     {
20163 	cid = cbucket->Ids[count];
20164 	ajStrAssignS(&cid->id,bid->id);
20165 	cid->dbno = bid->dbno;
20166 	cid->dups = bid->dups;
20167 	cid->offset = bid->offset;
20168 
20169         if(cache->refcount)
20170         {
20171             for(iref=0; iref < cache->refcount; iref++)
20172                 cid->refoffsets[iref] = bid->refoffsets[iref];
20173 	}
20174 
20175 	++cbucket->Nentries;
20176 	++count;
20177 	ajBtreeIdDel(&bid);
20178     }
20179 
20180 
20181     if((iold < order) && ptrs[iold])
20182         newptrs[i] = ptrs[iold++];
20183     else
20184         newptrs[i] = cache->totsize;
20185     btreeWriteIdbucket(cache,cbucket,newptrs[i]);
20186 
20187     btreeIdbucketDel(&cbucket);
20188 
20189     /# Now write out a modified leaf with new keys/ptrs #/
20190 
20191     nkeys = bucketlimit;
20192 
20193     btreeWriteNode(cache,leaf,newkeys,newptrs,nkeys);
20194 
20195     leaf->dirty = BT_DIRTY;
20196 
20197     if(nodetype == BT_ROOT)
20198     {
20199 	leaf->dirty = BT_LOCK;
20200         leaf->lockfor = 1512;
20201     }
20202 
20203     btreeDeallocPriArray(cache,arrays);
20204     btreeDeallocPriArray(cache,newarrays);
20205 
20206     ajListFree(&idlist);
20207 
20208     return ajTrue;
20209 }
20210 */
20211 #endif
20212 
20213 
20214 
20215 
20216 /* @funcstatic btreeIdSplitleaf **********************************************
20217 **
20218 ** Split a leaf and propagate up if necessary
20219 **
20220 ** @param [u] cache [AjPBtcache] cache
20221 ** @param [u] spage [AjPBtpage] page
20222 **
20223 ** @return [AjPBtpage] pointer to a parent page
20224 **
20225 ** @release 4.0.0
20226 ** @@
20227 ******************************************************************************/
20228 
btreeIdSplitleaf(AjPBtcache cache,AjPBtpage spage)20229 static AjPBtpage btreeIdSplitleaf(AjPBtcache cache, AjPBtpage spage)
20230 {
20231     ajuint nkeys     = 0U;
20232     ajuint order     = 0U;
20233     ajuint totalkeys = 0U;
20234     ajuint keylimit  = 0U;
20235     ajuint nodetype  = 0U;
20236 
20237     ajuint rootnodetype  = 0U;
20238 
20239     ajuint i;
20240     ajuint j;
20241     ajuint iref;
20242 
20243     AjPBtpage lpage = NULL;
20244     AjPBtpage rpage = NULL;
20245     AjPBtpage page  = NULL;
20246 
20247     AjPStr mediankey  = NULL;
20248     ajulong mediangtr  = 0UL;
20249     ajulong medianless = 0UL;
20250 
20251 
20252     AjPBtId bid = NULL;
20253     AjPBtId cid = NULL;
20254 
20255     unsigned char *buf  = NULL;
20256     unsigned char *lbuf = NULL;
20257     unsigned char *rbuf = NULL;
20258 
20259     AjPList idlist = NULL;
20260 
20261     AjPIdbucket cbucket  = NULL;
20262 
20263     ajulong *parray = NULL;
20264     AjPStr *newkarray  = NULL;
20265     ajulong *newparray  = NULL;
20266     AjPBtMem arrays     = NULL;
20267     AjPBtMem newarrays = NULL;
20268 
20269     ajuint lno    = 0U;
20270     ajuint rno    = 0U;
20271 
20272     ajuint lbucketlimit   = 0U;
20273     ajuint lmaxnperbucket = 0U;
20274     ajuint rbucketlimit   = 0U;
20275     ajuint rmaxnperbucket = 0U;
20276     ajuint count         = 0U;
20277 
20278     ajulong lblockno = 0UL;
20279     ajulong rblockno = 0UL;
20280     ajulong prev     = 0UL;
20281     ajulong overflow = 0UL;
20282     ajulong prevsave = 0UL;
20283 
20284     ajulong zero = 0UL;
20285     ajulong join = 0UL;
20286 
20287     ajulong lv = 0UL;
20288     ajuint  v  = 0U;
20289 
20290     ajuint iold = 0U;
20291     ajuint refskip = cache->refcount*BT_EXTRA;
20292 
20293 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
20294     ajDebug("btreeIdSplitleaf '%S' %Lu id '%S'\n",
20295             cache->basename, spage->pagepos, indexId);
20296 #endif
20297     ++statCallIdSplitleaf;
20298 
20299     /*ajDebug("In btreeIdSplitleaf page:%Lu\n", spage->pagepos);*/
20300 
20301     order = cache->porder;
20302 
20303     arrays    = btreeAllocPriArray(cache);
20304     parray    = arrays->parray;
20305 
20306     newarrays    = btreeAllocPriArray(cache);
20307     newkarray    = newarrays->karray;
20308     newparray    = newarrays->parray;
20309 
20310     buf = spage->buf;
20311     lbuf = buf;
20312 
20313     GBT_NKEYS(buf,&nkeys);
20314     GBT_NODETYPE(buf,&rootnodetype);
20315 
20316     /*ajDebug("btreeIdSplitleaf start page %Lu nkeys:%u\n",
20317       spage->pagepos,  nkeys);*/
20318 
20319     for(i=nkeys+1; i<order; ++i)
20320 	parray[i] = 0UL;
20321 
20322     if(rootnodetype == BT_ROOT)
20323     {
20324 #if AJINDEX_DEBUG
20325         ajDebug("    splitting root node %Lu new %Lu\n",
20326                 spage->pagepos, cache->totsize);
20327 #endif
20328 	lblockno = cache->totsize;
20329 	lpage = btreePricacheNodenew(cache);
20330 	lbuf = lpage->buf;
20331 	lv = prev;
20332 	SBT_PREV(lbuf,lv);
20333     }
20334     else
20335     {
20336 	lblockno = spage->pagepos;
20337 	lpage = spage;
20338 #if AJINDEX_DEBUG
20339         ajDebug("    keeping lpage %Lu\n", lpage->pagepos);
20340 #endif
20341     }
20342 
20343     lpage->dirty = BT_LOCK;
20344     lpage->lockfor = 1521;
20345 
20346 
20347 #if AJINDEX_DEBUG
20348         ajDebug("    new right node %Lu\n",
20349                 cache->totsize);
20350 #endif
20351 
20352     rblockno = cache->totsize;
20353     rpage = btreePricacheNodenew(cache);
20354     rpage->dirty = BT_LOCK;
20355     rpage->lockfor = 1522;
20356     rbuf = rpage->buf;
20357 
20358     if(rootnodetype == BT_ROOT)
20359     {
20360 	lv = zero;
20361 	SBT_RIGHT(rbuf,lv);
20362 	lv = zero;
20363 	SBT_LEFT(lbuf,lv);
20364     }
20365     else
20366     {
20367 	GBT_RIGHT(lbuf,&join);
20368 	lv = join;
20369 	SBT_RIGHT(rbuf,lv);
20370     }
20371 
20372     lv = lblockno;
20373     SBT_LEFT(rbuf,lv);
20374     lv = rblockno;
20375     SBT_RIGHT(lbuf,lv);
20376 
20377 
20378     btreeGetPointers(cache,buf,&parray);
20379 
20380     keylimit = nkeys+1;
20381 
20382     idlist = ajListNew();
20383 
20384     for(i=0;i<keylimit;++i)
20385         btreeIdbucketIdlist(cache,parray[i],idlist);
20386 
20387     ajListSort(idlist, &btreeIdCompare);
20388 
20389     totalkeys = (ajuint) ajListGetLength(idlist);
20390 
20391     btreeBucketSplitCalc(totalkeys, keylimit, cache->pnperbucket,
20392                          &lbucketlimit,&lmaxnperbucket,&lno,
20393                          &rbucketlimit,&rmaxnperbucket,&rno);
20394 
20395     cbucket = btreeIdbucketNew(cache->pnperbucket, cache->refcount);
20396 
20397     count = 0;
20398     iold=0;
20399 
20400     for(i=0;i<lbucketlimit;++i)
20401     {
20402 	cbucket->Nentries = 0;
20403 
20404 	for(j=0;j<lmaxnperbucket;++j)
20405 	{
20406 	    ajListPop(idlist,(void **)&bid);
20407 
20408 	    cid = cbucket->Ids[j];
20409 	    ajStrAssignS(&cid->id,bid->id);
20410 	    cid->dbno = bid->dbno;
20411 	    cid->dups = bid->dups;
20412 	    cid->offset = bid->offset;
20413 
20414             if(cache->refcount)
20415             {
20416                 for(iref=0; iref < cache->refcount; iref++)
20417                     cid->refoffsets[iref] = bid->refoffsets[iref];
20418 	    }
20419 
20420 	    cbucket->keylen[j] = BT_BUCKIDLEN(bid->id) + refskip;
20421 	    ++count;
20422 	    ++cbucket->Nentries;
20423 	    ajBtreeIdDel(&bid);
20424 	}
20425 
20426 	ajListPeek(idlist,(void **)&bid);
20427 
20428 	ajStrAssignS(&newkarray[i],bid->id);
20429 
20430 	if(iold < keylimit)
20431             newparray[i] = parray[iold++];
20432         else
20433 	    newparray[i] = cache->totsize;
20434 	btreeWriteIdbucket(cache,cbucket,newparray[i]);
20435     }
20436 
20437     cbucket->Nentries = 0;
20438 
20439     j = 0;
20440 
20441     while(count != lno)
20442     {
20443 	ajListPop(idlist,(void **)&bid);
20444 	cid = cbucket->Ids[j];
20445 	++j;
20446 	++count;
20447 
20448 	ajStrAssignS(&cid->id,bid->id);
20449 	cid->dbno = bid->dbno;
20450 	cid->dups = bid->dups;
20451 	cid->offset = bid->offset;
20452 
20453         if(cache->refcount)
20454         {
20455             for(iref=0; iref < cache->refcount; iref++)
20456                 cid->refoffsets[iref] = bid->refoffsets[iref];
20457 	}
20458 
20459 	++cbucket->Nentries;
20460 	ajBtreeIdDel(&bid);
20461     }
20462 
20463     if(iold < keylimit)
20464         newparray[i] = parray[iold++];
20465     else
20466         newparray[i] = cache->totsize;
20467     btreeWriteIdbucket(cache,cbucket,newparray[i]);
20468 
20469     nkeys = lbucketlimit;
20470 
20471     nodetype = BT_LEAF;
20472     v = nodetype;
20473     SBT_NODETYPE(lbuf,v);
20474     lpage->dirty = BT_DIRTY;
20475 
20476     GBT_PREV(lbuf,&prevsave);
20477 #if AJINDEX_DEBUG
20478     ajDebug("    lpage %Lu prevsave %Lu\n",
20479             lpage->pagepos, prevsave);
20480 #endif
20481     btreeWriteNode(cache,lpage,newkarray,newparray,nkeys);
20482 
20483     ajListPeek(idlist,(void **)&bid);
20484     mediankey = ajStrNewS(bid->id);
20485 
20486     for(i=0;i<rbucketlimit;++i)
20487     {
20488 	cbucket->Nentries = 0;
20489 	for(j=0;j<rmaxnperbucket;++j)
20490 	{
20491 	    ajListPop(idlist,(void **)&bid);
20492 
20493 	    cid = cbucket->Ids[j];
20494 	    ajStrAssignS(&cid->id,bid->id);
20495 	    cid->dbno = bid->dbno;
20496 	    cid->dups = bid->dups;
20497 	    cid->offset = bid->offset;
20498 
20499             if(cache->refcount)
20500             {
20501                 for(iref=0; iref < cache->refcount; iref++)
20502                     cid->refoffsets[iref] = bid->refoffsets[iref];
20503 	    }
20504 
20505 	    cbucket->keylen[j] = BT_BUCKIDLEN(bid->id) + refskip;
20506 	    ++cbucket->Nentries;
20507 	    ajBtreeIdDel(&bid);
20508 	}
20509 
20510 	ajListPeek(idlist,(void **)&bid);
20511 	ajStrAssignS(&newkarray[i],bid->id);
20512 
20513 	if(iold < keylimit)
20514             newparray[i] = parray[iold++];
20515         else
20516             newparray[i] = cache->totsize;
20517 	btreeWriteIdbucket(cache,cbucket,newparray[i]);
20518     }
20519 
20520     cbucket->Nentries = 0;
20521 
20522     j = 0;
20523 
20524     while(ajListPop(idlist,(void**)&bid))
20525     {
20526 	cid = cbucket->Ids[j];
20527 	++j;
20528 
20529 	ajStrAssignS(&cid->id,bid->id);
20530 	cid->dbno = bid->dbno;
20531 	cid->dups = bid->dups;
20532 	cid->offset = bid->offset;
20533 
20534         if(cache->refcount)
20535         {
20536             for(iref=0; iref < cache->refcount; iref++)
20537                 cid->refoffsets[iref] = bid->refoffsets[iref];
20538 	}
20539 
20540 	++cbucket->Nentries;
20541 	ajBtreeIdDel(&bid);
20542     }
20543 
20544     if(iold < keylimit)
20545         newparray[i] = parray[iold++];
20546     else
20547         newparray[i] = cache->totsize;
20548     btreeWriteIdbucket(cache,cbucket,newparray[i]);
20549 
20550     nkeys = rbucketlimit;
20551 
20552     nodetype = BT_LEAF;
20553     v = nodetype;
20554     SBT_NODETYPE(rbuf,v);
20555     lv = prevsave;
20556     SBT_PREV(rbuf,lv);
20557     lv = overflow;
20558     SBT_OVERFLOW(rbuf,lv);
20559 
20560 #if AJINDEX_DEBUG
20561     ajDebug("    right %Lu prev: %Lu\n",
20562             rpage->pagepos, prevsave);
20563 #endif
20564     btreeWriteNode(cache,rpage,newkarray,newparray,nkeys);
20565     rpage->dirty = BT_DIRTY;
20566 
20567     btreeIdbucketDel(&cbucket);
20568     ajListFree(&idlist);
20569 
20570 
20571 
20572     medianless = lblockno;
20573     mediangtr  = rblockno;
20574 
20575     btreeDeallocPriArray(cache,arrays);
20576     btreeDeallocPriArray(cache,newarrays);
20577 
20578     if(rootnodetype == BT_ROOT)
20579     {
20580 #if AJINDEX_DEBUG
20581         ajDebug("    rootnode new node for mediankey '%S' %Lu\n",
20582                 mediankey, spage->pagepos);
20583 #endif
20584 	btreeWriteNodeSingle(cache,spage,mediankey,lblockno,rblockno);
20585 	spage->dirty = BT_LOCK;
20586         spage->lockfor = 1523;
20587 
20588 	ajStrDel(&mediankey);
20589 	++cache->plevel;
20590 
20591 	return spage;
20592     }
20593 
20594 
20595     page = btreePricacheRead(cache,prevsave);
20596     btreeIdInsertKey(cache,page,mediankey,medianless,mediangtr);
20597     ajStrDel(&mediankey);
20598 
20599     page = btreePricacheRead(cache,prevsave);
20600 
20601     return page;
20602 }
20603 
20604 
20605 
20606 
20607 /* @funcstatic btreeIdInsertKey ***********************************************
20608 **
20609 ** Insert an identifier into a potentially full node
20610 **
20611 ** @param [u] cache [AjPBtcache] cache
20612 ** @param [u] page [AjPBtpage] original page
20613 ** @param [r] key [const AjPStr] key to insert
20614 ** @param [r] less [ajulong] less-than pointer
20615 ** @param [r] greater [ajulong] greater-than pointer
20616 **
20617 ** @return [void]
20618 **
20619 ** @release 6.5.0
20620 ** @@
20621 ******************************************************************************/
20622 
btreeIdInsertKey(AjPBtcache cache,AjPBtpage page,const AjPStr key,ajulong less,ajulong greater)20623 static void btreeIdInsertKey(AjPBtcache cache, AjPBtpage page,
20624                              const AjPStr key, ajulong less, ajulong greater)
20625 {
20626     unsigned char *lbuf = NULL;
20627     unsigned char *rbuf = NULL;
20628     unsigned char *tbuf = NULL;
20629     AjPStr *karray      = NULL;
20630     ajulong *parray      = NULL;
20631     AjPStr *tkarray     = NULL;
20632     ajulong *tparray     = NULL;
20633 
20634     AjPBtMem arrays1    = NULL;
20635     AjPBtMem arrays2    = NULL;
20636 
20637     ajuint nkeys    = 0U;
20638     ajuint order    = 0U;
20639     ajuint keypos   = 0U;
20640     ajuint rkeyno   = 0U;
20641 
20642     ajuint i = 0U;
20643     ajuint n = 0U;
20644 
20645     ajuint nodetype  = 0U;
20646     AjPBtpage ipage = NULL;
20647     AjPBtpage lpage = NULL;
20648     AjPBtpage rpage = NULL;
20649     AjPBtpage tpage = NULL;
20650 
20651     ajulong blockno  = 0UL;
20652     ajulong rblockno = 0UL;
20653     ajulong lblockno = 0UL;
20654     ajulong ibn      = 0UL;
20655 
20656     AjPStr mediankey  = NULL;
20657     ajulong medianless = 0UL;
20658     ajulong mediangtr  = 0UL;
20659     ajulong overflow   = 0UL;
20660     ajulong prev       = 0UL;
20661     ajuint totlen     = 0U;
20662 
20663     ajulong lv = 0UL;
20664     ajuint  v  = 0U;
20665 
20666     if(!btreeNodeIsFull(cache,page))
20667     {
20668 	btreePrimaryInsertNonfull(cache,page,key,less,greater);
20669 
20670 	return;
20671     }
20672 
20673     order = cache->porder;
20674     lbuf = page->buf;
20675     GBT_NODETYPE(lbuf,&nodetype);
20676     page->dirty = BT_LOCK;
20677     page->lockfor = 1531;
20678 
20679     if(nodetype == BT_ROOT)
20680     {
20681 	arrays1   = btreeAllocPriArray(cache);
20682 	karray    = arrays1->karray;
20683 	parray    = arrays1->parray;
20684 
20685 	btreeIdSplitroot(cache);
20686 
20687 	if(page->pagepos)
20688 	    page->dirty = BT_DIRTY;
20689 
20690 	btreeGetKeys(cache,lbuf,&karray,&parray);
20691 
20692 	if(MAJSTRCMPS(key,karray[0])<0)
20693 	    blockno = parray[0];
20694 	else
20695 	    blockno = parray[1];
20696 
20697 	ipage = btreePricacheRead(cache,blockno);
20698 	btreePrimaryInsertNonfull(cache,ipage,key,less,greater);
20699 
20700 	btreeDeallocPriArray(cache,arrays1);
20701 
20702 	return;
20703     }
20704 
20705 
20706     arrays1   = btreeAllocPriArray(cache);
20707     karray    = arrays1->karray;
20708     parray    = arrays1->parray;
20709 
20710     arrays2   = btreeAllocPriArray(cache);
20711     tkarray   = arrays2->karray;
20712     tparray   = arrays2->parray;
20713 
20714     mediankey = ajStrNew();
20715 
20716     lpage = page;
20717     lbuf = lpage->buf;
20718 
20719     btreeGetKeys(cache,lbuf,&karray,&parray);
20720 
20721     GBT_BLOCKNUMBER(lbuf,&lblockno);
20722 
20723     rblockno = cache->totsize;
20724     rpage = btreePricacheNodenew(cache);
20725     rpage->dirty = BT_LOCK;
20726     rpage->lockfor = 1532;
20727     rbuf = rpage->buf;
20728 
20729 
20730     GBT_PREV(lbuf,&prev);
20731     lv = prev;
20732     SBT_PREV(rbuf,lv);
20733 
20734     nkeys = order - 1;
20735     keypos = nkeys / 2;
20736 
20737     if(!(nkeys % 2))
20738 	--keypos;
20739 
20740     ajStrAssignS(&mediankey,karray[keypos]);
20741     medianless = lblockno;
20742     mediangtr  = rblockno;
20743 
20744 
20745     GBT_NODETYPE(lbuf,&nodetype);
20746     v = nodetype;
20747     SBT_NODETYPE(rbuf,v);
20748     lv = overflow;
20749     SBT_OVERFLOW(rbuf,lv);
20750 
20751 
20752     totlen = 0;
20753 
20754     for(i=0;i<keypos;++i)
20755     {
20756 	ajStrAssignS(&tkarray[i],karray[i]);
20757 	totlen += ajStrGetLen(karray[i]);
20758 	tparray[i] = parray[i];
20759     }
20760 
20761     tparray[i] = parray[i];
20762     v = totlen;
20763     SBT_TOTLEN(lbuf,v);
20764     n = i;
20765     v = n;
20766     SBT_NKEYS(lbuf,v);
20767     btreeWriteNode(cache,lpage,tkarray,tparray,i);
20768     lpage->dirty = BT_LOCK;
20769     lpage->lockfor = 1533;
20770 
20771 
20772     for(i=0;i<n+1;++i)
20773     {
20774 	tpage = btreePricacheRead(cache,tparray[i]);
20775 	tbuf = tpage->buf;
20776 	lv = lblockno;
20777 	SBT_PREV(tbuf,lv);
20778 	tpage->dirty = BT_DIRTY;
20779     }
20780 
20781 
20782     totlen = 0;
20783 
20784     for(i=keypos+1;i<nkeys;++i)
20785     {
20786 	ajStrAssignS(&tkarray[i-(keypos+1)],karray[i]);
20787 	totlen += ajStrGetLen(karray[i]);
20788 	tparray[i-(keypos+1)] = parray[i];
20789     }
20790     tparray[i-(keypos+1)] = parray[i];
20791     v = totlen;
20792     SBT_TOTLEN(rbuf,v);
20793     rkeyno = (nkeys-keypos) - 1;
20794     v = rkeyno;
20795     SBT_NKEYS(rbuf,v);
20796     rpage->dirty = BT_DIRTY;
20797 
20798     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
20799     rpage->dirty = BT_LOCK;
20800     rpage->lockfor = 1534;
20801 
20802     for(i=0;i<rkeyno+1;++i)
20803     {
20804 	tpage = btreePricacheRead(cache,tparray[i]);
20805 	tbuf = tpage->buf;
20806 	lv = rblockno;
20807 	SBT_PREV(tbuf,lv);
20808 	tpage->dirty = BT_DIRTY;
20809     }
20810 
20811 
20812     ibn = rblockno;
20813 
20814     if(MAJSTRCMPS(key,mediankey)<0)
20815 	ibn = lblockno;
20816 
20817     lpage->dirty = BT_DIRTY;
20818     rpage->dirty = BT_DIRTY;
20819 
20820     ipage = btreePricacheRead(cache,ibn);
20821 
20822     btreePrimaryInsertNonfull(cache,ipage,key,less,greater);
20823 
20824 
20825     btreeDeallocPriArray(cache,arrays1);
20826     btreeDeallocPriArray(cache,arrays2);
20827 
20828     ipage = btreePricacheRead(cache,prev);
20829 
20830     btreeIdInsertKey(cache,ipage,mediankey,medianless,mediangtr);
20831     ajStrDel(&mediankey);
20832 
20833     return;
20834 }
20835 
20836 
20837 
20838 
20839 /* @funcstatic btreeIdSplitroot ***********************************************
20840 **
20841 ** Split the root node
20842 **
20843 ** @param [u] cache [AjPBtcache] cache
20844 **
20845 ** @return [void]
20846 **
20847 ** @release 4.0.0
20848 ** @@
20849 ******************************************************************************/
20850 
btreeIdSplitroot(AjPBtcache cache)20851 static void btreeIdSplitroot(AjPBtcache cache)
20852 {
20853     AjPBtpage rootpage = NULL;
20854     AjPBtpage rpage    = NULL;
20855     AjPBtpage lpage    = NULL;
20856     AjPBtpage tpage    = NULL;
20857 
20858     AjPStr *karray     = NULL;
20859     AjPStr *tkarray    = NULL;
20860     ajulong *parray     = NULL;
20861     ajulong *tparray    = NULL;
20862     AjPBtMem arrays1   = NULL;
20863     AjPBtMem arrays2   = NULL;
20864 
20865     ajuint order     = 0U;
20866     ajuint nkeys     = 0U;
20867     ajuint keypos    = 0U;
20868 
20869     ajulong rblockno = 0UL;
20870     ajulong lblockno = 0UL;
20871 
20872     AjPStr key = NULL;
20873     ajuint  i;
20874     ajuint  j;
20875 
20876     unsigned char *rootbuf = NULL;
20877     unsigned char *rbuf    = NULL;
20878     unsigned char *lbuf    = NULL;
20879     unsigned char *tbuf    = NULL;
20880 
20881     ajuint nodetype  = 0U;
20882     ajulong overflow = 0UL;
20883     ajulong zero     = 0UL;
20884     ajuint rkeyno    = 0U;
20885 
20886     ajulong lv = 0UL;
20887     ajuint  v  = 0U;
20888 
20889 
20890 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
20891     ajDebug("btreeIdSplitroot '%S' zero id '%S' key '%S'\n",
20892             cache->basename, indexId, indexKeyword);
20893 #endif
20894     ++statCallIdSplitroot;
20895 
20896     /* ajDebug("In btreeIdSplitroot\n"); */
20897 
20898     order = cache->porder;
20899 
20900     arrays1   = btreeAllocPriArray(cache);
20901     karray    = arrays1->karray;
20902     parray    = arrays1->parray;
20903 
20904     arrays2   = btreeAllocPriArray(cache);
20905     tkarray   = arrays2->karray;
20906     tparray   = arrays2->parray;
20907 
20908     key = ajStrNew();
20909 
20910     rootpage = btreePricacheLocate(cache,0UL);
20911 
20912     if(!rootpage)
20913 	ajFatal("Root page has been unlocked 1");
20914 
20915     rootbuf = rootpage->buf;
20916     nkeys   = order - 1;
20917     keypos  = nkeys / 2U;
20918 
20919     if(!(nkeys % 2U))
20920 	--keypos;
20921 
20922 
20923     rblockno = cache->totsize;
20924     rpage = btreePricacheNodenew(cache);
20925     rpage->dirty = BT_LOCK;
20926     rpage->lockfor = 1541;
20927 
20928     lblockno = cache->totsize;
20929     lpage = btreePricacheNodenew(cache);
20930 
20931     if(!cache->plevel)
20932     {
20933 	lv = zero;
20934 	SBT_LEFT(lpage->buf,lv);
20935 	lv = rblockno;
20936 	SBT_RIGHT(lpage->buf,lv);
20937 	lv = lblockno;
20938 	SBT_LEFT(rpage->buf,lv);
20939 	lv = zero;
20940 	SBT_RIGHT(rpage->buf,lv);
20941     }
20942 
20943     btreeGetKeys(cache,rootbuf,&karray,&parray);
20944 
20945     /* write new root node */
20946 
20947     btreeWriteNodeSingle(cache,rootpage,karray[keypos],lblockno,rblockno);
20948     rootpage->dirty = BT_LOCK;
20949     rootpage->lockfor = 1542;
20950 
20951     rbuf = rpage->buf;
20952     lbuf = lpage->buf;
20953 
20954     if(cache->plevel)
20955 	nodetype = BT_INTERNAL;
20956     else
20957 	nodetype = BT_LEAF;
20958 
20959     v = nodetype;
20960     SBT_NODETYPE(rbuf,v);
20961     v = nodetype;
20962     SBT_NODETYPE(lbuf,v);
20963     lv = overflow;
20964     SBT_OVERFLOW(rbuf,lv);
20965     lv = overflow;
20966     SBT_PREV(rbuf,lv);
20967     lv = overflow;
20968     SBT_OVERFLOW(lbuf,lv);
20969     lv = overflow;
20970     SBT_PREV(lbuf,lv);
20971 
20972     for(i=0;i<keypos;++i)
20973     {
20974 	ajStrAssignS(&tkarray[i],karray[i]);
20975 	tparray[i] = parray[i];
20976     }
20977 
20978     tparray[i] = parray[i];
20979     btreeWriteNode(cache,lpage,tkarray,tparray,keypos);
20980 
20981     for(i=0;i<=keypos;++i)
20982     {
20983 	tpage = btreePricacheRead(cache,tparray[i]);
20984 	tbuf = tpage->buf;
20985 	lv = lblockno;
20986 	SBT_PREV(tbuf,lv);
20987 	tpage->dirty = BT_DIRTY;
20988 #if AJINDEX_DEBUG
20989         ajDebug("    new lpage %Lu prev %Lu\n",
20990                 tpage->pagepos, lblockno);
20991 #endif
20992     }
20993 
20994     j=0;
20995 
20996     for(i=keypos+1;i<nkeys;++i)
20997     {
20998 	ajStrAssignS(&tkarray[j],karray[i]);
20999 	tparray[j++] = parray[i];
21000     }
21001 
21002     tparray[j] = parray[i];
21003     rkeyno = (nkeys-keypos) - 1;
21004     rpage->dirty = BT_DIRTY;
21005 
21006     btreeWriteNode(cache,rpage,tkarray,tparray,rkeyno);
21007 
21008     for(i=0;i<=rkeyno;++i)
21009     {
21010 	tpage = btreePricacheRead(cache,tparray[i]);
21011 	tbuf = tpage->buf;
21012 	lv = rblockno;
21013 	SBT_PREV(tbuf,lv);
21014 	tpage->dirty = BT_DIRTY;
21015 #if AJINDEX_DEBUG
21016         ajDebug("    new rpage %Lu prev %Lu\n",
21017                 tpage->pagepos, rblockno);
21018 #endif
21019     }
21020 
21021 
21022     btreeDeallocPriArray(cache,arrays1);
21023     btreeDeallocPriArray(cache,arrays2);
21024 
21025     ++cache->plevel;
21026 
21027     ajStrDel(&key);
21028 
21029     return;
21030 }
21031 
21032 
21033 
21034 
21035 /* @func ajBtreeIdentIndex ****************************************************
21036 **
21037 ** Insert an ID structure into the tree
21038 **
21039 ** @param [u] cache [AjPBtcache] cache
21040 ** @param [u] id [AjPBtId] Id object
21041 **
21042 ** @return [void] pointer to a page
21043 **
21044 ** @release 6.5.0
21045 ** @@
21046 ******************************************************************************/
21047 
ajBtreeIdentIndex(AjPBtcache cache,AjPBtId id)21048 void ajBtreeIdentIndex(AjPBtcache cache, AjPBtId id)
21049 {
21050     AjPBtpage spage   = NULL;
21051     ajulong lblockno = 0UL;
21052     ajulong rblockno = 0UL;
21053     ajulong blockno  = 0UL;
21054     ajulong shift    = 0UL;
21055 
21056     ajuint nkeys = 0U;
21057 
21058     ajuint nodetype = 0U;
21059 
21060     AjPBtId curid = NULL;
21061 
21062     ajuint n;
21063     ajuint ientry;
21064 
21065     unsigned char *buf = NULL;
21066 
21067 #if AJINDEX_DEBUG
21068     ajDebug("btreeIdentIndex '%S'\n", id->id);
21069 #endif
21070 
21071     /* ajDebug("In ajBtreeIdentIndex\n"); */
21072 
21073     ajStrFmtQuery(&id->id);
21074     ajStrAssignS(&indexId, id->id);
21075     ajStrAssignC(&indexKeyword, "(none)");
21076 
21077     if(!MAJSTRGETLEN(id->id))
21078 	return;
21079 
21080     if(MAJSTRGETLEN(id->id) > cache->keylimit)
21081         ajStrTruncateLen(&id->id, cache->keylimit);
21082 
21083     spage = btreeIdentFind(cache,id->id);
21084     buf = spage->buf;
21085 
21086     GBT_NKEYS(buf,&nkeys);
21087     GBT_NODETYPE(buf,&nodetype);
21088 
21089     if(!nkeys)
21090     {
21091 	lblockno = cache->totsize;
21092 	btreeWriteIdbucketEmpty(cache,lblockno);
21093 
21094 	rblockno = cache->totsize;
21095 	btreeWriteIdbucketEmpty(cache,rblockno);
21096 
21097 	btreeWriteNodeSingle(cache,spage,id->id,lblockno,rblockno);
21098 
21099 	btreeIdbucketAdd(cache,rblockno,id);
21100         ++cache->countunique;
21101         ++cache->countall;
21102 
21103 	return;
21104     }
21105 
21106 
21107     /* Search to see whether entry exists */
21108 
21109     blockno = btreeGetBlockS(cache,buf,id->id);
21110 
21111     curid = btreeIdbucketFindDupId(cache,blockno,id->id,&ientry);
21112 
21113     if(curid)
21114     {
21115         ++cache->countall;
21116 
21117 	btreeIdentDupInsert(cache,id,curid);
21118         btreeWriteIdbucketId(cache,blockno,curid,ientry);
21119         ajBtreeIdDel(&curid);
21120 
21121 	return;
21122     }
21123 
21124     if(nodetype != BT_ROOT)
21125 	if((shift = btreeIdentInsertShift(cache,&spage,id->id)))
21126 	    blockno = shift;
21127 
21128     buf = spage->buf;
21129 
21130     n = btreeIdbucketCount(cache,blockno);
21131 
21132     if(n == cache->pnperbucket)
21133     {
21134 	if(btreeIdbucketsReorder(cache,spage))
21135 	{
21136             blockno = btreeGetBlockS(cache,buf,id->id);
21137 	}
21138 	else
21139 	{
21140 	    btreeIdSplitleaf(cache,spage);
21141 	    spage  = btreeIdentFind(cache,id->id);
21142 	    buf = spage->buf;
21143 
21144             blockno = btreeGetBlockS(cache,buf,id->id);
21145 	}
21146     }
21147 
21148 
21149     btreeIdbucketAdd(cache,blockno,id);
21150 
21151     ++cache->countunique;
21152     ++cache->countall;
21153 
21154     return;
21155 }
21156 
21157 
21158 
21159 
21160 /* @funcstatic btreeIdentDupInsert ********************************************
21161 **
21162 ** Insert a known duplicate ID structure into the tree
21163 **
21164 ** @param [u] cache [AjPBtcache] cache
21165 ** @param [r] newid [const AjPBtId] New ID object
21166 ** @param [u] curid [AjPBtId] Id object from current index
21167 **
21168 ** @return [void]
21169 **
21170 ** @release 6.5.0
21171 ** @@
21172 ******************************************************************************/
21173 
btreeIdentDupInsert(AjPBtcache cache,const AjPBtId newid,AjPBtId curid)21174 static void btreeIdentDupInsert(AjPBtcache cache, const AjPBtId newid,
21175                                 AjPBtId curid)
21176 {
21177     AjPBtpage page;
21178     AjPBtpage rpage;
21179     ajulong secrootpage = 0UL;
21180     unsigned char *buf;
21181     ajulong right = 0UL;
21182     ajulong refoffsets[1000];
21183     AjPBtNumId num = NULL;
21184     AjOBtNumId numobj = {0UL, refoffsets, 0U, 0U};
21185 
21186     ajuint iref;
21187 
21188     /* ajDebug("In btreeIdentDupInsert\n"); */
21189 
21190     if(!curid->dups)
21191     {
21192         /* first duplicate: could use SIMPLEBUCKET */
21193 
21194 	curid->dups = 1;
21195 	num = &numobj;
21196 
21197 	num->dbno      = curid->dbno;
21198 	num->offset    = curid->offset;
21199 
21200         if(cache->refcount)
21201         {
21202             if(cache->refcount > 1000)
21203                 ajFatal("btreeIdentDupInsert refcount %u", cache->refcount);
21204             num->refcount = cache->refcount;
21205             for(iref=0; iref < cache->refcount; iref++)
21206                 num->refoffsets[iref] = curid->refoffsets[iref];
21207         }
21208 
21209 	secrootpage = cache->totsize;
21210 
21211 	curid->offset = secrootpage;
21212 
21213 	btreeSecrootCreate(cache,secrootpage);
21214 	cache->secrootblock = secrootpage;
21215 	page = btreeSeccacheWrite(cache,secrootpage);
21216 	page->dirty = BT_DIRTY;
21217 
21218         if(btreeDoRootSync)
21219             btreeCacheRootSync(cache,secrootpage);
21220 	page->dirty = BT_LOCK;
21221 	page->lockfor = 1551;
21222 
21223 	rpage = btreePricacheLocate(cache, 0UL);
21224 	rpage->dirty = BT_LOCK;
21225         rpage->lockfor = 1552;
21226 
21227 	cache->slevel = 0U;
21228 
21229         btreeNumInsert(cache,num,page);
21230 
21231 	num->dbno      = newid->dbno;
21232 	num->offset    = newid->offset;
21233 
21234         if(cache->refcount)
21235         {
21236             num->refcount = cache->refcount;
21237             for(iref=0; iref < cache->refcount; iref++)
21238                 num->refoffsets[iref] = newid->refoffsets[iref];
21239         }
21240 
21241 	btreeNumInsert(cache,num,page);
21242 	++curid->dups;
21243 
21244 	return;
21245     }
21246     else
21247     {
21248 	cache->secrootblock = curid->offset;
21249 	page = btreeSeccacheWrite(cache,cache->secrootblock);
21250 	page->dirty = BT_LOCK;
21251         page->lockfor = 1553;
21252 	buf = page->buf;
21253 	GBT_RIGHT(buf,&right);
21254 	cache->slevel = (ajuint) right;
21255 
21256 	num = &numobj;
21257 
21258 	num->dbno      = newid->dbno;
21259 	num->offset    = newid->offset;
21260 
21261         if(cache->refcount)
21262         {
21263             for(iref=0; iref < cache->refcount; iref++)
21264                 num->refoffsets[iref] = newid->refoffsets[iref];
21265 	}
21266 
21267 	btreeNumInsert(cache,num,page);
21268 
21269 	++curid->dups;
21270     }
21271 
21272 
21273     page->dirty = BT_DIRTY;
21274 
21275     return;
21276 }
21277 
21278 
21279 
21280 
21281 #if 0
21282 /* #funcstatic btreeHybbucketIdlist *******************************************
21283 **
21284 ** Copies all hybrid IDs into a list
21285 **
21286 ** #param [u] cache [AjPBtcache] cache
21287 ** #param [r] pagepos [ajulong] page number
21288 ** #param [u] idlist [AjPList] list to hold hybrid IDs
21289 **
21290 ** #return [ajulong] Overflow
21291 **
21292 ** #release 6.4.0
21293 ** ##
21294 ******************************************************************************/
21295 /*
21296 static ajulong btreeHybbucketIdlist(AjPBtcache cache, ajulong pagepos,
21297                                     AjPList idlist)
21298 {
21299     AjPBtpage page      = NULL;
21300     AjPBtpage lpage     = NULL;
21301     unsigned char *buf  = NULL;
21302     unsigned char *kptr = NULL;
21303     AjPBtId id          = NULL;
21304 
21305     unsigned char *idptr = NULL;
21306 
21307     ajuint  nodetype  = 0U;
21308     ajuint  nentries  = 0U;
21309     ajulong overflow  = 0UL;
21310     ajulong pageoverflow = 0UL;
21311     ajuint  dirtysave = 0U;
21312 
21313     ajuint  i;
21314     ajuint  iref;
21315     ajuint  len = 0U;
21316     ajuint idlen;
21317     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
21318 
21319     /# ajDebug("In btreeHybbucketIdlist\n"); #/
21320 
21321     if(!pagepos)
21322 	ajFatal("HybbucketIdlist: cannot read bucket from root page cache %S",
21323                 cache->filename);
21324 
21325     page  = btreePricacheRead(cache,pagepos);
21326     lpage = page;
21327     dirtysave = lpage->dirty;
21328     lpage->dirty = BT_LOCK;
21329     lpage->lockfor = 1561;
21330 
21331     buf = lpage->buf;
21332 
21333     GBT_BUCKNODETYPE(buf,&nodetype);
21334     if(nodetype != BT_IDBUCKET)
21335 	ajFatal("HybbucketIdlist: NodeType mismatch. "
21336                 "Not bucket (%u) cache %S",
21337                 nodetype, cache->filename);
21338 
21339     GBT_BUCKNENTRIES(buf,&nentries);
21340     if(nentries > cache->pnperbucket)
21341 	ajFatal("HybbucketIdlist: Bucket too full page: %Lu "
21342                 "entries: %u max: %u cache %S",
21343                 pagepos, nentries, cache->pnperbucket, cache->filename);
21344 
21345 
21346     GBT_BUCKOVERFLOW(buf,&overflow);
21347     pageoverflow = overflow;
21348 
21349     kptr  = PBT_BUCKKEYLEN(buf);
21350     idptr = kptr + (nentries * sizeof(ajuint));
21351 
21352     for(i=0;i<nentries;++i)
21353     {
21354 	BT_GETAJUINT(kptr,&len);
21355         idlen = len - keyskip;
21356 
21357 /#
21358 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
21359 //	{
21360 //	    /# ajDebug("HybbucketIdlist: Overflow\n"); #/
21361 //	    page  = btreePricacheRead(cache,overflow);
21362 //	    buf = page->buf;
21363 //	    GBT_BUCKNODETYPE(buf,&nodetype);
21364 //	    if(nodetype != BT_IDBUCKET)
21365 //		ajFatal("HybbucketIdlist: NodeType mismatch. "
21366 //                        "Not bucket (%u) cache %S",
21367 //			nodetype, cache->filename);
21368 //	    GBT_BUCKOVERFLOW(buf,&overflow);
21369 //	    /# overflow bucket ids start at the keylen position #/
21370 //	    idptr = PBT_BUCKKEYLEN(buf);
21371 //	}
21372 #/
21373 
21374 	id = ajBtreeIdNew(cache->refcount);
21375 
21376 	/# Fill ID objects #/
21377 	ajStrAssignLenC(&id->id,(const char *)idptr,idlen);
21378 	idptr += (idlen + 1);
21379 	BT_GETAJUINT(idptr,&id->dbno);
21380 	idptr += sizeof(ajuint);
21381 	BT_GETAJUINT(idptr,&id->dups);
21382 	idptr += sizeof(ajuint);
21383 	BT_GETAJULONG(idptr,&id->offset);
21384 	idptr += sizeof(ajulong);
21385 
21386         if(cache->refcount)
21387         {
21388             for(iref=0; iref < cache->refcount; iref++)
21389             {
21390                 BT_GETAJULONG(idptr,&id->refoffsets[iref]);
21391                 idptr += sizeof(ajulong);
21392             }
21393         }
21394 
21395 	kptr += sizeof(ajuint);
21396         ajListPushAppend(idlist, id);
21397     }
21398 
21399     lpage->dirty = dirtysave;
21400 
21401     return pageoverflow;
21402 }
21403 */
21404 #endif
21405 
21406 
21407 
21408 
21409 /* @funcstatic btreeIdbucketIdlistAll *****************************************
21410 **
21411 ** Copies all ID objects into a list, following duplicates
21412 **
21413 ** @param [u] cache [AjPBtcache] cache
21414 ** @param [r] pagepos [ajulong] page number
21415 ** @param [u] idlist [AjPList] list to hold hybrid IDs
21416 **
21417 ** @return [ajulong] Overflow
21418 **
21419 ** @release 6.5.0
21420 ** @@
21421 ******************************************************************************/
21422 
btreeIdbucketIdlistAll(AjPBtcache cache,ajulong pagepos,AjPList idlist)21423 static ajulong btreeIdbucketIdlistAll(AjPBtcache cache, ajulong pagepos,
21424                                       AjPList idlist)
21425 {
21426     AjPBtpage page      = NULL;
21427     AjPBtpage lpage     = NULL;
21428     unsigned char *buf  = NULL;
21429     unsigned char *kptr = NULL;
21430     AjPBtId id          = NULL;
21431 
21432     unsigned char *idptr = NULL;
21433 
21434     ajuint  nodetype  = 0U;
21435     ajuint  nentries  = 0U;
21436     ajulong overflow  = 0UL;
21437     ajulong pageoverflow = 0UL;
21438     ajuint  dirtysave = 0U;
21439 
21440     ajuint  i;
21441     ajuint  iref;
21442     ajuint  len = 0U;
21443     ajuint idlen;
21444     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA + 1U;
21445 
21446     /* ajDebug("In btreeIdbucketIdlistAll\n"); */
21447 
21448     if(!pagepos)
21449 	ajFatal("IdbucketIdlistAll: cannot read bucket from root page cache %S",
21450                 cache->filename);
21451 
21452     page  = btreePricacheRead(cache,pagepos);
21453     lpage = page;
21454     dirtysave = lpage->dirty;
21455     lpage->dirty = BT_LOCK;
21456     lpage->lockfor = 1561;
21457 
21458     buf = lpage->buf;
21459 
21460     GBT_BUCKNODETYPE(buf,&nodetype);
21461     if(nodetype != BT_IDBUCKET)
21462 	ajFatal("IdbucketIdlistAll: NodeType mismatch. "
21463                 "Not bucket (%u) cache %S",
21464                 nodetype, cache->filename);
21465 
21466     GBT_BUCKNENTRIES(buf,&nentries);
21467     if(nentries > cache->pnperbucket)
21468 	ajFatal("IdbucketIdlistAll: Bucket too full page: %Lu "
21469                 "entries: %u max: %u cache %S",
21470                 pagepos, nentries, cache->pnperbucket, cache->filename);
21471 
21472 
21473     GBT_BUCKOVERFLOW(buf,&overflow);
21474     pageoverflow = overflow;
21475 
21476     kptr  = PBT_BUCKKEYLEN(buf);
21477     idptr = kptr + (nentries * sizeof(ajuint));
21478 
21479     for(i=0;i<nentries;++i)
21480     {
21481 	BT_GETAJUINT(kptr,&len);
21482         idlen = len - keyskip;
21483 
21484 /*
21485 //	if((idptr-buf+1) + len > cache->pagesize)	/# overflow #/
21486 //	{
21487 //	    /# ajDebug("IdbucketIdlistAll: Overflow\n"); #/
21488 //	    page  = btreePricacheRead(cache,overflow);
21489 //	    buf = page->buf;
21490 //	    GBT_BUCKNODETYPE(buf,&nodetype);
21491 //	    if(nodetype != BT_IDBUCKET)
21492 //		ajFatal("IdbucketIdlistAll: NodeType mismatch. "
21493 //                        "Not bucket (%u) cache %S",
21494 //			nodetype, cache->filename);
21495 //	    GBT_BUCKOVERFLOW(buf,&overflow);
21496 //	    /# overflow bucket ids start at the keylen position #/
21497 //	    idptr = PBT_BUCKKEYLEN(buf);
21498 //	}
21499 */
21500 
21501 	id = ajBtreeIdNew(cache->refcount);
21502 
21503 	/* Fill ID objects */
21504 	ajStrAssignLenC(&id->id,(const char *)idptr,idlen);
21505 	idptr += (idlen + 1);
21506 	BT_GETAJUINT(idptr,&id->dbno);
21507 	idptr += sizeof(ajuint);
21508 	BT_GETAJUINT(idptr,&id->dups);
21509 	idptr += sizeof(ajuint);
21510 	BT_GETAJULONG(idptr,&id->offset);
21511 	idptr += sizeof(ajulong);
21512 
21513         if(cache->refcount)
21514         {
21515             for(iref=0; iref < cache->refcount; iref++)
21516             {
21517                 BT_GETAJULONG(idptr,&id->refoffsets[iref]);
21518                 idptr += sizeof(ajulong);
21519             }
21520         }
21521 
21522 	kptr += sizeof(ajuint);
21523         if(!id->dups)
21524             ajListPushAppend(idlist, id);
21525         else
21526         {
21527             btreeIdentFetchMulti(cache, id->id, id->offset, idlist);
21528             ajBtreeIdDel(&id);
21529         }
21530     }
21531 
21532     lpage->dirty = dirtysave;
21533 
21534     return pageoverflow;
21535 }
21536 
21537 
21538 
21539 
21540 /* @funcstatic btreeGetNumKeys ************************************************
21541 **
21542 ** Get Keys and Pointers from an internal node
21543 **
21544 ** @param [u] cache [AjPBtcache] cache
21545 ** @param [u] buf [unsigned char *] page buffer
21546 ** @param [w] keys [ajulong **] keys
21547 ** @param [w] ptrs [ajulong**] ptrs
21548 **
21549 ** @return [void]
21550 **
21551 ** @release 4.0.0
21552 ** @@
21553 ******************************************************************************/
21554 
btreeGetNumKeys(AjPBtcache cache,unsigned char * buf,ajulong ** keys,ajulong ** ptrs)21555 static void btreeGetNumKeys(AjPBtcache cache, unsigned char *buf,
21556 			    ajulong **keys, ajulong **ptrs)
21557 {
21558     ajulong *karray = NULL;
21559     ajulong *parray = NULL;
21560 
21561     ajuint nkeys = 0;
21562     unsigned char *pptr = NULL;
21563     ajuint    i;
21564 
21565     /* ajDebug("In btreeGetNumKeys\n"); */
21566 
21567     (void) cache;			/* make it used */
21568 
21569     karray = *keys;
21570     parray = *ptrs;
21571 
21572     pptr = PBT_KEYLEN(buf);
21573     GBT_NKEYS(buf,&nkeys);
21574 
21575     if(!nkeys)
21576 	ajFatal("GetNumKeys: No keys in node");
21577 
21578     for(i=0;i<nkeys;++i)
21579     {
21580 	BT_GETAJULONG(pptr,&karray[i]);
21581 	pptr += sizeof(ajulong);
21582     }
21583 
21584     for(i=0;i<nkeys;++i)
21585     {
21586 	BT_GETAJULONG(pptr,&parray[i]);
21587 	pptr += sizeof(ajulong);
21588     }
21589 
21590     BT_GETAJULONG(pptr,&parray[i]);
21591 
21592     return;
21593 }
21594 
21595 
21596 
21597 
21598 /* @funcstatic btreeGetNumPointers ********************************************
21599 **
21600 ** Get Pointers from an internal numeric node
21601 **
21602 ** @param [u] cache [AjPBtcache] cache
21603 ** @param [u] buf [unsigned char *] page buffer
21604 ** @param [w] ptrs [ajulong**] ptrs
21605 **
21606 ** @return [void]
21607 **
21608 ** @release 6.4.0
21609 ** @@
21610 ******************************************************************************/
21611 
btreeGetNumPointers(AjPBtcache cache,unsigned char * buf,ajulong ** ptrs)21612 static void btreeGetNumPointers(AjPBtcache cache, unsigned char *buf,
21613                                 ajulong **ptrs)
21614 {
21615     ajulong *parray = NULL;
21616 
21617     ajuint nkeys = 0;
21618     unsigned char *pptr = NULL;
21619     ajuint    i;
21620 
21621     (void) cache;			/* make it used */
21622 
21623     parray = *ptrs;
21624 
21625     pptr = PBT_KEYLEN(buf);
21626     GBT_NKEYS(buf,&nkeys);
21627     if(!nkeys)
21628 	ajFatal("GetNumKeys: No keys in node cache %S", cache->filename);
21629 
21630     pptr += nkeys*sizeof(ajulong);
21631     for(i=0;i<nkeys;++i)
21632     {
21633 	BT_GETAJULONG(pptr,&parray[i]);
21634 	pptr += sizeof(ajulong);
21635     }
21636 
21637     BT_GETAJULONG(pptr,&parray[i]);
21638 
21639     return;
21640 }
21641 
21642 
21643 
21644 
21645 /* @funcstatic btreeWriteNumNode **********************************************
21646 **
21647 ** Write an internal numeric node
21648 **
21649 ** @param [u] cache [AjPBtcache] cache
21650 ** @param [u] spage [AjPBtpage] buffer
21651 ** @param [r] keys [const ajulong *] keys
21652 ** @param [r] ptrs [const ajulong*] page pointers
21653 ** @param [r] nkeys [ajuint] number of keys
21654 
21655 **
21656 ** @return [void]
21657 **
21658 ** @release 4.0.0
21659 ** @@
21660 ******************************************************************************/
21661 
btreeWriteNumNode(AjPBtcache cache,AjPBtpage spage,const ajulong * keys,const ajulong * ptrs,ajuint nkeys)21662 static void btreeWriteNumNode(AjPBtcache cache, AjPBtpage spage,
21663 			      const ajulong *keys, const ajulong *ptrs,
21664 			      ajuint nkeys)
21665 {
21666     unsigned char *pptr   = NULL;
21667     unsigned char *buf;
21668 
21669     ajuint tnkeys = 0U;
21670     ajuint totlen = 0U;
21671 
21672     ajulong lv       = 0UL;
21673     ajulong overflow = 0UL;
21674     ajuint i;
21675 
21676 /*  ajulong aspace   = 0UL;*/
21677 
21678     (void) cache;               /* make it used */
21679 
21680 #if AJINDEX_DEBUG
21681     ajDebug("btreeWriteNumNode %Lu nkeys:%u\n", spage->pagepos, nkeys);
21682 #endif
21683 
21684     buf = spage->buf;
21685 
21686     tnkeys = nkeys;
21687     SBT_NKEYS(buf,tnkeys);
21688     totlen = 0U;
21689     SBT_TOTLEN(buf, totlen);
21690 
21691     pptr = PBT_KEYLEN(buf);
21692 
21693     /* drop check - tested before node write - page size dependency */
21694 /*
21695 //    aspace = 2 * nkeys * sizeof(ajulong) + sizeof(ajulong);
21696 //
21697 //    if((ajuint) ((pptr+aspace)-buf) > cache->pagesize)
21698 //	ajFatal("WriteNumNode: too many keys for available pagesize");
21699 */
21700 
21701     for(i=0U;i<nkeys;++i)
21702     {
21703 	lv = keys[i];
21704 	BT_SETAJULONG(pptr,lv);
21705 	pptr += sizeof(ajulong);
21706     }
21707 
21708     for(i=0;i<nkeys;++i)
21709     {
21710 	lv = ptrs[i];
21711 	BT_SETAJULONG(pptr,lv);
21712 	pptr += sizeof(ajulong);
21713     }
21714 
21715     lv = ptrs[i];
21716     BT_SETAJULONG(pptr,lv);
21717     pptr += sizeof(ajulong);
21718 
21719     overflow = 0UL;
21720     SBT_OVERFLOW(buf,overflow);
21721 
21722     spage->dirty = BT_DIRTY;
21723 
21724     return;
21725 }
21726 
21727 
21728 
21729 
21730 /* @funcstatic btreeWriteNumNodeSingle ****************************************
21731 **
21732 ** Write an internal numeric node with a single key
21733 **
21734 ** @param [u] cache [AjPBtcache] cache
21735 ** @param [u] spage [AjPBtpage] buffer
21736 ** @param [r] numkey [ajulong] key
21737 ** @param [r] lptr [ajulong] left page pointer
21738 ** @param [r] rptr [ajulong] rightpage pointer
21739 **
21740 ** @return [void]
21741 **
21742 ** @release 6.4.0
21743 ** @@
21744 ******************************************************************************/
21745 
btreeWriteNumNodeSingle(AjPBtcache cache,AjPBtpage spage,ajulong numkey,ajulong lptr,ajulong rptr)21746 static void btreeWriteNumNodeSingle(AjPBtcache cache, AjPBtpage spage,
21747                                     ajulong numkey, ajulong lptr,
21748                                     ajulong rptr)
21749 {
21750     unsigned char *pptr   = NULL;
21751     unsigned char *buf;
21752 
21753     ajuint tnkeys = 0U;
21754     ajuint totlen = 0U;
21755 
21756     ajulong lv       = 0UL;
21757     ajulong overflow = 0UL;
21758 
21759 /*  ajulong aspace   = 0L;*/
21760 
21761     (void) cache;             /* make it used */
21762 
21763     /* ajDebug("In btreeWriteNumNode\n"); */
21764 
21765     buf = spage->buf;
21766 
21767     tnkeys = 1U;
21768     SBT_NKEYS(buf,tnkeys);
21769     totlen = 0U;
21770     SBT_TOTLEN(buf,totlen);
21771 
21772     pptr = PBT_KEYLEN(buf);
21773 
21774 /* drop check - tested before call - pagesize dependency */
21775 /*
21776 //    aspace = 2 * sizeof(ajulong) + sizeof(ajulong);
21777 //
21778 //    if((ajuint) ((pptr+aspace)-buf) > cache->pagesize)
21779 //	ajFatal("WriteNumNodeSingle: "
21780 //                "too many keys for available pagesize cache %S",
21781 //                cache->filename);
21782 */
21783 
21784     lv = numkey;
21785     BT_SETAJULONG(pptr,lv);
21786     pptr += sizeof(ajulong);
21787 
21788     lv = lptr;
21789     BT_SETAJULONG(pptr,lv);
21790     pptr += sizeof(ajulong);
21791 
21792     lv = rptr;
21793     BT_SETAJULONG(pptr,lv);
21794     pptr += sizeof(ajulong);
21795 
21796     overflow = 0UL;
21797     SBT_OVERFLOW(buf,overflow);
21798 
21799     spage->dirty = BT_DIRTY;    /* test in caller */
21800 
21801     return;
21802 }
21803 
21804 
21805 
21806 
21807 /* @funcstatic btreeWriteNumbucket ********************************************
21808 **
21809 ** Write index bucket object to the cache given a disc page number
21810 **
21811 ** @param [u] cache [AjPBtcache] cache
21812 ** @param [r] bucket [const AjPNumbucket] bucket
21813 ** @param [r] pagepos [ajulong] page number
21814 **
21815 ** @return [void]
21816 **
21817 ** @release 4.0.0
21818 ** @@
21819 ******************************************************************************/
21820 
btreeWriteNumbucket(AjPBtcache cache,const AjPNumbucket bucket,ajulong pagepos)21821 static void btreeWriteNumbucket(AjPBtcache cache, const AjPNumbucket bucket,
21822                                 ajulong pagepos)
21823 {
21824     AjPBtpage page      = NULL;
21825     unsigned char *buf  = NULL;
21826     unsigned char *pptr = NULL;
21827 
21828     ajuint  uv  = 0U;
21829     ajuint   v  = 0U;
21830     ajuint  i    = 0U;
21831     ajuint  iref = 0U;
21832     ajulong lv   = 0UL;
21833     ajuint nentries = 0U;
21834     ajulong overflow = 0UL;
21835 
21836     /* ajDebug("In btreeWriteNumbucket\n"); */
21837 
21838     if(pagepos == cache->totsize)
21839     {
21840 	page = btreeSeccacheBucketnew(cache);
21841 	buf = page->buf;
21842 	overflow = 0UL;
21843     }
21844     else
21845     {
21846 	page = btreeSeccacheRead(cache,pagepos);
21847 	buf = page->buf;
21848 	GBT_BUCKOVERFLOW(buf,&overflow);
21849     }
21850 
21851     v = BT_NUMBUCKET;
21852     SBT_BUCKNODETYPE(buf,v);
21853     page->dirty = BT_LOCK;
21854     page->lockfor = 1571;
21855 
21856     nentries = bucket->Nentries;
21857     v = nentries;
21858     SBT_BUCKNENTRIES(buf,v);
21859 
21860     pptr = PBT_BUCKKEYLEN(buf);
21861 
21862     for(i=0;i<nentries;++i)
21863     {
21864 	lv = bucket->NumId[i]->offset;
21865 	BT_SETAJULONG(pptr,lv);
21866 	pptr += sizeof(ajulong);
21867 
21868         if(cache->refcount)
21869         {
21870             for(iref=0; iref < cache->refcount; iref++)
21871             {
21872                 lv = bucket->NumId[i]->refoffsets[iref];
21873                 BT_SETAJULONG(pptr,lv);
21874                 pptr += sizeof(ajulong);
21875             }
21876         }
21877 
21878 	uv = bucket->NumId[i]->dbno;
21879 	BT_SETAJUINT(pptr,uv);
21880 	pptr += sizeof(ajuint);
21881     }
21882 
21883     lv = 0UL;
21884     SBT_BUCKOVERFLOW(buf,lv);
21885 
21886     page->dirty = BT_DIRTY;
21887 
21888     return;
21889 }
21890 
21891 
21892 
21893 
21894 /* @funcstatic btreeWriteNumbucketEmpty ***************************************
21895 **
21896 ** Write empty index numeric bucket object to the cache given a disc page number
21897 **
21898 ** @param [u] cache [AjPBtcache] cache
21899 ** @param [r] pagepos [ajulong] page number
21900 **
21901 ** @return [void]
21902 **
21903 ** @release 6.4.0
21904 ** @@
21905 ******************************************************************************/
21906 
btreeWriteNumbucketEmpty(AjPBtcache cache,ajulong pagepos)21907 static void btreeWriteNumbucketEmpty(AjPBtcache cache, ajulong pagepos)
21908 {
21909     AjPBtpage page      = NULL;
21910     unsigned char *buf  = NULL;
21911 
21912     ajuint   v   = 0U;
21913     ajulong lv   = 0UL;
21914     ajulong overflow = 0UL;
21915 
21916     /* ajDebug("In btreeWriteNumbucketEmpty\n"); */
21917 
21918     if(pagepos == cache->totsize)
21919     {
21920 	page = btreeSeccacheBucketnew(cache);
21921 	buf = page->buf;
21922 	overflow = 0UL;
21923 	lv = overflow;
21924 	SBT_BUCKOVERFLOW(buf,lv);
21925     }
21926     else
21927     {
21928 	page = btreeSeccacheRead(cache,pagepos);
21929 	buf = page->buf;
21930 	GBT_BUCKOVERFLOW(buf,&overflow);
21931     }
21932 
21933     v = BT_NUMBUCKET;
21934     SBT_BUCKNODETYPE(buf,v);
21935     page->dirty = BT_LOCK;      /* clear at end */
21936     page->lockfor = 1581;
21937 
21938     v = 0U;
21939     SBT_BUCKNENTRIES(buf,v);
21940 
21941     lv = 0UL;
21942     SBT_BUCKOVERFLOW(buf,lv);
21943 
21944     page->dirty = BT_DIRTY;     /* clear the lock */
21945 
21946     return;
21947 }
21948 
21949 
21950 
21951 
21952 /* @funcstatic btreeNumbucketIdlist *******************************************
21953 **
21954 ** Copies all numeric IDs into a list
21955 **
21956 ** @param [u] cache [AjPBtcache] cache
21957 ** @param [r] pagepos [ajulong] page number
21958 ** @param [u] idlist [AjPList] list to hold numeric ID strings
21959 **
21960 ** @return [ajulong] Overflow
21961 **
21962 ** @release 6.4.0
21963 ** @@
21964 ******************************************************************************/
21965 
btreeNumbucketIdlist(AjPBtcache cache,ajulong pagepos,AjPList idlist)21966 static ajulong btreeNumbucketIdlist(AjPBtcache cache, ajulong pagepos,
21967                                     AjPList idlist)
21968 {
21969     AjPBtNumId num      = NULL;
21970     AjPBtpage page      = NULL;
21971     unsigned char *buf  = NULL;
21972     unsigned char *pptr = NULL;
21973     ajuint  nodetype  = 0U;
21974     ajuint  nentries  = 0U;
21975     ajulong overflow  = 0UL;
21976     ajulong pageoverflow = 0UL;
21977     ajuint  dirtysave = 0U;
21978 
21979     ajuint  i;
21980     ajuint iref;
21981 
21982     /* ajDebug("In btreeNumbucketIdlist\n"); */
21983 
21984     if(pagepos == cache->secrootblock)
21985 	ajFatal("NumbucketIdlist: cannot read bucket from root page cache %S",
21986                 cache->filename);
21987 
21988     page = btreeSeccacheRead(cache,pagepos);
21989     dirtysave = page->dirty;
21990     page->dirty = BT_LOCK;
21991     page->lockfor = 1591;
21992     buf = page->buf;
21993 
21994     GBT_BUCKNODETYPE(buf,&nodetype);
21995     if(nodetype != BT_NUMBUCKET && nodetype != BT_IDBUCKET)
21996 	ajFatal("NumbucketIdlist: Nodetype mismatch. "
21997                 "Not bucket (%u) cache %S",
21998                 nodetype, cache->filename);
21999 
22000     GBT_BUCKNENTRIES(buf,&nentries);
22001     if(nentries > cache->snperbucket)
22002 	ajFatal("NumbucketIdlist: Bucket too full page: %Lu "
22003                 "entries: %u max: %u cache %S",
22004                 pagepos, nentries, cache->snperbucket, cache->filename);
22005 
22006     GBT_BUCKOVERFLOW(buf,&overflow);
22007     pageoverflow = overflow;
22008 
22009     pptr = PBT_BUCKKEYLEN(buf);
22010 
22011     for(i=0;i<nentries;++i)
22012     {
22013         AJNEW0(num);
22014         if(cache->refcount)
22015         {
22016             AJCNEW0(num->refoffsets, cache->refcount);
22017             num->refcount = cache->refcount;
22018         }
22019 
22020 	BT_GETAJULONG(pptr,&num->offset);
22021 	pptr += sizeof(ajulong);
22022 
22023         if(cache->refcount)
22024         {
22025             for(iref=0; iref < cache->refcount; iref++)
22026             {
22027                 BT_GETAJULONG(pptr,&num->refoffsets[iref]);
22028                 pptr += sizeof(ajulong);
22029             }
22030         }
22031 
22032 	BT_GETAJUINT(pptr,&num->dbno);
22033 	pptr += sizeof(ajuint);
22034         ajListPushAppend(idlist, num);
22035         num = NULL;
22036     }
22037 
22038     page->dirty = dirtysave;
22039 
22040     return pageoverflow;
22041 }
22042 
22043 
22044 
22045 
22046 /* @funcstatic btreeNumbucketBtidlist *****************************************
22047 **
22048 ** Copies all numeric IDs into a list of btree Ids
22049 **
22050 ** @param [u] cache [AjPBtcache] cache
22051 ** @param [r] pagepos [ajulong] page number
22052 ** @param [r] idname [const AjPStr] id name
22053 ** @param [u] idlist [AjPList] list to hold numeric ID strings
22054 **
22055 ** @return [ajulong] Overflow
22056 **
22057 ** @release 6.4.0
22058 ** @@
22059 ******************************************************************************/
22060 
btreeNumbucketBtidlist(AjPBtcache cache,ajulong pagepos,const AjPStr idname,AjPList idlist)22061 static ajulong btreeNumbucketBtidlist(AjPBtcache cache, ajulong pagepos,
22062                                       const AjPStr idname, AjPList idlist)
22063 {
22064     AjPBtId btid      = NULL;
22065     AjPBtpage page      = NULL;
22066     unsigned char *buf  = NULL;
22067     unsigned char *pptr = NULL;
22068     ajuint  nodetype  = 0U;
22069     ajuint  nentries  = 0U;
22070     ajulong overflow  = 0UL;
22071     ajulong pageoverflow = 0UL;
22072     ajuint  dirtysave = 0U;
22073 
22074     ajuint  i;
22075     ajuint  iref;
22076 
22077     /* ajDebug("In btreeNumbucketBtidlist\n"); */
22078 
22079     if(pagepos == cache->secrootblock)
22080 	ajFatal("NumbucketBtidlist: cannot read bucket "
22081                 "from root page cache %S",
22082                 cache->filename);
22083 
22084     page = btreeSeccacheRead(cache,pagepos);
22085     dirtysave = page->dirty;
22086     page->dirty = BT_LOCK;
22087     page->lockfor = 1601;
22088     buf = page->buf;
22089 
22090     GBT_BUCKNODETYPE(buf,&nodetype);
22091     if(nodetype != BT_NUMBUCKET && nodetype != BT_IDBUCKET)
22092 	ajFatal("NumbucketBtidlist: Nodetype mismatch. "
22093                 "Not bucket (%u) cache %S",
22094                 nodetype, cache->filename);
22095 
22096     GBT_BUCKNENTRIES(buf,&nentries);
22097     if(nentries > cache->snperbucket)
22098 	ajFatal("NumbucketBtidlist: Bucket too full page: %Lu "
22099                 "entries: %u max: %u cache %S",
22100                 pagepos, nentries, cache->snperbucket,
22101                 cache->filename);
22102 
22103     GBT_BUCKOVERFLOW(buf,&overflow);
22104     pageoverflow = overflow;
22105 
22106     pptr = PBT_BUCKKEYLEN(buf);
22107 
22108     for(i=0;i<nentries;++i)
22109     {
22110         btid = ajBtreeIdNew(cache->refcount);
22111         ajStrAssignS(&btid->id,idname);
22112 	BT_GETAJULONG(pptr,&btid->offset);
22113 	pptr += sizeof(ajulong);
22114 
22115         if(cache->refcount)
22116         {
22117             for(iref=0; iref < cache->refcount; iref++)
22118             {
22119                 BT_GETAJULONG(pptr,&btid->refoffsets[iref]);
22120                 pptr += sizeof(ajulong);
22121             }
22122         }
22123 
22124 	BT_GETAJUINT(pptr,&btid->dbno);
22125 	pptr += sizeof(ajuint);
22126         ajListPushAppend(idlist, btid);
22127         btid = NULL;
22128     }
22129 
22130     page->dirty = dirtysave;
22131 
22132     return pageoverflow;
22133 }
22134 
22135 
22136 
22137 
22138 /* @funcstatic btreeNumbucketBthitlist ****************************************
22139 **
22140 ** Copies all numeric IDs into a list of btree Hits
22141 **
22142 ** @param [u] cache [AjPBtcache] cache
22143 ** @param [r] pagepos [ajulong] page number
22144 ** @param [u] hitlist [AjPList] list to hold numeric hits
22145 **
22146 ** @return [ajulong] Overflow
22147 **
22148 ** @release 6.5.0
22149 ** @@
22150 ******************************************************************************/
22151 
btreeNumbucketBthitlist(AjPBtcache cache,ajulong pagepos,AjPList hitlist)22152 static ajulong btreeNumbucketBthitlist(AjPBtcache cache, ajulong pagepos,
22153                                        AjPList hitlist)
22154 {
22155     AjPBtHit bthit      = NULL;
22156     AjPBtpage page      = NULL;
22157     unsigned char *buf  = NULL;
22158     unsigned char *pptr = NULL;
22159     ajuint  nodetype  = 0U;
22160     ajuint  nentries  = 0U;
22161     ajulong overflow  = 0UL;
22162     ajulong pageoverflow = 0UL;
22163     ajuint  dirtysave = 0U;
22164 
22165     ajuint  i;
22166 
22167     /* ajDebug("In btreeNumbucketBthitlist\n"); */
22168 
22169     if(cache->refcount)
22170         ajWarn("btreeNumbucketbthitlist called for cache '%S' "
22171                "with %u reference files",
22172                cache->filename, cache->refcount);
22173 
22174     if(pagepos == cache->secrootblock)
22175 	ajFatal("NumbucketBthitlist: cannot read bucket "
22176                 "from root page cache %S",
22177                 cache->filename);
22178 
22179     page = btreeSeccacheRead(cache,pagepos);
22180     dirtysave = page->dirty;
22181     page->dirty = BT_LOCK;
22182     page->lockfor = 1601;
22183     buf = page->buf;
22184 
22185     GBT_BUCKNODETYPE(buf,&nodetype);
22186     if(nodetype != BT_NUMBUCKET && nodetype != BT_IDBUCKET)
22187 	ajFatal("NumbucketBthitlist: Nodetype mismatch. "
22188                 "Not bucket (%u) cache %S",
22189                 nodetype, cache->filename);
22190 
22191     GBT_BUCKNENTRIES(buf,&nentries);
22192     if(nentries > cache->snperbucket)
22193 	ajFatal("NumbucketBthitlist: Bucket too full page: %Lu "
22194                 "entries: %u max: %u cache %S",
22195                 pagepos, nentries, cache->snperbucket,
22196                 cache->filename);
22197 
22198     GBT_BUCKOVERFLOW(buf,&overflow);
22199     pageoverflow = overflow;
22200 
22201     pptr = PBT_BUCKKEYLEN(buf);
22202 
22203     for(i=0;i<nentries;++i)
22204     {
22205         bthit = ajBtreeHitNew();
22206 	BT_GETAJULONG(pptr,&bthit->offset);
22207 	pptr += sizeof(ajulong);
22208 
22209         if(cache->refcount)
22210             pptr += cache->refcount*sizeof(ajulong);
22211 
22212 	BT_GETAJUINT(pptr,&bthit->dbno);
22213 	pptr += sizeof(ajuint);
22214         ajListPushAppend(hitlist, bthit);
22215         bthit = NULL;
22216     }
22217 
22218     page->dirty = dirtysave;
22219 
22220     return pageoverflow;
22221 }
22222 
22223 
22224 
22225 
22226 /* @funcstatic btreeNumbucketBthitreflist *************************************
22227 **
22228 ** Copies all numeric IDs into a list of btree Hitrefs
22229 **
22230 ** @param [u] cache [AjPBtcache] cache
22231 ** @param [r] pagepos [ajulong] page number
22232 ** @param [u] hitlist [AjPList] list to hold numeric hitrefs
22233 **
22234 ** @return [ajulong] Overflow
22235 **
22236 ** @release 6.5.0
22237 ** @@
22238 ******************************************************************************/
22239 
btreeNumbucketBthitreflist(AjPBtcache cache,ajulong pagepos,AjPList hitlist)22240 static ajulong btreeNumbucketBthitreflist(AjPBtcache cache, ajulong pagepos,
22241                                           AjPList hitlist)
22242 {
22243     AjPBtHitref bthitref = NULL;
22244     AjPBtpage page      = NULL;
22245     unsigned char *buf  = NULL;
22246     unsigned char *pptr = NULL;
22247     ajuint  nodetype  = 0U;
22248     ajuint  nentries  = 0U;
22249     ajulong overflow  = 0UL;
22250     ajulong pageoverflow = 0UL;
22251     ajuint  dirtysave = 0U;
22252 
22253     ajuint  i;
22254 
22255     /* ajDebug("In btreeNumbucketBthitreflist\n"); */
22256 
22257     if(cache->refcount != 1)
22258         ajWarn("btreeNumbucketbthitreflist called for cache '%S' "
22259                "with %u reference files",
22260                cache->filename, cache->refcount);
22261 
22262     if(pagepos == cache->secrootblock)
22263 	ajFatal("NumbucketBthitreflist: cannot read bucket "
22264                 "from root page cache %S",
22265                 cache->filename);
22266 
22267     page = btreeSeccacheRead(cache,pagepos);
22268     dirtysave = page->dirty;
22269     page->dirty = BT_LOCK;
22270     page->lockfor = 1601;
22271     buf = page->buf;
22272 
22273     GBT_BUCKNODETYPE(buf,&nodetype);
22274     if(nodetype != BT_NUMBUCKET && nodetype != BT_IDBUCKET)
22275 	ajFatal("NumbucketBthitreflist: Nodetype mismatch. "
22276                 "Not bucket (%u) cache %S",
22277                 nodetype, cache->filename);
22278 
22279     GBT_BUCKNENTRIES(buf,&nentries);
22280     if(nentries > cache->snperbucket)
22281 	ajFatal("NumbucketBthitreflist: Bucket too full page: %Lu "
22282                 "entries: %u max: %u cache %S",
22283                 pagepos, nentries, cache->snperbucket,
22284                 cache->filename);
22285 
22286     GBT_BUCKOVERFLOW(buf,&overflow);
22287     pageoverflow = overflow;
22288 
22289     pptr = PBT_BUCKKEYLEN(buf);
22290 
22291     for(i=0;i<nentries;++i)
22292     {
22293         bthitref = ajBtreeHitrefNew();
22294 	BT_GETAJULONG(pptr,&bthitref->offset);
22295 	pptr += sizeof(ajulong);
22296 
22297         if(cache->refcount)
22298         {
22299             BT_GETAJULONG(pptr,&bthitref->refoffset);
22300             pptr += cache->refcount*sizeof(ajulong);
22301         }
22302 
22303 	BT_GETAJUINT(pptr,&bthitref->dbno);
22304 	pptr += sizeof(ajuint);
22305         ajListPushAppend(hitlist, bthitref);
22306         bthitref = NULL;
22307     }
22308 
22309     page->dirty = dirtysave;
22310 
22311     return pageoverflow;
22312 }
22313 
22314 
22315 
22316 
22317 /* @funcstatic btreeReadNumbucket *********************************************
22318 **
22319 ** Constructor for index bucket given a disc page number
22320 ** Creates one empty key slot for possible addition
22321 **
22322 ** @param [u] cache [AjPBtcache] cache
22323 ** @param [r] pagepos [ajulong] page number
22324 **
22325 ** @return [AjPNumbucket] bucket
22326 **
22327 ** @release 4.0.0
22328 ** @@
22329 ******************************************************************************/
22330 
btreeReadNumbucket(AjPBtcache cache,ajulong pagepos)22331 static AjPNumbucket btreeReadNumbucket(AjPBtcache cache, ajulong pagepos)
22332 {
22333     AjPNumbucket bucket    = NULL;
22334     AjPBtpage page      = NULL;
22335     unsigned char *buf  = NULL;
22336     unsigned char *pptr = NULL;
22337     ajuint  nodetype  = 0U;
22338     ajuint nentries  = 0U;
22339     ajulong overflow  = 0UL;
22340     ajuint  dirtysave = 0U;
22341 
22342     ajuint  i;
22343     ajuint  iref;
22344 
22345     /* ajDebug("In btreeReadNumbucket\n"); */
22346 
22347     if(pagepos == cache->secrootblock)
22348 	ajFatal("ReadNumbucket: cannot read bucket from a root page");
22349 
22350     page = btreeSeccacheRead(cache,pagepos);
22351     dirtysave = page->dirty;
22352     page->dirty = BT_LOCK;
22353     page->lockfor = 1611;
22354     buf = page->buf;
22355 
22356     GBT_BUCKNODETYPE(buf,&nodetype);
22357 
22358     if(nodetype != BT_NUMBUCKET && nodetype !=  BT_IDBUCKET)
22359 	ajFatal("ReadNumbucket: Nodetype mismatch. Not bucket (%u) cache %S",
22360                 nodetype, cache->filename);
22361 
22362     GBT_BUCKNENTRIES(buf,&nentries);
22363 
22364     if(nentries > cache->snperbucket)
22365 	ajFatal("ReadNumbucket: Bucket too full page: %Lu "
22366                 "entries: %u max: %u cache %S",
22367                 pagepos, nentries, cache->snperbucket,
22368                 cache->filename);
22369 
22370     GBT_BUCKOVERFLOW(buf,&overflow);
22371 
22372     bucket = btreeNumbucketNew(cache->snperbucket, cache->refcount);
22373     bucket->Nentries = nentries;
22374 
22375     pptr = PBT_BUCKKEYLEN(buf);
22376 
22377     for(i=0;i<nentries;++i)
22378     {
22379 	BT_GETAJULONG(pptr,&bucket->NumId[i]->offset);
22380 	pptr += sizeof(ajulong);
22381 
22382         if(cache->refcount)
22383         {
22384             for(iref=0; iref < cache->refcount; iref++)
22385             {
22386                 BT_GETAJULONG(pptr,&bucket->NumId[i]->refoffsets[iref]);
22387                 pptr += sizeof(ajulong);
22388             }
22389         }
22390 
22391 	BT_GETAJUINT(pptr,&bucket->NumId[i]->dbno);
22392 	pptr += sizeof(ajuint);
22393     }
22394 
22395     page->dirty = dirtysave;
22396 
22397     return bucket;
22398 }
22399 
22400 
22401 
22402 
22403 /* @funcstatic btreeNumbucketDel **********************************************
22404 **
22405 ** Delete a bucket object
22406 **
22407 ** @param [w] thys [AjPNumbucket*] bucket
22408 **
22409 ** @return [void]
22410 **
22411 ** @release 4.0.0
22412 ** @@
22413 ******************************************************************************/
22414 
btreeNumbucketDel(AjPNumbucket * thys)22415 static void btreeNumbucketDel(AjPNumbucket *thys)
22416 {
22417     AjPNumbucket pthis = NULL;
22418     ajuint newmax;
22419 
22420     /* ajDebug("In btreeNumbucketDel\n"); */
22421 
22422     if(!thys || !*thys)
22423 	return;
22424 
22425     pthis = *thys;
22426 
22427     if(!statSaveNumbucket)
22428     {
22429         statSaveNumbucketMax=2048;
22430         statSaveNumbucketNext=0;
22431         AJCNEW0(statSaveNumbucket,statSaveNumbucketMax);
22432     }
22433 
22434     if(statSaveNumbucketNext >= statSaveNumbucketMax)
22435     {
22436         newmax = statSaveNumbucketMax + statSaveNumbucketMax;
22437         AJCRESIZE0(statSaveNumbucket,statSaveNumbucketMax,newmax);
22438         statSaveNumbucketMax = newmax;
22439     }
22440 
22441     statSaveNumbucket[statSaveNumbucketNext++] = pthis;
22442 
22443     *thys = NULL;
22444 
22445     return;
22446 }
22447 
22448 
22449 
22450 
22451 /* @funcstatic btreeNumbucketFree *********************************************
22452 **
22453 ** Delete a bucket object
22454 **
22455 ** @param [w] thys [AjPNumbucket*] bucket
22456 **
22457 ** @return [void]
22458 **
22459 ** @release 6.4.0
22460 ** @@
22461 ******************************************************************************/
22462 
btreeNumbucketFree(AjPNumbucket * thys)22463 static void btreeNumbucketFree(AjPNumbucket *thys)
22464 {
22465     AjPNumbucket pthis = NULL;
22466     ajuint i;
22467 
22468 
22469     /* ajDebug("In btreeNumbucketFree\n"); */
22470 
22471     if(!thys || !*thys)
22472 	return;
22473 
22474     pthis = *thys;
22475 
22476     if(pthis->NumId)
22477     {
22478 	for(i=0;i<pthis->Maxentries;++i)
22479 	    AJFREE(pthis->NumId[i]);
22480 	AJFREE(pthis->NumId);
22481     }
22482 
22483     AJFREE(pthis);
22484 
22485     *thys = NULL;
22486 
22487     return;
22488 }
22489 
22490 
22491 
22492 
22493 /* @funcstatic btreeNumFind ***************************************************
22494 **
22495 ** Find the node that should contain a new key for insertion
22496 **
22497 ** @param [u] cache [AjPBtcache] cache
22498 ** @param [r] key [const ajulong] key to search for
22499 **
22500 ** @return [AjPBtpage] leaf node where item should be inserted
22501 **
22502 ** @release 6.5.0
22503 ** @@
22504 ******************************************************************************/
22505 
btreeNumFind(AjPBtcache cache,const ajulong key)22506 static AjPBtpage btreeNumFind(AjPBtcache cache, const ajulong key)
22507 {
22508     AjPBtpage root = NULL;
22509     AjPBtpage ret  = NULL;
22510 
22511     /* ajDebug("In btreeNumFind\n"); */
22512 
22513     /* The root node should always be in the cache (BT_LOCKed) */
22514     root = btreeSeccacheLocate(cache,cache->secrootblock);
22515 
22516     /* ajDebug("cache->slevel = %u root=%u\n",cache->slevel,(ajuint)root); */
22517 
22518 
22519     if(!cache->slevel)
22520 	return root;
22521 
22522     ret = btreeNumFindINode(cache,root,key);
22523 
22524     return ret;
22525 }
22526 
22527 
22528 
22529 
22530 /* @funcstatic btreeNumFindINode **********************************************
22531 **
22532 ** Recursive search for insert node in a secondary tree
22533 **
22534 ** @param [u] cache [AjPBtcache] cache
22535 ** @param [u] page [AjPBtpage] page
22536 ** @param [r] item [ajulong] key to search for
22537 **
22538 ** @return [AjPBtpage] leaf node where item should be inserted
22539 **
22540 ** @release 4.0.0
22541 ** @@
22542 ******************************************************************************/
22543 
btreeNumFindINode(AjPBtcache cache,AjPBtpage page,ajulong item)22544 static AjPBtpage btreeNumFindINode(AjPBtcache cache, AjPBtpage page,
22545 				   ajulong item)
22546 {
22547     AjPBtpage ret = NULL;
22548     AjPBtpage pg  = NULL;
22549 
22550     unsigned char *buf = NULL;
22551     ajuint status       = 0;
22552     ajuint ival         = 0;
22553 
22554     /* ajDebug("In btreeNumFindINode\n"); */
22555 
22556     ret = page;
22557     buf = page->buf;
22558     GBT_NODETYPE(buf,&ival);
22559 
22560     if(ival != BT_SECLEAF)
22561     {
22562 	status = ret->dirty;
22563 	ret->dirty = BT_LOCK;	/* Lock in case of lots of overflow pages */
22564         ret->lockfor = 1621;
22565 	pg = btreeNumPageFromKey(cache,buf,item);
22566 	ret->dirty = status;
22567 	ret = btreeNumFindINode(cache,pg,item);
22568     }
22569 
22570     return ret;
22571 }
22572 
22573 
22574 
22575 
22576 /* @funcstatic btreeNumPageFromKey ********************************************
22577 **
22578 ** Return next lower index page given a key in a numeric secondary tree
22579 **
22580 ** @param [u] cache [AjPBtcache] cache
22581 ** @param [u] buf [unsigned char *] page buffer
22582 ** @param [r] key [ajulong] key to search for
22583 **
22584 ** @return [AjPBtpage] pointer to a page
22585 **
22586 ** @release 4.0.0
22587 ** @@
22588 ******************************************************************************/
22589 
btreeNumPageFromKey(AjPBtcache cache,unsigned char * buf,ajulong key)22590 static AjPBtpage btreeNumPageFromKey(AjPBtcache cache, unsigned char *buf,
22591 				     ajulong key)
22592 {
22593     unsigned char *rootbuf = NULL;
22594     ajuint nkeys = 0U;
22595     ajuint i;
22596 
22597     ajulong blockno = 0UL;
22598     ajulong *karray = NULL;
22599     ajulong *parray = NULL;
22600     AjPBtpage page = NULL;
22601     AjPBtMem array = NULL;
22602 
22603     /* ajDebug("In btreeNumPageFromKey\n"); */
22604 
22605     rootbuf = buf;
22606 
22607 
22608     GBT_NKEYS(rootbuf,&nkeys);
22609 
22610     array = btreeAllocSecArray(cache);
22611     karray = array->overflows;
22612     parray = array->parray;
22613 
22614     btreeGetNumKeys(cache,rootbuf,&karray,&parray);
22615 
22616     i = 0;
22617 
22618     while(i!=nkeys && key >= karray[i])
22619 	++i;
22620 
22621     if(i==nkeys)
22622     {
22623 	if(key < karray[i-1])
22624 	    blockno = parray[i-1];
22625 	else
22626 	    blockno = parray[i];
22627     }
22628     else
22629 	blockno = parray[i];
22630 
22631     btreeDeallocSecArray(cache,array);
22632 
22633     page =  btreeSeccacheRead(cache,blockno);
22634 
22635     return page;
22636 }
22637 
22638 
22639 
22640 
22641 /* @funcstatic btreeNumbucketAdd **********************************************
22642 **
22643 ** Add offset info to a numeric bucket
22644 ** Only called if there is room in the bucket
22645 **
22646 ** @param [u] cache [AjPBtcache] cache
22647 ** @param [r] pagepos [ajulong] page number of bucket
22648 ** @param [r] num [const AjPBtNumId] ID info
22649 **
22650 ** @return [void]
22651 **
22652 ** @release 4.0.0
22653 ** @@
22654 ******************************************************************************/
22655 
btreeNumbucketAdd(AjPBtcache cache,ajulong pagepos,const AjPBtNumId num)22656 static void btreeNumbucketAdd(AjPBtcache cache, ajulong pagepos,
22657                               const AjPBtNumId num)
22658 {
22659     AjPBtpage page      = NULL;
22660     unsigned char *buf  = NULL;
22661     unsigned char *pptr = NULL;
22662     ajuint  nodetype  = 0U;
22663     ajuint  nentries  = 0U;
22664     ajulong overflow  = 0UL;
22665 
22666     ajulong lv = 0UL;
22667     ajuint uv = 0U;
22668 
22669     ajuint  i;
22670     ajuint  iref;
22671 
22672     /* ajDebug("In btreeReadNumbucket\n"); */
22673 
22674     if(pagepos == cache->secrootblock)
22675 	ajFatal("NumbucketAdd: cannot read bucket from root page cache %S",
22676                 cache->filename);
22677 
22678     page = btreeSeccacheRead(cache,pagepos);
22679 
22680     page->dirty = BT_LOCK;      /* reset at end */
22681     page->lockfor = 1631;
22682     buf = page->buf;
22683 
22684     GBT_BUCKNODETYPE(buf,&nodetype);
22685     if(nodetype != BT_NUMBUCKET)
22686 	ajFatal("NumbucketAdd: Nodetype mismatch. "
22687                 "Not bucket (%u) cache %S",
22688                 nodetype, cache->filename);
22689 
22690     GBT_BUCKNENTRIES(buf,&nentries);
22691     if(nentries > cache->snperbucket)
22692 	ajFatal("NumbucketAdd: Bucket too full page: %Lu "
22693                 "entries: %u max: %u cache %S",
22694                 pagepos, nentries, cache->snperbucket, cache->filename);
22695 
22696     GBT_BUCKOVERFLOW(buf,&overflow);
22697 
22698     pptr = PBT_BUCKKEYLEN(buf);
22699 
22700     for(i=0;i<nentries;++i)
22701     {
22702         pptr += sizeof(ajulong);
22703         if(cache->refcount)
22704         {
22705             for(iref=0; iref < cache->refcount; iref++)
22706                 pptr += sizeof(ajulong);
22707         }
22708 	pptr += sizeof(ajuint);
22709     }
22710 
22711     lv = num->offset;
22712     BT_SETAJULONG(pptr,lv);
22713     pptr += sizeof(ajulong);
22714 
22715     if(cache->refcount)
22716     {
22717         for(iref=0; iref < cache->refcount; iref++)
22718         {
22719             lv = num->refoffsets[iref];
22720             BT_SETAJULONG(pptr,lv);
22721             pptr += sizeof(ajulong);
22722         }
22723     }
22724 
22725     uv = num->dbno;
22726     BT_SETAJUINT(pptr,uv);
22727     pptr += sizeof(ajuint);
22728 
22729     nentries++;
22730     SBT_BUCKNENTRIES(buf,nentries);
22731 
22732     page->dirty = BT_DIRTY;
22733 
22734     return;
22735 }
22736 
22737 
22738 
22739 
22740 
22741 /* @funcstatic btreeNumInNumbucket ********************************************
22742 **
22743 ** Return number of entries in a bucket
22744 **
22745 ** @param [u] cache [AjPBtcache] cache
22746 ** @param [r] pagepos [ajulong] page number
22747 **
22748 ** @return [ajuint] Number of entries in bucket
22749 **
22750 ** @release 4.0.0
22751 ** @@
22752 ******************************************************************************/
22753 
btreeNumInNumbucket(AjPBtcache cache,ajulong pagepos)22754 static ajuint btreeNumInNumbucket(AjPBtcache cache, ajulong pagepos)
22755 {
22756     AjPBtpage page     = NULL;
22757     unsigned char *buf = NULL;
22758     ajuint  nodetype    = 0;
22759     ajuint nentries    = 0;
22760 
22761     /* ajDebug("In btreeNumInNumbucket\n"); */
22762 
22763     if(pagepos == cache->secrootblock)
22764 	ajFatal("NumInNumbucket: Attempt to read bucket from root page\n");
22765 
22766     page  = btreeSeccacheRead(cache,pagepos);
22767 
22768     buf = page->buf;
22769 
22770     GBT_BUCKNODETYPE(buf,&nodetype);
22771 
22772     if(nodetype != BT_NUMBUCKET)
22773 	ajFatal("NumInNumbucket: NodeType mismatch. Not numbucket (%u) "
22774                 "for cache '%S'",
22775 		nodetype, cache->filename);
22776 
22777     GBT_BUCKNENTRIES(buf,&nentries);
22778 
22779     return nentries;
22780 }
22781 
22782 
22783 
22784 
22785 /* @funcstatic btreeNumbucketNew **********************************************
22786 **
22787 ** Construct a bucket object
22788 **
22789 ** @param [r] n [ajuint] Number of IDs
22790 ** @param [r] refcount [ajuint] Number of reference files per entry
22791 **
22792 ** @return [AjPNumbucket] initialised disc block cache structure
22793 **
22794 ** @release 4.0.0
22795 ** @@
22796 ******************************************************************************/
22797 
btreeNumbucketNew(ajuint n,ajuint refcount)22798 static AjPNumbucket btreeNumbucketNew(ajuint n, ajuint refcount)
22799 {
22800     AjPNumbucket bucket = NULL;
22801     ajuint i;
22802     ajuint iref;
22803 
22804     /* ajDebug("In btreeNumbucketNew\n"); */
22805     if(statSaveNumbucketNext)
22806     {
22807        bucket = statSaveNumbucket[--statSaveNumbucketNext];
22808        for(i=0;i<bucket->Maxentries;++i)
22809        {
22810            bucket->NumId[i]->offset = 0UL;
22811            bucket->NumId[i]->dbno = 0U;
22812            if(refcount != bucket->NumId[i]->refcount)
22813            {
22814                if(refcount)
22815                    AJCRESIZE0(bucket->NumId[i]->refoffsets,
22816                              bucket->NumId[i]->refcount, refcount);
22817                else
22818                    AJFREE(bucket->NumId[i]->refoffsets);
22819 
22820                bucket->NumId[i]->refcount = refcount;
22821            }
22822            else if(refcount)
22823            {
22824                for(iref=0; iref < refcount; iref++)
22825                    bucket->NumId[i]->refoffsets[iref] = 0UL;
22826            }
22827        }
22828        if(n > bucket->Maxentries)
22829        {
22830            if(bucket->Maxentries)
22831                AJCRESIZE0(bucket->NumId,bucket->Maxentries+1,n+1);
22832            else
22833                AJCNEW0(bucket->NumId,n+1);
22834 
22835            for(i=bucket->Maxentries;i<n;++i)
22836            {
22837                AJNEW0(bucket->NumId[i]);
22838                if(refcount)
22839                    AJCNEW0(bucket->NumId[i]->refoffsets, refcount);
22840            }
22841 
22842            bucket->Maxentries = n;
22843        }
22844    }
22845    else
22846    {
22847        AJNEW0(bucket);
22848        AJCNEW0(bucket->NumId,n+1);
22849 
22850        for(i=0U;i<n;++i)
22851        {
22852            AJNEW0(bucket->NumId[i]);
22853            if(refcount)
22854                AJCNEW0(bucket->NumId[i]->refoffsets, refcount);
22855        }
22856 
22857        bucket->Maxentries = n;
22858    }
22859 
22860     bucket->NodeType = BT_NUMBUCKET;
22861     bucket->Nentries = n;
22862     bucket->Overflow = 0UL;
22863 
22864     return bucket;
22865 }
22866 
22867 
22868 
22869 
22870 /* @funcstatic btreeReorderNumbuckets *****************************************
22871 **
22872 ** Re-order leaf buckets
22873 ** Must only be called if one of the buckets is full
22874 **
22875 ** @param [u] cache [AjPBtcache] cache
22876 ** @param [u] leaf [AjPBtpage] leaf page
22877 **
22878 ** @return [AjBool] true if reorder was successful i.e. leaf not full
22879 **
22880 ** @release 4.0.0
22881 ** @@
22882 ******************************************************************************/
22883 
btreeReorderNumbuckets(AjPBtcache cache,AjPBtpage leaf)22884 static AjBool btreeReorderNumbuckets(AjPBtcache cache, AjPBtpage leaf)
22885 {
22886     ajuint nkeys = 0;
22887     unsigned char *lbuf = NULL;
22888 
22889     AjPBtMem array = NULL;
22890     AjPBtMem newarray = NULL;
22891     ajulong *ptrs        = NULL;
22892     ajulong *newkeys        = NULL;
22893     ajulong *newptrs        = NULL;
22894 
22895     ajuint i = 0;
22896     ajuint iref = 0;
22897 
22898     ajuint order;
22899     ajuint totalkeys     = 0;
22900     ajuint nperbucket    = 0;
22901     ajuint maxnperbucket = 0;
22902     ajuint count         = 0;
22903     ajuint keylimit      = 0;
22904     ajuint bucketlimit   = 0;
22905     ajuint nodetype      = 0;
22906 
22907     AjPList idlist    = NULL;
22908     ajuint   dirtysave = 0;
22909     AjPBtNumId bid       = NULL;
22910     AjPNumbucket cbucket = NULL;
22911     AjPBtNumId cid       = NULL;
22912 
22913     ajuint   iold = 0;
22914 
22915 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
22916     ajDebug("btreeReorderNumbuckets '%S' %Lu id '%S' key '%S'\n",
22917             cache->basename, leaf->pagepos, indexId, indexKeyword);
22918 #endif
22919     ++statCallNumbucketsReorder;
22920 
22921     /* ajDebug("In btreeReorderNumbuckets\n"); */
22922 
22923     dirtysave = leaf->dirty;
22924 
22925     leaf->dirty = BT_LOCK;
22926     leaf->lockfor = 1641;
22927     lbuf = leaf->buf;
22928 
22929     GBT_NODETYPE(lbuf,&nodetype);
22930 
22931     order = cache->sorder;
22932     nperbucket = cache->snperbucket;
22933 
22934 
22935     array = btreeAllocSecArray(cache);
22936     ptrs  = array->parray;
22937 
22938     /* Read keys/ptrs */
22939 
22940     btreeGetNumPointers(cache,lbuf,&ptrs);
22941 
22942     GBT_NKEYS(lbuf,&nkeys);
22943     keylimit = nkeys + 1;
22944 
22945 
22946     if(!nkeys)
22947 	ajFatal("ReorderNumbuckets: Attempt to reorder empty leaf");
22948 
22949     for(i=0;i<nkeys;++i)
22950 	totalkeys += btreeNumInNumbucket(cache,ptrs[i]);
22951 
22952     totalkeys += btreeNumInNumbucket(cache,ptrs[i]);
22953 
22954     btreeBucketCalc(totalkeys, keylimit, nperbucket,
22955                     &bucketlimit, &maxnperbucket);
22956 
22957 #if AJINDEX_DEBUG
22958     ajDebug("   bucketlimit:%u order: %u\n", bucketlimit, order);
22959 #endif
22960     if(bucketlimit >= order)
22961     {
22962 	btreeDeallocSecArray(cache,array);
22963 
22964 	leaf->dirty = dirtysave;
22965 
22966 #if AJINDEX_DEBUG
22967     ajDebug("   return ajFalse\n");
22968 #endif
22969 	return ajFalse;
22970     }
22971 
22972 
22973     newarray = btreeAllocSecArray(cache);
22974     newkeys  = newarray->overflows;
22975     newptrs  = newarray->parray;
22976 
22977     /* Read buckets */
22978 
22979     idlist  = ajListNew();
22980 
22981     /* Read IDs from all buckets and push to list and sort (increasing id) */
22982     for(i=0;i<keylimit;++i)
22983 	btreeNumbucketIdlist(cache,ptrs[i],idlist);
22984 
22985     ajListSort(idlist, &btreeNumIdCompare);
22986 
22987     cbucket = btreeNumbucketNew(maxnperbucket, cache->refcount);
22988     iold = 0;
22989     for(i=0;i<bucketlimit;++i)
22990     {
22991 	cbucket->Nentries = 0;
22992 
22993 	count = 0;
22994 
22995 	while(count!=maxnperbucket)
22996 	{
22997 	    ajListPop(idlist,(void **)&bid);
22998 
22999 	    cid = cbucket->NumId[count];
23000 	    cid->dbno = bid->dbno;
23001 	    cid->offset = bid->offset;
23002 
23003             if(cache->refcount)
23004             {
23005                 for(iref=0; iref < cache->refcount; iref++)
23006                     cid->refoffsets[iref] = bid->refoffsets[iref];
23007 	    }
23008 
23009 	    ++cbucket->Nentries;
23010 	    ++count;
23011 	    AJFREE(bid);
23012 	}
23013 
23014 
23015 	ajListPeek(idlist,(void **)&bid);
23016 	newkeys[i] = bid->offset;
23017 
23018 	if((iold < order) && ptrs[iold])
23019             newptrs[i] = ptrs[iold++];
23020         else
23021 	    newptrs[i] = cache->totsize;
23022 	btreeWriteNumbucket(cache,cbucket,newptrs[i]);
23023     }
23024 
23025 
23026     /* Deal with greater-than bucket */
23027 
23028     cbucket->Nentries = 0;
23029 
23030     count = 0;
23031 
23032     while(ajListPop(idlist,(void **)&bid))
23033     {
23034 	cid = cbucket->NumId[count];
23035 	cid->dbno = bid->dbno;
23036 	cid->offset = bid->offset;
23037 
23038         if(cache->refcount)
23039         {
23040             for(iref=0; iref < cache->refcount; iref++)
23041                 cid->refoffsets[iref] = bid->refoffsets[iref];
23042 	}
23043 
23044 	++cbucket->Nentries;
23045 	++count;
23046 	AJFREE(bid);
23047     }
23048 
23049     if((iold < order) && ptrs[iold])
23050         newptrs[i] = ptrs[iold++];
23051     else
23052         newptrs[i] = cache->totsize;
23053     btreeWriteNumbucket(cache,cbucket,newptrs[i]);
23054 
23055     btreeNumbucketDel(&cbucket);
23056 
23057 #if AJINDEX_DEBUG
23058     if(iold < keylimit)
23059         ajDebug("btreeReorderNumbuckets '%S' %u -> %u",
23060                 cache->filename, keylimit, iold);
23061 #endif
23062 
23063     for(i = iold+1; i <= keylimit; i++)
23064     {
23065         btreeSecpageSetfree(cache, ptrs[i]);
23066     }
23067 
23068     /* Now write out a modified leaf with new keys/ptrs */
23069 
23070     nkeys = bucketlimit;
23071 
23072     btreeWriteNumNode(cache,leaf,newkeys,newptrs,nkeys);
23073 
23074     leaf->dirty = BT_DIRTY;
23075 
23076     if(nodetype == BT_SECROOT)
23077     {
23078 	leaf->dirty = BT_LOCK;
23079         leaf->lockfor = 1642;
23080     }
23081 
23082     btreeDeallocSecArray(cache,array);
23083     btreeDeallocSecArray(cache,newarray);
23084 
23085     ajListFree(&idlist);
23086 
23087 #if AJINDEX_DEBUG
23088     ajDebug("   return ajTrue\n");
23089 #endif
23090 
23091     return ajTrue;
23092 }
23093 
23094 
23095 
23096 
23097 /* @funcstatic btreeNumNodeIsFull *********************************************
23098 **
23099 ** Tests whether a node is full of keys
23100 **
23101 ** @param [r] cache [const AjPBtcache] cache
23102 ** @param [u] page [AjPBtpage] original page
23103 **
23104 ** @return [AjBool] true if full
23105 **
23106 ** @release 4.0.0
23107 ** @@
23108 ******************************************************************************/
23109 
btreeNumNodeIsFull(const AjPBtcache cache,AjPBtpage page)23110 static AjBool btreeNumNodeIsFull(const AjPBtcache cache, AjPBtpage page)
23111 {
23112     unsigned char *buf = NULL;
23113     ajuint nkeys = 0;
23114 
23115     /* ajDebug("In btreeNumNodeIsFull\n"); */
23116 
23117     buf = page->buf;
23118     GBT_NKEYS(buf,&nkeys);
23119 
23120     if(nkeys == cache->sorder - 1)
23121 	return ajTrue;
23122 
23123     return ajFalse;
23124 }
23125 
23126 
23127 
23128 
23129 /* @funcstatic btreeNumInsertNonfull ******************************************
23130 **
23131 ** Insert a key into a non-full numeric secondary node
23132 **
23133 ** @param [u] cache [AjPBtcache] cache
23134 ** @param [u] page [AjPBtpage] original page
23135 ** @param [r] key [ajulong] key to insert
23136 ** @param [r] less [ajulong] less-than pointer
23137 ** @param [r] greater [ajulong] greater-than pointer
23138 **
23139 ** @return [void]
23140 **
23141 ** @release 4.0.0
23142 ** @@
23143 ******************************************************************************/
23144 
btreeNumInsertNonfull(AjPBtcache cache,AjPBtpage page,ajulong key,ajulong less,ajulong greater)23145 static void btreeNumInsertNonfull(AjPBtcache cache, AjPBtpage page,
23146 				  ajulong key, ajulong less,
23147 				  ajulong greater)
23148 {
23149     unsigned char *buf = NULL;
23150     ajulong *karray     = NULL;
23151     ajulong *parray     = NULL;
23152     ajuint nkeys  = 0U;
23153     ajuint ipos   = 0U;
23154     ajuint i;
23155     ajuint count  = 0U;
23156 
23157     ajulong lv = 0UL;
23158     ajuint  v  = 0U;
23159     AjPBtMem array = NULL;
23160 
23161     AjPBtpage ppage = NULL;
23162     ajulong pagepos   = 0UL;
23163 
23164     ajuint nodetype = 0U;
23165 
23166     /* ajDebug("In btreeNumInsertNonfull\n"); */
23167 
23168     array = btreeAllocSecArray(cache);
23169     karray  = array->overflows;
23170     parray  = array->parray;
23171 
23172 
23173     buf = page->buf;
23174     GBT_NKEYS(buf,&nkeys);
23175     GBT_NODETYPE(buf,&nodetype);
23176 
23177     btreeGetNumKeys(cache,buf,&karray,&parray);
23178 
23179     i = 0;
23180 
23181     while(i!=nkeys && key >= karray[i])
23182 	++i;
23183 
23184     ipos = i;
23185 
23186     count = nkeys - ipos;
23187 
23188 
23189     if(ipos == nkeys)
23190     {
23191 	karray[ipos] = key;
23192 	parray[ipos+1] = greater;
23193 	parray[ipos]   = less;
23194     }
23195     else
23196     {
23197 	parray[nkeys+1] = parray[nkeys];
23198 
23199 	for(i=nkeys-1; count>0; --count, --i)
23200 	{
23201 	    karray[i+1] = karray[i];
23202 	    parray[i+1] = parray[i];
23203 	}
23204 
23205 	karray[ipos] = key;
23206 	parray[ipos] = less;
23207 	parray[ipos+1] = greater;
23208     }
23209 
23210     ++nkeys;
23211     v = nkeys;
23212     SBT_NKEYS(buf,v);
23213 
23214     btreeWriteNumNode(cache,page,karray,parray,nkeys);
23215     if(nodetype == BT_SECROOT)
23216     {
23217 	page->dirty = BT_LOCK;
23218         page->lockfor = 1651;
23219     }
23220 
23221     pagepos = page->pagepos;
23222     ppage = btreeSeccacheRead(cache,less);
23223     lv = pagepos;
23224     SBT_PREV(ppage->buf,lv);
23225     ppage->dirty = BT_DIRTY;
23226 
23227     ppage = btreeSeccacheRead(cache,greater);
23228     lv = pagepos;
23229     SBT_PREV(ppage->buf,lv);
23230     ppage->dirty = BT_DIRTY;
23231 
23232     btreeDeallocSecArray(cache,array);
23233 
23234     if(nodetype != BT_SECROOT)
23235 	btreeNumKeyShift(cache,page);
23236 
23237     return;
23238 }
23239 
23240 
23241 
23242 
23243 /* @funcstatic btreeNumInsertKey **********************************************
23244 **
23245 ** Insert a key into a potentially full node
23246 **
23247 ** @param [u] cache [AjPBtcache] cache
23248 ** @param [u] page [AjPBtpage] original page
23249 ** @param [r] key [ajulong] key to insert
23250 ** @param [r] less [ajulong] less-than pointer
23251 ** @param [r] greater [ajulong] greater-than pointer
23252 **
23253 ** @return [void]
23254 **
23255 ** @release 4.0.0
23256 ** @@
23257 ******************************************************************************/
23258 
btreeNumInsertKey(AjPBtcache cache,AjPBtpage page,ajulong key,ajulong less,ajulong greater)23259 static void btreeNumInsertKey(AjPBtcache cache, AjPBtpage page,
23260 			      ajulong key, ajulong less, ajulong greater)
23261 {
23262     unsigned char *lbuf = NULL;
23263     unsigned char *rbuf = NULL;
23264     unsigned char *tbuf = NULL;
23265     ajulong *narray      = NULL;
23266     ajulong *parray      = NULL;
23267     ajulong *tnarray     = NULL;
23268     ajulong *tparray     = NULL;
23269     ajuint nkeys    = 0U;
23270     ajuint order    = 0U;
23271     ajuint keypos   = 0U;
23272     ajuint rkeyno   = 0U;
23273 
23274     ajuint i = 0U;
23275     ajuint n = 0U;
23276 
23277     ajuint nodetype  = 0U;
23278     AjPBtpage ipage = NULL;
23279     AjPBtpage lpage = NULL;
23280     AjPBtpage rpage = NULL;
23281     AjPBtpage tpage = NULL;
23282 
23283     ajulong blockno  = 0UL;
23284     ajulong rblockno = 0UL;
23285     ajulong lblockno = 0UL;
23286     ajulong ibn      = 0UL;
23287 
23288     ajulong mediankey  = 0UL;
23289     ajulong medianless = 0UL;
23290     ajulong mediangtr  = 0UL;
23291     ajulong overflow   = 0UL;
23292     ajulong prev       = 0UL;
23293 
23294     ajulong lv = 0UL;
23295     ajuint  v  = 0U;
23296     AjPBtMem savekeyarray  = NULL;
23297     AjPBtMem array2 = NULL;
23298 
23299     /* ajDebug("In btreeNumInsertKey\n"); */
23300 
23301     if(!btreeNumNodeIsFull(cache,page))
23302     {
23303 	btreeNumInsertNonfull(cache,page,key,less,greater);
23304 	return;
23305     }
23306 
23307     order = cache->sorder;
23308     lbuf = page->buf;
23309     GBT_NODETYPE(lbuf,&nodetype);
23310     page->dirty = BT_LOCK;
23311     page->lockfor = 1661;
23312 
23313     if(nodetype == BT_SECROOT)
23314     {
23315 	btreeNumSplitroot(cache);
23316 
23317 	if(page->pagepos)
23318 	    page->dirty = BT_DIRTY;
23319 
23320         blockno = btreeGetBlockFirstN(cache,lbuf,key);
23321 
23322 	ipage = btreeSeccacheRead(cache,blockno);
23323 	btreeNumInsertNonfull(cache,ipage,key,less,greater);
23324 
23325 	return;
23326     }
23327 
23328 
23329     savekeyarray = btreeAllocSecArray(cache);
23330     narray  = savekeyarray->overflows;
23331     parray  = savekeyarray->parray;
23332 
23333     array2 = btreeAllocSecArray(cache);
23334     tnarray  = array2->overflows;
23335     tparray  = array2->parray;
23336 
23337 
23338     lpage = page;
23339     lbuf = lpage->buf;
23340 
23341     btreeGetNumKeys(cache,lbuf,&narray,&parray);
23342 
23343     GBT_BLOCKNUMBER(lbuf,&lblockno);
23344 
23345     rblockno = cache->totsize;
23346     rpage = btreeSeccacheNodenew(cache);
23347     rpage->dirty = BT_LOCK;
23348     rpage->lockfor = 1662;
23349     rbuf = rpage->buf;
23350 
23351 
23352     GBT_PREV(lbuf,&prev);
23353     lv = prev;
23354     SBT_PREV(rbuf,lv);
23355 
23356     nkeys = order - 1;
23357     keypos = nkeys / 2;
23358 
23359     if(!(nkeys % 2))
23360 	--keypos;
23361 
23362     mediankey = narray[keypos];
23363     medianless = lblockno;
23364     mediangtr  = rblockno;
23365 
23366 
23367     GBT_NODETYPE(lbuf,&nodetype);
23368     v = nodetype;
23369     SBT_NODETYPE(rbuf,v);
23370     lv = overflow;
23371     SBT_OVERFLOW(rbuf,lv);
23372 
23373 
23374     for(i=0;i<keypos;++i)
23375     {
23376 	tnarray[i] = narray[i];
23377 	tparray[i] = parray[i];
23378     }
23379 
23380     tparray[i] = parray[i];
23381 
23382     btreeWriteNumNode(cache,lpage,tnarray,tparray,i);
23383     lpage->dirty = BT_LOCK;
23384     lpage->lockfor = 1663;
23385 
23386 
23387     for(i=0;i<n+1;++i)
23388     {
23389 	tpage = btreeSeccacheRead(cache,tparray[i]);
23390 	tbuf = tpage->buf;
23391 	lv = lblockno;
23392 	SBT_PREV(tbuf,lv);
23393 	tpage->dirty = BT_DIRTY;
23394     }
23395 
23396 
23397     for(i=keypos+1;i<nkeys;++i)
23398     {
23399 	tnarray[i-(keypos+1)] = narray[i];
23400 	tparray[i-(keypos+1)] = parray[i];
23401     }
23402 
23403     tparray[i-(keypos+1)] = parray[i];
23404 
23405     rkeyno = (nkeys-keypos) - 1;
23406     v = rkeyno;
23407     SBT_NKEYS(rbuf,v);
23408     rpage->dirty = BT_DIRTY;
23409 
23410     btreeWriteNumNode(cache,rpage,tnarray,tparray,rkeyno);
23411     rpage->dirty = BT_LOCK;
23412     rpage->lockfor = 1664;
23413 
23414     for(i=0;i<rkeyno+1;++i)
23415     {
23416 	tpage = btreeSeccacheRead(cache,tparray[i]);
23417 	tbuf = tpage->buf;
23418 	lv = rblockno;
23419 	SBT_PREV(tbuf,lv);
23420 	tpage->dirty = BT_DIRTY;
23421     }
23422 
23423     btreeDeallocSecArray(cache,array2);
23424 
23425     ibn = rblockno;
23426     if(key < mediankey)
23427 	ibn = lblockno;
23428 
23429     lpage->dirty = BT_DIRTY;
23430     rpage->dirty = BT_DIRTY;
23431 
23432     ipage = btreeSeccacheRead(cache,ibn);
23433 
23434     btreeNumInsertNonfull(cache,ipage,key,less,greater);
23435 
23436     ipage = btreeSeccacheRead(cache,prev);
23437 
23438     btreeNumInsertKey(cache,ipage,mediankey,medianless,mediangtr);
23439 
23440     btreeDeallocSecArray(cache,savekeyarray);
23441 
23442     return;
23443 }
23444 
23445 
23446 
23447 
23448 /* @funcstatic btreeNumSplitroot **********************************************
23449 **
23450 ** Split a numeric root node
23451 **
23452 ** @param [u] cache [AjPBtcache] cache
23453 **
23454 ** @return [void]
23455 **
23456 ** @release 4.0.0
23457 ** @@
23458 ******************************************************************************/
23459 
btreeNumSplitroot(AjPBtcache cache)23460 static void btreeNumSplitroot(AjPBtcache cache)
23461 {
23462     AjPBtpage rootpage = NULL;
23463     AjPBtpage rpage    = NULL;
23464     AjPBtpage lpage    = NULL;
23465     AjPBtpage tpage    = NULL;
23466 
23467     ajulong *karray  = NULL;
23468     ajulong *tkarray = NULL;
23469     ajulong *parray  = NULL;
23470     ajulong *tparray = NULL;
23471 
23472     ajuint order  = 0U;
23473     ajuint nkeys  = 0U;
23474     ajuint keypos = 0U;
23475 
23476     ajulong rblockno = 0UL;
23477     ajulong lblockno = 0UL;
23478 
23479     ajulong right;
23480 
23481     ajuint i;
23482 
23483     unsigned char *rootbuf = NULL;
23484     unsigned char *rbuf    = NULL;
23485     unsigned char *lbuf    = NULL;
23486     unsigned char *tbuf    = NULL;
23487 
23488     ajuint nodetype  = 0U;
23489     ajulong overflow = 0UL;
23490     ajulong zero     = 0UL;
23491     ajuint rkeyno   = 0U;
23492     ajuint n        = 0U;
23493 
23494     ajulong lv = 0UL;
23495     ajuint  v  = 0U;
23496 
23497     AjPBtMem array  = NULL;
23498     AjPBtMem array2 = NULL;
23499 
23500 
23501 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
23502     ajDebug("btreeNumSplitroot '%S' %Lu id '%S' key '%S'\n",
23503             cache->basename, cache->secrootblock, indexId, indexKeyword);
23504 #endif
23505     ++statCallNumSplitroot;
23506 
23507     /* ajDebug("In btreeNumSplitroot\n"); */
23508 
23509     order = cache->sorder;
23510 
23511     array = btreeAllocSecArray(cache);
23512     karray  = array->overflows;
23513     parray  = array->parray;
23514 
23515     array2 = btreeAllocSecArray(cache);
23516     tkarray  = array2->overflows;
23517     tparray  = array2->parray;
23518 
23519 
23520     rootpage = btreeSeccacheLocate(cache,cache->secrootblock);
23521     rootbuf = rootpage->buf;
23522 
23523     nkeys = order - 1;
23524 
23525     keypos = nkeys / 2;
23526 
23527     if(!(nkeys % 2))
23528 	--keypos;
23529 
23530 
23531     rblockno = cache->totsize;
23532     rpage = btreeSeccacheNodenew(cache);
23533     rpage->dirty = BT_LOCK;
23534     rpage->lockfor = 1671;
23535 
23536     lblockno = cache->totsize;
23537     lpage = btreeSeccacheNodenew(cache);
23538 
23539 
23540     /* Comment this next block out after the beta test */
23541     if(!cache->slevel)
23542     {
23543 	fprintf(stderr,"btreeNumSplitroot: Shouldn't get here\n");
23544 	exit(0);
23545 	lv = zero;
23546 	SBT_LEFT(lpage->buf,lv);
23547 	lv = rblockno;
23548 	SBT_RIGHT(lpage->buf,lv);
23549 	lv = lblockno;
23550 	SBT_LEFT(rpage->buf,lv);
23551 	lv = zero;
23552 	SBT_RIGHT(rpage->buf,lv);
23553     }
23554 
23555     btreeGetNumKeys(cache,rootbuf,&karray,&parray);
23556 
23557     /* Get key for root node and write new root node */
23558     tkarray[0] = karray[keypos];
23559     tparray[0] = lblockno;
23560     tparray[1] = rblockno;
23561 
23562 
23563     n = 1;
23564     v = n;
23565     SBT_NKEYS(rootbuf,v);
23566     btreeWriteNumNode(cache,rootpage,tkarray,tparray,1);
23567     right = (ajulong)(cache->slevel + 1);
23568     SBT_RIGHT(rootbuf,right);
23569 
23570     rootpage->dirty = BT_LOCK;
23571     rootpage->lockfor = 1672;
23572 
23573     rbuf = rpage->buf;
23574     lbuf = lpage->buf;
23575 
23576     if(cache->slevel)
23577 	nodetype = BT_SECINTERNAL;
23578     else
23579 	nodetype = BT_SECLEAF;
23580 
23581     v = nodetype;
23582     SBT_NODETYPE(rbuf,v);
23583     v = nodetype;
23584     SBT_NODETYPE(lbuf,v);
23585     lv = overflow;
23586     SBT_OVERFLOW(rbuf,lv);
23587     lv = cache->secrootblock;
23588     SBT_PREV(rbuf,lv);
23589     lv = overflow;
23590     SBT_OVERFLOW(lbuf,lv);
23591     lv = cache->secrootblock;
23592     SBT_PREV(lbuf,lv);
23593 
23594     for(i=0;i<keypos;++i)
23595     {
23596 	tkarray[i] = karray[i];
23597 	tparray[i] = parray[i];
23598     }
23599 
23600     tparray[i] = parray[i];
23601 
23602     n = i;
23603     v = n;
23604     SBT_NKEYS(lbuf,v);
23605     btreeWriteNumNode(cache,lpage,tkarray,tparray,i);
23606 
23607     for(i=0;i<n+1;++i)
23608     {
23609 	tpage = btreeSeccacheRead(cache,tparray[i]);
23610 	tbuf = tpage->buf;
23611 	lv = lblockno;
23612 	SBT_PREV(tbuf,lv);
23613 	tpage->dirty = BT_DIRTY;
23614     }
23615 
23616     for(i=keypos+1;i<nkeys;++i)
23617     {
23618 	tkarray[i-(keypos+1)] = karray[i];
23619 	tparray[i-(keypos+1)] = parray[i];
23620     }
23621 
23622     tparray[i-(keypos+1)] = parray[i];
23623 
23624     rkeyno = (nkeys-keypos) - 1;
23625     v = rkeyno;
23626     SBT_NKEYS(rbuf,v);
23627     rpage->dirty = BT_DIRTY;
23628 
23629     btreeWriteNumNode(cache,rpage,tkarray,tparray,rkeyno);
23630 
23631     for(i=0;i<rkeyno+1;++i)
23632     {
23633 	tpage = btreeSeccacheRead(cache,tparray[i]);
23634 	tbuf = tpage->buf;
23635 	lv = rblockno;
23636 	SBT_PREV(tbuf,lv);
23637 	tpage->dirty = BT_DIRTY;
23638     }
23639 
23640 
23641     ++cache->slevel;
23642 
23643     btreeDeallocSecArray(cache,array);
23644     btreeDeallocSecArray(cache,array2);
23645 
23646     return;
23647 }
23648 
23649 
23650 
23651 
23652 /* @funcstatic btreeNumKeyShift ***********************************************
23653 **
23654 ** Rebalance Nodes on insertion
23655 **
23656 ** @param [u] cache [AjPBtcache] cache
23657 ** @param [u] tpage [AjPBtpage] page
23658 **
23659 ** @return [void]
23660 **
23661 ** @release 4.0.0
23662 ** @@
23663 ******************************************************************************/
23664 
btreeNumKeyShift(AjPBtcache cache,AjPBtpage tpage)23665 static void btreeNumKeyShift(AjPBtcache cache, AjPBtpage tpage)
23666 {
23667     unsigned char *tbuf = NULL;
23668     unsigned char *pbuf = NULL;
23669     unsigned char *sbuf = NULL;
23670     unsigned char *buf  = NULL;
23671 
23672     AjPBtpage ppage = NULL;
23673     AjPBtpage spage = NULL;
23674     AjPBtpage page  = NULL;
23675 
23676     ajuint tkeys = 0U;
23677     ajuint pkeys = 0U;
23678     ajuint skeys = 0U;
23679     ajuint order = 0U;
23680 
23681     ajuint i;
23682     ajint ii;
23683 
23684     ajulong parent  = 0UL;
23685 
23686     AjPBtMem array  = NULL;
23687     AjPBtMem array2 = NULL;
23688     AjPBtMem array3 = NULL;
23689     ajulong *kTarray = NULL;
23690     ajulong *kParray = NULL;
23691     ajulong *kSarray = NULL;
23692     ajulong *pTarray = NULL;
23693     ajulong *pParray = NULL;
23694     ajulong *pSarray = NULL;
23695 
23696     ajuint pkeypos = 0U;
23697     ajuint minsize = 0U;
23698 
23699     ajulong lv = 0UL;
23700 
23701     /* ajDebug("In btreeNumKeyShift\n"); */
23702 
23703     tbuf = tpage->buf;
23704 
23705     GBT_PREV(tbuf,&parent);
23706     GBT_NKEYS(tbuf,&tkeys);
23707 
23708     order = cache->sorder;
23709     minsize = order / 2;
23710 
23711     if(order % 2)
23712 	++minsize;
23713 
23714     if(tkeys <= minsize)
23715 	return;
23716 
23717 
23718     ppage = btreeSeccacheRead(cache,parent);
23719     pbuf = ppage->buf;
23720     GBT_NKEYS(pbuf,&pkeys);
23721 
23722     array = btreeAllocSecArray(cache);
23723     kParray  = array->overflows;
23724     pParray  = array->parray;
23725 
23726     array2 = btreeAllocSecArray(cache);
23727     kSarray  = array2->overflows;
23728     pSarray  = array2->parray;
23729 
23730     array3 = btreeAllocSecArray(cache);
23731     kTarray  = array3->overflows;
23732     pTarray  = array3->parray;
23733 
23734 
23735     btreeGetNumKeys(cache,tbuf,&kTarray,&pTarray);
23736     GBT_NKEYS(tbuf,&tkeys);
23737 
23738 
23739     btreeGetNumKeys(cache,pbuf,&kParray,&pParray);
23740 
23741     i=0;
23742 
23743     while(pParray[i] != tpage->pagepos)
23744 	++i;
23745 
23746     if(i) /* There is another leaf to the left */
23747     {
23748 	pkeypos = i-1;
23749 	spage = btreeSeccacheRead(cache,pParray[pkeypos]);
23750 	sbuf = spage->buf;
23751 	GBT_NKEYS(sbuf,&skeys);
23752 
23753     }
23754 
23755     if(i && skeys != order-1) /* There is space in the left leaf */
23756     {
23757 	if(skeys)
23758 	    btreeGetNumKeys(cache,sbuf,&kSarray,&pSarray);
23759 
23760 	kSarray[skeys] = kParray[pkeypos];
23761 	pSarray[skeys+1] = pTarray[0];
23762 	++skeys;
23763 	--tkeys;
23764 	kParray[pkeypos] = kTarray[0];
23765 
23766 	for(i=0;i<tkeys;++i)
23767 	{
23768 	    kTarray[i] = kTarray[i+1];
23769 	    pTarray[i] = pTarray[i+1];
23770 	}
23771 
23772 	pTarray[i] = pTarray[i+1];
23773 	pTarray[i+1] = 0UL;
23774 
23775 	btreeWriteNumNode(cache,spage,kSarray,pSarray,skeys);
23776 	btreeWriteNumNode(cache,tpage,kTarray,pTarray,tkeys);
23777 	btreeWriteNumNode(cache,ppage,kParray,pParray,pkeys);
23778 	if(ppage->pagepos == cache->secrootblock)
23779         {
23780 	    ppage->dirty = BT_LOCK;
23781             ppage->lockfor = 1681;
23782         }
23783 
23784 	page = btreeSeccacheRead(cache,pSarray[skeys]);
23785 	buf = page->buf;
23786 	lv = spage->pagepos;
23787 	SBT_PREV(buf,lv);
23788 	page->dirty = BT_DIRTY;
23789 
23790 	btreeDeallocSecArray(cache,array);
23791 	btreeDeallocSecArray(cache,array2);
23792 	btreeDeallocSecArray(cache,array3);
23793 
23794 	return;
23795     }
23796 
23797 
23798 
23799     if(i != pkeys)	/* There is a right node */
23800     {
23801 	pkeypos = i;
23802 	spage = btreeSeccacheRead(cache,pParray[pkeypos+1]);
23803 	sbuf = spage->buf;
23804 	GBT_NKEYS(sbuf,&skeys);
23805     }
23806 
23807 
23808     if(i != pkeys && skeys != order-1) /* Space in the right node */
23809     {
23810 	if(skeys)
23811 	    btreeGetNumKeys(cache,sbuf,&kSarray,&pSarray);
23812 
23813 	pSarray[skeys+1] = pSarray[skeys];
23814 
23815 	for(ii=skeys-1;ii>-1;--ii)
23816 	{
23817 	    kSarray[ii+1] = kSarray[ii];
23818 	    pSarray[ii+1] = pSarray[ii];
23819 	}
23820 
23821 	kSarray[0] = kParray[pkeypos];
23822 	pSarray[0] = pTarray[tkeys];
23823 	kParray[pkeypos] = kTarray[tkeys-1];
23824 	++skeys;
23825 	--tkeys;
23826 	pTarray[tkeys+1] = 0UL;
23827 
23828 	btreeWriteNumNode(cache,spage,kSarray,pSarray,skeys);
23829 	btreeWriteNumNode(cache,tpage,kTarray,pTarray,tkeys);
23830 	btreeWriteNumNode(cache,ppage,kParray,pParray,pkeys);
23831 
23832 	if(ppage->pagepos == cache->secrootblock)
23833         {
23834 	    ppage->dirty = BT_LOCK;
23835             ppage->lockfor = 1682;
23836         }
23837 
23838 	page = btreeSeccacheRead(cache,pSarray[0]);
23839 	buf = page->buf;
23840 	lv = spage->pagepos;
23841 	SBT_PREV(buf,lv);
23842 	page->dirty = BT_DIRTY;
23843 
23844 	btreeDeallocSecArray(cache,array);
23845 	btreeDeallocSecArray(cache,array2);
23846 	btreeDeallocSecArray(cache,array3);
23847 
23848 	return;
23849     }
23850 
23851 
23852     btreeDeallocSecArray(cache,array);
23853     btreeDeallocSecArray(cache,array2);
23854     btreeDeallocSecArray(cache,array3);
23855 
23856     return;
23857 }
23858 
23859 
23860 
23861 
23862 /* @funcstatic btreeNumInsertShift ********************************************
23863 **
23864 ** Rebalance buckets on insertion
23865 **
23866 ** @param [u] cache [AjPBtcache] cache
23867 ** @param [u] retpage [AjPBtpage*] page
23868 ** @param [r] key [ajulong] key
23869 **
23870 ** @return [ajulong] bucket block or 0UL if shift not possible
23871 **
23872 ** @release 4.0.0
23873 ** @@
23874 ******************************************************************************/
23875 
btreeNumInsertShift(AjPBtcache cache,AjPBtpage * retpage,ajulong key)23876 static ajulong btreeNumInsertShift(AjPBtcache cache, AjPBtpage *retpage,
23877                                    ajulong key)
23878 {
23879     unsigned char *tbuf = NULL;
23880     unsigned char *pbuf = NULL;
23881     unsigned char *sbuf = NULL;
23882 
23883     AjPBtpage ppage = NULL;
23884     AjPBtpage spage = NULL;
23885     AjPBtpage tpage = NULL;
23886 
23887     ajuint tkeys = 0U;
23888     ajuint pkeys = 0U;
23889     ajuint skeys = 0U;
23890     ajuint order = 0U;
23891 
23892     ajuint i;
23893     ajuint n;
23894     ajint ii;
23895 
23896     ajulong parent  = 0UL;
23897     ajulong blockno = 0UL;
23898 
23899     ajulong *kTarray = NULL;
23900     ajulong *kParray = NULL;
23901     ajulong *kSarray = NULL;
23902     ajulong *pTarray = NULL;
23903     ajulong *pParray = NULL;
23904     ajulong *pSarray = NULL;
23905 
23906     ajulong *karray = NULL;
23907     ajulong *parray = NULL;
23908 
23909     ajuint ppos    = 0U;
23910     ajuint pkeypos = 0U;
23911     ajuint minsize = 0U;
23912 
23913     AjPBtMem array  = NULL;
23914     AjPBtMem array2 = NULL;
23915     AjPBtMem array3 = NULL;
23916 
23917 
23918 
23919     /* ajDebug("In btreeNumInsertShift\n"); */
23920 
23921 
23922     tpage = *retpage;
23923 
23924     tbuf = tpage->buf;
23925 
23926     GBT_PREV(tbuf,&parent);
23927     GBT_NKEYS(tbuf,&tkeys);
23928 
23929 
23930     order = cache->sorder;
23931     minsize = order / 2U;
23932 
23933     if(order % 2U)
23934 	++minsize;
23935 
23936     if(tkeys <= minsize)
23937 	return 0UL;
23938 
23939     ppage = btreeSeccacheRead(cache,parent);
23940 
23941 
23942 
23943     pbuf = ppage->buf;
23944     GBT_NKEYS(pbuf,&pkeys);
23945 
23946 
23947     array = btreeAllocSecArray(cache);
23948     kParray  = array->overflows;
23949     pParray  = array->parray;
23950 
23951     array2 = btreeAllocSecArray(cache);
23952     kSarray  = array2->overflows;
23953     pSarray  = array2->parray;
23954 
23955     array3 = btreeAllocSecArray(cache);
23956     kTarray  = array3->overflows;
23957     pTarray  = array3->parray;
23958 
23959 
23960     btreeGetNumKeys(cache,pbuf,&kParray,&pParray);
23961 
23962     i=0;
23963 
23964     while(i!=pkeys && key >= kParray[i])
23965 	++i;
23966 
23967     pkeypos = i;
23968 
23969     if(i==pkeys)
23970     {
23971 	if(key < kParray[i-1])
23972 	    ppos = i-1;
23973 	else
23974 	    ppos = i;
23975     }
23976     else
23977 	ppos = i;
23978 
23979 
23980     if(ppos) /* There is another leaf to the left */
23981     {
23982 	spage = btreeSeccacheRead(cache,pParray[ppos-1]);
23983 	sbuf = spage->buf;
23984 	GBT_NKEYS(sbuf,&skeys);
23985     }
23986 
23987     if(i && skeys != order-1) /* There is space in the left leaf */
23988     {
23989 	/* ajDebug("Left shift\n"); */
23990 	btreeGetNumKeys(cache,tbuf,&kTarray,&pTarray);
23991 	if(skeys)
23992 	    btreeGetNumKeys(cache,sbuf,&kSarray,&pSarray);
23993 
23994 	i = 0;
23995 
23996 	while(pParray[i] != tpage->pagepos)
23997 	    ++i;
23998 
23999 	--i;
24000 
24001 	pkeypos = i;
24002 
24003 	kSarray[skeys] = kParray[pkeypos];
24004 	pSarray[skeys+1] = pTarray[0];
24005 	++skeys;
24006 	--tkeys;
24007 	kParray[pkeypos] = kTarray[0];
24008 
24009 	for(i=0;i<tkeys;++i)
24010 	{
24011 	    kTarray[i] = kTarray[i+1];
24012 	    pTarray[i] = pTarray[i+1];
24013 	}
24014 
24015 	pTarray[i] = pTarray[i+1];
24016 	pTarray[i+1] = 0UL;
24017 
24018 	btreeWriteNumNode(cache,spage,kSarray,pSarray,skeys);
24019 	btreeWriteNumNode(cache,tpage,kTarray,pTarray,tkeys);
24020 	btreeWriteNumNode(cache,ppage,kParray,pParray,pkeys);
24021 	if(ppage->pagepos == cache->secrootblock)
24022         {
24023 	    ppage->dirty = BT_LOCK;
24024             ppage->lockfor = 1691;
24025        }
24026 
24027 	i = 0;
24028 
24029 	while(i!=pkeys && key >= kParray[i])
24030 	    ++i;
24031 
24032 	if(i==pkeys)
24033 	{
24034 	    if(key < kParray[i-1])
24035 		blockno = pParray[i-1];
24036 	    else
24037 		blockno = pParray[i];
24038 	}
24039 	else
24040 	    blockno = pParray[i];
24041 
24042 	if(blockno == spage->pagepos)
24043 	{
24044 	    *retpage = spage;
24045 	    karray = kSarray;
24046 	    parray = pSarray;
24047 	    n = skeys;
24048 	}
24049 	else
24050 	{
24051 	    karray = kTarray;
24052 	    parray = pTarray;
24053 	    n = tkeys;
24054 	}
24055 
24056 
24057 	i = 0;
24058 
24059 	while(i!=n && key >= karray[i])
24060 	    ++i;
24061 
24062 	if(i==n)
24063 	{
24064 	    if(key < karray[i-1])
24065 		blockno = parray[i-1];
24066 	    else
24067 		blockno = parray[i];
24068 	}
24069 	else
24070 	    blockno = parray[i];
24071 
24072 	btreeDeallocSecArray(cache,array);
24073 	btreeDeallocSecArray(cache,array2);
24074 	btreeDeallocSecArray(cache,array3);
24075 
24076 	/* ajDebug("... returns blockno (a) %Lu\n",blockno); */
24077 
24078 	return blockno;
24079     }
24080 
24081 
24082     if(ppos != pkeys)	/* There is a right node */
24083     {
24084 	spage = btreeSeccacheRead(cache,pParray[ppos+1]);
24085 	sbuf = spage->buf;
24086 	GBT_NKEYS(sbuf,&skeys);
24087     }
24088 
24089 
24090     /* Space in the right leaf */
24091     if(ppos != pkeys && skeys != order-1)
24092     {
24093 	/* ajDebug("Right shift\n"); */
24094 	btreeGetNumKeys(cache,tbuf,&kTarray,&pTarray);
24095 	btreeGetNumKeys(cache,sbuf,&kSarray,&pSarray);
24096 
24097 	i = 0;
24098 
24099 	while(pParray[i] != tpage->pagepos)
24100 	    ++i;
24101 
24102 	pkeypos = i;
24103 
24104 	pSarray[skeys+1] = pSarray[skeys];
24105 
24106 	for(ii=skeys-1;ii>-1;--ii)
24107 	{
24108 	    kSarray[ii+1] = kSarray[ii];
24109 	    pSarray[ii+1] = pSarray[ii];
24110 	}
24111 
24112 	kSarray[0] = kParray[pkeypos];
24113 	pSarray[0] = pTarray[tkeys];
24114 	kParray[pkeypos] = kTarray[tkeys-1];
24115 	++skeys;
24116 	--tkeys;
24117 	pTarray[tkeys+1] = 0UL;
24118 
24119 	btreeWriteNumNode(cache,spage,kSarray,pSarray,skeys);
24120 	btreeWriteNumNode(cache,tpage,kTarray,pTarray,tkeys);
24121 	btreeWriteNumNode(cache,ppage,kParray,pParray,pkeys);
24122 	if(ppage->pagepos == cache->secrootblock)
24123         {
24124             ppage->dirty = BT_LOCK;
24125             ppage->lockfor = 1692;
24126         }
24127 
24128 	i = 0;
24129 
24130 	while(i!=pkeys && key >= kParray[i])
24131 	    ++i;
24132 
24133 	if(i==pkeys)
24134 	{
24135 	    if(key < kParray[i-1])
24136 		blockno = pParray[i-1];
24137 	    else
24138 		blockno = pParray[i];
24139 	}
24140 	else
24141 	    blockno = pParray[i];
24142 
24143 	if(blockno == spage->pagepos)
24144 	{
24145 	    *retpage = spage;
24146 	    karray = kSarray;
24147 	    parray = pSarray;
24148 	    n = skeys;
24149 	}
24150 	else
24151 	{
24152 	    karray = kTarray;
24153 	    parray = pTarray;
24154 	    n = tkeys;
24155 	}
24156 
24157 	i = 0;
24158 
24159 	while(i!=n && key >= karray[i])
24160 	    ++i;
24161 
24162 	if(i==n)
24163 	{
24164 	    if(key < karray[i-1])
24165 		blockno = parray[i-1];
24166 	    else
24167 		blockno = parray[i];
24168 	}
24169 	else
24170 	    blockno = parray[i];
24171 
24172 	btreeDeallocSecArray(cache,array);
24173 	btreeDeallocSecArray(cache,array2);
24174 	btreeDeallocSecArray(cache,array3);
24175 
24176 	/* ajDebug("... returns blockno (b) %Lu\n",blockno); */
24177 
24178 	return blockno;
24179     }
24180 
24181 
24182     btreeDeallocSecArray(cache,array);
24183     btreeDeallocSecArray(cache,array2);
24184     btreeDeallocSecArray(cache,array3);
24185 
24186     /* ajDebug("... returns 0UL\n"); */
24187 
24188     return 0UL;
24189 }
24190 
24191 
24192 
24193 
24194 /* @funcstatic btreeNumInsert *************************************************
24195 **
24196 ** Insert a file offset key into a secondary tree
24197 **
24198 ** @param [u] cache [AjPBtcache] cache
24199 ** @param [r] num [const AjPBtNumId] Id object
24200 ** @param [w] page [AjPBtpage] cache page
24201 **
24202 ** @return [void] pointer to a page
24203 **
24204 ** @release 6.5.0
24205 ** @@
24206 ******************************************************************************/
24207 
btreeNumInsert(AjPBtcache cache,const AjPBtNumId num,AjPBtpage page)24208 static void btreeNumInsert(AjPBtcache cache, const AjPBtNumId num,
24209                            AjPBtpage page)
24210 {
24211     unsigned char *buf = NULL;
24212     ajulong numkey;
24213     AjPBtpage spage  = NULL;
24214     ajuint nkeys = 0;
24215     ajuint nodetype = 0;
24216 
24217     ajulong lblockno;
24218     ajulong rblockno;
24219     ajulong blockno;
24220     ajulong shift;
24221     ajuint n;
24222 
24223     /* ajDebug("In btreeNumInsert\n"); */
24224 
24225     (void) page;			/* make it used */
24226 
24227     numkey = num->offset;
24228 
24229     spage = btreeNumFind(cache,numkey);
24230     buf = spage->buf;
24231 
24232     GBT_NKEYS(buf,&nkeys);
24233     GBT_NODETYPE(buf,&nodetype);
24234 
24235     if(!nkeys)
24236     {
24237 	lblockno = cache->totsize;
24238 	btreeWriteNumbucketEmpty(cache,lblockno);
24239 	rblockno = cache->totsize;
24240 	btreeWriteNumbucketEmpty(cache,rblockno);
24241 
24242 	btreeWriteNumNodeSingle(cache,spage,numkey,lblockno,rblockno);
24243 
24244 	btreeNumbucketAdd(cache,rblockno,num);
24245 
24246 	return;
24247     }
24248 
24249     blockno = btreeGetBlockN(cache,buf,numkey);
24250 
24251     if(nodetype != BT_SECROOT)
24252 	if((shift = btreeNumInsertShift(cache,&spage,numkey)))
24253 	    blockno = shift;
24254 
24255 
24256     buf = spage->buf;
24257     n = btreeNumInNumbucket(cache,blockno);
24258 
24259     if(n == cache->snperbucket)
24260     {
24261 	if(btreeReorderNumbuckets(cache,spage))
24262 	{
24263             blockno = btreeGetBlockN(cache,buf,numkey);
24264 	}
24265 	else
24266 	{
24267 	    btreeNumSplitleaf(cache,spage);
24268 	    spage = btreeNumFind(cache,numkey);
24269 	    buf = spage->buf;
24270 
24271 	    blockno = btreeGetBlockN(cache,buf,numkey);
24272 	}
24273     }
24274 
24275     btreeNumbucketAdd(cache,blockno,num);
24276 
24277     return;
24278 }
24279 
24280 
24281 
24282 
24283 /* @funcstatic btreeNumSplitleaf **********************************************
24284 **
24285 ** Split a leaf and propagate up if necessary
24286 **
24287 ** @param [u] cache [AjPBtcache] cache
24288 ** @param [u] spage [AjPBtpage] page
24289 **
24290 ** @return [AjPBtpage] pointer to a parent page
24291 **
24292 ** @release 4.0.0
24293 ** @@
24294 ******************************************************************************/
24295 
btreeNumSplitleaf(AjPBtcache cache,AjPBtpage spage)24296 static AjPBtpage btreeNumSplitleaf(AjPBtcache cache, AjPBtpage spage)
24297 {
24298     ajuint nkeys     = 0U;
24299     ajuint totalkeys = 0U;
24300     ajuint keylimit  = 0U;
24301     ajuint order     = 0U;
24302     ajuint nodetype  = 0U;
24303 
24304     ajuint rootnodetype = 0U;
24305 
24306     ajuint i;
24307     ajuint j;
24308     ajuint iref;
24309 
24310     AjPBtpage lpage = NULL;
24311     AjPBtpage rpage = NULL;
24312     AjPBtpage page  = NULL;
24313 
24314     ajulong mediankey  = 0UL;
24315     ajulong mediangtr  = 0UL;
24316     ajulong medianless = 0UL;
24317 
24318     AjPBtNumId bid = NULL;
24319 
24320     unsigned char *buf  = NULL;
24321     unsigned char *lbuf = NULL;
24322     unsigned char *rbuf = NULL;
24323 
24324     AjPList idlist = NULL;
24325 
24326     AjPNumbucket cbucket  = NULL;
24327 
24328     AjPBtMem array = NULL;
24329     AjPBtMem newarray = NULL;
24330     ajulong *newnarray = NULL;
24331     ajulong *parray = NULL;
24332     ajulong *newparray = NULL;
24333 
24334     ajuint lno    = 0U;
24335     ajuint rno    = 0U;
24336 
24337     ajuint lbucketlimit   = 0U;
24338     ajuint rbucketlimit   = 0U;
24339     ajuint lmaxnperbucket = 0U;
24340     ajuint rmaxnperbucket = 0U;
24341     ajuint count         = 0U;
24342 
24343     ajulong lblockno = 0UL;
24344     ajulong rblockno = 0UL;
24345     ajulong overflow = 0UL;
24346     ajulong prevsave = 0UL;
24347 
24348     ajulong zero = 0UL;
24349     ajulong join = 0UL;
24350 
24351     ajuint iold = 0U;
24352 
24353     ajulong lv = 0UL;
24354     ajuint  v  = 0U;
24355 
24356 
24357 #if AJINDEX_DEBUG || AJINDEX_DOSTATS
24358     ajDebug("btreeNumSplitleaf '%S' %Lu id '%S' key '%S' dirty:%u lockfor:%u\n",
24359             cache->basename, spage->pagepos, indexId, indexKeyword,
24360             spage->dirty, spage->lockfor);
24361 #endif
24362     ++statCallNumSplitleaf;
24363 
24364     /* ajDebug("In btreeNumSplitleaf\n"); */
24365 
24366     order = cache->sorder;
24367 
24368     array = btreeAllocSecArray(cache);
24369     parray = array->parray;
24370     newarray = btreeAllocSecArray(cache);
24371     newparray = newarray->parray;
24372     newnarray = newarray->overflows;
24373 
24374     buf = spage->buf;
24375     lbuf = buf;
24376 
24377     GBT_NKEYS(buf,&nkeys);
24378     GBT_NODETYPE(buf,&rootnodetype);
24379 
24380 #if AJINDEX_DEBUG
24381     ajDebug("    nkeys:%u nodetype:%u\n", nkeys, rootnodetype);
24382 #endif
24383 
24384     if(rootnodetype == BT_SECROOT)
24385     {
24386 #if AJINDEX_DEBUG
24387 	ajDebug("   Root leaf splitting page:%Lu secroot:%Lu\n",
24388                 cache->totsize, cache->secrootblock);
24389 #endif
24390 	lblockno = cache->totsize;
24391 	lpage = btreeSeccacheNodenew(cache);
24392 	lbuf = lpage->buf;
24393 	lv = cache->secrootblock;
24394 	SBT_PREV(lbuf,lv);
24395     }
24396     else
24397     {
24398 	lblockno = spage->pagepos;
24399 	lpage = spage;
24400     }
24401 
24402     lpage->dirty = BT_LOCK;
24403     lpage->lockfor = 1701;
24404 
24405     rblockno = cache->totsize;
24406     rpage = btreeSeccacheNodenew(cache);
24407     rbuf = rpage->buf;
24408     rpage->dirty = BT_LOCK;
24409     rpage->lockfor = 1702;
24410 
24411     if(rootnodetype == BT_SECROOT)
24412     {
24413 	lv = zero;
24414 	SBT_RIGHT(rbuf,lv);
24415 	lv = zero;
24416 	SBT_LEFT(lbuf,lv);
24417     }
24418     else
24419     {
24420 	GBT_RIGHT(lbuf,&join);
24421 	lv = join;
24422 	SBT_RIGHT(rbuf,lv);
24423     }
24424 
24425     lv = lblockno;
24426     SBT_LEFT(rbuf,lv);
24427     lv = rblockno;
24428     SBT_RIGHT(lbuf,lv);
24429 
24430 
24431     btreeGetNumPointers(cache,buf,&parray);
24432 
24433 
24434     keylimit = nkeys+1;
24435     idlist = ajListNew();
24436     for(i=0;i<keylimit;++i)
24437 	btreeNumbucketIdlist(cache,parray[i], idlist);
24438 
24439     ajListSort(idlist, &btreeNumIdCompare);
24440 
24441     totalkeys = (ajuint) ajListGetLength(idlist);
24442 
24443     btreeBucketSplitCalc(totalkeys, keylimit, cache->snperbucket,
24444                          &lbucketlimit,&lmaxnperbucket,&lno,
24445                          &rbucketlimit,&rmaxnperbucket,&rno);
24446 
24447     cbucket = btreeNumbucketNew(cache->snperbucket, cache->refcount);
24448 
24449     count = 0;
24450     iold = 0;
24451 
24452     for(i=0;i<lbucketlimit;++i)
24453     {
24454 	cbucket->Nentries = 0;
24455 
24456 	for(j=0;j<lmaxnperbucket;++j)
24457 	{
24458 	    ajListPop(idlist,(void **)&bid);
24459 
24460 	    cbucket->NumId[j]->dbno      = bid->dbno;
24461 	    cbucket->NumId[j]->offset    = bid->offset;
24462 
24463             if(cache->refcount)
24464             {
24465                 for(iref=0; iref < cache->refcount; iref++)
24466                     cbucket->NumId[j]->refoffsets[iref] = bid->refoffsets[iref];
24467 	    }
24468 
24469 	    ++count;
24470 	    ++cbucket->Nentries;
24471 	    AJFREE(bid);
24472 	}
24473 
24474 	ajListPeek(idlist,(void **)&bid);
24475 
24476 	newnarray[i] = bid->offset;
24477 
24478         if((iold < order) && parray[iold])
24479             newparray[i] = parray[iold++];
24480         else
24481             newparray[i] = cache->totsize;
24482 
24483 	btreeWriteNumbucket(cache,cbucket,newparray[i]);
24484     }
24485 
24486     cbucket->Nentries = 0;
24487 
24488     j = 0;
24489 
24490     while(count != lno)
24491     {
24492 	ajListPop(idlist,(void **)&bid);
24493 
24494 	cbucket->NumId[j]->dbno      = bid->dbno;
24495 	cbucket->NumId[j]->offset    = bid->offset;
24496 
24497         if(cache->refcount)
24498         {
24499             for(iref=0; iref < cache->refcount; iref++)
24500                 cbucket->NumId[j]->refoffsets[iref] = bid->refoffsets[iref];
24501         }
24502 
24503 	++j;
24504 	++count;
24505 
24506 
24507 	++cbucket->Nentries;
24508 	AJFREE(bid);
24509     }
24510 
24511     if((iold < order) && parray[iold])
24512         newparray[i] = parray[iold++];
24513     else
24514         newparray[i] = cache->totsize;
24515 
24516     btreeWriteNumbucket(cache,cbucket,newparray[i]);
24517 
24518     nkeys = lbucketlimit;
24519     nodetype = BT_SECLEAF;
24520     v = nodetype;
24521     SBT_NODETYPE(lbuf,v);
24522     GBT_PREV(lbuf,&prevsave);
24523     lpage->dirty = BT_DIRTY;
24524 
24525     btreeWriteNumNode(cache,lpage,newnarray,newparray,nkeys);
24526 
24527     ajListPeek(idlist,(void **)&bid);
24528     mediankey = bid->offset;
24529 
24530     for(i=0;i<rbucketlimit;++i)
24531     {
24532 	cbucket->Nentries = 0;
24533 
24534 	for(j=0;j<rmaxnperbucket;++j)
24535 	{
24536 	    ajListPop(idlist,(void **)&bid);
24537 
24538 	    cbucket->NumId[j]->dbno      = bid->dbno;
24539 	    cbucket->NumId[j]->offset    = bid->offset;
24540 
24541             if(cache->refcount)
24542             {
24543                 for(iref=0; iref < cache->refcount; iref++)
24544                     cbucket->NumId[j]->refoffsets[iref] = bid->refoffsets[iref];
24545             }
24546 
24547 	    ++cbucket->Nentries;
24548 	    AJFREE(bid);
24549 	}
24550 
24551 	ajListPeek(idlist,(void **)&bid);
24552 	newnarray[i] = bid->offset;
24553 
24554         if((iold < order) && parray[iold])
24555             newparray[i] = parray[iold++];
24556         else
24557             newparray[i] = cache->totsize;
24558 
24559 	btreeWriteNumbucket(cache,cbucket,newparray[i]);
24560     }
24561 
24562     cbucket->Nentries = 0;
24563 
24564     j = 0;
24565 
24566     while(ajListPop(idlist,(void**)&bid))
24567     {
24568 	cbucket->NumId[j]->dbno      = bid->dbno;
24569 	cbucket->NumId[j]->offset    = bid->offset;
24570 
24571         if(cache->refcount)
24572         {
24573             for(iref=0; iref < cache->refcount; iref++)
24574                 cbucket->NumId[j]->refoffsets[iref] = bid->refoffsets[iref];
24575         }
24576 
24577 	++j;
24578 
24579 
24580 	++cbucket->Nentries;
24581 	AJFREE(bid);
24582     }
24583 
24584     if((iold < order) && parray[iold])
24585         newparray[i] = parray[iold++];
24586     else
24587         newparray[i] = cache->totsize;
24588 
24589     btreeWriteNumbucket(cache,cbucket,newparray[i]);
24590 
24591 #if AJINDEX_DEBUG
24592     ajDebug("end of NumSplitleaf i:%u iold:%u, order:%u\n", i, iold, order);
24593 #endif
24594     nkeys = rbucketlimit;
24595 
24596     nodetype = BT_SECLEAF;
24597     v = nodetype;
24598     SBT_NODETYPE(rbuf,v);
24599     lv = prevsave;
24600     SBT_PREV(rbuf,lv);
24601     lv = overflow;
24602     SBT_OVERFLOW(rbuf,lv);
24603 
24604     btreeWriteNumNode(cache,rpage,newnarray,newparray,nkeys);
24605     rpage->dirty = BT_DIRTY;
24606 
24607     btreeNumbucketDel(&cbucket);
24608     ajListFree(&idlist);
24609 
24610 
24611 
24612     medianless = lblockno;
24613     mediangtr  = rblockno;
24614 
24615 
24616     if(rootnodetype == BT_SECROOT)
24617     {
24618 	spage->dirty = BT_DIRTY;
24619 	btreeWriteNumNodeSingle(cache,spage,mediankey,lblockno,rblockno);
24620 	++cache->slevel;
24621 	lv = cache->slevel;
24622 	SBT_RIGHT(buf,lv);
24623 	spage->dirty = BT_LOCK;
24624         spage->lockfor = 1703;
24625 
24626 	btreeDeallocSecArray(cache,array);
24627         btreeDeallocSecArray(cache,newarray);
24628 
24629 	return spage;
24630     }
24631 
24632     btreeDeallocSecArray(cache,array);
24633     btreeDeallocSecArray(cache,newarray);
24634 
24635 
24636     page = btreeSeccacheRead(cache,prevsave);
24637     btreeNumInsertKey(cache,page,mediankey,medianless,mediangtr);
24638 
24639 
24640     page = btreeSeccacheRead(cache,prevsave);
24641 
24642     return page;
24643 }
24644 
24645 
24646 
24647 
24648 /* @funcstatic btreeFreePriArray **********************************************
24649 **
24650 ** Free karray and parray arrays for a primary key
24651 **
24652 ** @param [u] cache [AjPBtcache] cache
24653 **
24654 ** @return [void]
24655 **
24656 ** @release 6.5.0
24657 ** @@
24658 ******************************************************************************/
24659 
btreeFreePriArray(AjPBtcache cache)24660 static void btreeFreePriArray(AjPBtcache cache)
24661 {
24662     AjPBtMem p;
24663     AjPBtMem next;
24664     ajuint i;
24665 
24666     /* ajDebug("In btreeFreePriArray\n"); */
24667 
24668     if(!cache->bmem)
24669 	return;
24670 
24671     p = cache->bmem;
24672 
24673     while(p)
24674     {
24675         next = p->next;
24676 
24677         AJFREE(p->parray);
24678 	AJFREE(p->overflows);
24679 
24680 	for(i=0;i<cache->porder;++i)
24681 	    ajStrDel(&p->karray[i]);
24682 
24683 	AJFREE(p->karray);
24684 	AJFREE(p);
24685 	p = next;
24686     }
24687 
24688 
24689     cache->bmem = NULL;
24690     cache->tmem = NULL;
24691 
24692     return;
24693 }
24694 
24695 
24696 
24697 
24698 /* @funcstatic  btreeFreeSecArray *********************************************
24699 **
24700 ** Free karray and parray arrays for a secondary key
24701 **
24702 ** @param [u] cache [AjPBtcache] cache
24703 **
24704 ** @return [void]
24705 **
24706 ** @release 6.5.0
24707 ** @@
24708 ******************************************************************************/
24709 
btreeFreeSecArray(AjPBtcache cache)24710 static void btreeFreeSecArray(AjPBtcache cache)
24711 {
24712     AjPBtMem p;
24713     AjPBtMem next;
24714     ajuint i;
24715 
24716     /* ajDebug("In btreeFreeSecArray\n"); */
24717 
24718     if(!cache->bsmem)
24719 	return;
24720 
24721     p = cache->bsmem;
24722 
24723     while(p)
24724     {
24725         next = p->next;
24726 
24727 	AJFREE(p->parray);
24728 	AJFREE(p->overflows);
24729 
24730 	for(i=0;i<cache->sorder;++i)
24731 	    ajStrDel(&p->karray[i]);
24732 
24733 	AJFREE(p->karray);
24734 	AJFREE(p);
24735 
24736 	p = next;
24737     }
24738 
24739 
24740     cache->bsmem = NULL;
24741     cache->tsmem = NULL;
24742 
24743     return;
24744 }
24745 
24746 
24747 
24748 
24749 /* @funcstatic btreeIdentFetchMulti *******************************************
24750 **
24751 ** Read the leaves of a secondary hybrid tree
24752 **
24753 ** @param [u] cache [AjPBtcache] cache
24754 ** @param [r] idname [const AjPStr] id name
24755 ** @param [r] rootblock [ajulong] root page of secondary tree
24756 ** @param [u] list [AjPList] list to add BtIDs to
24757 **
24758 ** @return [void]
24759 **
24760 ** @release 6.5.0
24761 ** @@
24762 ******************************************************************************/
24763 
btreeIdentFetchMulti(AjPBtcache cache,const AjPStr idname,ajulong rootblock,AjPList list)24764 static void btreeIdentFetchMulti(AjPBtcache cache, const AjPStr idname,
24765                                  ajulong rootblock,
24766                                  AjPList list)
24767 {
24768     AjPBtMem array = NULL;
24769     ajulong *parray;
24770 
24771     AjPBtpage page;
24772     unsigned char *buf;
24773 
24774     ajuint nodetype;
24775     ajuint i;
24776     ajulong level = 0UL;
24777 
24778     ajuint nkeys;
24779     ajulong right;
24780 
24781     array = btreeAllocSecArray(cache);
24782     parray = array->parray;
24783 
24784     page = btreeSeccacheRead(cache, rootblock);
24785     buf = page->buf;
24786 
24787     GBT_RIGHT(buf,&level);
24788     cache->slevel = (ajuint) level;
24789 
24790     btreeGetNumPointers(cache,buf,&parray);
24791     GBT_NODETYPE(buf,&nodetype);
24792 
24793     while(nodetype != BT_SECLEAF && cache->slevel != 0)
24794     {
24795 	page = btreeSeccacheRead(cache,parray[0]);
24796 	buf = page->buf;
24797 	btreeGetNumPointers(cache,buf,&parray);
24798 	GBT_NODETYPE(buf,&nodetype);
24799     }
24800 
24801     do
24802     {
24803 	GBT_NKEYS(buf,&nkeys);
24804 
24805 	for(i=0;i<=nkeys;++i)
24806 	    btreeNumbucketBtidlist(cache,parray[i],idname,list);
24807 
24808 	right = 0UL;
24809 	if(cache->slevel)
24810 	{
24811 	    GBT_RIGHT(buf,&right);
24812 
24813 	    if(right)
24814 	    {
24815 		page = btreeSeccacheRead(cache,right);
24816 		buf = page->buf;
24817 		btreeGetNumPointers(cache,buf,&parray);
24818 	    }
24819 	}
24820     } while(right);
24821 
24822     btreeDeallocSecArray(cache,array);
24823 
24824     return;
24825 }
24826 
24827 
24828 
24829 
24830 /* @funcstatic btreeIdentFetchMultiHit ****************************************
24831 **
24832 ** Read the leaves of a secondary hybrid tree
24833 **
24834 ** @param [u] cache [AjPBtcache] cache
24835 ** @param [r] rootblock [ajulong] root page of secondary tree
24836 ** @param [u] list [AjPList] list to add BtHits to
24837 **
24838 ** @return [void]
24839 **
24840 ** @release 6.5.0
24841 ** @@
24842 ******************************************************************************/
24843 
btreeIdentFetchMultiHit(AjPBtcache cache,ajulong rootblock,AjPList list)24844 static void btreeIdentFetchMultiHit(AjPBtcache cache,
24845                                     ajulong rootblock,
24846                                     AjPList list)
24847 {
24848     AjPBtMem array = NULL;
24849     ajulong *parray;
24850 
24851     AjPBtpage page;
24852     unsigned char *buf;
24853 
24854     ajuint nodetype;
24855     ajuint i;
24856     ajulong level = 0UL;
24857 
24858     ajuint nkeys;
24859     ajulong right;
24860 
24861     array = btreeAllocSecArray(cache);
24862     parray = array->parray;
24863 
24864     page = btreeSeccacheRead(cache, rootblock);
24865     buf = page->buf;
24866 
24867     GBT_RIGHT(buf,&level);
24868     cache->slevel = (ajuint) level;
24869 
24870     btreeGetNumPointers(cache,buf,&parray);
24871     GBT_NODETYPE(buf,&nodetype);
24872 
24873     while(nodetype != BT_SECLEAF && cache->slevel != 0)
24874     {
24875 	page = btreeSeccacheRead(cache,parray[0]);
24876 	buf = page->buf;
24877 	btreeGetNumPointers(cache,buf,&parray);
24878 	GBT_NODETYPE(buf,&nodetype);
24879     }
24880 
24881     do
24882     {
24883 	GBT_NKEYS(buf,&nkeys);
24884 
24885 	for(i=0;i<=nkeys;++i)
24886 	    btreeNumbucketBthitlist(cache,parray[i],list);
24887 
24888 	right = 0UL;
24889 	if(cache->slevel)
24890 	{
24891 	    GBT_RIGHT(buf,&right);
24892 
24893 	    if(right)
24894 	    {
24895 		page = btreeSeccacheRead(cache,right);
24896 		buf = page->buf;
24897 		btreeGetNumPointers(cache,buf,&parray);
24898 	    }
24899 	}
24900     } while(right);
24901 
24902     btreeDeallocSecArray(cache,array);
24903 
24904     return;
24905 }
24906 
24907 
24908 
24909 
24910 /* @funcstatic btreeIdentFetchMultiHitref *************************************
24911 **
24912 ** Read the leaves of a secondary hybrid tree
24913 **
24914 ** @param [u] cache [AjPBtcache] cache
24915 ** @param [r] rootblock [ajulong] root page of secondary tree
24916 ** @param [u] list [AjPList] list to add BtHitrefs to
24917 **
24918 ** @return [void]
24919 **
24920 ** @release 6.5.0
24921 ** @@
24922 ******************************************************************************/
24923 
btreeIdentFetchMultiHitref(AjPBtcache cache,ajulong rootblock,AjPList list)24924 static void btreeIdentFetchMultiHitref(AjPBtcache cache,
24925                                        ajulong rootblock,
24926                                        AjPList list)
24927 {
24928     AjPBtMem array = NULL;
24929     ajulong *parray;
24930 
24931     AjPBtpage page;
24932     unsigned char *buf;
24933 
24934     ajuint nodetype;
24935     ajuint i;
24936     ajulong level = 0UL;
24937 
24938     ajuint nkeys;
24939     ajulong right;
24940 
24941     array = btreeAllocSecArray(cache);
24942     parray = array->parray;
24943 
24944     page = btreeSeccacheRead(cache, rootblock);
24945     buf = page->buf;
24946 
24947     GBT_RIGHT(buf,&level);
24948     cache->slevel = (ajuint) level;
24949 
24950     btreeGetNumPointers(cache,buf,&parray);
24951     GBT_NODETYPE(buf,&nodetype);
24952 
24953     while(nodetype != BT_SECLEAF && cache->slevel != 0)
24954     {
24955 	page = btreeSeccacheRead(cache,parray[0]);
24956 	buf = page->buf;
24957 	btreeGetNumPointers(cache,buf,&parray);
24958 	GBT_NODETYPE(buf,&nodetype);
24959     }
24960 
24961     do
24962     {
24963 	GBT_NKEYS(buf,&nkeys);
24964 
24965 	for(i=0;i<=nkeys;++i)
24966 	    btreeNumbucketBthitreflist(cache,parray[i],list);
24967 
24968 	right = 0UL;
24969 	if(cache->slevel)
24970 	{
24971 	    GBT_RIGHT(buf,&right);
24972 
24973 	    if(right)
24974 	    {
24975 		page = btreeSeccacheRead(cache,right);
24976 		buf = page->buf;
24977 		btreeGetNumPointers(cache,buf,&parray);
24978 	    }
24979 	}
24980     } while(right);
24981 
24982     btreeDeallocSecArray(cache,array);
24983 
24984     return;
24985 }
24986 
24987 
24988 
24989 
24990 /* @func ajBtreeDumpIdentifiers ***********************************************
24991 **
24992 ** Read the leaves of an identifier tree
24993 **
24994 ** @param [u] cache [AjPBtcache] cache
24995 ** @param [r] dmin [ajuint] minimum number of times the key should appear
24996 ** @param [r] dmax [ajuint] maximum number of times the key should appear
24997 ** @param [u] outf [AjPFile] output file
24998 **
24999 ** @return [void]
25000 **
25001 ** @release 5.0.0
25002 ** @@
25003 ******************************************************************************/
25004 
ajBtreeDumpIdentifiers(AjPBtcache cache,ajuint dmin,ajuint dmax,AjPFile outf)25005 void ajBtreeDumpIdentifiers(AjPBtcache cache, ajuint dmin, ajuint dmax,
25006                             AjPFile outf)
25007 {
25008     AjPBtMem array = NULL;
25009     AjPStr *karray;
25010     ajulong *parray;
25011 
25012     AjPBtpage page;
25013     unsigned char *buf;
25014 
25015     ajuint nodetype;
25016     ajuint i;
25017     ajuint j;
25018     ajuint dups;
25019 
25020     AjPIdbucket bucket;
25021     ajuint nkeys;
25022     ajulong right;
25023 
25024     if(cache->secondary)
25025     {
25026         ajBtreeDumpKeywords(cache, dmin, dmax, outf);
25027         return;
25028     }
25029 
25030     array = btreeAllocPriArray(cache);
25031     karray = array->karray;
25032     parray = array->parray;
25033 
25034     page = btreePricacheRead(cache, 0UL);
25035     buf = page->buf;
25036 
25037     btreeGetKeys(cache,buf,&karray,&parray);
25038     GBT_NODETYPE(buf,&nodetype);
25039 
25040     while(nodetype != BT_LEAF && cache->plevel != 0)
25041     {
25042 	page = btreePricacheRead(cache,parray[0]);
25043 	buf = page->buf;
25044 	btreeGetKeys(cache,buf,&karray,&parray);
25045 	GBT_NODETYPE(buf,&nodetype);
25046     }
25047 
25048     do
25049     {
25050 	GBT_NKEYS(buf,&nkeys);
25051 	for(i=0;i<=nkeys;++i)
25052 	{
25053 	    bucket = btreeReadIdbucket(cache,parray[i]);
25054             btreeIdbucketSort(bucket);
25055 
25056 	    for(j=0;j<bucket->Nentries;++j)
25057 	    {
25058 		dups = bucket->Ids[j]->dups;
25059 
25060 		if(!dups)
25061 		    dups = 1;
25062 
25063 		if(dups < dmin)
25064 		    continue;
25065 
25066 		if(dmax && dups > dmax)
25067 		    continue;
25068 
25069 		ajFmtPrintF(outf,"%10d %S\n",
25070 			    dups, bucket->Ids[j]->id);
25071 	    }
25072 
25073 	    btreeIdbucketDel(&bucket);
25074 	}
25075 
25076 	right = 0UL;
25077 
25078 	if(cache->plevel)
25079 	{
25080 	    GBT_RIGHT(buf,&right);
25081 
25082 	    if(right)
25083 	    {
25084 		page = btreePricacheRead(cache,right);
25085 		buf = page->buf;
25086 		btreeGetKeys(cache,buf,&karray,&parray);
25087 	    }
25088 	}
25089     } while(right);
25090 
25091     btreeDeallocPriArray(cache,array);
25092 
25093     return;
25094 }
25095 
25096 
25097 
25098 
25099 /* @func ajBtreeDeleteIdent ***************************************************
25100 **
25101 ** Entry point for hybrid ID deletion.
25102 **
25103 ** Deletion software
25104 **
25105 ** @param [u] cache [AjPBtcache] cache
25106 ** @param [r] btid [const AjPBtId] hybrid object
25107 **
25108 ** @return [AjBool] True if found and deleted
25109 **
25110 ** @release 6.5.0
25111 ** @@
25112 ******************************************************************************/
25113 
ajBtreeDeleteIdent(AjPBtcache cache,const AjPBtId btid)25114 AjBool ajBtreeDeleteIdent(AjPBtcache cache, const AjPBtId btid)
25115 {
25116     AjPBtpage rootpage = NULL;
25117     AjPBtpage spage   = NULL;
25118     AjPStr key        = NULL;
25119     AjPIdbucket bucket  = NULL;
25120     ajulong blockno  = 0UL;
25121 
25122     ajuint nkeys = 0U;
25123 
25124     ajuint nodetype = 0U;
25125     ajuint nentries = 0U;
25126 
25127     AjPStr *karray = NULL;
25128     ajulong *parray = NULL;
25129     AjPBtMem arrays = NULL;
25130     AjBool found = ajFalse;
25131     ajuint dups = 0U;
25132 
25133     ajuint i;
25134 
25135     unsigned char *buf = NULL;
25136     AjPBtId did = NULL;
25137     ajulong  secrootpage = 0UL;
25138     AjBool ret = ajFalse;
25139 
25140     /* ajDebug("In ajBtreeDeleteIdent\n"); */
25141 
25142 
25143     if(!ajStrGetLen(btid->id))
25144 	return ajFalse;
25145 
25146     spage = btreeIdentFind(cache,btid->id);
25147     buf = spage->buf;
25148 
25149     GBT_NKEYS(buf,&nkeys);
25150     GBT_NODETYPE(buf,&nodetype);
25151 
25152     arrays = btreeAllocPriArray(cache);
25153     karray = arrays->karray;
25154     parray = arrays->parray;
25155 
25156     if(!nkeys)
25157     {
25158 	btreeDeallocPriArray(cache,arrays);
25159 	ajStrDel(&key);
25160 
25161 	return ajFalse;
25162     }
25163 
25164 
25165     /* Search to see whether entry exists */
25166 
25167     btreeGetKeys(cache,buf,&karray,&parray);
25168 
25169     i=0;
25170 
25171     while(i!=nkeys && MAJSTRCMPS(btid->id,karray[i])>=0)
25172 	++i;
25173 
25174     blockno = parray[i];
25175 
25176 
25177     bucket = btreeReadIdbucket(cache,blockno);
25178 
25179     nentries = bucket->Nentries;
25180 
25181     found = ajFalse;
25182 
25183     for(i=0;i<nentries;++i)
25184 	if(ajStrMatchS(btid->id,bucket->Ids[i]->id))
25185 	{
25186 	    found = ajTrue;
25187 	    break;
25188 	}
25189 
25190 
25191     if(!found)
25192     {
25193 	btreeDeallocPriArray(cache,arrays);
25194 
25195 	return ajFalse;
25196     }
25197 
25198 
25199     dups = bucket->Ids[i]->dups;
25200 
25201 
25202     if(!dups)
25203     {
25204         /* ajDebug("No secondary tree\n"); */
25205         rootpage = btreePricacheLocate(cache,0UL);
25206 
25207         if(!rootpage)
25208             ajFatal("Rootpage has been unlocked (ajBtreeDeleteHybId)");
25209 
25210         rootpage->dirty = BT_LOCK;
25211         rootpage->lockfor = 1711;
25212 
25213         btreeFindIdentBalanceOne(cache,0UL,BTNO_NODE,BTNO_NODE,BTNO_NODE,
25214                                  BTNO_NODE,btid);
25215 
25216 
25217         btreeDeallocPriArray(cache,arrays);
25218 
25219         if(cache->dodelete)
25220             ret = ajTrue;
25221         else
25222             ret = ajFalse;
25223     }
25224     else
25225     {
25226         did = bucket->Ids[i];
25227         secrootpage = did->offset;
25228         cache->secrootblock = secrootpage;
25229 
25230         ret = btreeDeleteIdentIdTwo(cache,btid,did);
25231 
25232         btreeWriteIdbucket(cache,bucket,blockno);
25233     }
25234 
25235 
25236     return ret;
25237 }
25238 
25239 
25240 
25241 
25242 /* @funcstatic btreeFindIdentBalanceOne ***************************************
25243 **
25244 ** Master routine for entry deletion from level 1 identifier tree.
25245 **
25246 ** Deletion software.
25247 **
25248 ** @param [u] cache [AjPBtcache] cache
25249 ** @param [r] thisNode [ajulong] Current node
25250 ** @param [r] leftNode [ajulong] Node to left
25251 ** @param [r] rightNode [ajulong] Node to right
25252 ** @param [r] lAnchor [ajulong] Left anchor
25253 ** @param [r] rAnchor [ajulong] Right anchor
25254 ** @param [r] btid [const AjPBtId] Id object
25255 **
25256 ** @return [ajulong] page number or BTNO_NODE
25257 **
25258 ** @release 6.1.0
25259 ** @@
25260 ******************************************************************************/
25261 
btreeFindIdentBalanceOne(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor,const AjPBtId btid)25262 static ajulong btreeFindIdentBalanceOne(AjPBtcache cache, ajulong thisNode,
25263                                         ajulong leftNode, ajulong rightNode,
25264                                         ajulong lAnchor, ajulong rAnchor,
25265                                         const AjPBtId btid)
25266 {
25267     unsigned char *buf  = NULL;
25268     unsigned char *buf1 = NULL;
25269 
25270     ajulong nextNode   = BTNO_NODE;
25271     ajulong nextLeft   = BTNO_NODE;
25272     ajulong nextRight  = BTNO_NODE;
25273     ajulong nextAncL   = BTNO_NODE;
25274     ajulong nextAncR   = BTNO_NODE;
25275     ajulong done       = 0UL;
25276 
25277     ajuint  nkeys      = 0U;
25278     ajuint  order      = 0U;
25279     ajuint  minkeys    = 0U;
25280     ajuint  i;
25281     ajuint   nodetype   = 0U;
25282 
25283     ajuint n1keys      = 0U;
25284 
25285     AjPBtpage page  = NULL;
25286     AjPBtpage page1 = NULL;
25287 
25288     ajulong balanceNode = 0UL;
25289     ajulong blockno     = 0UL;
25290     ajulong ptrSave     = 0UL;
25291 
25292     AjPStr *karray  = NULL;
25293     ajulong *parray  = NULL;
25294     AjPStr *k1array = NULL;
25295     ajulong *p1array = NULL;
25296 
25297     AjPBtMem arrays  = NULL;
25298     AjPBtMem arrays1 = NULL;
25299 
25300     AjBool existed = ajFalse;
25301 
25302     /* ajDebug("In btreeFindBalance\n"); */
25303 
25304     if(thisNode)
25305 	page = btreePricacheRead(cache,thisNode);
25306     else
25307     {   /* It's the root node of the primary hyb tree */
25308         /* Needs altering for secondary tree          */
25309 	page = btreePricacheLocate(cache,thisNode);
25310 	page->dirty = BT_LOCK;
25311         page->lockfor = 1721;
25312     }
25313 
25314     cache->dodelete = ajFalse;
25315 
25316     buf = page->buf;
25317     GBT_NKEYS(buf,&nkeys);
25318 
25319     order = cache->porder;
25320     /* order-1 is the number of keys in the node */
25321     minkeys = (order-1) / 2;
25322 
25323     if((order-1)%2)
25324 	++minkeys;
25325 
25326     /*
25327     ** If thisNode contains >= minkeys then it is not a candidate
25328     ** for balancing
25329     */
25330     if(nkeys >= minkeys)
25331 	balanceNode = BTNO_BALANCE;
25332     else
25333 	balanceNode = page->pagepos;
25334 
25335     arrays  = btreeAllocPriArray(cache);
25336     arrays1 = btreeAllocPriArray(cache);
25337 
25338     karray = arrays->karray;
25339     parray = arrays->parray;
25340 
25341     k1array = arrays1->karray;
25342     p1array = arrays1->parray;
25343 
25344     btreeGetKeys(cache,buf,&karray,&parray);
25345 
25346     i=0;
25347 
25348     while(i!=nkeys && MAJSTRCMPS(btid->id,karray[i])>=0)
25349 	++i;
25350 
25351     blockno = parray[i];
25352 
25353     nextNode = blockno;
25354     ptrSave = i;
25355 
25356     GBT_NODETYPE(buf,&nodetype);
25357 
25358     if(!(nodetype == BT_LEAF) && !(nodetype == BT_ROOT && !cache->plevel))
25359     {
25360 	if(nextNode == parray[0])
25361 	{
25362 	    if(leftNode != BTNO_NODE)
25363 	    {
25364 		page1 = btreePricacheRead(cache,leftNode);
25365 		buf1 = page1->buf;
25366 		GBT_NKEYS(buf1,&n1keys);
25367 		btreeGetKeys(cache,buf1,&k1array,&p1array);
25368 		nextLeft = p1array[n1keys];
25369 	    }
25370 	    else
25371 		nextLeft = BTNO_NODE;
25372 
25373 	    if(!thisNode)
25374 		nextAncL = thisNode;
25375 	    else
25376 		nextAncL = lAnchor;
25377 	}
25378 	else
25379 	{
25380 	    nextLeft = parray[ptrSave-1];
25381 	    nextAncL = thisNode;
25382 	}
25383 
25384 	if(nextNode == parray[nkeys])
25385 	{
25386 	    if(rightNode != BTNO_NODE)
25387 	    {
25388 		page1 = btreePricacheRead(cache,rightNode);
25389 		buf1 = page1->buf;
25390 		GBT_NKEYS(buf1,&n1keys);
25391 		btreeGetKeys(cache,buf1,&k1array,&p1array);
25392 		nextRight = p1array[0];
25393 	    }
25394 	    else
25395 		nextRight = BTNO_NODE;
25396 
25397 	    if(!thisNode)
25398 		nextAncR = thisNode;
25399 	    else
25400 		nextAncR = rAnchor;
25401 	}
25402 	else
25403 	{
25404 	    nextRight = parray[ptrSave+1];
25405 	    nextAncR  = thisNode;
25406 	}
25407 
25408 
25409 
25410 	/* Check to see whether key exists in an internal node */
25411 	if(nodetype != BT_LEAF && cache->plevel)
25412 	{
25413 	    i=0;
25414 
25415 	    while(i!=nkeys && MAJSTRCMPS(btid->id,karray[i]))
25416 		++i;
25417 
25418 	    if(i!=nkeys)
25419 	    {
25420 		btreeFindHybMinOne(cache,parray[i+1],btid->id);
25421 		ajStrAssignS(&karray[i],cache->replace);
25422 		btreeWriteNode(cache,page,karray,parray,nkeys);
25423 	    }
25424 
25425 	}
25426 
25427 	btreeFindIdentBalanceOne(cache,nextNode,nextLeft,nextRight,
25428                                  nextAncL,nextAncR,btid);
25429 
25430 	if(thisNode)
25431 	    page = btreePricacheRead(cache,thisNode);
25432 	else
25433 	{
25434 	    page = btreePricacheLocate(cache,thisNode);
25435 	    page->dirty = BT_LOCK;
25436             page->lockfor = 1722;
25437 	}
25438 	buf = page->buf;
25439 
25440     }
25441     else
25442     {
25443 	if(nodetype == BT_LEAF || (nodetype==BT_ROOT && !cache->plevel))
25444 	{
25445 	    existed = btreeRemoveIdentEntryOne(cache,thisNode,btid);
25446 
25447 	    if(existed)
25448 		cache->dodelete = ajTrue;
25449 
25450 	    GBT_NKEYS(buf,&nkeys);
25451 
25452 	    if(nkeys >= minkeys || (nodetype==BT_ROOT && !cache->plevel))
25453 		balanceNode = BTNO_BALANCE;
25454 	    else
25455 		balanceNode = page->pagepos;
25456 	}
25457     }
25458 
25459 
25460     if(balanceNode == BTNO_BALANCE || thisNode == 0UL)
25461 	done = BTNO_NODE;
25462     else
25463 	done = btreeRebalanceHybOne(cache,thisNode,leftNode,rightNode,
25464                                     lAnchor,rAnchor);
25465 
25466 
25467     btreeDeallocPriArray(cache,arrays);
25468     btreeDeallocPriArray(cache,arrays1);
25469 
25470     return done;
25471 }
25472 
25473 
25474 
25475 
25476 /* @funcstatic btreeFindHybMinOne *********************************************
25477 **
25478 ** Find minimum key in hybrid level 1 subtree and store in cache.
25479 **
25480 ** Deletion software.
25481 **
25482 ** @param [u] cache [AjPBtcache] cache
25483 ** @param [r] pagepos [ajulong] page
25484 ** @param [r] key [const AjPStr] key
25485 **
25486 ** @return [void]
25487 **
25488 ** @release 6.1.0
25489 ** @@
25490 ******************************************************************************/
25491 
btreeFindHybMinOne(AjPBtcache cache,ajulong pagepos,const AjPStr key)25492 static void btreeFindHybMinOne(AjPBtcache cache, ajulong pagepos,
25493                                const AjPStr key)
25494 {
25495     AjPBtpage page   = NULL;
25496     AjPIdbucket bucket = NULL;
25497     AjPStr *karray   = NULL;
25498     ajulong *parray   = NULL;
25499 
25500     ajuint nkeys     = 0;
25501     ajuint  nodetype = 0;
25502     ajuint nentries  = 0;
25503     ajuint i;
25504     AjPBtMem arrays = NULL;
25505 
25506     unsigned char *buf = NULL;
25507 
25508     /* ajDebug("In btreeFindHybMinOne\n"); */
25509 
25510     arrays = btreeAllocPriArray(cache);
25511     karray = arrays->karray;
25512     parray = arrays->parray;
25513 
25514     page = btreePricacheRead(cache,pagepos);
25515     buf  = page->buf;
25516     GBT_NODETYPE(buf,&nodetype);
25517     GBT_NKEYS(buf,&nkeys);
25518 
25519     btreeGetKeys(cache,buf,&karray,&parray);
25520 
25521     if(nodetype == BT_LEAF)
25522     {
25523 	bucket = btreeReadIdbucket(cache,parray[0]);
25524 	nentries = bucket->Nentries;
25525 
25526         /*
25527         ** If there's only one entry then it must be the key marked
25528         ** for deletion
25529         */
25530         if(nentries<2)
25531 	{
25532 	    btreeIdbucketDel(&bucket);
25533 	    bucket = btreeReadIdbucket(cache,parray[1]);
25534 	    nentries = bucket->Nentries;
25535 	}
25536 
25537         /* Check for empty bucket - shouldn't happen */
25538         /* Checking solely out of interest */
25539 	if(nentries<1)
25540 	    ajFatal("FindHybMinOne: Too few entries in bucket Nkeys=%u\n",
25541                     nkeys);
25542 
25543 
25544         /* Find lowest value key in the bucket and store in cache */
25545         ajStrAssignS(&cache->replace,bucket->Ids[0]->id);
25546 	if(!MAJSTRCMPS(cache->replace,key))
25547 	    ajStrAssignS(&cache->replace,bucket->Ids[1]->id);
25548 
25549 	for(i=1;i<nentries;++i)
25550 	    if(MAJSTRCMPS(bucket->Ids[i]->id,cache->replace)<0 &&
25551 	       MAJSTRCMPS(bucket->Ids[i]->id,key))
25552 		ajStrAssignS(&cache->replace,bucket->Ids[i]->id);
25553 	btreeIdbucketDel(&bucket);
25554     }
25555     else
25556     {
25557 	pagepos = parray[0];
25558 	btreeFindHybMinOne(cache,pagepos,key);
25559 
25560     }
25561 
25562     btreeDeallocPriArray(cache,arrays);
25563 
25564     return;
25565 }
25566 
25567 
25568 
25569 
25570 /* @funcstatic btreeRemoveIdentEntryOne ***************************************
25571 **
25572 ** Find and delete an ID from a given identifier tree level 1 leaf node if
25573 ** necessary.
25574 **
25575 ** Deletion software
25576 **
25577 ** @param [u] cache [AjPBtcache] cache
25578 ** @param [r] pagepos [ajulong] leaf node page
25579 ** @param [r] btid [const AjPBtId] id object
25580 **
25581 ** @return [AjBool] True if found (and deleted)
25582 **
25583 ** @release 6.5.0
25584 ** @@
25585 ******************************************************************************/
25586 
btreeRemoveIdentEntryOne(AjPBtcache cache,ajulong pagepos,const AjPBtId btid)25587 static AjBool btreeRemoveIdentEntryOne(AjPBtcache cache, ajulong pagepos,
25588                                        const AjPBtId btid)
25589 {
25590     AjPBtpage page   = NULL;
25591     AjPIdbucket bucket = NULL;
25592 
25593     AjPStr *karray = NULL;
25594     ajulong *parray = NULL;
25595     ajulong blockno = 0UL;
25596 
25597     ajuint nkeys    = 0U;
25598     ajuint nentries = 0U;
25599     ajuint i;
25600 
25601     ajuint dirtysave = 0U;
25602 
25603     AjBool found = ajFalse;
25604 
25605     unsigned char *buf = NULL;
25606     AjPBtMem arrays = NULL;
25607 
25608     /* ajDebug("In btreeRemoveIdentEntryOne\n"); */
25609 
25610     page = btreePricacheRead(cache,pagepos);
25611     buf = page->buf;
25612     dirtysave = page->dirty;
25613     page->dirty = BT_LOCK;
25614     page->lockfor = 1731;
25615 
25616     GBT_NKEYS(buf,&nkeys);
25617 
25618     if(!nkeys)
25619 	return ajFalse;
25620 
25621 
25622     arrays = btreeAllocPriArray(cache);
25623     karray = arrays->karray;
25624     parray = arrays->parray;
25625 
25626     btreeGetKeys(cache,buf,&karray,&parray);
25627 
25628     i=0;
25629 
25630     while(i!=nkeys && MAJSTRCMPS(btid->id,karray[i])>=0)
25631 	++i;
25632 
25633     blockno = parray[i];
25634 
25635     bucket = btreeReadIdbucket(cache,blockno);
25636 
25637     nentries = bucket->Nentries;
25638     found = ajFalse;
25639 
25640     for(i=0;i<nentries;++i)
25641 	if(!MAJSTRCMPS(btid->id,bucket->Ids[i]->id))
25642 	{
25643 	    found = ajTrue;
25644 	    break;
25645 	}
25646 
25647 
25648     if(found)
25649     {
25650 	/* Perform the deletion */
25651 	if(nentries == 1)
25652 	{
25653 	    bucket->Nentries = 0;
25654 	    ajBtreeIdDel(&bucket->Ids[0]);
25655 	}
25656 	else
25657 	{
25658 	    ajBtreeIdDel(&bucket->Ids[i]);
25659 	    bucket->Ids[i] = bucket->Ids[nentries-1];
25660 	    --bucket->Nentries;
25661 	}
25662 
25663 
25664 	btreeWriteIdbucket(cache,bucket,blockno);
25665 
25666 	btreeAdjustHybbucketsOne(cache,page);
25667 
25668 	page->dirty = BT_DIRTY;
25669     }
25670     else
25671 	page->dirty = dirtysave;
25672 
25673     btreeIdbucketDel(&bucket);
25674 
25675     btreeDeallocPriArray(cache,arrays);
25676 
25677     if(!found)
25678 	return ajFalse;
25679 
25680     return ajTrue;
25681 }
25682 
25683 
25684 
25685 
25686 /* @funcstatic btreeAdjustHybbucketsOne ***************************************
25687 **
25688 ** Re-order leaf buckets
25689 ** Can be called whatever the state of a leaf.
25690 **
25691 ** Deletion software
25692 **
25693 ** @param [u] cache [AjPBtcache] cache
25694 ** @param [u] leaf [AjPBtpage] leaf page
25695 **
25696 ** @return [void]
25697 **
25698 ** @release 6.1.0
25699 ** @@
25700 ******************************************************************************/
25701 
btreeAdjustHybbucketsOne(AjPBtcache cache,AjPBtpage leaf)25702 static void btreeAdjustHybbucketsOne(AjPBtcache cache, AjPBtpage leaf)
25703 {
25704     ajuint nkeys = 0;
25705     unsigned char *lbuf = NULL;
25706     AjPIdbucket *buckets  = NULL;
25707 
25708     AjPBtMem arrays = NULL;
25709     AjPStr *keys        = NULL;
25710     ajulong *ptrs        = NULL;
25711     ajulong *overflows   = NULL;
25712 
25713     ajuint i = 0;
25714     ajuint j = 0;
25715     ajuint iref = 0;
25716 
25717     ajuint order;
25718     ajuint bentries      = 0;
25719     ajuint totalkeys     = 0;
25720     ajuint nperbucket    = 0;
25721     ajuint maxnperbucket = 0;
25722     ajuint count         = 0;
25723     ajuint totkeylen     = 0;
25724     ajuint keylimit      = 0;
25725     ajuint bucketn       = 0;
25726     ajuint bucketlimit   = 0;
25727     ajuint nodetype      = 0;
25728     ajuint nids          = 0;
25729     ajuint totnids       = 0;
25730 
25731     AjPList idlist    = NULL;
25732     ajuint  dirtysave = 0;
25733     AjPBtId bid       = NULL;
25734     AjPIdbucket cbucket = NULL;
25735     AjPBtId cid       = NULL;
25736 
25737     ajuint v = 0;
25738     ajuint refskip = cache->refcount*BT_EXTRA;
25739 
25740     /* ajDebug("In btreeAdjustHybbucketsOne\n"); */
25741 
25742     dirtysave = leaf->dirty;
25743 
25744     leaf->dirty = BT_LOCK;
25745     leaf->lockfor = 1741;
25746     lbuf = leaf->buf;
25747 
25748     GBT_NKEYS(lbuf,&nkeys);
25749 
25750     if(!nkeys)
25751     {
25752 	leaf->dirty = dirtysave;
25753 	return;
25754     }
25755 
25756 
25757     GBT_NODETYPE(lbuf,&nodetype);
25758 
25759     order = cache->porder;
25760     nperbucket = cache->pnperbucket;
25761 
25762 
25763     /* Read keys/ptrs */
25764 
25765     arrays = btreeAllocPriArray(cache);
25766     keys = arrays->karray;
25767     ptrs = arrays->parray;
25768     overflows = arrays->overflows;
25769 
25770     btreeGetKeys(cache,lbuf,&keys,&ptrs);
25771 
25772     for(i=0;i<=nkeys;++i)
25773 	totalkeys += btreeIdbucketCount(cache,ptrs[i]);
25774 
25775     /* Set the number of entries per bucket to approximately half full */
25776     maxnperbucket = nperbucket >> 1;
25777 
25778     if(!maxnperbucket)
25779 	++maxnperbucket;
25780 
25781     if(!leaf->pagepos)
25782 	maxnperbucket = nperbucket;
25783 
25784     /* Work out the number of new buckets needed */
25785     bucketn = (totalkeys / maxnperbucket);
25786     if(totalkeys % maxnperbucket)
25787 	++bucketn;
25788 
25789     if(bucketn == 1)
25790 	++bucketn;
25791 
25792     while(bucketn > order)
25793     {
25794         ++maxnperbucket;
25795         bucketn = (totalkeys / maxnperbucket);
25796 
25797         if(totalkeys % maxnperbucket)
25798             ++bucketn;
25799     }
25800 
25801     /* Read buckets */
25802     AJCNEW0(buckets,nkeys+1);
25803     keylimit = nkeys + 1;
25804 
25805     for(i=0;i<keylimit;++i)
25806 	buckets[i] = btreeReadIdbucket(cache,ptrs[i]);
25807 
25808 
25809     /* Read IDs from all buckets and push to list and sort (increasing id) */
25810     idlist  = ajListNew();
25811 
25812     for(i=0;i<keylimit;++i)
25813     {
25814 	overflows[i] = buckets[i]->Overflow;
25815 	bentries = buckets[i]->Nentries;
25816 
25817 	for(j=0;j<bentries;++j)
25818 	    ajListPush(idlist,(void *)buckets[i]->Ids[j]);
25819 
25820 	AJFREE(buckets[i]->keylen);
25821 	AJFREE(buckets[i]->Ids);
25822 	AJFREE(buckets[i]);
25823     }
25824 
25825     ajListSort(idlist, &btreeIdCompare);
25826     AJFREE(buckets);
25827 
25828     cbucket = btreeIdbucketNew(maxnperbucket, cache->refcount);
25829     bucketlimit = bucketn - 1;
25830 
25831     totnids = 0;
25832     nids = (ajuint) ajListGetLength(idlist);
25833 
25834 
25835     if(!totalkeys)
25836     {
25837 	v = totalkeys;
25838 	SBT_NKEYS(lbuf,v);
25839 
25840         btreeDeallocPriArray(cache,arrays);
25841 
25842 	ajListFree(&idlist);
25843 	leaf->dirty = BT_DIRTY;
25844 
25845 	return;
25846     }
25847 
25848     if(nids <= maxnperbucket)
25849     {
25850 	cbucket->Overflow = overflows[1];
25851 	cbucket->Nentries = 0;
25852 	ajListPeek(idlist,(void **)&bid);
25853 	ajStrAssignS(&keys[0],bid->id);
25854 
25855 	count = 0;
25856 
25857 	while(count!=maxnperbucket && totnids != nids)
25858 	{
25859 	    ajListPop(idlist,(void **)&bid);
25860 
25861 	    cid = cbucket->Ids[count];
25862 	    ajStrAssignS(&cid->id,bid->id);
25863 	    cid->dbno = bid->dbno;
25864 	    cid->dups = bid->dups;
25865 	    cid->offset = bid->offset;
25866 
25867 	    if(cache->refcount)
25868             {
25869                 for(iref=0; iref < cache->refcount; iref++)
25870                     cid->refoffsets[iref] = bid->refoffsets[iref];
25871             }
25872 
25873 	    cbucket->keylen[count] = BT_BUCKIDLEN(bid->id) + refskip;
25874 	    ++cbucket->Nentries;
25875 	    ++count;
25876 	    ++totnids;
25877 	    ajBtreeIdDel(&bid);
25878 	}
25879 
25880 
25881 	totkeylen += ajStrGetLen(keys[0]);
25882 
25883 	if(!ptrs[1])
25884 	    ptrs[1] = cache->totsize;
25885 
25886 	btreeWriteIdbucket(cache,cbucket,ptrs[1]);
25887 
25888 	cbucket->Overflow = overflows[0];
25889 	cbucket->Nentries = 0;
25890 
25891 	if(!ptrs[0])
25892 	    ptrs[0] = cache->totsize;
25893 
25894 	btreeWriteIdbucket(cache,cbucket,ptrs[0]);
25895     }
25896     else
25897     {
25898 	for(i=0;i<bucketlimit;++i)
25899 	{
25900 	    cbucket->Overflow = overflows[i];
25901 	    cbucket->Nentries = 0;
25902 
25903 	    count = 0;
25904 
25905 	    while(count!=maxnperbucket && totnids != nids)
25906 	    {
25907 		ajListPop(idlist,(void **)&bid);
25908 
25909 		cid = cbucket->Ids[count];
25910 		ajStrAssignS(&cid->id,bid->id);
25911 		cid->dbno = bid->dbno;
25912 		cid->dups = bid->dups;
25913 		cid->offset = bid->offset;
25914 
25915                 if(cache->refcount)
25916                 {
25917                     for(iref=0; iref < cache->refcount; iref++)
25918                         cid->refoffsets[iref] = bid->refoffsets[iref];
25919                 }
25920 
25921 		cbucket->keylen[count] = BT_BUCKIDLEN(bid->id) + refskip;
25922 		++cbucket->Nentries;
25923 		++count;
25924 		ajBtreeIdDel(&bid);
25925 	    }
25926 
25927 
25928 	    ajListPeek(idlist,(void **)&bid);
25929 	    ajStrAssignS(&keys[i],bid->id);
25930 
25931 
25932 	    totkeylen += ajStrGetLen(bid->id);
25933 
25934 	    if(!ptrs[i])
25935 		ptrs[i] = cache->totsize;
25936 	    btreeWriteIdbucket(cache,cbucket,ptrs[i]);
25937 	}
25938 
25939 
25940 	/* Deal with greater-than bucket */
25941 
25942 	cbucket->Overflow = overflows[i];
25943 	cbucket->Nentries = 0;
25944 
25945 
25946 
25947 	count = 0;
25948 
25949 	while(ajListPop(idlist,(void **)&bid))
25950 	{
25951 	    cid = cbucket->Ids[count];
25952 	    ajStrAssignS(&cid->id,bid->id);
25953 	    cid->dbno = bid->dbno;
25954 	    cid->dups = bid->dups;
25955 	    cid->offset = bid->offset;
25956 
25957             if(cache->refcount)
25958             {
25959                 for(iref=0; iref < cache->refcount; iref++)
25960                     cid->refoffsets[iref] = bid->refoffsets[iref];
25961             }
25962 
25963 	    ++cbucket->Nentries;
25964 	    ++count;
25965 	    ajBtreeIdDel(&bid);
25966 	}
25967 
25968 
25969 	if(!ptrs[i])
25970 	    ptrs[i] = cache->totsize;
25971 
25972 	btreeWriteIdbucket(cache,cbucket,ptrs[i]);
25973     }
25974 
25975 
25976     cbucket->Nentries = maxnperbucket;
25977     btreeIdbucketDel(&cbucket);
25978 
25979     /* Now write out a modified leaf with new keys/ptrs */
25980 
25981     nkeys = bucketn - 1;
25982 
25983     btreeWriteNode(cache,leaf,keys,ptrs,nkeys);
25984 
25985     leaf->dirty = dirtysave;
25986     if(nodetype == BT_ROOT)
25987     {
25988 	leaf->dirty = BT_LOCK;
25989         leaf->lockfor = 1742;
25990     }
25991 
25992     btreeDeallocPriArray(cache,arrays);
25993 
25994     btreeIdbucketDel(&cbucket);
25995     ajListFree(&idlist);
25996 
25997     return;
25998 }
25999 
26000 
26001 
26002 
26003 /* @funcstatic btreeRebalanceHybOne *******************************************
26004 **
26005 ** Rebalance Hybrid level 1 tree after deletion
26006 **
26007 ** Deletion software
26008 **
26009 ** @param [u] cache [AjPBtcache] cache
26010 ** @param [r] thisNode [ajulong] Node to rebalance
26011 ** @param [r] leftNode [ajulong] left node
26012 ** @param [r] rightNode [ajulong] right node
26013 ** @param [r] lAnchor [ajulong] left anchor
26014 ** @param [r] rAnchor [ajulong] right anchor
26015 **
26016 ** @return [ajulong] page number or BTNO_NODE
26017 **
26018 ** @release 6.1.0
26019 ** @@
26020 ******************************************************************************/
26021 
btreeRebalanceHybOne(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor)26022 static ajulong btreeRebalanceHybOne(AjPBtcache cache, ajulong thisNode,
26023                                     ajulong leftNode, ajulong rightNode,
26024                                     ajulong lAnchor, ajulong rAnchor)
26025 {
26026     unsigned char *lbuf = NULL;
26027     unsigned char *rbuf = NULL;
26028     unsigned char *tbuf = NULL;
26029 
26030     ajulong anchorNode   = 0UL;
26031     ajulong balanceNode  = 0UL;
26032     ajulong mergeNode    = 0UL;
26033     ajulong done         = 0UL;
26034     ajulong parent       = 0UL;
26035 
26036     AjPBtpage lpage = NULL;
26037     AjPBtpage rpage = NULL;
26038     AjPBtpage tpage = NULL;
26039 
26040     ajuint lnkeys  = 0U;
26041     ajuint rnkeys  = 0U;
26042     ajuint size    = 0U;
26043     ajuint order   = 0U;
26044     ajuint minsize = 0U;
26045 
26046     AjBool leftok  = ajFalse;
26047     AjBool rightok = ajFalse;
26048 
26049 
26050     /* ajDebug("In btreeRebalanceHybOne\n"); */
26051 
26052     if(leftNode!=BTNO_NODE && lAnchor!=BTNO_NODE)
26053 	leftok = ajTrue;
26054 
26055     if(rightNode!=BTNO_NODE && rAnchor!=BTNO_NODE)
26056 	rightok = ajTrue;
26057 
26058     if(!leftok && !rightok)
26059 	return BTNO_NODE;
26060 
26061 
26062     if(leftok)
26063     {
26064 	lpage = btreePricacheRead(cache,leftNode);
26065 	lbuf  = lpage->buf;
26066 	GBT_NKEYS(lbuf,&lnkeys);
26067     }
26068 
26069 
26070     if(rightok)
26071     {
26072 	rpage = btreePricacheRead(cache,rightNode);
26073 	rbuf  = rpage->buf;
26074 	GBT_NKEYS(rbuf,&rnkeys);
26075     }
26076 
26077 
26078 
26079     if(leftok && rightok)
26080     {
26081 	size = (lnkeys >= rnkeys) ? lnkeys : rnkeys;
26082 	balanceNode = (lnkeys >= rnkeys) ? leftNode : rightNode;
26083     }
26084     else if(leftok)
26085     {
26086 	size = lnkeys;
26087 	balanceNode = leftNode;
26088     }
26089     else
26090     {
26091 	size = rnkeys;
26092 	balanceNode = rightNode;
26093     }
26094 
26095 
26096     order = cache->porder;
26097     minsize = (order-1) / 2;
26098 
26099     if((order-1)%2)
26100 	++minsize;
26101 
26102     if(size >= minsize)
26103     {
26104 	if(leftok && rightok)
26105 	    anchorNode = (lnkeys >= rnkeys) ? lAnchor : rAnchor;
26106 	else if(leftok)
26107 	    anchorNode = lAnchor;
26108 	else
26109 	    anchorNode = rAnchor;
26110 
26111 	done = btreeShiftHybOne(cache,thisNode,balanceNode,anchorNode);
26112     }
26113 
26114     else
26115     {
26116 	tpage = btreePricacheRead(cache,thisNode);
26117 	tbuf  = tpage->buf;
26118 	GBT_PREV(tbuf,&parent);
26119 
26120 	if(leftok && rightok)
26121 	{
26122 	    anchorNode = (parent == lAnchor) ? lAnchor : rAnchor;
26123 	    mergeNode  = (anchorNode == lAnchor) ? leftNode : rightNode;
26124 	}
26125 	else if(leftok)
26126 	{
26127 	    anchorNode = lAnchor;
26128 	    mergeNode  = leftNode;
26129 	}
26130 	else
26131 	{
26132 	    anchorNode = rAnchor;
26133 	    mergeNode  = rightNode;
26134 	}
26135 
26136 	done = btreeMergeHybOne(cache,thisNode,mergeNode,anchorNode);
26137     }
26138 
26139     return done;
26140 }
26141 
26142 
26143 
26144 
26145 /* @funcstatic btreeShiftHybOne ***********************************************
26146 **
26147 ** Shift spare entries from one hybrid tree level 1 node to another.
26148 **
26149 ** Deletion software.
26150 **
26151 ** @param [u] cache [AjPBtcache] cache
26152 ** @param [r] thisNode [ajulong] master node
26153 ** @param [r] balanceNode [ajulong] balance node
26154 ** @param [r] anchorNode [ajulong] anchor node
26155 **
26156 ** @return [ajulong] page number or BTNO_NODE
26157 **
26158 ** @release 6.1.0
26159 ** @@
26160 ******************************************************************************/
26161 
btreeShiftHybOne(AjPBtcache cache,ajulong thisNode,ajulong balanceNode,ajulong anchorNode)26162 static ajulong btreeShiftHybOne(AjPBtcache cache, ajulong thisNode,
26163                                 ajulong balanceNode, ajulong anchorNode)
26164 {
26165     unsigned char *tbuf = NULL;
26166     unsigned char *abuf = NULL;
26167     unsigned char *bbuf = NULL;
26168     unsigned char *buf  = NULL;
26169 
26170     AjPStr *kTarray = NULL;
26171     AjPStr *kAarray = NULL;
26172     AjPStr *kBarray = NULL;
26173     ajulong *pTarray = NULL;
26174     ajulong *pAarray = NULL;
26175     ajulong *pBarray = NULL;
26176 
26177     ajuint  nAkeys = 0U;
26178     ajuint  nBkeys = 0U;
26179     ajuint  nTkeys = 0U;
26180     ajuint  i;
26181     ajint ii;
26182 
26183     AjPBtpage pageA = NULL;
26184     AjPBtpage pageB = NULL;
26185     AjPBtpage pageT = NULL;
26186     AjPBtpage page  = NULL;
26187 
26188     AjPBtpage leftpage = NULL;
26189 
26190     ajuint anchorPos   = 0U;
26191     ajulong prev        = 0UL;
26192     ajuint  nodetype    = 0U;
26193 
26194     ajulong lv = 0UL;
26195 
26196     AjPBtMem arraysA = NULL;
26197     AjPBtMem arraysB = NULL;
26198     AjPBtMem arraysT = NULL;
26199 
26200     /* ajDebug("In btreeShiftHybOne\n"); */
26201 
26202 
26203     arraysA = btreeAllocPriArray(cache);
26204     kAarray = arraysA->karray;
26205     pAarray = arraysA->parray;
26206 
26207     arraysB = btreeAllocPriArray(cache);
26208     kBarray = arraysB->karray;
26209     pBarray = arraysB->parray;
26210 
26211     arraysT = btreeAllocPriArray(cache);
26212     kTarray = arraysT->karray;
26213     pTarray = arraysT->parray;
26214 
26215 
26216     pageA = btreePricacheRead(cache,anchorNode);
26217     pageA->dirty = BT_LOCK;
26218     pageA->lockfor = 1751;
26219     abuf = pageA->buf;
26220     pageB = btreePricacheRead(cache,balanceNode);
26221     pageB->dirty = BT_LOCK;
26222     pageB->lockfor = 1752;
26223     bbuf = pageB->buf;
26224     pageT = btreePricacheRead(cache,thisNode);
26225     pageT->dirty = BT_LOCK;
26226     pageT->lockfor = 1753;
26227     tbuf = pageT->buf;
26228 
26229     GBT_NKEYS(abuf,&nAkeys);
26230     GBT_NKEYS(bbuf,&nBkeys);
26231     GBT_NKEYS(tbuf,&nTkeys);
26232 
26233     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
26234     btreeGetKeys(cache,bbuf,&kBarray,&pBarray);
26235     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
26236 
26237     if(MAJSTRCMPS(kTarray[nTkeys-1],kBarray[nBkeys-1])<0)
26238 	leftpage = pageT;
26239     else
26240 	leftpage = pageB;
26241 
26242 
26243     if(leftpage == pageT)
26244     {
26245 	/* Find anchor key position */
26246 	i=0;
26247 
26248 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
26249 	    ++i;
26250 
26251 	anchorPos = i;
26252 
26253 	/* Move down anchor key to thisNode */
26254 	ajStrAssignS(&kTarray[nTkeys],kAarray[anchorPos]);
26255 	++nTkeys;
26256 
26257 	/* Shift extra */
26258 	while(nTkeys < nBkeys)
26259 	{
26260 	    ajStrAssignS(&kTarray[nTkeys],kBarray[0]);
26261 	    pTarray[nTkeys] = pBarray[0];
26262 	    ++nTkeys;
26263 	    --nBkeys;
26264 
26265 	    for(i=0;i<nBkeys;++i)
26266 	    {
26267 		ajStrAssignS(&kBarray[i],kBarray[i+1]);
26268 		pBarray[i] = pBarray[i+1];
26269 	    }
26270 	    pBarray[i] = pBarray[i+1];
26271 	}
26272 
26273 	/* Adjust anchor key */
26274 	ajStrAssignS(&kAarray[anchorPos],kTarray[nTkeys-1]);
26275 	--nTkeys;
26276     }
26277     else	/* thisNode on the right */
26278     {
26279 	/* Find anchor key position */
26280 	i=0;
26281 
26282 	while(i!=nAkeys && MAJSTRCMPS(kBarray[nBkeys-1],kAarray[i])>=0)
26283 	    ++i;
26284 
26285 	anchorPos = i;
26286 
26287 	/* Move down anchor key to thisNode */
26288 	pTarray[nTkeys+1] = pTarray[nTkeys];
26289 
26290 	for(ii=nTkeys-1;ii>-1;--ii)
26291 	{
26292 	    ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
26293 	    pTarray[ii+1] = pTarray[ii];
26294 	}
26295 
26296 	ajStrAssignS(&kTarray[0],kAarray[anchorPos]);
26297 	++nTkeys;
26298 
26299 	/* Shift extra */
26300 	while(nTkeys < nBkeys)
26301 	{
26302 	    pTarray[nTkeys+1] = pTarray[nTkeys];
26303 
26304 	    for(ii=nTkeys-1;ii>-1;--ii)
26305 	    {
26306 		ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
26307 		pTarray[ii+1] = pTarray[ii];
26308 	    }
26309 
26310 	    ajStrAssignS(&kTarray[0],kBarray[nBkeys-1]);
26311 	    pTarray[1] = pBarray[nBkeys];
26312 	    ++nTkeys;
26313 	    --nBkeys;
26314 	}
26315 
26316 
26317 	/* Adjust anchor key */
26318 	ajStrAssignS(&kAarray[anchorPos],kTarray[0]);
26319 	--nTkeys;
26320 
26321 	for(i=0;i<nTkeys;++i)
26322 	{
26323 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
26324 	    pTarray[i] = pTarray[i+1];
26325 	}
26326 
26327 	pTarray[i] = pTarray[i+1];
26328     }
26329 
26330 
26331     /* Adjust PREV pointers for thisNode */
26332     prev = pageT->pagepos;
26333 
26334     for(i=0;i<nTkeys+1;++i)
26335     {
26336 	page = btreePricacheRead(cache,pTarray[i]);
26337 	buf = page->buf;
26338 	GBT_NODETYPE(buf,&nodetype);
26339 
26340 	if(nodetype != BT_IDBUCKET)
26341 	{
26342 	    lv = prev;
26343 	    SBT_PREV(buf,lv);
26344 	    page->dirty = BT_DIRTY;
26345 	}
26346     }
26347 
26348     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
26349     btreeWriteNode(cache,pageB,kBarray,pBarray,nBkeys);
26350     btreeWriteNode(cache,pageT,kTarray,pTarray,nTkeys);
26351 
26352     if(!anchorNode)
26353     {
26354 	pageA->dirty = BT_LOCK;
26355         pageA->lockfor = 1754;
26356     }
26357 
26358     btreeDeallocPriArray(cache,arraysA);
26359     btreeDeallocPriArray(cache,arraysB);
26360     btreeDeallocPriArray(cache,arraysT);
26361 
26362     return BTNO_NODE;
26363 }
26364 
26365 
26366 
26367 
26368 /* @funcstatic btreeMergeHybOne ***********************************************
26369 **
26370 ** Merge two nodes.
26371 **
26372 ** Deletion software
26373 **
26374 ** @param [u] cache [AjPBtcache] cache
26375 ** @param [r] thisNode [ajulong] master node
26376 ** @param [r] mergeNode [ajulong] merge node
26377 ** @param [r] anchorNode [ajulong] anchor node
26378 **
26379 ** @return [ajulong] page number or BTNO_NODE
26380 **
26381 ** @release 6.1.0
26382 ** @@
26383 ******************************************************************************/
26384 
btreeMergeHybOne(AjPBtcache cache,ajulong thisNode,ajulong mergeNode,ajulong anchorNode)26385 static ajulong btreeMergeHybOne(AjPBtcache cache, ajulong thisNode,
26386                                 ajulong mergeNode, ajulong anchorNode)
26387 {
26388     unsigned char *tbuf = NULL;
26389     unsigned char *abuf = NULL;
26390     unsigned char *nbuf = NULL;
26391     unsigned char *buf  = NULL;
26392 
26393     AjPStr *kTarray = NULL;
26394     AjPStr *kAarray = NULL;
26395     AjPStr *kNarray = NULL;
26396     ajulong *pTarray = NULL;
26397     ajulong *pAarray = NULL;
26398     ajulong *pNarray = NULL;
26399 
26400     ajulong thisprev  = 0UL;
26401     ajulong mergeprev = 0UL;
26402 
26403 
26404     ajuint  nAkeys = 0U;
26405     ajuint  nNkeys = 0U;
26406     ajuint  nTkeys = 0U;
26407     ajuint  count  = 0U;
26408     ajuint  i;
26409     ajint ii;
26410 
26411     ajuint   nodetype = 0U;
26412 
26413     ajuint saveA = 0U;
26414     ajuint saveN = 0U;
26415     ajuint saveT = 0U;
26416 
26417     AjPBtpage pageA = NULL;
26418     AjPBtpage pageN = NULL;
26419     AjPBtpage pageT = NULL;
26420     AjPBtpage page  = NULL;
26421 
26422     AjPBtpage leftpage = NULL;
26423 
26424     ajuint anchorPos = 0U;
26425     ajulong prev      = 0UL;
26426 
26427     ajulong lv = 0UL;
26428 
26429     AjPBtMem arraysA = NULL;
26430     AjPBtMem arraysN = NULL;
26431     AjPBtMem arraysT = NULL;
26432 
26433     AjBool collapse = ajFalse;
26434 
26435     /* ajDebug("In btreeMergeHybOne\n"); */
26436 
26437     pageA = btreePricacheRead(cache,anchorNode);
26438     saveA = pageA->dirty;
26439     pageA->dirty = BT_LOCK;
26440     pageA->lockfor = 1761;
26441     abuf = pageA->buf;
26442     pageN = btreePricacheRead(cache,mergeNode);
26443     saveN = pageN->dirty;
26444     pageN->dirty = BT_LOCK;
26445     pageN->lockfor = 1762;
26446     nbuf = pageN->buf;
26447     pageT = btreePricacheRead(cache,thisNode);
26448     saveT = pageT->dirty;
26449     pageT->dirty = BT_LOCK;
26450     pageT->lockfor = 1763;
26451     tbuf = pageT->buf;
26452 
26453     GBT_PREV(tbuf,&thisprev);
26454     GBT_PREV(nbuf,&mergeprev);
26455 
26456     GBT_NKEYS(abuf,&nAkeys);
26457     GBT_NKEYS(nbuf,&nNkeys);
26458     GBT_NKEYS(tbuf,&nTkeys);
26459 
26460     GBT_NODETYPE(nbuf,&nodetype);
26461 
26462 
26463     if(nAkeys == 1)
26464     {
26465 	if(!anchorNode && !thisprev && !mergeprev)
26466 	    collapse = ajTrue;
26467 	else
26468 	{
26469 	    pageA->dirty = saveA;
26470 	    pageN->dirty = saveN;
26471 	    pageT->dirty = saveT;
26472 	    return thisNode;
26473 	}
26474     }
26475 
26476     arraysA = btreeAllocPriArray(cache);
26477     kAarray = arraysA->karray;
26478     pAarray = arraysA->parray;
26479 
26480     arraysN = btreeAllocPriArray(cache);
26481     kNarray = arraysN->karray;
26482     pNarray = arraysN->parray;
26483 
26484     arraysT = btreeAllocPriArray(cache);
26485     kTarray = arraysT->karray;
26486     pTarray = arraysT->parray;
26487 
26488     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
26489     btreeGetKeys(cache,nbuf,&kNarray,&pNarray);
26490     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
26491 
26492     if(MAJSTRCMPS(kTarray[nTkeys-1],kNarray[nNkeys-1])<0)
26493 	leftpage = pageT;
26494     else
26495 	leftpage = pageN;
26496 
26497 
26498     if(leftpage == pageT)
26499     {
26500 	/* Find anchor key position */
26501 	i=0;
26502 
26503 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
26504 	    ++i;
26505 
26506 	anchorPos = i;
26507 
26508 	/* Move down anchor key to neighbour Node */
26509 	pNarray[nNkeys+1] = pNarray[nNkeys];
26510 
26511 	for(ii=nNkeys-1;ii>-1;--ii)
26512 	{
26513 	    ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
26514 	    pNarray[ii+1] = pNarray[ii];
26515 	}
26516 	ajStrAssignS(&kNarray[0],kAarray[anchorPos]);
26517 	++nNkeys;
26518 
26519 
26520 	/* Adjust anchor node keys/ptrs */
26521 	++anchorPos;
26522 
26523 	if(anchorPos==nAkeys)
26524 	    pAarray[nAkeys-1] = pAarray[nAkeys];
26525 	else
26526 	{
26527 	    for(i=anchorPos;i<nAkeys;++i)
26528 	    {
26529 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
26530 		pAarray[i-1] = pAarray[i];
26531 	    }
26532 
26533 	    pAarray[i-1] = pAarray[i];
26534 	}
26535 	--nAkeys;
26536 
26537 
26538 	/* Merge this to neighbour */
26539 
26540 	while(nTkeys)
26541 	{
26542 	    pNarray[nNkeys+1] = pNarray[nNkeys];
26543 
26544 	    for(ii=nNkeys-1;ii>-1;--ii)
26545 	    {
26546 		ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
26547 		pNarray[ii+1] = pNarray[ii];
26548 	    }
26549 
26550 	    ajStrAssignS(&kNarray[0],kTarray[nTkeys-1]);
26551 	    pNarray[1] = pTarray[nTkeys];
26552 	    pNarray[0] = pTarray[nTkeys-1];
26553 	    --nTkeys;
26554 	    ++nNkeys;
26555 	}
26556 
26557 	/* At this point the 'this' node could be added to a free list */
26558     }
26559     else
26560     {
26561 	/* Find anchor key position */
26562 	i=0;
26563 
26564 	while(i!=nAkeys && MAJSTRCMPS(kNarray[nNkeys-1],kAarray[i])>=0)
26565 	    ++i;
26566 
26567 	anchorPos = i;
26568 
26569 	/* Move down anchor key to neighbourNode */
26570 	ajStrAssignS(&kNarray[nNkeys],kAarray[anchorPos]);
26571 	++nNkeys;
26572 
26573 	/* Adjust anchor node keys/ptrs */
26574 	++anchorPos;
26575 
26576 	if(anchorPos!=nAkeys)
26577 	    for(i=anchorPos;i<nAkeys;++i)
26578 	    {
26579 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
26580 		pAarray[i] = pAarray[i+1];
26581 	    }
26582 
26583 	--nAkeys;
26584 
26585 	/* merge extra */
26586 	count = 0;
26587 
26588 	while(nTkeys)
26589 	{
26590 	    ajStrAssignS(&kNarray[nNkeys],kTarray[count]);
26591 	    pNarray[nNkeys] = pTarray[count];
26592 	    ++nNkeys;
26593 	    ++count;
26594 	    --nTkeys;
26595 	    pNarray[nNkeys] = pTarray[count];
26596 
26597 	}
26598 
26599 	/* At this point the 'this' node could be added to a free list */
26600     }
26601 
26602 
26603     /* Adjust PREV pointers for neighbour Node */
26604     prev = pageN->pagepos;
26605 
26606     for(i=0;i<=nNkeys;++i)
26607     {
26608 	page = btreePricacheRead(cache,pNarray[i]);
26609 	buf = page->buf;
26610 	GBT_NODETYPE(buf,&nodetype);
26611 
26612 	if(nodetype != BT_IDBUCKET)
26613 	{
26614 	    lv = prev;
26615 	    SBT_PREV(buf,lv);
26616 	    page->dirty = BT_DIRTY;
26617 	}
26618     }
26619 
26620     pageT->dirty = BT_CLEAN;
26621     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
26622     btreeWriteNode(cache,pageN,kNarray,pNarray,nNkeys);
26623 
26624     if(!anchorNode)
26625     {
26626 	pageA->dirty = BT_LOCK;
26627         pageA->lockfor = 1764;
26628     }
26629 
26630     btreeDeallocPriArray(cache,arraysA);
26631     btreeDeallocPriArray(cache,arraysN);
26632     btreeDeallocPriArray(cache,arraysT);
26633 
26634     if(collapse)
26635 	btreeCollapseRootHybOne(cache,mergeNode);
26636 
26637     return thisNode;
26638 }
26639 
26640 
26641 
26642 
26643 /* @funcstatic btreeCollapseRootHybOne ****************************************
26644 **
26645 ** Collapse root page for hybrid level 1 tree.
26646 **
26647 ** Deletion software.
26648 **
26649 ** @param [u] cache [AjPBtcache] cache
26650 ** @param [r] pagepos [ajulong] page number to make new root
26651 **
26652 ** @return [ajulong] page number or BTNO_NODE
26653 **
26654 ** @release 6.1.0
26655 ** @@
26656 ******************************************************************************/
26657 
btreeCollapseRootHybOne(AjPBtcache cache,ajulong pagepos)26658 static ajulong btreeCollapseRootHybOne(AjPBtcache cache, ajulong pagepos)
26659 {
26660     unsigned char *buf  = NULL;
26661     unsigned char *lbuf = NULL;
26662 
26663     AjPStr *karray = NULL;
26664     ajulong *parray = NULL;
26665 
26666     AjPBtpage rootpage = NULL;
26667     AjPBtpage page     = NULL;
26668 
26669     ajuint nodetype = 0U;
26670     ajuint nkeys    = 0U;
26671     ajuint i;
26672 
26673     ajulong prev = 0UL;
26674     AjPBtMem arrays = NULL;
26675 
26676     /* ajDebug("In btreeCollapseRootHybOne\n"); */
26677 
26678     if(!cache->plevel)
26679 	return BTNO_NODE;
26680 
26681     rootpage = btreePricacheLocate(cache,0UL);
26682     buf = rootpage->buf;
26683     page = btreePricacheRead(cache,pagepos);
26684 
26685 
26686     arrays = btreeAllocPriArray(cache);
26687     karray = arrays->karray;
26688     parray = arrays->parray;
26689 
26690     /*
26691     ** Swap pagepos values to make root the child and child the root
26692     ** Update node types and mark the original root as a clean page
26693     */
26694 
26695     /* At this point page->pagepos could be added to a free list */
26696 
26697     rootpage->pagepos = page->pagepos;
26698     rootpage->dirty = BT_CLEAN;
26699     nodetype = BT_INTERNAL;
26700     SBT_NODETYPE(buf,nodetype);
26701 
26702     page->pagepos = 0;
26703     page->dirty = BT_LOCK;
26704     page->lockfor = 1771;
26705     buf = page->buf;
26706     nodetype = BT_ROOT;
26707     SBT_NODETYPE(buf,nodetype);
26708 
26709     --cache->plevel;
26710 
26711     if(cache->plevel)
26712     {
26713 	/*
26714 	 ** Update the PREV pointers of the new root's children
26715 	 */
26716 	GBT_NKEYS(buf,&nkeys);
26717 	btreeGetKeys(cache,buf,&karray,&parray);
26718 	for(i=0;i<=nkeys;++i)
26719 	{
26720 	    page = btreePricacheRead(cache,parray[i]);
26721 	    lbuf = page->buf;
26722 	    SBT_PREV(lbuf,prev);
26723 	    page->dirty = BT_DIRTY;
26724 	}
26725     }
26726 
26727     btreeDeallocPriArray(cache,arrays);
26728 
26729     return 0UL;
26730 }
26731 
26732 
26733 
26734 
26735 /* @funcstatic btreeDeleteIdentIdTwo ******************************************
26736 **
26737 ** Entry point for secondary hybrid tree ID deletion.
26738 ** Assumes cache->secrootblock has been initialised.
26739 **
26740 ** Deletion software
26741 **
26742 ** @param [u] cache [AjPBtcache] cache
26743 ** @param [r] btid [const AjPBtId] ID object
26744 ** @param [u] did [AjPBtId] ID object in primary tree
26745 **
26746 ** @return [AjBool] True if found and deleted
26747 **
26748 ** @release 6.1.0
26749 ** @@
26750 ******************************************************************************/
26751 
btreeDeleteIdentIdTwo(AjPBtcache cache,const AjPBtId btid,AjPBtId did)26752 static AjBool btreeDeleteIdentIdTwo(AjPBtcache cache, const AjPBtId btid,
26753                                     AjPBtId did)
26754 {
26755     AjPBtpage rpage = NULL;
26756     AjPBtpage page  = NULL;
26757 
26758     ajulong sval = 0UL;
26759     ajulong key  = 0UL;
26760     ajulong *karray = NULL;
26761     ajulong *parray = NULL;
26762     ajulong blockno = 0UL;
26763 
26764     unsigned char *rbuf = NULL;
26765     unsigned char *buf  = NULL;
26766 
26767     ajuint nkeys    = 0U;
26768     ajuint nentries = 0U;
26769 
26770     ajuint i;
26771     ajuint iref;
26772 
26773     AjBool found = ajFalse;
26774 
26775     AjPBtNumId num      = NULL;
26776     AjPNumbucket bucket = NULL;
26777 
26778     AjPBtMem array = NULL;
26779 
26780     rpage = btreeSeccacheWrite(cache,cache->secrootblock);
26781 
26782     rpage->dirty = BT_LOCK;
26783     rpage->lockfor = 1781;
26784     rbuf = rpage->buf;
26785     GBT_RIGHT(rbuf,&sval);
26786     cache->slevel = (ajuint) sval;
26787 
26788     key  = btid->offset;
26789 
26790     page = btreeNumFind(cache,key);
26791     buf  = page->buf;
26792 
26793     GBT_NKEYS(buf,&nkeys);
26794 
26795     array  = btreeAllocSecArray(cache);
26796 
26797     karray = array->overflows;
26798     parray = array->parray;
26799 
26800     if(!nkeys)
26801     {
26802         ajWarn("btreeDeleteIdentIdTwo: No keys in findinsert node");
26803         btreeDeallocSecArray(cache,array);
26804         rpage->dirty = BT_CLEAN;
26805 
26806         return ajFalse;
26807     }
26808 
26809     btreeGetNumKeys(cache,buf,&karray,&parray);
26810 
26811     i = 0;
26812 
26813     while(i != nkeys && key >= karray[i])
26814         ++i;
26815 
26816     blockno = parray[i];
26817 
26818     bucket = btreeReadNumbucket(cache,blockno);
26819     nentries = bucket->Nentries;
26820 
26821     found = ajFalse;
26822 
26823     for(i=0; i < nentries; ++i)
26824     {
26825         num = bucket->NumId[i];
26826 
26827         if(num->offset == key)
26828         {
26829             found = ajTrue;
26830             break;
26831         }
26832     }
26833 
26834     if(!found)
26835     {
26836         ajWarn("btreeDeleteIdentIdTwo: Numeric key not in bucket");
26837         btreeDeallocSecArray(cache,array);
26838         rpage->dirty = BT_CLEAN;
26839 
26840         return ajFalse;
26841     }
26842 
26843     btreeFindHybBalanceTwo(cache,cache->secrootblock,BTNO_NODE,BTNO_NODE,
26844                            BTNO_NODE,BTNO_NODE,key);
26845 
26846     if(!cache->dodelete)
26847     {
26848         ajWarn("btreeDeleteIdentIdTwo: entry %Lu not deleted",key);
26849         rpage->dirty = BT_CLEAN;
26850         btreeDeallocSecArray(cache,array);
26851 
26852         return ajFalse;
26853     }
26854 
26855     --did->dups;
26856 
26857     if(did->dups != 1)
26858     {
26859         rpage->dirty = BT_DIRTY;
26860         btreeDeallocSecArray(cache,array);
26861 
26862         return ajTrue;
26863     }
26864 
26865     /*
26866     ** Need to find remaining 2ry key here. Should be in root node.
26867     */
26868     buf = rpage->buf;
26869     btreeGetNumKeys(cache,buf,&karray,&parray);
26870     bucket = btreeReadNumbucket(cache,parray[0]);
26871 
26872     if(!bucket->Nentries)
26873         bucket = btreeReadNumbucket(cache,parray[1]);
26874 
26875     if(bucket->Nentries != 1)
26876         ajFatal("Expected only one remaining entry in btreeDeleteIdentIdTwo");
26877 
26878     num = bucket->NumId[0];
26879 
26880     if(cache->refcount)
26881     {
26882         for(iref=0; iref < cache->refcount; iref++)
26883             did->refoffsets[iref] = num->refoffsets[iref];
26884     }
26885 
26886     did->offset    = num->offset;
26887     did->dups      = 0;
26888     rpage->dirty = BT_CLEAN; /* Doesn't matter as page is now defunct */
26889 
26890     /*
26891     ** At this point pages parray[0], parray[1] and cache->secrootblock
26892     ** could be reused.
26893     **/
26894 
26895 
26896     btreeDeallocSecArray(cache,array);
26897 
26898     return ajTrue;
26899 }
26900 
26901 
26902 
26903 
26904 /* @funcstatic btreeFindHybBalanceTwo *****************************************
26905 **
26906 ** Master routine for entry deletion from level 2 hybrid tree.
26907 **
26908 ** Deletion software.
26909 **
26910 ** @param [u] cache [AjPBtcache] cache
26911 ** @param [r] thisNode [ajulong] Current node
26912 ** @param [r] leftNode [ajulong] Node to left
26913 ** @param [r] rightNode [ajulong] Node to right
26914 ** @param [r] lAnchor [ajulong] Left anchor
26915 ** @param [r] rAnchor [ajulong] Right anchor
26916 ** @param [r] key [ajulong] key
26917 **
26918 ** @return [ajulong] page number or BTNO_NODE
26919 **
26920 ** @release 6.1.0
26921 ** @@
26922 ******************************************************************************/
26923 
btreeFindHybBalanceTwo(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor,ajulong key)26924 static ajulong btreeFindHybBalanceTwo(AjPBtcache cache, ajulong thisNode,
26925                                       ajulong leftNode, ajulong rightNode,
26926                                       ajulong lAnchor, ajulong rAnchor,
26927                                       ajulong key)
26928 {
26929     unsigned char *buf  = NULL;
26930     unsigned char *buf1 = NULL;
26931 
26932     ajulong nextNode   = BTNO_NODE;
26933     ajulong nextLeft   = BTNO_NODE;
26934     ajulong nextRight  = BTNO_NODE;
26935     ajulong nextAncL   = BTNO_NODE;
26936     ajulong nextAncR   = BTNO_NODE;
26937     ajulong done       = 0UL;
26938 
26939     ajuint  nkeys      = 0U;
26940     ajuint  order      = 0U;
26941     ajuint  minkeys    = 0U;
26942     ajuint  i;
26943     ajuint   nodetype   = 0U;
26944 
26945     ajuint n1keys      = 0U;
26946 
26947     AjPBtpage page  = NULL;
26948     AjPBtpage page1 = NULL;
26949 
26950     ajulong balanceNode = 0UL;
26951     ajulong blockno     = 0UL;
26952     ajulong ptrSave     = 0UL;
26953 
26954     AjPBtMem arrays  = NULL;
26955     AjPBtMem arrays1 = NULL;
26956     ajulong *karray  = NULL;
26957     ajulong *parray  = NULL;
26958     ajulong *k1array = NULL;
26959     ajulong *p1array = NULL;
26960 
26961     AjBool existed = ajFalse;
26962 
26963     /* ajDebug("In btreeFindHybBalanceTwo\n"); */
26964 
26965     if(thisNode != cache->secrootblock)
26966 	page = btreeSeccacheRead(cache,thisNode);
26967     else
26968     {   /* It's the root node of the primary hyb tree */
26969 	page = btreeSeccacheLocate(cache,thisNode);
26970 	page->dirty = BT_LOCK;
26971         page->lockfor = 1791;
26972     }
26973 
26974     cache->dodelete = ajFalse;
26975 
26976     buf = page->buf;
26977     GBT_NKEYS(buf,&nkeys);
26978 
26979     order = cache->sorder;
26980     /* order-1 is the number of keys in the node */
26981     minkeys = (order-1) / 2;
26982 
26983     if((order-1)%2)
26984 	++minkeys;
26985 
26986     /*
26987     ** If thisNode contains >= minkeys then it is not a candidate
26988     ** for balancing
26989     */
26990     if(nkeys >= minkeys)
26991 	balanceNode = BTNO_BALANCE;
26992     else
26993 	balanceNode = page->pagepos;
26994 
26995     arrays  = btreeAllocSecArray(cache);
26996     arrays1 = btreeAllocSecArray(cache);
26997 
26998     karray = arrays->overflows;
26999     parray = arrays->parray;
27000 
27001     k1array = arrays1->overflows;
27002     p1array = arrays1->parray;
27003 
27004     btreeGetNumKeys(cache,buf,&karray,&parray);
27005 
27006     i=0;
27007 
27008     while(i!=nkeys && key >= karray[i])
27009 	++i;
27010 
27011     blockno = parray[i];
27012 
27013     nextNode = blockno;
27014     ptrSave = i;
27015 
27016     GBT_NODETYPE(buf,&nodetype);
27017 
27018     if(!(nodetype == BT_SECLEAF) && !(nodetype == BT_SECROOT && !cache->slevel))
27019     {
27020 	if(nextNode == parray[0])
27021 	{
27022 	    if(leftNode != BTNO_NODE)
27023 	    {
27024 		page1 = btreeSeccacheRead(cache,leftNode);
27025 		buf1 = page1->buf;
27026 		GBT_NKEYS(buf1,&n1keys);
27027 		btreeGetNumKeys(cache,buf1,&k1array,&p1array);
27028 		nextLeft = p1array[n1keys];
27029 	    }
27030 	    else
27031 		nextLeft = BTNO_NODE;
27032 
27033 	    if(thisNode == cache->secrootblock)
27034 		nextAncL = thisNode;
27035 	    else
27036 		nextAncL = lAnchor;
27037 	}
27038 	else
27039 	{
27040 	    nextLeft = parray[ptrSave-1];
27041 	    nextAncL = thisNode;
27042 	}
27043 
27044 	if(nextNode == parray[nkeys])
27045 	{
27046 	    if(rightNode != BTNO_NODE)
27047 	    {
27048 		page1 = btreeSeccacheRead(cache,rightNode);
27049 		buf1 = page1->buf;
27050 		GBT_NKEYS(buf1,&n1keys);
27051 		btreeGetNumKeys(cache,buf1,&k1array,&p1array);
27052 		nextRight = p1array[0];
27053 	    }
27054 	    else
27055 		nextRight = BTNO_NODE;
27056 
27057 	    if(thisNode == cache->secrootblock)
27058 		nextAncR = thisNode;
27059 	    else
27060 		nextAncR = rAnchor;
27061 	}
27062 	else
27063 	{
27064 	    nextRight = parray[ptrSave+1];
27065 	    nextAncR  = thisNode;
27066 	}
27067 
27068 
27069 
27070 	/* Check to see whether key exists in an internal node */
27071 	if(nodetype != BT_SECLEAF && cache->slevel)
27072 	{
27073 	    i=0;
27074 
27075 	    while(i!=nkeys && key > karray[i])
27076 		++i;
27077 
27078 	    if(i!=nkeys)
27079 	    {
27080 		btreeFindHybMinTwo(cache,parray[i+1],key);
27081 		karray[i] = cache->numreplace;
27082 		btreeWriteNumNode(cache,page,karray,parray,nkeys);
27083 	    }
27084 
27085 	}
27086 
27087 	btreeFindHybBalanceTwo(cache,nextNode,nextLeft,nextRight,
27088                                nextAncL,nextAncR,key);
27089 
27090 	if(thisNode != cache->secrootblock)
27091 	    page = btreeSeccacheRead(cache,thisNode);
27092 	else
27093 	{
27094 	    page = btreeSeccacheLocate(cache,thisNode);
27095 	    page->dirty = BT_LOCK;
27096             page->lockfor = 1792;
27097 	}
27098 	buf = page->buf;
27099 
27100     }
27101     else
27102     {
27103 	if(nodetype == BT_SECLEAF || (nodetype==BT_SECROOT && !cache->slevel))
27104 	{
27105 	    existed = btreeRemoveHybEntryTwo(cache,thisNode,key);
27106 
27107 	    if(existed)
27108 		cache->dodelete = ajTrue;
27109 
27110 	    GBT_NKEYS(buf,&nkeys);
27111 
27112 	    if(nkeys >= minkeys || (nodetype==BT_SECROOT && !cache->slevel))
27113 		balanceNode = BTNO_BALANCE;
27114 	    else
27115 		balanceNode = page->pagepos;
27116 	}
27117     }
27118 
27119 
27120     if(balanceNode == BTNO_BALANCE || thisNode == cache->secrootblock)
27121 	done = BTNO_NODE;
27122     else
27123 	done = btreeRebalanceHybTwo(cache,thisNode,leftNode,rightNode,
27124                                     lAnchor,rAnchor);
27125 
27126 
27127     btreeDeallocSecArray(cache,arrays);
27128     btreeDeallocSecArray(cache,arrays1);
27129 
27130     return done;
27131 }
27132 
27133 
27134 
27135 
27136 /* @funcstatic btreeFindHybMinTwo *********************************************
27137 **
27138 ** Find minimum key in hybrid level 2 subtree and store in cache.
27139 **
27140 ** Deletion software.
27141 **
27142 ** @param [u] cache [AjPBtcache] cache
27143 ** @param [r] pagepos [ajulong] page
27144 ** @param [r] key [ajulong] key
27145 **
27146 ** @return [void]
27147 **
27148 ** @release 6.1.0
27149 ** @@
27150 ******************************************************************************/
27151 
btreeFindHybMinTwo(AjPBtcache cache,ajulong pagepos,ajulong key)27152 static void btreeFindHybMinTwo(AjPBtcache cache, ajulong pagepos,
27153                                ajulong key)
27154 {
27155     AjPBtpage page   = NULL;
27156 
27157     AjPBtMem arrays = NULL;
27158     ajulong *karray   = NULL;
27159     ajulong *parray   = NULL;
27160 
27161     ajuint nkeys    = 0;
27162     ajuint nodetype = 0;
27163     ajuint nentries = 0;
27164     ajuint i;
27165 
27166     AjPNumbucket bucket = NULL;
27167 
27168     unsigned char *buf = NULL;
27169 
27170     /* ajDebug("In btreeFindHybMinTwo\n"); */
27171 
27172     arrays = btreeAllocSecArray(cache);
27173     karray = arrays->overflows;
27174     parray = arrays->parray;
27175 
27176     page = btreeSeccacheRead(cache,pagepos);
27177     buf  = page->buf;
27178     GBT_NODETYPE(buf,&nodetype);
27179     GBT_NKEYS(buf,&nkeys);
27180 
27181     btreeGetNumKeys(cache,buf,&karray,&parray);
27182 
27183     if(nodetype == BT_SECLEAF)
27184     {
27185 	bucket = btreeReadNumbucket(cache,parray[0]);
27186 	nentries = bucket->Nentries;
27187 
27188         /*
27189         ** If there's only one entry then it must be the key marked
27190         ** for deletion
27191         */
27192         if(nentries<2)
27193 	{
27194 	    btreeNumbucketDel(&bucket);
27195 	    bucket = btreeReadNumbucket(cache,parray[1]);
27196 	    nentries = bucket->Nentries;
27197 	}
27198 
27199         /* Check for empty bucket - shouldn't happen */
27200         /* Checking solely out of interest */
27201 	if(nentries<1)
27202 	    ajFatal("FindHybMinTwo: Too few entries in bucket Nkeys=%u\n",
27203                     nkeys);
27204 
27205 
27206         /* Find lowest value key in the bucket and store in cache */
27207         cache->numreplace = bucket->NumId[0]->offset;
27208 
27209 	if(cache->numreplace == key)
27210 	    cache->numreplace = bucket->NumId[1]->offset;
27211 
27212         for(i=1;i<nentries;++i)
27213             if(bucket->NumId[i]->offset < cache->numreplace  &&
27214                bucket->NumId[i]->offset != key)
27215                 cache->numreplace = bucket->NumId[i]->offset;
27216 
27217 	btreeNumbucketDel(&bucket);
27218     }
27219     else
27220     {
27221 	pagepos = parray[0];
27222 	btreeFindHybMinTwo(cache,pagepos,key);
27223 
27224     }
27225 
27226     btreeDeallocSecArray(cache,arrays);
27227 
27228     return;
27229 }
27230 
27231 
27232 
27233 
27234 /* @funcstatic btreeRemoveHybEntryTwo *****************************************
27235 **
27236 ** Find and delete an ID from a given hybrid tree level 2 leaf node if
27237 ** necessary.
27238 **
27239 ** Deletion software
27240 **
27241 ** @param [u] cache [AjPBtcache] cache
27242 ** @param [r] pagepos [ajulong] leaf node page
27243 ** @param [r] key [ajulong] key
27244 **
27245 ** @return [AjBool] True if found (and deleted)
27246 **
27247 ** @release 6.1.0
27248 ** @@
27249 ******************************************************************************/
27250 
btreeRemoveHybEntryTwo(AjPBtcache cache,ajulong pagepos,ajulong key)27251 static AjBool btreeRemoveHybEntryTwo(AjPBtcache cache, ajulong pagepos,
27252                                      ajulong key)
27253 {
27254     AjPBtpage page   = NULL;
27255     AjPNumbucket bucket = NULL;
27256 
27257     ajulong *karray = NULL;
27258     ajulong *parray = NULL;
27259     ajulong blockno = 0UL;
27260 
27261     ajuint nkeys    = 0U;
27262     ajuint nentries = 0U;
27263     ajuint i;
27264 
27265     ajuint dirtysave = 0U;
27266 
27267     AjBool found = ajFalse;
27268 
27269     unsigned char *buf = NULL;
27270     AjPBtMem arrays = NULL;
27271 
27272     /* ajDebug("In btreeRemoveHybEntryTwo\n"); */
27273 
27274     page = btreeSeccacheRead(cache,pagepos);
27275     buf = page->buf;
27276     dirtysave = page->dirty;
27277     page->dirty = BT_LOCK;
27278     page->lockfor = 1801;
27279 
27280     GBT_NKEYS(buf,&nkeys);
27281 
27282     if(!nkeys)
27283 	return ajFalse;
27284 
27285 
27286     arrays = btreeAllocSecArray(cache);
27287     karray = arrays->overflows;
27288     parray = arrays->parray;
27289 
27290     btreeGetNumKeys(cache,buf,&karray,&parray);
27291 
27292     i=0;
27293 
27294     while(i!=nkeys && key >= karray[i])
27295 	++i;
27296 
27297     blockno = parray[i];
27298 
27299     bucket = btreeReadNumbucket(cache,blockno);
27300 
27301 
27302     nentries = bucket->Nentries;
27303     found = ajFalse;
27304 
27305     for(i=0;i<nentries;++i)
27306 	if(key == bucket->NumId[i]->offset)
27307 	{
27308 	    found = ajTrue;
27309 	    break;
27310 	}
27311 
27312 
27313     if(found)
27314     {
27315 	/* Perform the deletion */
27316 	if(nentries == 1)
27317 	{
27318 	    bucket->Nentries = 0;
27319             AJFREE(bucket->NumId[0]);
27320 	}
27321 	else
27322 	{
27323             AJFREE(bucket->NumId[i]);
27324 	    bucket->NumId[i] = bucket->NumId[nentries-1];
27325 	    --bucket->Nentries;
27326 	}
27327 
27328 	btreeWriteNumbucket(cache,bucket,blockno);
27329 	btreeAdjustHybbucketsTwo(cache,page);
27330 	page->dirty = BT_DIRTY;
27331     }
27332     else
27333 	page->dirty = dirtysave;
27334 
27335     btreeNumbucketDel(&bucket);
27336 
27337     btreeDeallocSecArray(cache,arrays);
27338 
27339     if(!found)
27340 	return ajFalse;
27341 
27342     return ajTrue;
27343 }
27344 
27345 
27346 
27347 
27348 /* @funcstatic btreeAdjustHybbucketsTwo ***************************************
27349 **
27350 ** Re-order leaf buckets in 2ry hybrid tree
27351 ** Can be called whatever the state of a leaf.
27352 **
27353 ** Deletion software
27354 **
27355 ** @param [u] cache [AjPBtcache] cache
27356 ** @param [u] leaf [AjPBtpage] leaf page
27357 **
27358 ** @return [void]
27359 **
27360 ** @release 6.1.0
27361 ** @@
27362 ******************************************************************************/
27363 
btreeAdjustHybbucketsTwo(AjPBtcache cache,AjPBtpage leaf)27364 static void btreeAdjustHybbucketsTwo(AjPBtcache cache, AjPBtpage leaf)
27365 {
27366     ajuint nkeys = 0;
27367     unsigned char *lbuf = NULL;
27368     AjPNumbucket *buckets  = NULL;
27369 
27370     AjPBtMem arrays = NULL;
27371     AjPBtMem overarrays = NULL;
27372 
27373     ajulong *keys           = NULL;
27374     ajulong *ptrs           = NULL;
27375     ajulong *overflows      = NULL;
27376 
27377     ajuint i = 0;
27378     ajuint j = 0;
27379     ajuint iref = 0;
27380 
27381     ajuint order;
27382     ajuint bentries      = 0;
27383     ajuint totalkeys     = 0;
27384     ajuint nperbucket    = 0;
27385     ajuint maxnperbucket = 0;
27386     ajuint count         = 0;
27387     ajuint keylimit      = 0;
27388     ajuint bucketn       = 0;
27389     ajuint bucketlimit   = 0;
27390     ajuint nodetype      = 0;
27391     ajuint nids          = 0;
27392     ajuint totnids       = 0;
27393 
27394     AjPList idlist    = NULL;
27395     ajuint   dirtysave = 0;
27396     AjPBtNumId bid       = NULL;
27397     AjPNumbucket cbucket = NULL;
27398     AjPBtNumId cid       = NULL;
27399 
27400     ajuint v = 0;
27401 
27402     /* ajDebug("In btreeAdjustHybbucketsTwo\n"); */
27403 
27404     dirtysave = leaf->dirty;
27405 
27406     leaf->dirty = BT_LOCK;
27407     leaf->lockfor = 1811;
27408     lbuf = leaf->buf;
27409 
27410     GBT_NKEYS(lbuf,&nkeys);
27411 
27412     if(!nkeys)
27413     {
27414 	leaf->dirty = dirtysave;
27415 	return;
27416     }
27417 
27418 
27419     GBT_NODETYPE(lbuf,&nodetype);
27420 
27421     order = cache->sorder;
27422     nperbucket = cache->snperbucket;
27423 
27424 
27425     /* Read keys/ptrs */
27426 
27427     arrays = btreeAllocSecArray(cache);
27428     keys = arrays->overflows;
27429     ptrs = arrays->parray;
27430 
27431     overarrays = btreeAllocSecArray(cache);
27432     overflows = overarrays->overflows;
27433 
27434 
27435     btreeGetNumKeys(cache,lbuf,&keys,&ptrs);
27436 
27437 
27438     for(i=0;i<nkeys;++i)
27439 	totalkeys += btreeNumInNumbucket(cache,ptrs[i]);
27440 
27441     totalkeys += btreeNumInNumbucket(cache,ptrs[i]);
27442 
27443 
27444     /* Set the number of entries per bucket to approximately half full */
27445     maxnperbucket = nperbucket >> 1;
27446 
27447     if(!maxnperbucket)
27448 	++maxnperbucket;
27449 
27450     if(!leaf->pagepos)
27451 	maxnperbucket = nperbucket;
27452 
27453     /* Work out the number of new buckets needed */
27454     bucketn = (totalkeys / maxnperbucket);
27455 
27456     if(totalkeys % maxnperbucket)
27457 	++bucketn;
27458 
27459     if(bucketn == 1)
27460 	++bucketn;
27461 
27462 
27463     while(bucketn > order)
27464     {
27465         ++maxnperbucket;
27466         bucketn = (totalkeys / maxnperbucket);
27467         if(totalkeys % maxnperbucket)
27468             ++bucketn;
27469     }
27470 
27471     /* Read buckets */
27472     AJCNEW0(buckets,nkeys+1);
27473     keylimit = nkeys + 1;
27474 
27475     for(i=0;i<keylimit;++i)
27476 	buckets[i] = btreeReadNumbucket(cache,ptrs[i]);
27477 
27478 
27479     /* Read IDs from all buckets and push to list and sort (increasing id) */
27480     idlist  = ajListNew();
27481 
27482     for(i=0;i<keylimit;++i)
27483     {
27484 	overflows[i] = buckets[i]->Overflow;
27485 	bentries = buckets[i]->Nentries;
27486 	for(j=0;j<bentries;++j)
27487 	    ajListPush(idlist,(void *)buckets[i]->NumId[j]);
27488 
27489 	AJFREE(buckets[i]->NumId);
27490 	AJFREE(buckets[i]);
27491     }
27492 
27493     ajListSort(idlist, &btreeNumIdCompare);
27494     AJFREE(buckets);
27495 
27496     cbucket = btreeNumbucketNew(maxnperbucket, cache->refcount);
27497     bucketlimit = bucketn - 1;
27498 
27499     totnids = 0;
27500     nids = (ajuint) ajListGetLength(idlist);
27501 
27502 
27503     if(!totalkeys)
27504     {
27505 	v = totalkeys;
27506 	SBT_NKEYS(lbuf,v);
27507 
27508         btreeDeallocSecArray(cache,arrays);
27509         btreeDeallocSecArray(cache,overarrays);
27510 
27511 	ajListFree(&idlist);
27512 	leaf->dirty = BT_DIRTY;
27513 
27514 	return;
27515     }
27516 
27517     if(nids <= maxnperbucket)
27518     {
27519 	cbucket->Overflow = overflows[1];
27520 	cbucket->Nentries = 0;
27521 	ajListPeek(idlist,(void **)&bid);
27522 	keys[0] = bid->offset;
27523 
27524 	count = 0;
27525 
27526 	while(count!=maxnperbucket && totnids != nids)
27527 	{
27528 	    ajListPop(idlist,(void **)&bid);
27529 
27530 	    cid = cbucket->NumId[count];
27531 	    cid->dbno = bid->dbno;
27532 	    cid->offset = bid->offset;
27533             if(cache->refcount)
27534             {
27535                 for(iref=0; iref < cache->refcount; iref++)
27536                     cid->refoffsets[iref] = bid->refoffsets[iref];
27537             }
27538 
27539 	    ++cbucket->Nentries;
27540 	    ++count;
27541 	    ++totnids;
27542 	    AJFREE(bid);
27543 	}
27544 
27545 
27546 	if(!ptrs[1])
27547 	    ptrs[1] = cache->totsize;
27548 
27549 	btreeWriteNumbucket(cache,cbucket,ptrs[1]);
27550 
27551 	cbucket->Overflow = overflows[0];
27552 	cbucket->Nentries = 0;
27553 
27554 	if(!ptrs[0])
27555 	    ptrs[0] = cache->totsize;
27556 
27557 	btreeWriteNumbucket(cache,cbucket,ptrs[0]);
27558     }
27559     else
27560     {
27561 	for(i=0;i<bucketlimit;++i)
27562 	{
27563 	    cbucket->Overflow = overflows[i];
27564 	    cbucket->Nentries = 0;
27565 
27566 	    count = 0;
27567 
27568 	    while(count!=maxnperbucket && totnids != nids)
27569 	    {
27570 		ajListPop(idlist,(void **)&bid);
27571 
27572 		cid = cbucket->NumId[count];
27573 		cid->dbno = bid->dbno;
27574 		cid->offset = bid->offset;
27575 
27576                 if(cache->refcount)
27577                 {
27578                     for(iref=0; iref < cache->refcount; iref++)
27579                         cid->refoffsets[iref] = bid->refoffsets[iref];
27580 		}
27581 
27582 		++cbucket->Nentries;
27583 		++count;
27584 		AJFREE(bid);
27585 	    }
27586 
27587 
27588 	    ajListPeek(idlist,(void **)&bid);
27589 	    keys[i] = bid->offset;
27590 
27591 
27592 	    if(!ptrs[i])
27593 		ptrs[i] = cache->totsize;
27594 
27595 	    btreeWriteNumbucket(cache,cbucket,ptrs[i]);
27596 	}
27597 
27598 
27599 	/* Deal with greater-than bucket */
27600 
27601 	cbucket->Overflow = overflows[i];
27602 	cbucket->Nentries = 0;
27603 
27604 
27605 
27606 	count = 0;
27607 
27608 	while(ajListPop(idlist,(void **)&bid))
27609 	{
27610 	    cid = cbucket->NumId[count];
27611 	    cid->dbno = bid->dbno;
27612 	    cid->offset = bid->offset;
27613 
27614             if(cache->refcount)
27615             {
27616                 for(iref=0; iref < cache->refcount; iref++)
27617                     cid->refoffsets[iref] = bid->refoffsets[iref];
27618 	    }
27619 
27620 	    ++cbucket->Nentries;
27621 	    ++count;
27622 	    AJFREE(bid);
27623 	}
27624 
27625 
27626 	if(!ptrs[i])
27627 	    ptrs[i] = cache->totsize;
27628 
27629 	btreeWriteNumbucket(cache,cbucket,ptrs[i]);
27630     }
27631 
27632 
27633     cbucket->Nentries = maxnperbucket;
27634     btreeNumbucketDel(&cbucket);
27635 
27636     /* Now write out a modified leaf with new keys/ptrs */
27637 
27638     nkeys = bucketn - 1;
27639     v = nkeys;
27640     SBT_NKEYS(lbuf,v);
27641 
27642     btreeWriteNumNode(cache,leaf,keys,ptrs,nkeys);
27643 
27644     leaf->dirty = dirtysave;
27645     if(nodetype == BT_SECROOT)
27646     {
27647 	leaf->dirty = BT_LOCK;
27648         leaf->lockfor = 1812;
27649     }
27650 
27651     btreeDeallocSecArray(cache,arrays);
27652     btreeDeallocSecArray(cache,overarrays);
27653 
27654     btreeNumbucketDel(&cbucket);
27655     ajListFree(&idlist);
27656 
27657     return;
27658 }
27659 
27660 
27661 
27662 
27663 /* @funcstatic btreeRebalanceHybTwo *******************************************
27664 **
27665 ** Rebalance Hybrid level 2 tree after deletion
27666 **
27667 ** Deletion software
27668 **
27669 ** @param [u] cache [AjPBtcache] cache
27670 ** @param [r] thisNode [ajulong] Node to rebalance
27671 ** @param [r] leftNode [ajulong] left node
27672 ** @param [r] rightNode [ajulong] right node
27673 ** @param [r] lAnchor [ajulong] left anchor
27674 ** @param [r] rAnchor [ajulong] right anchor
27675 **
27676 ** @return [ajulong] page number or BTNO_NODE
27677 **
27678 ** @release 6.1.0
27679 ** @@
27680 ******************************************************************************/
27681 
btreeRebalanceHybTwo(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor)27682 static ajulong btreeRebalanceHybTwo(AjPBtcache cache, ajulong thisNode,
27683                                    ajulong leftNode, ajulong rightNode,
27684                                    ajulong lAnchor, ajulong rAnchor)
27685 {
27686     unsigned char *lbuf = NULL;
27687     unsigned char *rbuf = NULL;
27688     unsigned char *tbuf = NULL;
27689 
27690     ajulong anchorNode   = 0UL;
27691     ajulong balanceNode  = 0UL;
27692     ajulong mergeNode    = 0UL;
27693     ajulong done         = 0UL;
27694     ajulong parent       = 0UL;
27695 
27696     AjPBtpage lpage = NULL;
27697     AjPBtpage rpage = NULL;
27698     AjPBtpage tpage = NULL;
27699 
27700     ajuint lnkeys  = 0U;
27701     ajuint rnkeys  = 0U;
27702     ajuint size    = 0U;
27703     ajuint order   = 0U;
27704     ajuint minsize = 0U;
27705 
27706     AjBool leftok  = ajFalse;
27707     AjBool rightok = ajFalse;
27708 
27709 
27710     /* ajDebug("In btreeRebalanceHybTwo\n"); */
27711 
27712     if(leftNode!=BTNO_NODE && lAnchor!=BTNO_NODE)
27713 	leftok = ajTrue;
27714 
27715     if(rightNode!=BTNO_NODE && rAnchor!=BTNO_NODE)
27716 	rightok = ajTrue;
27717 
27718     if(!leftok && !rightok)
27719 	return BTNO_NODE;
27720 
27721 
27722     if(leftok)
27723     {
27724 	lpage = btreeSeccacheRead(cache,leftNode);
27725 	lbuf  = lpage->buf;
27726 	GBT_NKEYS(lbuf,&lnkeys);
27727     }
27728 
27729 
27730     if(rightok)
27731     {
27732 	rpage = btreeSeccacheRead(cache,rightNode);
27733 	rbuf  = rpage->buf;
27734 	GBT_NKEYS(rbuf,&rnkeys);
27735     }
27736 
27737 
27738 
27739     if(leftok && rightok)
27740     {
27741 	size = (lnkeys >= rnkeys) ? lnkeys : rnkeys;
27742 	balanceNode = (lnkeys >= rnkeys) ? leftNode : rightNode;
27743     }
27744     else if(leftok)
27745     {
27746 	size = lnkeys;
27747 	balanceNode = leftNode;
27748     }
27749     else
27750     {
27751 	size = rnkeys;
27752 	balanceNode = rightNode;
27753     }
27754 
27755 
27756     order = cache->sorder;
27757     minsize = (order-1) / 2;
27758 
27759     if((order-1)%2)
27760 	++minsize;
27761 
27762     if(size >= minsize)
27763     {
27764 	if(leftok && rightok)
27765 	    anchorNode = (lnkeys >= rnkeys) ? lAnchor : rAnchor;
27766 	else if(leftok)
27767 	    anchorNode = lAnchor;
27768 	else
27769 	    anchorNode = rAnchor;
27770 
27771 	done = btreeShiftHybTwo(cache,thisNode,balanceNode,anchorNode);
27772     }
27773 
27774     else
27775     {
27776 	tpage = btreeSeccacheRead(cache,thisNode);
27777 	tbuf  = tpage->buf;
27778 	GBT_PREV(tbuf,&parent);
27779 
27780 	if(leftok && rightok)
27781 	{
27782 	    anchorNode = (parent == lAnchor) ? lAnchor : rAnchor;
27783 	    mergeNode  = (anchorNode == lAnchor) ? leftNode : rightNode;
27784 	}
27785 	else if(leftok)
27786 	{
27787 	    anchorNode = lAnchor;
27788 	    mergeNode  = leftNode;
27789 	}
27790 	else
27791 	{
27792 	    anchorNode = rAnchor;
27793 	    mergeNode  = rightNode;
27794 	}
27795 
27796 	done = btreeMergeHybTwo(cache,thisNode,mergeNode,anchorNode);
27797     }
27798 
27799     return done;
27800 }
27801 
27802 
27803 
27804 
27805 /* @funcstatic btreeShiftHybTwo ***********************************************
27806 **
27807 ** Shift spare entries from one hybrid tree level 2 node to another.
27808 **
27809 ** Deletion software.
27810 **
27811 ** @param [u] cache [AjPBtcache] cache
27812 ** @param [r] thisNode [ajulong] master node
27813 ** @param [r] balanceNode [ajulong] balance node
27814 ** @param [r] anchorNode [ajulong] anchor node
27815 **
27816 ** @return [ajulong] page number or BTNO_NODE
27817 **
27818 ** @release 6.1.0
27819 ** @@
27820 ******************************************************************************/
27821 
btreeShiftHybTwo(AjPBtcache cache,ajulong thisNode,ajulong balanceNode,ajulong anchorNode)27822 static ajulong btreeShiftHybTwo(AjPBtcache cache, ajulong thisNode,
27823                                 ajulong balanceNode, ajulong anchorNode)
27824 {
27825     unsigned char *tbuf = NULL;
27826     unsigned char *abuf = NULL;
27827     unsigned char *bbuf = NULL;
27828     unsigned char *buf  = NULL;
27829 
27830     ajulong *kTarray = NULL;
27831     ajulong *kAarray = NULL;
27832     ajulong *kBarray = NULL;
27833     ajulong *pTarray = NULL;
27834     ajulong *pAarray = NULL;
27835     ajulong *pBarray = NULL;
27836 
27837     ajuint  nAkeys = 0U;
27838     ajuint  nBkeys = 0U;
27839     ajuint  nTkeys = 0U;
27840     ajuint  i;
27841     ajint  ii;
27842 
27843     AjPBtpage pageA = NULL;
27844     AjPBtpage pageB = NULL;
27845     AjPBtpage pageT = NULL;
27846     AjPBtpage page  = NULL;
27847 
27848     AjPBtpage leftpage = NULL;
27849 
27850     ajuint anchorPos   = 0U;
27851     ajulong prev        = 0L;
27852     ajuint  nodetype    = 0U;
27853 
27854     ajulong lv = 0UL;
27855 
27856     AjPBtMem arraysA = NULL;
27857     AjPBtMem arraysB = NULL;
27858     AjPBtMem arraysT = NULL;
27859 
27860     /* ajDebug("In btreeShiftHybTwo\n"); */
27861 
27862 
27863     arraysA = btreeAllocSecArray(cache);
27864     kAarray = arraysA->overflows;
27865     pAarray = arraysA->parray;
27866 
27867     arraysB = btreeAllocSecArray(cache);
27868     kBarray = arraysB->overflows;
27869     pBarray = arraysB->parray;
27870 
27871     arraysT = btreeAllocSecArray(cache);
27872     kTarray = arraysT->overflows;
27873     pTarray = arraysT->parray;
27874 
27875 
27876     pageA = btreeSeccacheRead(cache,anchorNode);
27877     pageA->dirty = BT_LOCK;
27878     pageA->lockfor = 1821;
27879     abuf = pageA->buf;
27880     pageB = btreeSeccacheRead(cache,balanceNode);
27881     pageB->dirty = BT_LOCK;
27882     pageB->lockfor = 1822;
27883     bbuf = pageB->buf;
27884     pageT = btreeSeccacheRead(cache,thisNode);
27885     pageT->dirty = BT_LOCK;
27886     pageT->lockfor = 1823;
27887     tbuf = pageT->buf;
27888 
27889     GBT_NKEYS(abuf,&nAkeys);
27890     GBT_NKEYS(bbuf,&nBkeys);
27891     GBT_NKEYS(tbuf,&nTkeys);
27892 
27893     btreeGetNumKeys(cache,abuf,&kAarray,&pAarray);
27894     btreeGetNumKeys(cache,bbuf,&kBarray,&pBarray);
27895     btreeGetNumKeys(cache,tbuf,&kTarray,&pTarray);
27896 
27897     if(kTarray[nTkeys-1] < kBarray[nBkeys-1])
27898 	leftpage = pageT;
27899     else
27900 	leftpage = pageB;
27901 
27902 
27903     if(leftpage == pageT)
27904     {
27905 	/* Find anchor key position */
27906 	i=0;
27907 
27908 	while(i!=nAkeys && kTarray[nTkeys-1] >= kAarray[i])
27909 	    ++i;
27910 
27911 	anchorPos = i;
27912 
27913 	/* Move down anchor key to thisNode */
27914         kTarray[nTkeys] = kAarray[anchorPos];
27915 	++nTkeys;
27916 
27917 	/* Shift extra */
27918 	while(nTkeys < nBkeys)
27919 	{
27920             kTarray[nTkeys] = kBarray[0];
27921 	    pTarray[nTkeys] = pBarray[0];
27922 	    ++nTkeys;
27923 	    --nBkeys;
27924 
27925 	    for(i=0;i<nBkeys;++i)
27926 	    {
27927                 kBarray[i] = kBarray[i+1];
27928 		pBarray[i] = pBarray[i+1];
27929 	    }
27930 
27931 	    pBarray[i] = pBarray[i+1];
27932 	}
27933 
27934 	/* Adjust anchor key */
27935         kAarray[anchorPos] = kTarray[nTkeys-1];
27936 	--nTkeys;
27937     }
27938     else	/* thisNode on the right */
27939     {
27940 	/* Find anchor key position */
27941 	i=0;
27942 
27943 	while(i!=nAkeys && kBarray[nBkeys-1] >= kAarray[i])
27944 	    ++i;
27945 
27946 	anchorPos = i;
27947 
27948 	/* Move down anchor key to thisNode */
27949 	pTarray[nTkeys+1] = pTarray[nTkeys];
27950 
27951 	for(ii=nTkeys-1;ii>-1;--ii)
27952 	{
27953             kTarray[ii+1] = kTarray[ii];
27954 	    pTarray[ii+1] = pTarray[ii];
27955 	}
27956 
27957         kTarray[0] = kAarray[anchorPos];
27958 	++nTkeys;
27959 
27960 	/* Shift extra */
27961 	while(nTkeys < nBkeys)
27962 	{
27963 	    pTarray[nTkeys+1] = pTarray[nTkeys];
27964 
27965 	    for(ii=nTkeys-1;ii>-1;--ii)
27966 	    {
27967                 kTarray[ii+1] = kTarray[ii];
27968 		pTarray[ii+1] = pTarray[ii];
27969 	    }
27970 
27971             kTarray[0] = kBarray[nBkeys-1];
27972 	    pTarray[1] = pBarray[nBkeys];
27973 	    ++nTkeys;
27974 	    --nBkeys;
27975 	}
27976 
27977 
27978 	/* Adjust anchor key */
27979         kAarray[anchorPos] = kTarray[0];
27980 	--nTkeys;
27981 
27982 	for(i=0;i<nTkeys;++i)
27983 	{
27984             kTarray[i] = kTarray[i+1];
27985 	    pTarray[i] = pTarray[i+1];
27986 	}
27987 
27988 	pTarray[i] = pTarray[i+1];
27989     }
27990 
27991 
27992     /* Adjust PREV pointers for thisNode */
27993     prev = pageT->pagepos;
27994 
27995     for(i=0;i<nTkeys+1;++i)
27996     {
27997 	page = btreeSeccacheRead(cache,pTarray[i]);
27998 	buf = page->buf;
27999 	GBT_NODETYPE(buf,&nodetype);
28000 
28001 	if(nodetype != BT_IDBUCKET)
28002 	{
28003 	    lv = prev;
28004 	    SBT_PREV(buf,lv);
28005 	    page->dirty = BT_DIRTY;
28006 	}
28007     }
28008 
28009     btreeWriteNumNode(cache,pageA,kAarray,pAarray,nAkeys);
28010     btreeWriteNumNode(cache,pageB,kBarray,pBarray,nBkeys);
28011     btreeWriteNumNode(cache,pageT,kTarray,pTarray,nTkeys);
28012 
28013     if(anchorNode == cache->secrootblock)
28014     {
28015 	pageA->dirty = BT_LOCK;
28016         pageA->lockfor = 1824;
28017     }
28018 
28019     btreeDeallocSecArray(cache,arraysA);
28020     btreeDeallocSecArray(cache,arraysB);
28021     btreeDeallocSecArray(cache,arraysT);
28022 
28023     return BTNO_NODE;
28024 }
28025 
28026 
28027 
28028 
28029 /* @funcstatic btreeMergeHybTwo ***********************************************
28030 **
28031 ** Merge two nodes in Hybrid 2ry tree
28032 **
28033 ** Deletion software
28034 **
28035 ** @param [u] cache [AjPBtcache] cache
28036 ** @param [r] thisNode [ajulong] master node
28037 ** @param [r] mergeNode [ajulong] merge node
28038 ** @param [r] anchorNode [ajulong] anchor node
28039 **
28040 ** @return [ajulong] page number or BTNO_NODE
28041 **
28042 ** @release 6.1.0
28043 ** @@
28044 ******************************************************************************/
28045 
btreeMergeHybTwo(AjPBtcache cache,ajulong thisNode,ajulong mergeNode,ajulong anchorNode)28046 static ajulong btreeMergeHybTwo(AjPBtcache cache, ajulong thisNode,
28047                                 ajulong mergeNode, ajulong anchorNode)
28048 {
28049     unsigned char *tbuf = NULL;
28050     unsigned char *abuf = NULL;
28051     unsigned char *nbuf = NULL;
28052     unsigned char *buf  = NULL;
28053 
28054     AjPBtMem arraysA = NULL;
28055     AjPBtMem arraysN = NULL;
28056     AjPBtMem arraysT = NULL;
28057 
28058     ajulong *kTarray = NULL;
28059     ajulong *kAarray = NULL;
28060     ajulong *kNarray = NULL;
28061     ajulong *pTarray = NULL;
28062     ajulong *pAarray = NULL;
28063     ajulong *pNarray = NULL;
28064 
28065     ajulong thisprev  = 0UL;
28066     ajulong mergeprev = 0UL;
28067 
28068 
28069     ajuint  nAkeys = 0U;
28070     ajuint  nNkeys = 0U;
28071     ajuint  nTkeys = 0U;
28072     ajuint  count  = 0U;
28073     ajuint  i;
28074     ajint ii;
28075 
28076     ajuint   nodetype = 0U;
28077 
28078     ajuint saveA = 0U;
28079     ajuint saveN = 0U;
28080     ajuint saveT = 0U;
28081 
28082     AjPBtpage pageA = NULL;
28083     AjPBtpage pageN = NULL;
28084     AjPBtpage pageT = NULL;
28085     AjPBtpage page  = NULL;
28086 
28087     AjPBtpage leftpage = NULL;
28088 
28089     ajuint anchorPos = 0U;
28090     ajulong prev      = 0UL;
28091 
28092     ajulong lv = 0UL;
28093 
28094     AjBool collapse = ajFalse;
28095     ajulong csrb     = 0UL;
28096 
28097     /* ajDebug("In btreeMergeHybTwo\n"); */
28098 
28099     pageA = btreeSeccacheRead(cache,anchorNode);
28100     saveA = pageA->dirty;
28101     pageA->dirty = BT_LOCK;
28102     pageA->lockfor = 1831;
28103     abuf = pageA->buf;
28104     pageN = btreeSeccacheRead(cache,mergeNode);
28105     saveN = pageN->dirty;
28106     pageN->dirty = BT_LOCK;
28107     pageN->lockfor = 1832;
28108     nbuf = pageN->buf;
28109     pageT = btreeSeccacheRead(cache,thisNode);
28110     saveT = pageT->dirty;
28111     pageT->dirty = BT_LOCK;
28112     pageT->lockfor = 1833;
28113     tbuf = pageT->buf;
28114 
28115     GBT_PREV(tbuf,&thisprev);
28116     GBT_PREV(nbuf,&mergeprev);
28117 
28118     GBT_NKEYS(abuf,&nAkeys);
28119     GBT_NKEYS(nbuf,&nNkeys);
28120     GBT_NKEYS(tbuf,&nTkeys);
28121 
28122     GBT_NODETYPE(nbuf,&nodetype);
28123 
28124     csrb = cache->secrootblock;
28125 
28126     if(nAkeys == 1)
28127     {
28128 	if(anchorNode==csrb && thisprev==csrb && mergeprev==csrb)
28129 	    collapse = ajTrue;
28130 	else
28131 	{
28132 	    pageA->dirty = saveA;
28133 	    pageN->dirty = saveN;
28134 	    pageT->dirty = saveT;
28135 
28136 	    return thisNode;
28137 	}
28138     }
28139 
28140     arraysA = btreeAllocSecArray(cache);
28141     kAarray = arraysA->overflows;
28142     pAarray = arraysA->parray;
28143 
28144     arraysN = btreeAllocSecArray(cache);
28145     kNarray = arraysN->overflows;
28146     pNarray = arraysN->parray;
28147 
28148     arraysT = btreeAllocSecArray(cache);
28149     kTarray = arraysT->overflows;
28150     pTarray = arraysT->parray;
28151 
28152     btreeGetNumKeys(cache,abuf,&kAarray,&pAarray);
28153     btreeGetNumKeys(cache,nbuf,&kNarray,&pNarray);
28154     btreeGetNumKeys(cache,tbuf,&kTarray,&pTarray);
28155 
28156     if(kTarray[nTkeys-1] < kNarray[nNkeys-1])
28157 	leftpage = pageT;
28158     else
28159 	leftpage = pageN;
28160 
28161 
28162     if(leftpage == pageT)
28163     {
28164 	/* Find anchor key position */
28165 	i=0;
28166 
28167 	while(i!=nAkeys && kTarray[nTkeys-1] >= kAarray[i])
28168 	    ++i;
28169 
28170 	anchorPos = i;
28171 
28172 	/* Move down anchor key to neighbour Node */
28173 	pNarray[nNkeys+1] = pNarray[nNkeys];
28174 
28175 	for(ii=nNkeys-1;ii>-1;--ii)
28176 	{
28177 	    kNarray[ii+1] = kNarray[ii];
28178 	    pNarray[ii+1] = pNarray[ii];
28179 	}
28180 
28181 	kNarray[0] = kAarray[anchorPos];
28182 	++nNkeys;
28183 
28184 
28185 	/* Adjust anchor node keys/ptrs */
28186 	++anchorPos;
28187 
28188 	if(anchorPos==nAkeys)
28189 	    pAarray[nAkeys-1] = pAarray[nAkeys];
28190 	else
28191 	{
28192 	    for(i=anchorPos;i<nAkeys;++i)
28193 	    {
28194 		kAarray[i-1] = kAarray[i];
28195 		pAarray[i-1] = pAarray[i];
28196 	    }
28197 	    pAarray[i-1] = pAarray[i];
28198 	}
28199 	--nAkeys;
28200 
28201 
28202 	/* Merge this to neighbour */
28203 
28204 	while(nTkeys)
28205 	{
28206 	    pNarray[nNkeys+1] = pNarray[nNkeys];
28207 
28208 	    for(ii=nNkeys-1;ii>-1;--ii)
28209 	    {
28210 		kNarray[ii+1] = kNarray[ii];
28211 		pNarray[ii+1] = pNarray[ii];
28212 	    }
28213 
28214 	    kNarray[0] = kTarray[nTkeys-1];
28215 	    pNarray[1] = pTarray[nTkeys];
28216 	    pNarray[0] = pTarray[nTkeys-1];
28217 	    --nTkeys;
28218 	    ++nNkeys;
28219 	}
28220 
28221 	/* At this point the 'this' node could be added to a free list */
28222     }
28223     else
28224     {
28225 	/* Find anchor key position */
28226 	i=0;
28227 
28228 	while(i!=nAkeys && kNarray[nNkeys-1] >= kAarray[i])
28229 	    ++i;
28230 
28231 	anchorPos = i;
28232 
28233 	/* Move down anchor key to neighbourNode */
28234 	kNarray[nNkeys] = kAarray[anchorPos];
28235 	++nNkeys;
28236 
28237 	/* Adjust anchor node keys/ptrs */
28238 	++anchorPos;
28239 
28240 	if(anchorPos!=nAkeys)
28241 	    for(i=anchorPos;i<nAkeys;++i)
28242 	    {
28243 		kAarray[i-1] = kAarray[i];
28244 		pAarray[i]   = pAarray[i+1];
28245 	    }
28246 
28247 	--nAkeys;
28248 
28249 	/* merge extra */
28250 	count = 0;
28251 
28252 	while(nTkeys)
28253 	{
28254 	    kNarray[nNkeys] = kTarray[count];
28255 	    pNarray[nNkeys] = pTarray[count];
28256 	    ++nNkeys;
28257 	    ++count;
28258 	    --nTkeys;
28259 	    pNarray[nNkeys] = pTarray[count];
28260 
28261 	}
28262 
28263 	/* At this point the 'this' node could be added to a free list */
28264     }
28265 
28266 
28267     /* Adjust PREV pointers for neighbour Node */
28268     prev = pageN->pagepos;
28269 
28270     for(i=0;i<=nNkeys;++i)
28271     {
28272 	page = btreeSeccacheRead(cache,pNarray[i]);
28273 	buf = page->buf;
28274 	GBT_NODETYPE(buf,&nodetype);
28275 
28276 	if(nodetype != BT_IDBUCKET)
28277 	{
28278 	    lv = prev;
28279 	    SBT_PREV(buf,lv);
28280 	    page->dirty = BT_DIRTY;
28281 	}
28282     }
28283 
28284     pageT->dirty = BT_CLEAN;
28285     btreeWriteNumNode(cache,pageA,kAarray,pAarray,nAkeys);
28286     btreeWriteNumNode(cache,pageN,kNarray,pNarray,nNkeys);
28287 
28288     if(anchorNode == csrb)
28289     {
28290 	pageA->dirty = BT_LOCK;
28291         pageA->lockfor = 1834;
28292     }
28293 
28294     btreeDeallocSecArray(cache,arraysA);
28295     btreeDeallocSecArray(cache,arraysN);
28296     btreeDeallocSecArray(cache,arraysT);
28297 
28298     if(collapse)
28299 	btreeCollapseRootHybTwo(cache,mergeNode);
28300 
28301     return thisNode;
28302 }
28303 
28304 
28305 
28306 
28307 /* @funcstatic btreeCollapseRootHybTwo ****************************************
28308 **
28309 ** Collapse root page for hybrid level 2 tree.
28310 **
28311 ** Deletion software.
28312 **
28313 ** @param [u] cache [AjPBtcache] cache
28314 ** @param [r] pagepos [ajulong] page number to make new root
28315 **
28316 ** @return [ajulong] page number or BTNO_NODE
28317 **
28318 ** @release 6.1.0
28319 ** @@
28320 ******************************************************************************/
28321 
btreeCollapseRootHybTwo(AjPBtcache cache,ajulong pagepos)28322 static ajulong btreeCollapseRootHybTwo(AjPBtcache cache, ajulong pagepos)
28323 {
28324     unsigned char *buf  = NULL;
28325     unsigned char *lbuf = NULL;
28326 
28327     ajulong *karray = NULL;
28328     ajulong *parray = NULL;
28329 
28330     AjPBtpage rootpage = NULL;
28331     AjPBtpage page     = NULL;
28332 
28333     ajuint nodetype = 0U;
28334     ajuint nkeys    = 0U;
28335     ajuint i;
28336 
28337     ajulong prev = 0UL;
28338     AjPBtMem arrays = NULL;
28339 
28340     /* ajDebug("In btreeCollapseRootHybTwo\n"); */
28341 
28342     if(!cache->slevel)
28343 	return BTNO_NODE;
28344 
28345     rootpage = btreeSeccacheLocate(cache,cache->secrootblock);
28346     buf = rootpage->buf;
28347     page = btreeSeccacheRead(cache,pagepos);
28348 
28349 
28350     arrays = btreeAllocSecArray(cache);
28351     karray = arrays->overflows;
28352     parray = arrays->parray;
28353 
28354     /*
28355     ** Swap pagepos values to make root the child and child the root
28356     ** Update node types and mark the original root as a clean page
28357     */
28358 
28359     /* At this point page->pagepos could be added to a free list */
28360 
28361     rootpage->pagepos = page->pagepos;
28362     rootpage->dirty = BT_CLEAN;
28363     nodetype = BT_SECINTERNAL;
28364     SBT_NODETYPE(buf,nodetype);
28365 
28366     page->pagepos = cache->secrootblock;
28367     page->dirty = BT_LOCK;
28368     page->lockfor = 1841;
28369     buf = page->buf;
28370     nodetype = BT_SECROOT;
28371     SBT_NODETYPE(buf,nodetype);
28372 
28373     --cache->slevel;
28374 
28375     if(cache->slevel)
28376     {
28377 	/*
28378 	 ** Update the PREV pointers of the new root's children
28379 	 */
28380 	GBT_NKEYS(buf,&nkeys);
28381 	btreeGetNumKeys(cache,buf,&karray,&parray);
28382 
28383 	for(i=0;i<=nkeys;++i)
28384 	{
28385 	    page = btreeSeccacheRead(cache,parray[i]);
28386 	    lbuf = page->buf;
28387 	    SBT_PREV(lbuf,prev);
28388 	    page->dirty = BT_DIRTY;
28389 	}
28390     }
28391 
28392     btreeDeallocSecArray(cache,arrays);
28393 
28394     cache->secrootblock = pagepos;
28395 
28396     return 0UL;
28397 }
28398 
28399 
28400 
28401 
28402 /* @func ajBtreeDeletePriId ***************************************************
28403 **
28404 ** Entry point for keyword tree ID deletion.
28405 **
28406 ** Deletion software
28407 **
28408 ** @param [u] cache [AjPBtcache] cache
28409 ** @param [r] pri [const AjPBtPri] keyword object
28410 **
28411 ** @return [AjBool] True if found and deleted
28412 **
28413 ** @release 6.1.0
28414 ** @@
28415 ******************************************************************************/
28416 
ajBtreeDeletePriId(AjPBtcache cache,const AjPBtPri pri)28417 AjBool ajBtreeDeletePriId(AjPBtcache cache, const AjPBtPri pri)
28418 {
28419     AjPBtpage rootpage = NULL;
28420     AjPBtpage spage    = NULL;
28421     AjPBtpage page     = NULL;
28422     AjPStr key         = NULL;
28423     AjPSecbucket bucket  = NULL;
28424     ajulong blockno  = 0UL;
28425 
28426     ajuint nkeys = 0U;
28427     ajulong slevel = 0UL;
28428 
28429     ajuint nentries = 0U;
28430 
28431     AjPStr *karray = NULL;
28432     ajulong *parray = NULL;
28433     AjPBtMem arrays = NULL;
28434     AjBool found = ajFalse;
28435 
28436 
28437     ajuint i;
28438 
28439     unsigned char *buf = NULL;
28440     ajulong  secrootpage = 0UL;
28441 
28442     AjBool empty = ajFalse;
28443     AjBool ret   = ajFalse;
28444 
28445     AjPBtpage prirootpage = NULL;
28446 
28447     ajulong treeblock = 0UL;
28448 
28449     /* ajDebug("In ajBtreeDeletePriId\n"); */
28450 
28451     key = ajStrNew();
28452 
28453 
28454     ajStrAssignS(&key,pri->keyword);
28455 
28456     if(!ajStrGetLen(key))
28457     {
28458 	ajStrDel(&key);
28459 
28460 	return ajFalse;
28461     }
28462 
28463     if(!btreeKeyFind(cache,key,&treeblock))
28464     {
28465         ajStrDel(&key);
28466         ajWarn("DeletePriId: Keyword %S not found",pri->keyword);
28467 
28468         return ajFalse;
28469     }
28470 
28471     secrootpage = treeblock;
28472     cache->secrootblock = treeblock;
28473 
28474     arrays = btreeAllocSecArray(cache);
28475     karray = arrays->karray;
28476     parray = arrays->parray;
28477 
28478     page = btreePricacheRead(cache,secrootpage);
28479     page->dirty = BT_LOCK;
28480     page->lockfor = 1851;
28481     buf = page->buf;
28482 
28483     GBT_RIGHT(buf,&slevel);
28484     cache->slevel = (ajuint)slevel;
28485 
28486     spage = btreeKeyidFind(cache,pri->id);
28487     buf = spage->buf;
28488 
28489     btreeGetKeys(cache,buf,&karray,&parray);
28490 
28491 
28492 
28493     GBT_NKEYS(buf,&nkeys);
28494 
28495     if(!nkeys)
28496     {
28497 	btreeDeallocSecArray(cache,arrays);
28498 	ajStrDel(&key);
28499         page->dirty = BT_CLEAN;
28500 
28501 	return ajFalse;
28502     }
28503 
28504 
28505     i=0;
28506 
28507     while(i!=nkeys && MAJSTRCMPS(pri->id,karray[i])>=0)
28508         ++i;
28509 
28510     blockno = parray[i];
28511 
28512     bucket = btreeReadSecbucket(cache,blockno);
28513 
28514     nentries = bucket->Nentries;
28515 
28516     found = ajFalse;
28517 
28518     for(i=0;i<nentries;++i)
28519 	if(ajStrMatchS(pri->id,bucket->SecIds[i]))
28520 	{
28521 	    found = ajTrue;
28522 	    break;
28523 	}
28524 
28525     if(!found)
28526     {
28527         ajWarn("DeletePriId: ID %S  not found for Keyword %S",pri->id,
28528                pri->keyword);
28529 	btreeDeallocPriArray(cache,arrays);
28530 	ajStrDel(&key);
28531         page->dirty = BT_CLEAN;
28532 
28533 	return ajFalse;
28534     }
28535 
28536 
28537 
28538     /*
28539     ** Have to delete ID from secondary tree. If that empties the
28540     ** tree then have to delete keyword from primary tree.
28541     ** Needs a little thought. Maybe use cache->dodelete for multiple
28542     ** purposes i.e. 0=not deleted  1=keyword deleted 2=tree deleted
28543     */
28544 
28545     rootpage = btreeSeccacheLocate(cache,secrootpage);
28546     if(!rootpage)
28547         ajFatal("DeletePriId: secondary root page became unlocked");
28548     rootpage->dirty = BT_LOCK;
28549     rootpage->lockfor = 1852;
28550     buf = rootpage->buf;
28551 
28552     GBT_RIGHT(buf,&slevel);
28553     cache->slevel = (ajuint)slevel;
28554 
28555     btreeFindPriBalanceTwo(cache,secrootpage,BTNO_NODE,BTNO_NODE,BTNO_NODE,
28556                            BTNO_NODE,pri);
28557 
28558     ret = cache->dodelete;
28559 
28560     if(!ret)
28561     {
28562         btreeDeallocSecArray(cache,arrays);
28563         ajStrDel(&key);
28564         page->dirty = BT_CLEAN;
28565 
28566         return ajFalse;
28567     }
28568 
28569     empty = btreeIsSecEmpty(cache);
28570 
28571 
28572     if(empty)
28573     {
28574         prirootpage = btreePricacheLocate(cache,0UL);
28575 
28576         if(!prirootpage)
28577             ajFatal("ajBtreeDeletePriId: prirootpage unlocked");
28578 
28579 
28580         btreeFindPriBalanceOne(cache,0UL,BTNO_NODE,BTNO_NODE,BTNO_NODE,
28581                                BTNO_NODE,pri);
28582 
28583         ret = cache->dodelete;
28584     }
28585 
28586 
28587     btreeDeallocSecArray(cache,arrays);
28588     ajStrDel(&key);
28589 
28590     return ret;
28591 }
28592 
28593 
28594 
28595 
28596 /* @funcstatic btreeFindPriBalanceTwo *****************************************
28597 **
28598 ** Master routine for entry deletion from level 2 keyword tree.
28599 **
28600 ** Deletion software.
28601 **
28602 ** @param [u] cache [AjPBtcache] cache
28603 ** @param [r] thisNode [ajulong] Current node
28604 ** @param [r] leftNode [ajulong] Node to left
28605 ** @param [r] rightNode [ajulong] Node to right
28606 ** @param [r] lAnchor [ajulong] Left anchor
28607 ** @param [r] rAnchor [ajulong] Right anchor
28608 ** @param [r] pri [const AjPBtPri] pri
28609 **
28610 ** @return [ajulong] page number or BTNO_NODE
28611 **
28612 ** @release 6.1.0
28613 ** @@
28614 ******************************************************************************/
28615 
btreeFindPriBalanceTwo(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor,const AjPBtPri pri)28616 static ajulong btreeFindPriBalanceTwo(AjPBtcache cache, ajulong thisNode,
28617                                       ajulong leftNode, ajulong rightNode,
28618                                       ajulong lAnchor, ajulong rAnchor,
28619                                       const AjPBtPri pri)
28620 {
28621     unsigned char *buf  = NULL;
28622     unsigned char *buf1 = NULL;
28623 
28624     ajulong nextNode   = BTNO_NODE;
28625     ajulong nextLeft   = BTNO_NODE;
28626     ajulong nextRight  = BTNO_NODE;
28627     ajulong nextAncL   = BTNO_NODE;
28628     ajulong nextAncR   = BTNO_NODE;
28629     ajulong done       = 0UL;
28630 
28631     ajuint  nkeys      = 0U;
28632     ajuint  order      = 0U;
28633     ajuint  minkeys    = 0U;
28634     ajuint  i;
28635     ajuint  nodetype   = 0U;
28636 
28637     ajuint n1keys      = 0U;
28638 
28639     AjPBtpage page  = NULL;
28640     AjPBtpage page1 = NULL;
28641 
28642     ajulong balanceNode = 0UL;
28643     ajulong blockno     = 0UL;
28644     ajulong ptrSave     = 0UL;
28645 
28646     AjPStr *karray  = NULL;
28647     ajulong *parray  = NULL;
28648     AjPStr *k1array = NULL;
28649     ajulong *p1array = NULL;
28650 
28651     AjPBtMem arrays  = NULL;
28652     AjPBtMem arrays1 = NULL;
28653 
28654     const AjPStr key = NULL;
28655     AjBool existed = ajFalse;
28656 
28657     /* ajDebug("In btreeFindPriBalanceTwo\n"); */
28658 
28659     if(thisNode != cache->secrootblock)
28660 	page = btreeSeccacheRead(cache,thisNode);
28661     else
28662     {
28663 	page = btreeSeccacheLocate(cache,thisNode);
28664 	page->dirty = BT_LOCK;
28665         page->lockfor = 1861;
28666     }
28667 
28668     cache->dodelete = ajFalse;
28669 
28670     buf = page->buf;
28671     GBT_NKEYS(buf,&nkeys);
28672 
28673     order = cache->sorder;
28674     /* order-1 is the number of keys in the node */
28675     minkeys = (order-1) / 2;
28676     if((order-1)%2)
28677 	++minkeys;
28678 
28679     /*
28680     ** If thisNode contains >= minkeys then it is not a candidate
28681     ** for balancing
28682     */
28683     if(nkeys >= minkeys)
28684 	balanceNode = BTNO_BALANCE;
28685     else
28686 	balanceNode = page->pagepos;
28687 
28688     arrays  = btreeAllocSecArray(cache);
28689     arrays1 = btreeAllocSecArray(cache);
28690 
28691     karray = arrays->karray;
28692     parray = arrays->parray;
28693 
28694     k1array = arrays1->karray;
28695     p1array = arrays1->parray;
28696 
28697     key = pri->id;
28698 
28699     btreeGetKeys(cache,buf,&karray,&parray);
28700 
28701     i=0;
28702 
28703     while(i!=nkeys && MAJSTRCMPS(key,karray[i])>=0)
28704 	++i;
28705 
28706     blockno = parray[i];
28707 
28708     nextNode = blockno;
28709     ptrSave = i;
28710 
28711     GBT_NODETYPE(buf,&nodetype);
28712 
28713     if(!(nodetype == BT_SECLEAF) && !(nodetype == BT_SECROOT && !cache->slevel))
28714     {
28715 	if(nextNode == parray[0])
28716 	{
28717 	    if(leftNode != BTNO_NODE)
28718 	    {
28719 		page1 = btreeSeccacheRead(cache,leftNode);
28720 		buf1 = page1->buf;
28721 		GBT_NKEYS(buf1,&n1keys);
28722 		btreeGetKeys(cache,buf1,&k1array,&p1array);
28723 		nextLeft = p1array[n1keys];
28724 	    }
28725 	    else
28726 		nextLeft = BTNO_NODE;
28727 
28728 	    if(thisNode == cache->secrootblock)
28729 		nextAncL = thisNode;
28730 	    else
28731 		nextAncL = lAnchor;
28732 	}
28733 	else
28734 	{
28735 	    nextLeft = parray[ptrSave-1];
28736 	    nextAncL = thisNode;
28737 	}
28738 
28739 	if(nextNode == parray[nkeys])
28740 	{
28741 	    if(rightNode != BTNO_NODE)
28742 	    {
28743 		page1 = btreeSeccacheRead(cache,rightNode);
28744 		buf1 = page1->buf;
28745 		GBT_NKEYS(buf1,&n1keys);
28746 		btreeGetKeys(cache,buf1,&k1array,&p1array);
28747 		nextRight = p1array[0];
28748 	    }
28749 	    else
28750 		nextRight = BTNO_NODE;
28751 
28752 	    if(thisNode == cache->secrootblock)
28753 		nextAncR = thisNode;
28754 	    else
28755 		nextAncR = rAnchor;
28756 	}
28757 	else
28758 	{
28759 	    nextRight = parray[ptrSave+1];
28760 	    nextAncR  = thisNode;
28761 	}
28762 
28763 
28764 
28765 	/* Check to see whether key exists in an internal node */
28766 	if(nodetype != BT_SECLEAF && cache->slevel)
28767 	{
28768 	    i=0;
28769 
28770 	    while(i!=nkeys && MAJSTRCMPS(key,karray[i]))
28771 		++i;
28772 
28773 	    if(i!=nkeys)
28774 	    {
28775 		btreeFindPriMinTwo(cache,parray[i+1],key);
28776 		ajStrAssignS(&karray[i],cache->replace);
28777 		btreeWriteNode(cache,page,karray,parray,nkeys);
28778 	    }
28779 
28780 	}
28781 
28782 	btreeFindPriBalanceTwo(cache,nextNode,nextLeft,nextRight,
28783                                nextAncL,nextAncR,pri);
28784 
28785 	if(thisNode != cache->secrootblock)
28786 	    page = btreeSeccacheRead(cache,thisNode);
28787 	else
28788 	{
28789 	    page = btreeSeccacheLocate(cache,thisNode);
28790 	    page->dirty = BT_LOCK;
28791             page->lockfor = 1862;
28792 	}
28793 	buf = page->buf;
28794 
28795     }
28796     else
28797     {
28798 	if(nodetype == BT_SECLEAF || (nodetype==BT_SECROOT && !cache->slevel))
28799 	{
28800 	    existed = btreeRemovePriEntryTwo(cache,thisNode,pri);
28801 
28802 	    if(existed)
28803 		cache->dodelete = ajTrue;
28804 	    GBT_NKEYS(buf,&nkeys);
28805 
28806 	    if(nkeys >= minkeys || (nodetype==BT_SECROOT && !cache->slevel))
28807 		balanceNode = BTNO_BALANCE;
28808 	    else
28809 		balanceNode = page->pagepos;
28810 	}
28811     }
28812 
28813 
28814     if(balanceNode == BTNO_BALANCE || thisNode == cache->secrootblock)
28815 	done = BTNO_NODE;
28816     else
28817 	done = btreeRebalancePriTwo(cache,thisNode,leftNode,rightNode,
28818                                     lAnchor,rAnchor);
28819 
28820     btreeDeallocSecArray(cache,arrays);
28821     btreeDeallocSecArray(cache,arrays1);
28822 
28823     return done;
28824 }
28825 
28826 
28827 
28828 
28829 /* @funcstatic btreeFindPriMinTwo *********************************************
28830 **
28831 ** Find minimum key in keyword level 2 subtree and store in cache.
28832 **
28833 ** Deletion software.
28834 **
28835 ** @param [u] cache [AjPBtcache] cache
28836 ** @param [r] pagepos [ajulong] page
28837 ** @param [r] key [const AjPStr] key
28838 **
28839 ** @return [void]
28840 **
28841 ** @release 6.1.0
28842 ** @@
28843 ******************************************************************************/
28844 
btreeFindPriMinTwo(AjPBtcache cache,ajulong pagepos,const AjPStr key)28845 static void btreeFindPriMinTwo(AjPBtcache cache, ajulong pagepos,
28846                                const AjPStr key)
28847 {
28848     AjPBtpage page   = NULL;
28849     AjPSecbucket bucket = NULL;
28850     AjPStr *karray   = NULL;
28851     ajulong *parray   = NULL;
28852 
28853     ajuint nkeys    = 0;
28854     ajuint nodetype = 0;
28855     ajuint nentries = 0;
28856     ajuint i;
28857     AjPBtMem arrays = NULL;
28858 
28859     unsigned char *buf = NULL;
28860 
28861     /* ajDebug("In btreeFindPriMinTwo\n"); */
28862 
28863     arrays = btreeAllocSecArray(cache);
28864     karray = arrays->karray;
28865     parray = arrays->parray;
28866 
28867     page = btreeSeccacheRead(cache,pagepos);
28868     buf  = page->buf;
28869     GBT_NODETYPE(buf,&nodetype);
28870     GBT_NKEYS(buf,&nkeys);
28871 
28872     btreeGetKeys(cache,buf,&karray,&parray);
28873 
28874     if(nodetype == BT_SECLEAF)
28875     {
28876 	bucket = btreeReadSecbucket(cache,parray[0]);
28877 	nentries = bucket->Nentries;
28878 
28879         /*
28880         ** If there's only one entry then it must be the key marked
28881         ** for deletion
28882         */
28883         if(nentries<2)
28884 	{
28885 	    btreeSecbucketDel(&bucket);
28886 	    bucket = btreeReadSecbucket(cache,parray[1]);
28887 	    nentries = bucket->Nentries;
28888 	}
28889 
28890         /* Check for empty bucket - shouldn't happen */
28891         /* Checking solely out of interest */
28892 	if(nentries<1)
28893 	    ajFatal("FindPriMinTwo: Too few entries in bucket Nkeys=%u\n",
28894                     nkeys);
28895 
28896 
28897         /* Find lowest value key in the bucket and store in cache */
28898         ajStrAssignS(&cache->replace,bucket->SecIds[0]);
28899 
28900 	if(!MAJSTRCMPS(cache->replace,key))
28901 	    ajStrAssignS(&cache->replace,bucket->SecIds[1]);
28902 
28903 	for(i=1;i<nentries;++i)
28904 	    if(MAJSTRCMPS(bucket->SecIds[i],cache->replace)<0 &&
28905 	       MAJSTRCMPS(bucket->SecIds[i],key))
28906 		ajStrAssignS(&cache->replace,bucket->SecIds[i]);
28907 
28908 	btreeSecbucketDel(&bucket);
28909     }
28910     else
28911     {
28912 	pagepos = parray[0];
28913 	btreeFindPriMinTwo(cache,pagepos,key);
28914 
28915     }
28916 
28917     btreeDeallocSecArray(cache,arrays);
28918 
28919     return;
28920 }
28921 
28922 
28923 
28924 
28925 /* @funcstatic btreeRemovePriEntryTwo *****************************************
28926 **
28927 ** Find and delete an ID from a given keyword tree level 2 leaf node if
28928 ** necessary.
28929 **
28930 ** Deletion software
28931 **
28932 ** @param [u] cache [AjPBtcache] cache
28933 ** @param [r] pagepos [ajulong] leaf node page
28934 ** @param [r] pri [const AjPBtPri] pri
28935 **
28936 ** @return [AjBool] True if found (and deleted)
28937 **
28938 ** @release 6.1.0
28939 ** @@
28940 ******************************************************************************/
28941 
btreeRemovePriEntryTwo(AjPBtcache cache,ajulong pagepos,const AjPBtPri pri)28942 static AjBool btreeRemovePriEntryTwo(AjPBtcache cache, ajulong pagepos,
28943                                      const AjPBtPri pri)
28944 {
28945     AjPBtpage page   = NULL;
28946     AjPSecbucket bucket = NULL;
28947 
28948     AjPStr *karray = NULL;
28949     ajulong *parray = NULL;
28950     ajulong blockno = 0UL;
28951 
28952     ajuint nkeys    = 0U;
28953     ajuint nentries = 0U;
28954     ajuint i;
28955 
28956     ajuint dirtysave = 0U;
28957 
28958     AjBool found = ajFalse;
28959     const AjPStr key  = NULL;
28960 
28961     unsigned char *buf = NULL;
28962     AjPBtMem arrays = NULL;
28963 
28964     /* ajDebug("In btreeRemovePriEntryTwo\n"); */
28965 
28966     page = btreeSeccacheRead(cache,pagepos);
28967     buf = page->buf;
28968     dirtysave = page->dirty;
28969     page->dirty = BT_LOCK;
28970     page->lockfor = 1871;
28971 
28972     GBT_NKEYS(buf,&nkeys);
28973 
28974     if(!nkeys)
28975 	return ajFalse;
28976 
28977 
28978     arrays = btreeAllocSecArray(cache);
28979     karray = arrays->karray;
28980     parray = arrays->parray;
28981 
28982     btreeGetKeys(cache,buf,&karray,&parray);
28983 
28984     key = pri->id;
28985 
28986     i=0;
28987 
28988     while(i!=nkeys && MAJSTRCMPS(key,karray[i])>=0)
28989 	++i;
28990 
28991     blockno = parray[i];
28992 
28993     bucket = btreeReadSecbucket(cache,blockno);
28994 
28995 
28996     nentries = bucket->Nentries;
28997     found = ajFalse;
28998 
28999     for(i=0;i<nentries;++i)
29000 	if(!MAJSTRCMPS(key,bucket->SecIds[i]))
29001 	{
29002 	    found = ajTrue;
29003 	    break;
29004 	}
29005 
29006 
29007     if(found)
29008     {
29009 	/* Perform the deletion */
29010 	if(nentries == 1)
29011 	{
29012 	    bucket->Nentries = 0;
29013 	    AJFREE(bucket->SecIds[0]);
29014 	}
29015 	else
29016 	{
29017 	    AJFREE(bucket->SecIds[i]);
29018 	    bucket->SecIds[i] = bucket->SecIds[nentries-1];
29019 	    --bucket->Nentries;
29020 	}
29021 
29022 	btreeWriteSecbucket(cache,bucket,blockno);
29023 	btreeAdjustPribucketsTwo(cache,page);
29024 	page->dirty = BT_DIRTY;
29025     }
29026     else
29027 	page->dirty = dirtysave;
29028 
29029     btreeSecbucketDel(&bucket);
29030 
29031     btreeDeallocSecArray(cache,arrays);
29032 
29033     if(!found)
29034 	return ajFalse;
29035 
29036     return ajTrue;
29037 }
29038 
29039 
29040 
29041 
29042 /* @funcstatic btreeAdjustPribucketsTwo ***************************************
29043 **
29044 ** Re-order leaf buckets in keyword level 2 tree
29045 ** Can be called whatever the state of a leaf.
29046 **
29047 ** Deletion software
29048 **
29049 ** @param [u] cache [AjPBtcache] cache
29050 ** @param [u] leaf [AjPBtpage] leaf page
29051 **
29052 ** @return [void]
29053 **
29054 ** @release 6.1.0
29055 ** @@
29056 ******************************************************************************/
29057 
btreeAdjustPribucketsTwo(AjPBtcache cache,AjPBtpage leaf)29058 static void btreeAdjustPribucketsTwo(AjPBtcache cache, AjPBtpage leaf)
29059 {
29060     ajuint nkeys = 0;
29061     unsigned char *lbuf = NULL;
29062     AjPSecbucket *buckets  = NULL;
29063 
29064     AjPBtMem arrays = NULL;
29065     AjPBtMem newarrays = NULL;
29066     AjPStr *keys        = NULL;
29067     ajulong *ptrs        = NULL;
29068     ajulong *overflows   = NULL;
29069 
29070     ajuint i = 0;
29071     ajuint j = 0;
29072 
29073     ajuint order;
29074     ajuint bentries      = 0;
29075     ajuint totalkeys     = 0;
29076     ajuint nperbucket    = 0;
29077     ajuint maxnperbucket = 0;
29078     ajuint count         = 0;
29079     ajuint totkeylen     = 0;
29080     ajuint keylimit      = 0;
29081     ajuint bucketn       = 0;
29082     ajuint bucketlimit   = 0;
29083     ajuint nodetype      = 0;
29084     ajuint nids          = 0;
29085     ajuint totnids       = 0;
29086 
29087     AjPList idlist    = NULL;
29088     ajuint   dirtysave = 0;
29089     AjPStr bid       = NULL;
29090     AjPSecbucket cbucket = NULL;
29091     AjPStr cid       = NULL;
29092 
29093     ajuint v = 0;
29094 
29095     /* ajDebug("In btreeAdjustPribucketsTwo\n"); */
29096 
29097     dirtysave = leaf->dirty;
29098 
29099     leaf->dirty = BT_LOCK;
29100     leaf->lockfor = 1881;
29101     lbuf = leaf->buf;
29102 
29103     GBT_NKEYS(lbuf,&nkeys);
29104 
29105     if(!nkeys)
29106     {
29107 	leaf->dirty = dirtysave;
29108 
29109 	return;
29110     }
29111 
29112 
29113     GBT_NODETYPE(lbuf,&nodetype);
29114 
29115     order = cache->sorder;
29116     nperbucket = cache->snperbucket;
29117 
29118 
29119     /* Read keys/ptrs */
29120 
29121     arrays = btreeAllocSecArray(cache);
29122     ptrs = arrays->parray;
29123 
29124     btreeGetPointers(cache,lbuf,&ptrs);
29125 
29126 
29127     for(i=0;i<nkeys;++i)
29128 	totalkeys += btreeNumInSecbucket(cache,ptrs[i]);
29129 
29130     totalkeys += btreeNumInSecbucket(cache,ptrs[i]);
29131 
29132 
29133     /* Set the number of entries per bucket to approximately half full */
29134     maxnperbucket = nperbucket >> 1;
29135 
29136     if(!maxnperbucket)
29137 	++maxnperbucket;
29138 
29139     if(!leaf->pagepos)
29140 	maxnperbucket = nperbucket;
29141 
29142     /* Work out the number of new buckets needed */
29143     bucketn = (totalkeys / maxnperbucket);
29144 
29145     if(totalkeys % maxnperbucket)
29146 	++bucketn;
29147 
29148     if(bucketn == 1)
29149 	++bucketn;
29150 
29151     while(bucketn > order)
29152     {
29153         ++maxnperbucket;
29154         bucketn = (totalkeys / maxnperbucket);
29155 
29156         if(totalkeys % maxnperbucket)
29157             ++bucketn;
29158     }
29159 
29160     /* Read buckets */
29161     AJCNEW0(buckets,nkeys+1);
29162     keylimit = nkeys + 1;
29163 
29164     for(i=0;i<keylimit;++i)
29165 	buckets[i] = btreeReadSecbucket(cache,ptrs[i]);
29166 
29167 
29168     newarrays = btreeAllocSecArray(cache);
29169     keys = newarrays->karray;
29170     overflows = newarrays->overflows;
29171 
29172     /* Read IDs from all buckets and push to list and sort (increasing id) */
29173     idlist  = ajListNew();
29174 
29175     for(i=0;i<keylimit;++i)
29176     {
29177 	overflows[i] = buckets[i]->Overflow;
29178 	bentries = buckets[i]->Nentries;
29179 
29180 	for(j=0;j<bentries;++j)
29181 	    ajListPush(idlist,(void *)buckets[i]->SecIds[j]);
29182 
29183 	AJFREE(buckets[i]->keylen);
29184 	AJFREE(buckets[i]->SecIds);
29185 	AJFREE(buckets[i]);
29186     }
29187 
29188 /* AJB: check this compares ajStr objects OK */
29189     ajListSort(idlist, &ajStrVcmp);
29190     AJFREE(buckets);
29191 
29192     cbucket = btreeSecbucketNew(maxnperbucket,cache->idlimit);
29193     bucketlimit = bucketn - 1;
29194 
29195     totnids = 0;
29196     nids = (ajuint) ajListGetLength(idlist);
29197 
29198 
29199     if(!totalkeys)
29200     {
29201 	v = totalkeys;
29202 	SBT_NKEYS(lbuf,v);
29203 
29204         btreeDeallocSecArray(cache,arrays);
29205         btreeDeallocSecArray(cache,newarrays);
29206 
29207 	ajListFree(&idlist);
29208 	leaf->dirty = BT_DIRTY;
29209 
29210 	return;
29211     }
29212 
29213     if(nids <= maxnperbucket)
29214     {
29215 	cbucket->Overflow = overflows[1];
29216 	cbucket->Nentries = 0;
29217 	ajListPeek(idlist,(void **)&bid);
29218 	ajStrAssignS(&keys[0],bid);
29219 
29220 	count = 0;
29221 
29222 	while(count!=maxnperbucket && totnids != nids)
29223 	{
29224 	    ajListPop(idlist,(void **)&bid);
29225 
29226 	    cid = cbucket->SecIds[count];
29227 	    ajStrAssignS(&cid,bid);
29228 
29229 	    cbucket->keylen[count] = BT_BUCKSECLEN(bid);
29230 	    ++cbucket->Nentries;
29231 	    ++count;
29232 	    ++totnids;
29233 	    ajStrDel(&bid);
29234 	}
29235 
29236 
29237 	totkeylen += ajStrGetLen(keys[0]);
29238 
29239 	if(!ptrs[1])
29240 	    ptrs[1] = cache->totsize;
29241 
29242 	btreeWriteSecbucket(cache,cbucket,ptrs[1]);
29243 
29244 	cbucket->Overflow = overflows[0];
29245 	cbucket->Nentries = 0;
29246 
29247 	if(!ptrs[0])
29248 	    ptrs[0] = cache->totsize;
29249 
29250 	btreeWriteSecbucket(cache,cbucket,ptrs[0]);
29251     }
29252     else
29253     {
29254 	for(i=0;i<bucketlimit;++i)
29255 	{
29256 	    cbucket->Overflow = overflows[i];
29257 	    cbucket->Nentries = 0;
29258 
29259 	    count = 0;
29260 
29261 	    while(count!=maxnperbucket && totnids != nids)
29262 	    {
29263 		ajListPop(idlist,(void **)&bid);
29264 
29265 		cid = cbucket->SecIds[count];
29266 		ajStrAssignS(&cid,bid);
29267 
29268 		cbucket->keylen[count] = BT_BUCKSECLEN(bid);
29269 		++cbucket->Nentries;
29270 		++count;
29271 		ajStrDel(&bid);
29272 	    }
29273 
29274 
29275 	    ajListPeek(idlist,(void **)&bid);
29276 	    ajStrAssignS(&keys[i],bid);
29277 
29278 
29279 	    totkeylen += ajStrGetLen(bid);
29280 
29281 	    if(!ptrs[i])
29282 		ptrs[i] = cache->totsize;
29283 
29284 	    btreeWriteSecbucket(cache,cbucket,ptrs[i]);
29285 	}
29286 
29287 
29288 	/* Deal with greater-than bucket */
29289 
29290 	cbucket->Overflow = overflows[i];
29291 	cbucket->Nentries = 0;
29292 
29293 	count = 0;
29294 
29295 	while(ajListPop(idlist,(void **)&bid))
29296 	{
29297 	    cid = cbucket->SecIds[count];
29298 	    ajStrAssignS(&cid,bid);
29299 
29300 	    ++cbucket->Nentries;
29301 	    ++count;
29302 	    ajStrDel(&bid);
29303 	}
29304 
29305 
29306 	if(!ptrs[i])
29307 	    ptrs[i] = cache->totsize;
29308 
29309 	btreeWriteSecbucket(cache,cbucket,ptrs[i]);
29310     }
29311 
29312 
29313     cbucket->Nentries = maxnperbucket;
29314     btreeSecbucketDel(&cbucket);
29315 
29316     /* Now write out a modified leaf with new keys/ptrs */
29317 
29318     nkeys = bucketn - 1;
29319     btreeWriteNode(cache,leaf,keys,ptrs,nkeys);
29320 
29321     leaf->dirty = dirtysave;
29322     if(nodetype == BT_SECROOT)
29323     {
29324 	leaf->dirty = BT_LOCK;
29325         leaf->lockfor = 1882;
29326     }
29327 
29328     btreeDeallocSecArray(cache,arrays);
29329     btreeDeallocSecArray(cache,newarrays);
29330 
29331     btreeSecbucketDel(&cbucket);
29332     ajListFree(&idlist);
29333 
29334     return;
29335 }
29336 
29337 
29338 
29339 
29340 /* @funcstatic btreeRebalancePriTwo *******************************************
29341 **
29342 ** Rebalance keyword level 2 tree after deletion
29343 **
29344 ** Deletion software
29345 **
29346 ** @param [u] cache [AjPBtcache] cache
29347 ** @param [r] thisNode [ajulong] Node to rebalance
29348 ** @param [r] leftNode [ajulong] left node
29349 ** @param [r] rightNode [ajulong] right node
29350 ** @param [r] lAnchor [ajulong] left anchor
29351 ** @param [r] rAnchor [ajulong] right anchor
29352 **
29353 ** @return [ajulong] page number or BTNO_NODE
29354 **
29355 ** @release 6.1.0
29356 ** @@
29357 ******************************************************************************/
29358 
btreeRebalancePriTwo(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor)29359 static ajulong btreeRebalancePriTwo(AjPBtcache cache, ajulong thisNode,
29360                                    ajulong leftNode, ajulong rightNode,
29361                                    ajulong lAnchor, ajulong rAnchor)
29362 {
29363     unsigned char *lbuf = NULL;
29364     unsigned char *rbuf = NULL;
29365     unsigned char *tbuf = NULL;
29366 
29367     ajulong anchorNode   = 0UL;
29368     ajulong balanceNode  = 0UL;
29369     ajulong mergeNode    = 0UL;
29370     ajulong done         = 0UL;
29371     ajulong parent       = 0UL;
29372 
29373     AjPBtpage lpage = NULL;
29374     AjPBtpage rpage = NULL;
29375     AjPBtpage tpage = NULL;
29376 
29377     ajuint lnkeys  = 0U;
29378     ajuint rnkeys  = 0U;
29379     ajuint size    = 0U;
29380     ajuint order   = 0U;
29381     ajuint minsize = 0U;
29382 
29383     AjBool leftok  = ajFalse;
29384     AjBool rightok = ajFalse;
29385 
29386 
29387     /* ajDebug("In btreeRebalancePriTwo\n"); */
29388 
29389     if(leftNode!=BTNO_NODE && lAnchor!=BTNO_NODE)
29390 	leftok = ajTrue;
29391 
29392     if(rightNode!=BTNO_NODE && rAnchor!=BTNO_NODE)
29393 	rightok = ajTrue;
29394 
29395     if(!leftok && !rightok)
29396 	return BTNO_NODE;
29397 
29398 
29399     if(leftok)
29400     {
29401 	lpage = btreeSeccacheRead(cache,leftNode);
29402 	lbuf  = lpage->buf;
29403 	GBT_NKEYS(lbuf,&lnkeys);
29404     }
29405 
29406 
29407     if(rightok)
29408     {
29409 	rpage = btreeSeccacheRead(cache,rightNode);
29410 	rbuf  = rpage->buf;
29411 	GBT_NKEYS(rbuf,&rnkeys);
29412     }
29413 
29414 
29415 
29416     if(leftok && rightok)
29417     {
29418 	size = (lnkeys >= rnkeys) ? lnkeys : rnkeys;
29419 	balanceNode = (lnkeys >= rnkeys) ? leftNode : rightNode;
29420     }
29421     else if(leftok)
29422     {
29423 	size = lnkeys;
29424 	balanceNode = leftNode;
29425     }
29426     else
29427     {
29428 	size = rnkeys;
29429 	balanceNode = rightNode;
29430     }
29431 
29432 
29433     order = cache->sorder;
29434     minsize = (order-1) / 2;
29435 
29436     if((order-1)%2)
29437 	++minsize;
29438 
29439     if(size >= minsize)
29440     {
29441 	if(leftok && rightok)
29442 	    anchorNode = (lnkeys >= rnkeys) ? lAnchor : rAnchor;
29443 	else if(leftok)
29444 	    anchorNode = lAnchor;
29445 	else
29446 	    anchorNode = rAnchor;
29447 
29448 	done = btreeShiftPriTwo(cache,thisNode,balanceNode,anchorNode);
29449     }
29450 
29451     else
29452     {
29453 	tpage = btreeSeccacheRead(cache,thisNode);
29454 	tbuf  = tpage->buf;
29455 	GBT_PREV(tbuf,&parent);
29456 	if(leftok && rightok)
29457 	{
29458 	    anchorNode = (parent == lAnchor) ? lAnchor : rAnchor;
29459 	    mergeNode  = (anchorNode == lAnchor) ? leftNode : rightNode;
29460 	}
29461 	else if(leftok)
29462 	{
29463 	    anchorNode = lAnchor;
29464 	    mergeNode  = leftNode;
29465 	}
29466 	else
29467 	{
29468 	    anchorNode = rAnchor;
29469 	    mergeNode  = rightNode;
29470 	}
29471 
29472 	done = btreeMergePriTwo(cache,thisNode,mergeNode,anchorNode);
29473     }
29474 
29475     return done;
29476 }
29477 
29478 
29479 
29480 
29481 /* @funcstatic btreeShiftPriTwo ***********************************************
29482 **
29483 ** Shift spare entries from one keyword tree level 2 node to another.
29484 **
29485 ** Deletion software.
29486 **
29487 ** @param [u] cache [AjPBtcache] cache
29488 ** @param [r] thisNode [ajulong] master node
29489 ** @param [r] balanceNode [ajulong] balance node
29490 ** @param [r] anchorNode [ajulong] anchor node
29491 **
29492 ** @return [ajulong] page number or BTNO_NODE
29493 **
29494 ** @release 6.1.0
29495 ** @@
29496 ******************************************************************************/
29497 
btreeShiftPriTwo(AjPBtcache cache,ajulong thisNode,ajulong balanceNode,ajulong anchorNode)29498 static ajulong btreeShiftPriTwo(AjPBtcache cache, ajulong thisNode,
29499                                ajulong balanceNode, ajulong anchorNode)
29500 {
29501     unsigned char *tbuf = NULL;
29502     unsigned char *abuf = NULL;
29503     unsigned char *bbuf = NULL;
29504     unsigned char *buf  = NULL;
29505 
29506     AjPStr *kTarray = NULL;
29507     AjPStr *kAarray = NULL;
29508     AjPStr *kBarray = NULL;
29509     ajulong *pTarray = NULL;
29510     ajulong *pAarray = NULL;
29511     ajulong *pBarray = NULL;
29512 
29513     ajuint  nAkeys = 0U;
29514     ajuint  nBkeys = 0U;
29515     ajuint  nTkeys = 0U;
29516     ajuint  i;
29517     ajint ii;
29518 
29519     AjPBtpage pageA = NULL;
29520     AjPBtpage pageB = NULL;
29521     AjPBtpage pageT = NULL;
29522     AjPBtpage page  = NULL;
29523 
29524     AjPBtpage leftpage = NULL;
29525 
29526     ajuint anchorPos   = 0U;
29527     ajulong prev        = 0UL;
29528     ajuint  nodetype    = 0U;
29529 
29530     ajulong lv = 0UL;
29531 
29532     AjPBtMem arraysA = NULL;
29533     AjPBtMem arraysB = NULL;
29534     AjPBtMem arraysT = NULL;
29535 
29536     /* ajDebug("In btreeShiftPriTwo\n"); */
29537 
29538 
29539     arraysA = btreeAllocSecArray(cache);
29540     kAarray = arraysA->karray;
29541     pAarray = arraysA->parray;
29542 
29543     arraysB = btreeAllocSecArray(cache);
29544     kBarray = arraysB->karray;
29545     pBarray = arraysB->parray;
29546 
29547     arraysT = btreeAllocSecArray(cache);
29548     kTarray = arraysT->karray;
29549     pTarray = arraysT->parray;
29550 
29551 
29552     pageA = btreeSeccacheRead(cache,anchorNode);
29553     pageA->dirty = BT_LOCK;
29554     pageA->lockfor = 1891;
29555     abuf = pageA->buf;
29556     pageB = btreeSeccacheRead(cache,balanceNode);
29557     pageB->dirty = BT_LOCK;
29558     pageB->lockfor = 1892;
29559     bbuf = pageB->buf;
29560     pageT = btreeSeccacheRead(cache,thisNode);
29561     pageT->dirty = BT_LOCK;
29562     pageT->lockfor = 1893;
29563     tbuf = pageT->buf;
29564 
29565     GBT_NKEYS(abuf,&nAkeys);
29566     GBT_NKEYS(bbuf,&nBkeys);
29567     GBT_NKEYS(tbuf,&nTkeys);
29568 
29569     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
29570     btreeGetKeys(cache,bbuf,&kBarray,&pBarray);
29571     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
29572 
29573     if(MAJSTRCMPS(kTarray[nTkeys-1],kBarray[nBkeys-1])<0)
29574 	leftpage = pageT;
29575     else
29576 	leftpage = pageB;
29577 
29578 
29579     if(leftpage == pageT)
29580     {
29581 	/* Find anchor key position */
29582 	i=0;
29583 
29584 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
29585 	    ++i;
29586 
29587 	anchorPos = i;
29588 
29589 	/* Move down anchor key to thisNode */
29590 	ajStrAssignS(&kTarray[nTkeys],kAarray[anchorPos]);
29591 	++nTkeys;
29592 
29593 	/* Shift extra */
29594 
29595 	while(nTkeys < nBkeys)
29596 	{
29597 	    ajStrAssignS(&kTarray[nTkeys],kBarray[0]);
29598 	    pTarray[nTkeys] = pBarray[0];
29599 	    ++nTkeys;
29600 	    --nBkeys;
29601 
29602 	    for(i=0;i<nBkeys;++i)
29603 	    {
29604 		ajStrAssignS(&kBarray[i],kBarray[i+1]);
29605 		pBarray[i] = pBarray[i+1];
29606 	    }
29607 	    pBarray[i] = pBarray[i+1];
29608 	}
29609 
29610 	/* Adjust anchor key */
29611 	ajStrAssignS(&kAarray[anchorPos],kTarray[nTkeys-1]);
29612 	--nTkeys;
29613     }
29614     else	/* thisNode on the right */
29615     {
29616 	/* Find anchor key position */
29617 	i=0;
29618 
29619 	while(i!=nAkeys && MAJSTRCMPS(kBarray[nBkeys-1],kAarray[i])>=0)
29620 	    ++i;
29621 
29622 	anchorPos = i;
29623 
29624 	/* Move down anchor key to thisNode */
29625 	pTarray[nTkeys+1] = pTarray[nTkeys];
29626 
29627 	for(ii=nTkeys-1;ii>-1;--ii)
29628 	{
29629 	    ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
29630 	    pTarray[ii+1] = pTarray[ii];
29631 	}
29632 
29633 	ajStrAssignS(&kTarray[0],kAarray[anchorPos]);
29634 	++nTkeys;
29635 
29636 	/* Shift extra */
29637 	while(nTkeys < nBkeys)
29638 	{
29639 	    pTarray[nTkeys+1] = pTarray[nTkeys];
29640 
29641 	    for(ii=nTkeys-1;ii>-1;--ii)
29642 	    {
29643 		ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
29644 		pTarray[ii+1] = pTarray[ii];
29645 	    }
29646 	    ajStrAssignS(&kTarray[0],kBarray[nBkeys-1]);
29647 	    pTarray[1] = pBarray[nBkeys];
29648 	    ++nTkeys;
29649 	    --nBkeys;
29650 	}
29651 
29652 
29653 	/* Adjust anchor key */
29654 	ajStrAssignS(&kAarray[anchorPos],kTarray[0]);
29655 	--nTkeys;
29656 
29657 	for(i=0;i<nTkeys;++i)
29658 	{
29659 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
29660 	    pTarray[i] = pTarray[i+1];
29661 	}
29662 	pTarray[i] = pTarray[i+1];
29663     }
29664 
29665 
29666     /* Adjust PREV pointers for thisNode */
29667     prev = pageT->pagepos;
29668 
29669     for(i=0;i<nTkeys+1;++i)
29670     {
29671 	page = btreeSeccacheRead(cache,pTarray[i]);
29672 	buf = page->buf;
29673 	GBT_NODETYPE(buf,&nodetype);
29674 
29675 	if(nodetype != BT_IDBUCKET)
29676 	{
29677 	    lv = prev;
29678 	    SBT_PREV(buf,lv);
29679 	    page->dirty = BT_DIRTY;
29680 	}
29681     }
29682 
29683     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
29684     btreeWriteNode(cache,pageB,kBarray,pBarray,nBkeys);
29685     btreeWriteNode(cache,pageT,kTarray,pTarray,nTkeys);
29686 
29687     if(anchorNode == cache->secrootblock)
29688     {
29689 	pageA->dirty = BT_LOCK;
29690         pageA->lockfor = 1894;
29691     }
29692 
29693     btreeDeallocSecArray(cache,arraysA);
29694     btreeDeallocSecArray(cache,arraysB);
29695     btreeDeallocSecArray(cache,arraysT);
29696 
29697     return BTNO_NODE;
29698 }
29699 
29700 
29701 
29702 
29703 /* @funcstatic btreeMergePriTwo ***********************************************
29704 **
29705 ** Merge two nodes.
29706 **
29707 ** Deletion software
29708 **
29709 ** @param [u] cache [AjPBtcache] cache
29710 ** @param [r] thisNode [ajulong] master node
29711 ** @param [r] mergeNode [ajulong] merge node
29712 ** @param [r] anchorNode [ajulong] anchor node
29713 **
29714 ** @return [ajulong] page number or BTNO_NODE
29715 **
29716 ** @release 6.1.0
29717 ** @@
29718 ******************************************************************************/
29719 
btreeMergePriTwo(AjPBtcache cache,ajulong thisNode,ajulong mergeNode,ajulong anchorNode)29720 static ajulong btreeMergePriTwo(AjPBtcache cache, ajulong thisNode,
29721                                 ajulong mergeNode, ajulong anchorNode)
29722 {
29723     unsigned char *tbuf = NULL;
29724     unsigned char *abuf = NULL;
29725     unsigned char *nbuf = NULL;
29726     unsigned char *buf  = NULL;
29727 
29728     AjPStr *kTarray = NULL;
29729     AjPStr *kAarray = NULL;
29730     AjPStr *kNarray = NULL;
29731     ajulong *pTarray = NULL;
29732     ajulong *pAarray = NULL;
29733     ajulong *pNarray = NULL;
29734 
29735     ajulong thisprev  = 0UL;
29736     ajulong mergeprev = 0UL;
29737 
29738 
29739     ajuint  nAkeys = 0U;
29740     ajuint  nNkeys = 0U;
29741     ajuint  nTkeys = 0U;
29742     ajuint  count  = 0U;
29743     ajuint  i;
29744     ajint ii;
29745 
29746     ajuint   nodetype = 0U;
29747 
29748     ajuint saveA = 0U;
29749     ajuint saveN = 0U;
29750     ajuint saveT = 0U;
29751 
29752     AjPBtpage pageA = NULL;
29753     AjPBtpage pageN = NULL;
29754     AjPBtpage pageT = NULL;
29755     AjPBtpage page  = NULL;
29756 
29757     AjPBtpage leftpage = NULL;
29758 
29759     ajuint anchorPos = 0U;
29760     ajulong prev      = 0UL;
29761 
29762     ajulong lv = 0UL;
29763 
29764     AjPBtMem arraysA = NULL;
29765     AjPBtMem arraysN = NULL;
29766     AjPBtMem arraysT = NULL;
29767 
29768     AjBool collapse = ajFalse;
29769     ajulong csrb = 0UL;
29770 
29771     /* ajDebug("In btreeMergePriTwo\n"); */
29772 
29773     pageA = btreeSeccacheRead(cache,anchorNode);
29774     saveA = pageA->dirty;
29775     pageA->dirty = BT_LOCK;
29776     pageA->lockfor = 1901;
29777     abuf = pageA->buf;
29778     pageN = btreeSeccacheRead(cache,mergeNode);
29779     saveN = pageN->dirty;
29780     pageN->dirty = BT_LOCK;
29781     pageN->lockfor = 1902;
29782     nbuf = pageN->buf;
29783     pageT = btreeSeccacheRead(cache,thisNode);
29784     saveT = pageT->dirty;
29785     pageT->dirty = BT_LOCK;
29786     pageT->lockfor = 1903;
29787     tbuf = pageT->buf;
29788 
29789     GBT_PREV(tbuf,&thisprev);
29790     GBT_PREV(nbuf,&mergeprev);
29791 
29792     GBT_NKEYS(abuf,&nAkeys);
29793     GBT_NKEYS(nbuf,&nNkeys);
29794     GBT_NKEYS(tbuf,&nTkeys);
29795 
29796     GBT_NODETYPE(nbuf,&nodetype);
29797 
29798     csrb = cache->secrootblock;
29799 
29800     if(nAkeys == 1)
29801     {
29802 	if(anchorNode==csrb && thisprev==csrb && mergeprev==csrb)
29803 	    collapse = ajTrue;
29804 	else
29805 	{
29806 	    pageA->dirty = saveA;
29807 	    pageN->dirty = saveN;
29808 	    pageT->dirty = saveT;
29809 
29810 	    return thisNode;
29811 	}
29812     }
29813 
29814     arraysA = btreeAllocSecArray(cache);
29815     kAarray = arraysA->karray;
29816     pAarray = arraysA->parray;
29817 
29818     arraysN = btreeAllocSecArray(cache);
29819     kNarray = arraysN->karray;
29820     pNarray = arraysN->parray;
29821 
29822     arraysT = btreeAllocSecArray(cache);
29823     kTarray = arraysT->karray;
29824     pTarray = arraysT->parray;
29825 
29826     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
29827     btreeGetKeys(cache,nbuf,&kNarray,&pNarray);
29828     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
29829 
29830     if(MAJSTRCMPS(kTarray[nTkeys-1],kNarray[nNkeys-1])<0)
29831 	leftpage = pageT;
29832     else
29833 	leftpage = pageN;
29834 
29835 
29836     if(leftpage == pageT)
29837     {
29838 	/* Find anchor key position */
29839 	i=0;
29840 
29841 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
29842 	    ++i;
29843 
29844 	anchorPos = i;
29845 
29846 	/* Move down anchor key to neighbour Node */
29847 	pNarray[nNkeys+1] = pNarray[nNkeys];
29848 
29849 	for(ii=nNkeys-1;ii>-1;--ii)
29850 	{
29851 	    ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
29852 	    pNarray[ii+1] = pNarray[ii];
29853 	}
29854 
29855 	ajStrAssignS(&kNarray[0],kAarray[anchorPos]);
29856 	++nNkeys;
29857 
29858 
29859 	/* Adjust anchor node keys/ptrs */
29860 	++anchorPos;
29861 	if(anchorPos==nAkeys)
29862 	    pAarray[nAkeys-1] = pAarray[nAkeys];
29863 	else
29864 	{
29865 	    for(i=anchorPos;i<nAkeys;++i)
29866 	    {
29867 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
29868 		pAarray[i-1] = pAarray[i];
29869 	    }
29870 	    pAarray[i-1] = pAarray[i];
29871 	}
29872 	--nAkeys;
29873 
29874 
29875 	/* Merge this to neighbour */
29876 
29877 	while(nTkeys)
29878 	{
29879 	    pNarray[nNkeys+1] = pNarray[nNkeys];
29880 
29881 	    for(ii=nNkeys-1;ii>-1;--ii)
29882 	    {
29883 		ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
29884 		pNarray[ii+1] = pNarray[ii];
29885 	    }
29886 
29887 	    ajStrAssignS(&kNarray[0],kTarray[nTkeys-1]);
29888 	    pNarray[1] = pTarray[nTkeys];
29889 	    pNarray[0] = pTarray[nTkeys-1];
29890 	    --nTkeys;
29891 	    ++nNkeys;
29892 	}
29893 
29894 	/* At this point the 'this' node could be added to a free list */
29895     }
29896     else
29897     {
29898 	/* Find anchor key position */
29899 	i=0;
29900 
29901 	while(i!=nAkeys && MAJSTRCMPS(kNarray[nNkeys-1],kAarray[i])>=0)
29902 	    ++i;
29903 
29904 	anchorPos = i;
29905 
29906 	/* Move down anchor key to neighbourNode */
29907 	ajStrAssignS(&kNarray[nNkeys],kAarray[anchorPos]);
29908 	++nNkeys;
29909 
29910 	/* Adjust anchor node keys/ptrs */
29911 	++anchorPos;
29912 
29913 	if(anchorPos!=nAkeys)
29914 	    for(i=anchorPos;i<nAkeys;++i)
29915 	    {
29916 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
29917 		pAarray[i] = pAarray[i+1];
29918 	    }
29919 	--nAkeys;
29920 
29921 	/* merge extra */
29922 	count = 0;
29923 
29924 	while(nTkeys)
29925 	{
29926 	    ajStrAssignS(&kNarray[nNkeys],kTarray[count]);
29927 	    pNarray[nNkeys] = pTarray[count];
29928 	    ++nNkeys;
29929 	    ++count;
29930 	    --nTkeys;
29931 	    pNarray[nNkeys] = pTarray[count];
29932 
29933 	}
29934 
29935 	/* At this point the 'this' node could be added to a free list */
29936     }
29937 
29938 
29939     /* Adjust PREV pointers for neighbour Node */
29940     prev = pageN->pagepos;
29941 
29942     for(i=0;i<=nNkeys;++i)
29943     {
29944 	page = btreeSeccacheRead(cache,pNarray[i]);
29945 	buf = page->buf;
29946 	GBT_NODETYPE(buf,&nodetype);
29947 
29948 	if(nodetype != BT_IDBUCKET)
29949 	{
29950 	    lv = prev;
29951 	    SBT_PREV(buf,lv);
29952 	    page->dirty = BT_DIRTY;
29953 	}
29954     }
29955 
29956     pageT->dirty = BT_CLEAN;
29957     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
29958     btreeWriteNode(cache,pageN,kNarray,pNarray,nNkeys);
29959 
29960     if(anchorNode == cache->secrootblock)
29961     {
29962 	pageA->dirty = BT_LOCK;
29963         pageA->lockfor = 1904;
29964     }
29965 
29966     btreeDeallocSecArray(cache,arraysA);
29967     btreeDeallocSecArray(cache,arraysN);
29968     btreeDeallocSecArray(cache,arraysT);
29969 
29970     if(collapse)
29971 	btreeCollapseRootPriTwo(cache,mergeNode);
29972 
29973     return thisNode;
29974 }
29975 
29976 
29977 
29978 
29979 /* @funcstatic btreeCollapseRootPriTwo ****************************************
29980 **
29981 ** Collapse root page for keyword level 2 tree.
29982 **
29983 ** Deletion software.
29984 **
29985 ** @param [u] cache [AjPBtcache] cache
29986 ** @param [r] pagepos [ajulong] page number to make new root
29987 **
29988 ** @return [ajulong] page number or BTNO_NODE
29989 **
29990 ** @release 6.1.0
29991 ** @@
29992 ******************************************************************************/
29993 
btreeCollapseRootPriTwo(AjPBtcache cache,ajulong pagepos)29994 static ajulong btreeCollapseRootPriTwo(AjPBtcache cache, ajulong pagepos)
29995 {
29996     unsigned char *buf  = NULL;
29997     unsigned char *lbuf = NULL;
29998 
29999     AjPStr *karray = NULL;
30000     ajulong *parray = NULL;
30001 
30002     AjPBtpage rootpage = NULL;
30003     AjPBtpage page     = NULL;
30004 
30005     ajuint nodetype = 0U;
30006     ajuint nkeys    = 0U;
30007     ajuint i;
30008 
30009     ajulong prev = 0UL;
30010     AjPBtMem arrays = NULL;
30011 
30012     /* ajDebug("In btreeCollapseRootPriTwo\n"); */
30013 
30014     if(!cache->slevel)
30015 	return BTNO_NODE;
30016 
30017     rootpage = btreeSeccacheLocate(cache,cache->secrootblock);
30018     buf = rootpage->buf;
30019     page = btreeSeccacheRead(cache,pagepos);
30020 
30021 
30022     arrays = btreeAllocSecArray(cache);
30023     karray = arrays->karray;
30024     parray = arrays->parray;
30025 
30026     /*
30027     ** Swap pagepos values to make root the child and child the root
30028     ** Update node types and mark the original root as a clean page
30029     */
30030 
30031     /* At this point page->pagepos could be added to a free list */
30032 
30033     rootpage->pagepos = page->pagepos;
30034     rootpage->dirty = BT_CLEAN;
30035     nodetype = BT_SECINTERNAL;
30036     SBT_NODETYPE(buf,nodetype);
30037 
30038     page->pagepos = cache->secrootblock;
30039     page->dirty = BT_LOCK;
30040     page->lockfor = 1911;
30041     buf = page->buf;
30042     nodetype = BT_SECROOT;
30043     SBT_NODETYPE(buf,nodetype);
30044 
30045     --cache->slevel;
30046 
30047     if(cache->slevel)
30048     {
30049 	/*
30050 	 ** Update the PREV pointers of the new root's children
30051 	 */
30052 	GBT_NKEYS(buf,&nkeys);
30053 	btreeGetKeys(cache,buf,&karray,&parray);
30054 
30055 	for(i=0;i<=nkeys;++i)
30056 	{
30057 	    page = btreeSeccacheRead(cache,parray[i]);
30058 	    lbuf = page->buf;
30059 	    SBT_PREV(lbuf,prev);
30060 	    page->dirty = BT_DIRTY;
30061 	}
30062     }
30063 
30064     btreeDeallocSecArray(cache,arrays);
30065 
30066     return 0UL;
30067 }
30068 
30069 
30070 
30071 
30072 /* @funcstatic btreeFindPriBalanceOne *****************************************
30073 **
30074 ** Master routine for entry deletion from level 1 keyword tree.
30075 **
30076 ** Deletion software.
30077 **
30078 ** @param [u] cache [AjPBtcache] cache
30079 ** @param [r] thisNode [ajulong] Current node
30080 ** @param [r] leftNode [ajulong] Node to left
30081 ** @param [r] rightNode [ajulong] Node to right
30082 ** @param [r] lAnchor [ajulong] Left anchor
30083 ** @param [r] rAnchor [ajulong] Right anchor
30084 ** @param [r] pri [const AjPBtPri] pri
30085 **
30086 ** @return [ajulong] page number or BTNO_NODE
30087 **
30088 ** @release 6.1.0
30089 ** @@
30090 ******************************************************************************/
30091 
btreeFindPriBalanceOne(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor,const AjPBtPri pri)30092 static ajulong btreeFindPriBalanceOne(AjPBtcache cache, ajulong thisNode,
30093                                       ajulong leftNode, ajulong rightNode,
30094                                       ajulong lAnchor, ajulong rAnchor,
30095                                       const AjPBtPri pri)
30096 {
30097     unsigned char *buf  = NULL;
30098     unsigned char *buf1 = NULL;
30099 
30100     ajulong nextNode   = BTNO_NODE;
30101     ajulong nextLeft   = BTNO_NODE;
30102     ajulong nextRight  = BTNO_NODE;
30103     ajulong nextAncL   = BTNO_NODE;
30104     ajulong nextAncR   = BTNO_NODE;
30105     ajulong done       = 0UL;
30106 
30107     ajuint  nkeys      = 0U;
30108     ajuint  order      = 0U;
30109     ajuint  minkeys    = 0U;
30110     ajuint  i;
30111     ajuint  nodetype   = 0U;
30112 
30113     ajuint n1keys      = 0U;
30114 
30115     AjPBtpage page  = NULL;
30116     AjPBtpage page1 = NULL;
30117 
30118     ajulong balanceNode = 0UL;
30119     ajulong blockno     = 0UL;
30120     ajulong ptrSave     = 0UL;
30121 
30122     AjPStr *karray  = NULL;
30123     ajulong *parray  = NULL;
30124     AjPStr *k1array = NULL;
30125     ajulong *p1array = NULL;
30126 
30127     AjPBtMem arrays  = NULL;
30128     AjPBtMem arrays1 = NULL;
30129 
30130     const AjPStr key = NULL;
30131     AjBool existed = ajFalse;
30132 
30133     /* ajDebug("In btreeFindPriBalanceOne\n"); */
30134 
30135     if(thisNode)
30136 	page = btreePricacheRead(cache,thisNode);
30137     else
30138     {   /* It's the root node of the primary hyb tree */
30139         /* Needs altering for secondary tree          */
30140 	page = btreePricacheLocate(cache,thisNode);
30141 	page->dirty = BT_LOCK;
30142         page->lockfor = 1921;
30143     }
30144 
30145     cache->dodelete = ajFalse;
30146 
30147     buf = page->buf;
30148     GBT_NKEYS(buf,&nkeys);
30149 
30150     order = cache->porder;
30151     /* order-1 is the number of keys in the node */
30152     minkeys = (order-1) / 2;
30153 
30154     if((order-1)%2)
30155 	++minkeys;
30156 
30157     /*
30158     ** If thisNode contains >= minkeys then it is not a candidate
30159     ** for balancing
30160     */
30161     if(nkeys >= minkeys)
30162 	balanceNode = BTNO_BALANCE;
30163     else
30164 	balanceNode = page->pagepos;
30165 
30166     arrays  = btreeAllocPriArray(cache);
30167     arrays1 = btreeAllocPriArray(cache);
30168 
30169     karray = arrays->karray;
30170     parray = arrays->parray;
30171 
30172     k1array = arrays1->karray;
30173     p1array = arrays1->parray;
30174 
30175     key = pri->keyword;
30176 
30177     btreeGetKeys(cache,buf,&karray,&parray);
30178 
30179     i=0;
30180 
30181     while(i!=nkeys && MAJSTRCMPS(key,karray[i])>=0)
30182 	++i;
30183 
30184     blockno = parray[i];
30185 
30186     nextNode = blockno;
30187     ptrSave = i;
30188 
30189     GBT_NODETYPE(buf,&nodetype);
30190 
30191     if(!(nodetype == BT_LEAF) && !(nodetype == BT_ROOT && !cache->plevel))
30192     {
30193 	if(nextNode == parray[0])
30194 	{
30195 	    if(leftNode != BTNO_NODE)
30196 	    {
30197 		page1 = btreePricacheRead(cache,leftNode);
30198 		buf1 = page1->buf;
30199 		GBT_NKEYS(buf1,&n1keys);
30200 		btreeGetKeys(cache,buf1,&k1array,&p1array);
30201 		nextLeft = p1array[n1keys];
30202 	    }
30203 	    else
30204 		nextLeft = BTNO_NODE;
30205 
30206 	    if(!thisNode)
30207 		nextAncL = thisNode;
30208 	    else
30209 		nextAncL = lAnchor;
30210 	}
30211 	else
30212 	{
30213 	    nextLeft = parray[ptrSave-1];
30214 	    nextAncL = thisNode;
30215 	}
30216 
30217 	if(nextNode == parray[nkeys])
30218 	{
30219 	    if(rightNode != BTNO_NODE)
30220 	    {
30221 		page1 = btreePricacheRead(cache,rightNode);
30222 		buf1 = page1->buf;
30223 		GBT_NKEYS(buf1,&n1keys);
30224 		btreeGetKeys(cache,buf1,&k1array,&p1array);
30225 		nextRight = p1array[0];
30226 	    }
30227 	    else
30228 		nextRight = BTNO_NODE;
30229 
30230 	    if(!thisNode)
30231 		nextAncR = thisNode;
30232 	    else
30233 		nextAncR = rAnchor;
30234 	}
30235 	else
30236 	{
30237 	    nextRight = parray[ptrSave+1];
30238 	    nextAncR  = thisNode;
30239 	}
30240 
30241 
30242 
30243 	/* Check to see whether key exists in an internal node */
30244 	if(nodetype != BT_LEAF && cache->plevel)
30245 	{
30246 	    i=0;
30247 
30248 	    while(i!=nkeys && MAJSTRCMPS(key,karray[i]))
30249 		++i;
30250 
30251 	    if(i!=nkeys)
30252 	    {
30253 		btreeFindPriMinOne(cache,parray[i+1],key);
30254 		ajStrAssignS(&karray[i],cache->replace);
30255 		btreeWriteNode(cache,page,karray,parray,nkeys);
30256 	    }
30257 
30258 	}
30259 
30260 	btreeFindPriBalanceOne(cache,nextNode,nextLeft,nextRight,
30261                                nextAncL,nextAncR,pri);
30262 
30263 	if(thisNode)
30264 	    page = btreePricacheRead(cache,thisNode);
30265 	else
30266 	{
30267 	    page = btreePricacheLocate(cache,thisNode);
30268 	    page->dirty = BT_LOCK;
30269             page->lockfor = 1922;
30270 	}
30271 	buf = page->buf;
30272 
30273     }
30274     else
30275     {
30276 	if(nodetype == BT_LEAF || (nodetype==BT_ROOT && !cache->plevel))
30277 	{
30278 	    existed = btreeRemovePriEntryOne(cache,thisNode,pri);
30279 
30280 	    if(existed)
30281 		cache->dodelete = ajTrue;
30282 	    GBT_NKEYS(buf,&nkeys);
30283 
30284 	    if(nkeys >= minkeys || (nodetype==BT_ROOT && !cache->plevel))
30285 		balanceNode = BTNO_BALANCE;
30286 	    else
30287 		balanceNode = page->pagepos;
30288 	}
30289     }
30290 
30291 
30292     if(balanceNode == BTNO_BALANCE || thisNode == 0UL)
30293 	done = BTNO_NODE;
30294     else
30295 	done = btreeRebalancePriOne(cache,thisNode,leftNode,rightNode,
30296                                     lAnchor,rAnchor);
30297 
30298 
30299     btreeDeallocPriArray(cache,arrays);
30300     btreeDeallocPriArray(cache,arrays1);
30301 
30302     return done;
30303 }
30304 
30305 
30306 
30307 
30308 /* @funcstatic btreeFindPriMinOne *********************************************
30309 **
30310 ** Find minimum key in keyword level 1 subtree and store in cache.
30311 **
30312 ** Deletion software.
30313 **
30314 ** @param [u] cache [AjPBtcache] cache
30315 ** @param [r] pagepos [ajulong] page
30316 ** @param [r] key [const AjPStr] key
30317 **
30318 ** @return [void]
30319 **
30320 ** @release 6.1.0
30321 ** @@
30322 ******************************************************************************/
30323 
btreeFindPriMinOne(AjPBtcache cache,ajulong pagepos,const AjPStr key)30324 static void btreeFindPriMinOne(AjPBtcache cache, ajulong pagepos,
30325                                const AjPStr key)
30326 {
30327     AjPBtpage page   = NULL;
30328     AjPPribucket bucket = NULL;
30329     AjPStr *karray   = NULL;
30330     ajulong *parray   = NULL;
30331 
30332     ajuint nkeys    = 0;
30333     ajuint nodetype = 0;
30334     ajuint nentries = 0;
30335     ajuint i;
30336     AjPBtMem arrays = NULL;
30337 
30338     unsigned char *buf = NULL;
30339 
30340     /* ajDebug("In btreeFindPriMinOne\n"); */
30341 
30342     arrays = btreeAllocPriArray(cache);
30343     karray = arrays->karray;
30344     parray = arrays->parray;
30345 
30346     page = btreePricacheRead(cache,pagepos);
30347     buf  = page->buf;
30348     GBT_NODETYPE(buf,&nodetype);
30349     GBT_NKEYS(buf,&nkeys);
30350 
30351     btreeGetKeys(cache,buf,&karray,&parray);
30352 
30353     if(nodetype == BT_LEAF)
30354     {
30355 	bucket = btreePribucketRead(cache,parray[0]);
30356 	nentries = bucket->Nentries;
30357 
30358         /*
30359         ** If there's only one entry then it must be the key marked
30360         ** for deletion
30361         */
30362         if(nentries<2)
30363 	{
30364 	    btreePribucketDel(&bucket);
30365 	    bucket = btreePribucketRead(cache,parray[1]);
30366 	    nentries = bucket->Nentries;
30367 	}
30368 
30369         /* Check for empty bucket - shouldn't happen */
30370         /* Checking solely out of interest */
30371 	if(nentries<1)
30372 	    ajFatal("FindPriMinOne: Too few entries in bucket Nkeys=%u\n",
30373                     nkeys);
30374 
30375 
30376         /* Find lowest value key in the bucket and store in cache */
30377         ajStrAssignS(&cache->replace,bucket->codes[0]->keyword);
30378 
30379 	if(!MAJSTRCMPS(cache->replace,key))
30380 	    ajStrAssignS(&cache->replace,bucket->codes[1]->keyword);
30381 
30382 	for(i=1;i<nentries;++i)
30383 	    if(MAJSTRCMPS(bucket->codes[i]->keyword,cache->replace)<0 &&
30384 	       MAJSTRCMPS(bucket->codes[i]->keyword,key))
30385 		ajStrAssignS(&cache->replace,bucket->codes[i]->keyword);
30386 	btreePribucketDel(&bucket);
30387     }
30388     else
30389     {
30390 	pagepos = parray[0];
30391 	btreeFindPriMinOne(cache,pagepos,key);
30392 
30393     }
30394 
30395     btreeDeallocPriArray(cache,arrays);
30396 
30397     return;
30398 }
30399 
30400 
30401 
30402 
30403 /* @funcstatic btreeRemovePriEntryOne *****************************************
30404 **
30405 ** Find and delete an ID from a given hybrid tree level 1 leaf node if
30406 ** necessary.
30407 **
30408 ** Deletion software
30409 **
30410 ** @param [u] cache [AjPBtcache] cache
30411 ** @param [r] pagepos [ajulong] leaf node page
30412 ** @param [r] pri [const AjPBtPri] keyword object
30413 **
30414 ** @return [AjBool] True if found (and deleted)
30415 **
30416 ** @release 6.1.0
30417 ** @@
30418 ******************************************************************************/
30419 
btreeRemovePriEntryOne(AjPBtcache cache,ajulong pagepos,const AjPBtPri pri)30420 static AjBool btreeRemovePriEntryOne(AjPBtcache cache, ajulong pagepos,
30421                                      const AjPBtPri pri)
30422 {
30423     AjPBtpage page   = NULL;
30424     AjPPribucket bucket = NULL;
30425 
30426     AjPStr *karray = NULL;
30427     ajulong *parray = NULL;
30428     ajulong blockno = 0UL;
30429 
30430     ajuint nkeys    = 0U;
30431     ajuint nentries = 0U;
30432     ajuint i;
30433 
30434     ajuint dirtysave = 0U;
30435 
30436     AjBool found = ajFalse;
30437     const AjPStr key  = NULL;
30438 
30439     unsigned char *buf = NULL;
30440     AjPBtMem arrays = NULL;
30441 
30442     /* ajDebug("In btreeRemovePriEntryOne\n"); */
30443 
30444     page = btreePricacheRead(cache,pagepos);
30445     buf = page->buf;
30446     dirtysave = page->dirty;
30447     page->dirty = BT_LOCK;
30448     page->lockfor = 1931;
30449 
30450     GBT_NKEYS(buf,&nkeys);
30451 
30452     if(!nkeys)
30453 	return ajFalse;
30454 
30455 
30456     arrays = btreeAllocPriArray(cache);
30457     karray = arrays->karray;
30458     parray = arrays->parray;
30459 
30460     btreeGetKeys(cache,buf,&karray,&parray);
30461 
30462     key = pri->keyword;
30463 
30464     i=0;
30465 
30466     while(i!=nkeys && MAJSTRCMPS(key,karray[i])>=0)
30467 	++i;
30468 
30469     blockno = parray[i];
30470 
30471     bucket = btreePribucketRead(cache,blockno);
30472 
30473 
30474     nentries = bucket->Nentries;
30475     found = ajFalse;
30476 
30477     for(i=0;i<nentries;++i)
30478 	if(!MAJSTRCMPS(key,bucket->codes[i]->keyword))
30479 	{
30480 	    found = ajTrue;
30481 	    break;
30482 	}
30483 
30484 
30485     if(found)
30486     {
30487 	/* Perform the deletion */
30488 	if(nentries == 1)
30489 	{
30490 	    bucket->Nentries = 0;
30491 	    ajBtreePriDel(&bucket->codes[0]);
30492 	}
30493 	else
30494 	{
30495 	    ajBtreePriDel(&bucket->codes[i]);
30496 	    bucket->codes[i] = bucket->codes[nentries-1];
30497 	    --bucket->Nentries;
30498 	}
30499 
30500 	btreeWritePribucket(cache,bucket,blockno);
30501 	btreeAdjustPribucketsOne(cache,page);
30502 	page->dirty = BT_DIRTY;
30503     }
30504     else
30505 	page->dirty = dirtysave;
30506 
30507     btreePribucketDel(&bucket);
30508 
30509     btreeDeallocPriArray(cache,arrays);
30510 
30511     if(!found)
30512 	return ajFalse;
30513 
30514     return ajTrue;
30515 }
30516 
30517 
30518 
30519 
30520 /* @funcstatic btreeAdjustPribucketsOne ***************************************
30521 **
30522 ** Re-order leaf buckets
30523 ** Can be called whatever the state of a leaf.
30524 **
30525 ** Deletion software
30526 **
30527 ** @param [u] cache [AjPBtcache] cache
30528 ** @param [u] leaf [AjPBtpage] leaf page
30529 **
30530 ** @return [void]
30531 **
30532 ** @release 6.1.0
30533 ** @@
30534 ******************************************************************************/
30535 
btreeAdjustPribucketsOne(AjPBtcache cache,AjPBtpage leaf)30536 static void btreeAdjustPribucketsOne(AjPBtcache cache, AjPBtpage leaf)
30537 {
30538     ajuint nkeys = 0;
30539     unsigned char *lbuf = NULL;
30540     AjPPribucket *buckets  = NULL;
30541 
30542     AjPBtMem arrays = NULL;
30543 
30544     AjPStr *keys        = NULL;
30545     ajulong *ptrs        = NULL;
30546     ajulong *overflows   = NULL;
30547 
30548     ajuint i = 0;
30549     ajuint j = 0;
30550 
30551     ajuint order;
30552     ajuint bentries      = 0;
30553     ajuint totalkeys     = 0;
30554     ajuint nperbucket    = 0;
30555     ajuint maxnperbucket = 0;
30556     ajuint count         = 0;
30557     ajuint totkeylen     = 0;
30558     ajuint keylimit      = 0;
30559     ajuint bucketn       = 0;
30560     ajuint bucketlimit   = 0;
30561     ajuint nodetype      = 0;
30562     ajuint nids          = 0;
30563     ajuint totnids       = 0;
30564 
30565     AjPList idlist    = NULL;
30566     ajuint   dirtysave = 0;
30567     AjPBtPri bid       = NULL;
30568     AjPPribucket cbucket = NULL;
30569     AjPBtPri cid       = NULL;
30570 
30571     ajuint v = 0;
30572 
30573     /* ajDebug("In btreeAdjustPribucketsOne\n"); */
30574 
30575     dirtysave = leaf->dirty;
30576 
30577     leaf->dirty = BT_LOCK;
30578     leaf->lockfor = 1941;
30579     lbuf = leaf->buf;
30580 
30581     GBT_NKEYS(lbuf,&nkeys);
30582 
30583     if(!nkeys)
30584     {
30585 	leaf->dirty = dirtysave;
30586 	return;
30587     }
30588 
30589     GBT_NODETYPE(lbuf,&nodetype);
30590 
30591     order = cache->porder;
30592     nperbucket = cache->pnperbucket;
30593 
30594 
30595     /* Read keys/ptrs */
30596 
30597     arrays = btreeAllocPriArray(cache);
30598     keys = arrays->karray;
30599     ptrs = arrays->parray;
30600     overflows = arrays->overflows;
30601 
30602     btreeGetPointers(cache,lbuf,&ptrs);
30603 
30604 
30605     for(i=0;i<nkeys;++i)
30606 	totalkeys += btreeNumInPribucket(cache,ptrs[i]);
30607 
30608     totalkeys += btreeNumInPribucket(cache,ptrs[i]);
30609 
30610 
30611     /* Set the number of entries per bucket to approximately half full */
30612     maxnperbucket = nperbucket >> 1;
30613 
30614     if(!maxnperbucket)
30615 	++maxnperbucket;
30616 
30617     if(!leaf->pagepos)
30618 	maxnperbucket = nperbucket;
30619 
30620     /* Work out the number of new buckets needed */
30621     bucketn = (totalkeys / maxnperbucket);
30622 
30623     if(totalkeys % maxnperbucket)
30624 	++bucketn;
30625 
30626     if(bucketn == 1)
30627 	++bucketn;
30628 
30629     while(bucketn > order)
30630     {
30631         ++maxnperbucket;
30632         bucketn = (totalkeys / maxnperbucket);
30633 
30634         if(totalkeys % maxnperbucket)
30635             ++bucketn;
30636     }
30637 
30638 
30639     /* Read buckets */
30640     AJCNEW0(buckets,nkeys+1);
30641     keylimit = nkeys + 1;
30642 
30643     for(i=0;i<keylimit;++i)
30644 	buckets[i] = btreePribucketRead(cache,ptrs[i]);
30645 
30646 
30647     /* Read IDs from all buckets and push to list and sort (increasing id) */
30648     idlist  = ajListNew();
30649 
30650     for(i=0;i<keylimit;++i)
30651     {
30652 	overflows[i] = buckets[i]->Overflow;
30653 	bentries = buckets[i]->Nentries;
30654 
30655 	for(j=0;j<bentries;++j)
30656 	    ajListPush(idlist,(void *)buckets[i]->codes[j]);
30657 
30658 	AJFREE(buckets[i]->keylen);
30659 	AJFREE(buckets[i]->codes);
30660 	AJFREE(buckets[i]);
30661     }
30662     ajListSort(idlist, &btreeKeywordCompare);
30663     AJFREE(buckets);
30664 
30665     cbucket = btreePribucketNew(maxnperbucket);
30666     bucketlimit = bucketn - 1;
30667 
30668     totnids = 0;
30669     nids = (ajuint) ajListGetLength(idlist);
30670 
30671 
30672     if(!totalkeys)
30673     {
30674 	v = totalkeys;
30675 	SBT_NKEYS(lbuf,v);
30676 
30677         btreeDeallocPriArray(cache,arrays);
30678 
30679 	ajListFree(&idlist);
30680 	leaf->dirty = BT_DIRTY;
30681 
30682 	return;
30683     }
30684 
30685     if(nids <= maxnperbucket)
30686     {
30687 	cbucket->Overflow = overflows[1];
30688 	cbucket->Nentries = 0;
30689 	ajListPeek(idlist,(void **)&bid);
30690 	ajStrAssignS(&keys[0],bid->keyword);
30691 
30692 	count = 0;
30693 
30694 	while(count!=maxnperbucket && totnids != nids)
30695 	{
30696 	    ajListPop(idlist,(void **)&bid);
30697 
30698 	    cid = cbucket->codes[count];
30699 	    ajStrAssignS(&cid->keyword,bid->keyword);
30700 	    cid->treeblock = bid->treeblock;
30701 
30702 	    cbucket->keylen[count] = BT_BUCKPRILEN(bid->keyword);
30703 	    ++cbucket->Nentries;
30704 	    ++count;
30705 	    ++totnids;
30706 	    ajBtreePriDel(&bid);
30707 	}
30708 
30709 
30710 	totkeylen += ajStrGetLen(keys[0]);
30711 
30712 	if(!ptrs[1])
30713 	    ptrs[1] = cache->totsize;
30714 	btreeWritePribucket(cache,cbucket,ptrs[1]);
30715 
30716 	cbucket->Overflow = overflows[0];
30717 	cbucket->Nentries = 0;
30718 	if(!ptrs[0])
30719 	    ptrs[0] = cache->totsize;
30720 	btreeWritePribucket(cache,cbucket,ptrs[0]);
30721     }
30722     else
30723     {
30724 	for(i=0;i<bucketlimit;++i)
30725 	{
30726 	    cbucket->Overflow = overflows[i];
30727 	    cbucket->Nentries = 0;
30728 
30729 	    count = 0;
30730 
30731 	    while(count!=maxnperbucket && totnids != nids)
30732 	    {
30733 		ajListPop(idlist,(void **)&bid);
30734 
30735 		cid = cbucket->codes[count];
30736 		ajStrAssignS(&cid->keyword,bid->keyword);
30737 		cid->treeblock = bid->treeblock;
30738 
30739 		cbucket->keylen[count] = BT_BUCKPRILEN(bid->id);
30740 		++cbucket->Nentries;
30741 		++count;
30742 		ajBtreePriDel(&bid);
30743 	    }
30744 
30745 
30746 	    ajListPeek(idlist,(void **)&bid);
30747 	    ajStrAssignS(&keys[i],bid->keyword);
30748 
30749 
30750 	    totkeylen += ajStrGetLen(bid->keyword);
30751 
30752 	    if(!ptrs[i])
30753 		ptrs[i] = cache->totsize;
30754 
30755 	    btreeWritePribucket(cache,cbucket,ptrs[i]);
30756 	}
30757 
30758 
30759 	/* Deal with greater-than bucket */
30760 
30761 	cbucket->Overflow = overflows[i];
30762 	cbucket->Nentries = 0;
30763 
30764 
30765 
30766 	count = 0;
30767 
30768 	while(ajListPop(idlist,(void **)&bid))
30769 	{
30770 	    cid = cbucket->codes[count];
30771 	    ajStrAssignS(&cid->keyword,bid->keyword);
30772 	    cid->treeblock = bid->treeblock;
30773 
30774 	    ++cbucket->Nentries;
30775 	    ++count;
30776 	    ajBtreePriDel(&bid);
30777 	}
30778 
30779 
30780 	if(!ptrs[i])
30781 	    ptrs[i] = cache->totsize;
30782 
30783 	btreeWritePribucket(cache,cbucket,ptrs[i]);
30784     }
30785 
30786 
30787     cbucket->Nentries = maxnperbucket;
30788     btreePribucketDel(&cbucket);
30789 
30790     /* Now write out a modified leaf with new keys/ptrs */
30791 
30792     nkeys = bucketn - 1;
30793     v = nkeys;
30794     SBT_NKEYS(lbuf,v);
30795     v = totkeylen;
30796     SBT_TOTLEN(lbuf,v);
30797 
30798     btreeWriteNode(cache,leaf,keys,ptrs,nkeys);
30799 
30800     leaf->dirty = dirtysave;
30801 
30802     if(nodetype == BT_ROOT)
30803     {
30804 	leaf->dirty = BT_LOCK;
30805         leaf->lockfor = 1942;
30806     }
30807 
30808     btreeDeallocPriArray(cache,arrays);
30809 
30810     btreePribucketDel(&cbucket);
30811     ajListFree(&idlist);
30812 
30813     return;
30814 }
30815 
30816 
30817 
30818 
30819 /* @funcstatic btreeRebalancePriOne *******************************************
30820 **
30821 ** Rebalance keyword level 1 tree after deletion
30822 **
30823 ** Deletion software
30824 **
30825 ** @param [u] cache [AjPBtcache] cache
30826 ** @param [r] thisNode [ajulong] Node to rebalance
30827 ** @param [r] leftNode [ajulong] left node
30828 ** @param [r] rightNode [ajulong] right node
30829 ** @param [r] lAnchor [ajulong] left anchor
30830 ** @param [r] rAnchor [ajulong] right anchor
30831 **
30832 ** @return [ajulong] page number or BTNO_NODE
30833 **
30834 ** @release 6.1.0
30835 ** @@
30836 ******************************************************************************/
30837 
btreeRebalancePriOne(AjPBtcache cache,ajulong thisNode,ajulong leftNode,ajulong rightNode,ajulong lAnchor,ajulong rAnchor)30838 static ajulong btreeRebalancePriOne(AjPBtcache cache, ajulong thisNode,
30839                                     ajulong leftNode, ajulong rightNode,
30840                                     ajulong lAnchor, ajulong rAnchor)
30841 {
30842     unsigned char *lbuf = NULL;
30843     unsigned char *rbuf = NULL;
30844     unsigned char *tbuf = NULL;
30845 
30846     ajulong anchorNode   = 0UL;
30847     ajulong balanceNode  = 0UL;
30848     ajulong mergeNode    = 0UL;
30849     ajulong done         = 0UL;
30850     ajulong parent       = 0UL;
30851 
30852     AjPBtpage lpage = NULL;
30853     AjPBtpage rpage = NULL;
30854     AjPBtpage tpage = NULL;
30855 
30856     ajuint lnkeys  = 0U;
30857     ajuint rnkeys  = 0U;
30858     ajuint size    = 0U;
30859     ajuint order   = 0U;
30860     ajuint minsize = 0U;
30861 
30862     AjBool leftok  = ajFalse;
30863     AjBool rightok = ajFalse;
30864 
30865 
30866     /* ajDebug("In btreeRebalancePriOne\n"); */
30867 
30868     if(leftNode!=BTNO_NODE && lAnchor!=BTNO_NODE)
30869 	leftok = ajTrue;
30870 
30871     if(rightNode!=BTNO_NODE && rAnchor!=BTNO_NODE)
30872 	rightok = ajTrue;
30873 
30874     if(!leftok && !rightok)
30875 	return BTNO_NODE;
30876 
30877 
30878     if(leftok)
30879     {
30880 	lpage = btreePricacheRead(cache,leftNode);
30881 	lbuf  = lpage->buf;
30882 	GBT_NKEYS(lbuf,&lnkeys);
30883     }
30884 
30885 
30886     if(rightok)
30887     {
30888 	rpage = btreePricacheRead(cache,rightNode);
30889 	rbuf  = rpage->buf;
30890 	GBT_NKEYS(rbuf,&rnkeys);
30891     }
30892 
30893 
30894 
30895     if(leftok && rightok)
30896     {
30897 	size = (lnkeys >= rnkeys) ? lnkeys : rnkeys;
30898 	balanceNode = (lnkeys >= rnkeys) ? leftNode : rightNode;
30899     }
30900     else if(leftok)
30901     {
30902 	size = lnkeys;
30903 	balanceNode = leftNode;
30904     }
30905     else
30906     {
30907 	size = rnkeys;
30908 	balanceNode = rightNode;
30909     }
30910 
30911 
30912     order = cache->porder;
30913     minsize = (order-1) / 2;
30914 
30915     if((order-1)%2)
30916 	++minsize;
30917 
30918     if(size >= minsize)
30919     {
30920 	if(leftok && rightok)
30921 	    anchorNode = (lnkeys >= rnkeys) ? lAnchor : rAnchor;
30922 	else if(leftok)
30923 	    anchorNode = lAnchor;
30924 	else
30925 	    anchorNode = rAnchor;
30926 
30927 	done = btreeShiftPriOne(cache,thisNode,balanceNode,anchorNode);
30928     }
30929 
30930     else
30931     {
30932 	tpage = btreePricacheRead(cache,thisNode);
30933 	tbuf  = tpage->buf;
30934 	GBT_PREV(tbuf,&parent);
30935 
30936 	if(leftok && rightok)
30937 	{
30938 	    anchorNode = (parent == lAnchor) ? lAnchor : rAnchor;
30939 	    mergeNode  = (anchorNode == lAnchor) ? leftNode : rightNode;
30940 	}
30941 	else if(leftok)
30942 	{
30943 	    anchorNode = lAnchor;
30944 	    mergeNode  = leftNode;
30945 	}
30946 	else
30947 	{
30948 	    anchorNode = rAnchor;
30949 	    mergeNode  = rightNode;
30950 	}
30951 
30952 	done = btreeMergePriOne(cache,thisNode,mergeNode,anchorNode);
30953     }
30954 
30955     return done;
30956 }
30957 
30958 
30959 
30960 
30961 /* @funcstatic btreeShiftPriOne ***********************************************
30962 **
30963 ** Shift spare entries from one keyword tree level 1 node to another.
30964 ** Same as btreeShiftHybOne but duplicated for clarity
30965 **
30966 ** Deletion software.
30967 **
30968 ** @param [u] cache [AjPBtcache] cache
30969 ** @param [r] thisNode [ajulong] master node
30970 ** @param [r] balanceNode [ajulong] balance node
30971 ** @param [r] anchorNode [ajulong] anchor node
30972 **
30973 ** @return [ajulong] page number or BTNO_NODE
30974 **
30975 ** @release 6.1.0
30976 ** @@
30977 ******************************************************************************/
30978 
btreeShiftPriOne(AjPBtcache cache,ajulong thisNode,ajulong balanceNode,ajulong anchorNode)30979 static ajulong btreeShiftPriOne(AjPBtcache cache, ajulong thisNode,
30980                                ajulong balanceNode, ajulong anchorNode)
30981 {
30982     unsigned char *tbuf = NULL;
30983     unsigned char *abuf = NULL;
30984     unsigned char *bbuf = NULL;
30985     unsigned char *buf  = NULL;
30986 
30987     AjPBtMem arraysA = NULL;
30988     AjPBtMem arraysB = NULL;
30989     AjPBtMem arraysT = NULL;
30990 
30991     AjPStr *kTarray = NULL;
30992     AjPStr *kAarray = NULL;
30993     AjPStr *kBarray = NULL;
30994     ajulong *pTarray = NULL;
30995     ajulong *pAarray = NULL;
30996     ajulong *pBarray = NULL;
30997 
30998     ajuint  nAkeys = 0U;
30999     ajuint  nBkeys = 0U;
31000     ajuint  nTkeys = 0U;
31001     ajuint  i;
31002     ajint   ii;
31003 
31004     AjPBtpage pageA = NULL;
31005     AjPBtpage pageB = NULL;
31006     AjPBtpage pageT = NULL;
31007     AjPBtpage page  = NULL;
31008 
31009     AjPBtpage leftpage = NULL;
31010 
31011     ajuint anchorPos   = 0U;
31012     ajulong prev        = 0UL;
31013     ajuint  nodetype    = 0U;
31014 
31015     ajulong lv = 0UL;
31016 
31017     /* ajDebug("In btreeShiftPriOne\n"); */
31018 
31019 
31020     arraysA = btreeAllocPriArray(cache);
31021     kAarray = arraysA->karray;
31022     pAarray = arraysA->parray;
31023 
31024     arraysB = btreeAllocPriArray(cache);
31025     kBarray = arraysB->karray;
31026     pBarray = arraysB->parray;
31027 
31028     arraysT = btreeAllocPriArray(cache);
31029     kTarray = arraysT->karray;
31030     pTarray = arraysT->parray;
31031 
31032 
31033     pageA = btreePricacheRead(cache,anchorNode);
31034     pageA->dirty = BT_LOCK;
31035     pageA->lockfor = 1951;
31036     abuf = pageA->buf;
31037     pageB = btreePricacheRead(cache,balanceNode);
31038     pageB->dirty = BT_LOCK;
31039     pageB->lockfor = 1952;
31040     bbuf = pageB->buf;
31041     pageT = btreePricacheRead(cache,thisNode);
31042     pageT->dirty = BT_LOCK;
31043     pageT->lockfor = 1953;
31044     tbuf = pageT->buf;
31045 
31046     GBT_NKEYS(abuf,&nAkeys);
31047     GBT_NKEYS(bbuf,&nBkeys);
31048     GBT_NKEYS(tbuf,&nTkeys);
31049 
31050     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
31051     btreeGetKeys(cache,bbuf,&kBarray,&pBarray);
31052     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
31053 
31054     if(MAJSTRCMPS(kTarray[nTkeys-1],kBarray[nBkeys-1])<0)
31055 	leftpage = pageT;
31056     else
31057 	leftpage = pageB;
31058 
31059 
31060     if(leftpage == pageT)
31061     {
31062 	/* Find anchor key position */
31063 	i=0;
31064 
31065 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
31066 	    ++i;
31067 
31068 	anchorPos = i;
31069 
31070 	/* Move down anchor key to thisNode */
31071 	ajStrAssignS(&kTarray[nTkeys],kAarray[anchorPos]);
31072 	++nTkeys;
31073 
31074 	/* Shift extra */
31075 	while(nTkeys < nBkeys)
31076 	{
31077 	    ajStrAssignS(&kTarray[nTkeys],kBarray[0]);
31078 	    pTarray[nTkeys] = pBarray[0];
31079 	    ++nTkeys;
31080 	    --nBkeys;
31081 
31082 	    for(i=0;i<nBkeys;++i)
31083 	    {
31084 		ajStrAssignS(&kBarray[i],kBarray[i+1]);
31085 		pBarray[i] = pBarray[i+1];
31086 	    }
31087 
31088 	    pBarray[i] = pBarray[i+1];
31089 	}
31090 
31091 	/* Adjust anchor key */
31092 	ajStrAssignS(&kAarray[anchorPos],kTarray[nTkeys-1]);
31093 	--nTkeys;
31094     }
31095     else	/* thisNode on the right */
31096     {
31097 	/* Find anchor key position */
31098 	i=0;
31099 
31100 	while(i!=nAkeys && MAJSTRCMPS(kBarray[nBkeys-1],kAarray[i])>=0)
31101 	    ++i;
31102 
31103 	anchorPos = i;
31104 
31105 	/* Move down anchor key to thisNode */
31106 	pTarray[nTkeys+1] = pTarray[nTkeys];
31107 
31108 	for(ii=nTkeys-1;ii>-1;--ii)
31109 	{
31110 	    ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
31111 	    pTarray[ii+1] = pTarray[ii];
31112 	}
31113 
31114 	ajStrAssignS(&kTarray[0],kAarray[anchorPos]);
31115 	++nTkeys;
31116 
31117 	/* Shift extra */
31118 	while(nTkeys < nBkeys)
31119 	{
31120 	    pTarray[nTkeys+1] = pTarray[nTkeys];
31121 
31122 	    for(ii=nTkeys-1;ii>-1;--ii)
31123 	    {
31124 		ajStrAssignS(&kTarray[ii+1],kTarray[ii]);
31125 		pTarray[ii+1] = pTarray[ii];
31126 	    }
31127 
31128 	    ajStrAssignS(&kTarray[0],kBarray[nBkeys-1]);
31129 	    pTarray[1] = pBarray[nBkeys];
31130 	    ++nTkeys;
31131 	    --nBkeys;
31132 	}
31133 
31134 
31135 	/* Adjust anchor key */
31136 	ajStrAssignS(&kAarray[anchorPos],kTarray[0]);
31137 	--nTkeys;
31138 
31139 	for(i=0;i<nTkeys;++i)
31140 	{
31141 	    ajStrAssignS(&kTarray[i],kTarray[i+1]);
31142 	    pTarray[i] = pTarray[i+1];
31143 	}
31144 
31145 	pTarray[i] = pTarray[i+1];
31146     }
31147 
31148 
31149     /* Adjust PREV pointers for thisNode */
31150     prev = pageT->pagepos;
31151 
31152     for(i=0;i<nTkeys+1;++i)
31153     {
31154 	page = btreePricacheRead(cache,pTarray[i]);
31155 	buf = page->buf;
31156 	GBT_NODETYPE(buf,&nodetype);
31157 	if(nodetype != BT_IDBUCKET)
31158 	{
31159 	    lv = prev;
31160 	    SBT_PREV(buf,lv);
31161 	    page->dirty = BT_DIRTY;
31162 	}
31163     }
31164 
31165     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
31166     btreeWriteNode(cache,pageB,kBarray,pBarray,nBkeys);
31167     btreeWriteNode(cache,pageT,kTarray,pTarray,nTkeys);
31168 
31169     if(!anchorNode)
31170     {
31171 	pageA->dirty = BT_LOCK;
31172         pageA->lockfor = 1954;
31173     }
31174 
31175     btreeDeallocPriArray(cache,arraysA);
31176     btreeDeallocPriArray(cache,arraysB);
31177     btreeDeallocPriArray(cache,arraysT);
31178 
31179     return BTNO_NODE;
31180 }
31181 
31182 
31183 
31184 
31185 /* @funcstatic btreeMergePriOne ***********************************************
31186 **
31187 ** Merge two nodes.
31188 **
31189 ** Deletion software
31190 **
31191 ** @param [u] cache [AjPBtcache] cache
31192 ** @param [r] thisNode [ajulong] master node
31193 ** @param [r] mergeNode [ajulong] merge node
31194 ** @param [r] anchorNode [ajulong] anchor node
31195 **
31196 ** @return [ajulong] page number or BTNO_NODE
31197 **
31198 ** @release 6.1.0
31199 ** @@
31200 ******************************************************************************/
31201 
btreeMergePriOne(AjPBtcache cache,ajulong thisNode,ajulong mergeNode,ajulong anchorNode)31202 static ajulong btreeMergePriOne(AjPBtcache cache, ajulong thisNode,
31203                                 ajulong mergeNode, ajulong anchorNode)
31204 {
31205     unsigned char *tbuf = NULL;
31206     unsigned char *abuf = NULL;
31207     unsigned char *nbuf = NULL;
31208     unsigned char *buf  = NULL;
31209 
31210     AjPStr *kTarray = NULL;
31211     AjPStr *kAarray = NULL;
31212     AjPStr *kNarray = NULL;
31213     ajulong *pTarray = NULL;
31214     ajulong *pAarray = NULL;
31215     ajulong *pNarray = NULL;
31216 
31217     ajulong thisprev  = 0UL;
31218     ajulong mergeprev = 0UL;
31219 
31220 
31221     ajuint  nAkeys = 0U;
31222     ajuint  nNkeys = 0U;
31223     ajuint  nTkeys = 0U;
31224     ajuint  count  = 0U;
31225     ajuint  i;
31226     ajint   ii;
31227 
31228     ajuint   nodetype = 0U;
31229 
31230     ajuint saveA = 0U;
31231     ajuint saveN = 0U;
31232     ajuint saveT = 0U;
31233 
31234     AjPBtpage pageA = NULL;
31235     AjPBtpage pageN = NULL;
31236     AjPBtpage pageT = NULL;
31237     AjPBtpage page  = NULL;
31238 
31239     AjPBtpage leftpage = NULL;
31240 
31241     ajuint anchorPos = 0U;
31242     ajulong prev      = 0UL;
31243 
31244     ajulong lv = 0UL;
31245 
31246     AjPBtMem arraysA = NULL;
31247     AjPBtMem arraysN = NULL;
31248     AjPBtMem arraysT = NULL;
31249 
31250     AjBool collapse = ajFalse;
31251 
31252     /* ajDebug("In btreeMergePriOne\n"); */
31253 
31254     pageA = btreePricacheRead(cache,anchorNode);
31255     saveA = pageA->dirty;
31256     pageA->dirty = BT_LOCK;
31257     pageA->lockfor = 1961;
31258     abuf = pageA->buf;
31259     pageN = btreePricacheRead(cache,mergeNode);
31260     saveN = pageN->dirty;
31261     pageN->dirty = BT_LOCK;
31262     pageN->lockfor = 1962;
31263     nbuf = pageN->buf;
31264     pageT = btreePricacheRead(cache,thisNode);
31265     saveT = pageT->dirty;
31266     pageT->dirty = BT_LOCK;
31267     pageT->lockfor = 1963;
31268     tbuf = pageT->buf;
31269 
31270     GBT_PREV(tbuf,&thisprev);
31271     GBT_PREV(nbuf,&mergeprev);
31272 
31273     GBT_NKEYS(abuf,&nAkeys);
31274     GBT_NKEYS(nbuf,&nNkeys);
31275     GBT_NKEYS(tbuf,&nTkeys);
31276 
31277     GBT_NODETYPE(nbuf,&nodetype);
31278 
31279 
31280     if(nAkeys == 1)
31281     {
31282 	if(!anchorNode && !thisprev && !mergeprev)
31283 	    collapse = ajTrue;
31284 	else
31285 	{
31286 	    pageA->dirty = saveA;
31287 	    pageN->dirty = saveN;
31288 	    pageT->dirty = saveT;
31289 
31290 	    return thisNode;
31291 	}
31292     }
31293 
31294     arraysA = btreeAllocPriArray(cache);
31295     kAarray = arraysA->karray;
31296     pAarray = arraysA->parray;
31297 
31298     arraysN = btreeAllocPriArray(cache);
31299     kNarray = arraysN->karray;
31300     pNarray = arraysN->parray;
31301 
31302     arraysT = btreeAllocPriArray(cache);
31303     kTarray = arraysT->karray;
31304     pTarray = arraysT->parray;
31305 
31306     btreeGetKeys(cache,abuf,&kAarray,&pAarray);
31307     btreeGetKeys(cache,nbuf,&kNarray,&pNarray);
31308     btreeGetKeys(cache,tbuf,&kTarray,&pTarray);
31309 
31310     if(MAJSTRCMPS(kTarray[nTkeys-1],kNarray[nNkeys-1])<0)
31311 	leftpage = pageT;
31312     else
31313 	leftpage = pageN;
31314 
31315 
31316     if(leftpage == pageT)
31317     {
31318 	/* Find anchor key position */
31319 	i=0;
31320 
31321 	while(i!=nAkeys && MAJSTRCMPS(kTarray[nTkeys-1],kAarray[i])>=0)
31322 	    ++i;
31323 
31324 	anchorPos = i;
31325 
31326 	/* Move down anchor key to neighbour Node */
31327 	pNarray[nNkeys+1] = pNarray[nNkeys];
31328 
31329 	for(ii=nNkeys-1;ii>-1;--ii)
31330 	{
31331 	    ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
31332 	    pNarray[ii+1] = pNarray[ii];
31333 	}
31334 
31335 	ajStrAssignS(&kNarray[0],kAarray[anchorPos]);
31336 	++nNkeys;
31337 
31338 
31339 	/* Adjust anchor node keys/ptrs */
31340 	++anchorPos;
31341 
31342 	if(anchorPos==nAkeys)
31343 	    pAarray[nAkeys-1] = pAarray[nAkeys];
31344 	else
31345 	{
31346 	    for(i=anchorPos;i<nAkeys;++i)
31347 	    {
31348 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
31349 		pAarray[i-1] = pAarray[i];
31350 	    }
31351 	    pAarray[i-1] = pAarray[i];
31352 	}
31353 
31354 	--nAkeys;
31355 
31356 
31357 	/* Merge this to neighbour */
31358 
31359 	while(nTkeys)
31360 	{
31361 	    pNarray[nNkeys+1] = pNarray[nNkeys];
31362 
31363 	    for(ii=nNkeys-1;ii>-1;--ii)
31364 	    {
31365 		ajStrAssignS(&kNarray[ii+1],kNarray[ii]);
31366 		pNarray[ii+1] = pNarray[ii];
31367 	    }
31368 
31369 	    ajStrAssignS(&kNarray[0],kTarray[nTkeys-1]);
31370 	    pNarray[1] = pTarray[nTkeys];
31371 	    pNarray[0] = pTarray[nTkeys-1];
31372 	    --nTkeys;
31373 	    ++nNkeys;
31374 	}
31375 
31376 	/* At this point the 'this' node could be added to a free list */
31377     }
31378     else
31379     {
31380 	/* Find anchor key position */
31381 	i=0;
31382 
31383 	while(i!=nAkeys && MAJSTRCMPS(kNarray[nNkeys-1],kAarray[i])>=0)
31384 	    ++i;
31385 
31386 	anchorPos = i;
31387 
31388 	/* Move down anchor key to neighbourNode */
31389 	ajStrAssignS(&kNarray[nNkeys],kAarray[anchorPos]);
31390 	++nNkeys;
31391 
31392 	/* Adjust anchor node keys/ptrs */
31393 	++anchorPos;
31394 
31395 	if(anchorPos!=nAkeys)
31396 	    for(i=anchorPos;i<nAkeys;++i)
31397 	    {
31398 		ajStrAssignS(&kAarray[i-1],kAarray[i]);
31399 		pAarray[i] = pAarray[i+1];
31400 	    }
31401 
31402 	--nAkeys;
31403 
31404 	/* merge extra */
31405 	count = 0;
31406 
31407 	while(nTkeys)
31408 	{
31409 	    ajStrAssignS(&kNarray[nNkeys],kTarray[count]);
31410 	    pNarray[nNkeys] = pTarray[count];
31411 	    ++nNkeys;
31412 	    ++count;
31413 	    --nTkeys;
31414 	    pNarray[nNkeys] = pTarray[count];
31415 
31416 	}
31417 
31418 	/* At this point the 'this' node could be added to a free list */
31419     }
31420 
31421 
31422     /* Adjust PREV pointers for neighbour Node */
31423     prev = pageN->pagepos;
31424 
31425     for(i=0;i<=nNkeys;++i)
31426     {
31427 	page = btreePricacheRead(cache,pNarray[i]);
31428 	buf = page->buf;
31429 	GBT_NODETYPE(buf,&nodetype);
31430 
31431 	if(nodetype != BT_IDBUCKET)
31432 	{
31433 	    lv = prev;
31434 	    SBT_PREV(buf,lv);
31435 	    page->dirty = BT_DIRTY;
31436 	}
31437     }
31438 
31439     pageT->dirty = BT_CLEAN;
31440     btreeWriteNode(cache,pageA,kAarray,pAarray,nAkeys);
31441     btreeWriteNode(cache,pageN,kNarray,pNarray,nNkeys);
31442 
31443     if(!anchorNode)
31444     {
31445 	pageA->dirty = BT_LOCK;
31446         pageA->lockfor = 1964;
31447     }
31448 
31449     btreeDeallocPriArray(cache,arraysA);
31450     btreeDeallocPriArray(cache,arraysN);
31451     btreeDeallocPriArray(cache,arraysT);
31452 
31453     if(collapse)
31454 	btreeCollapseRootPriOne(cache,mergeNode);
31455 
31456     return thisNode;
31457 }
31458 
31459 
31460 
31461 
31462 /* @funcstatic btreeCollapseRootPriOne ****************************************
31463 **
31464 ** Collapse root page for keyword level 1 tree.
31465 **
31466 ** Deletion software.
31467 **
31468 ** @param [u] cache [AjPBtcache] cache
31469 ** @param [r] pagepos [ajulong] page number to make new root
31470 **
31471 ** @return [ajulong] page number or BTNO_NODE
31472 **
31473 ** @release 6.1.0
31474 ** @@
31475 ******************************************************************************/
31476 
btreeCollapseRootPriOne(AjPBtcache cache,ajulong pagepos)31477 static ajulong btreeCollapseRootPriOne(AjPBtcache cache, ajulong pagepos)
31478 {
31479     unsigned char *buf  = NULL;
31480     unsigned char *lbuf = NULL;
31481 
31482     AjPStr *karray = NULL;
31483     ajulong *parray = NULL;
31484 
31485     AjPBtpage rootpage = NULL;
31486     AjPBtpage page     = NULL;
31487 
31488     ajuint nodetype = 0U;
31489     ajuint nkeys    = 0U;
31490     ajuint i;
31491 
31492     ajulong prev = 0UL;
31493     AjPBtMem arrays = NULL;
31494 
31495     /* ajDebug("In btreeCollapseRootPriOne\n"); */
31496 
31497     if(!cache->plevel)
31498 	return BTNO_NODE;
31499 
31500     rootpage = btreePricacheLocate(cache,0UL);
31501     buf = rootpage->buf;
31502     page = btreePricacheRead(cache,pagepos);
31503 
31504 
31505     arrays = btreeAllocPriArray(cache);
31506     karray = arrays->karray;
31507     parray = arrays->parray;
31508 
31509     /*
31510     ** Swap pagepos values to make root the child and child the root
31511     ** Update node types and mark the original root as a clean page
31512     */
31513 
31514     /* At this point page->pagepos could be added to a free list */
31515 
31516     rootpage->pagepos = page->pagepos;
31517     rootpage->dirty = BT_CLEAN;
31518     nodetype = BT_INTERNAL;
31519     SBT_NODETYPE(buf,nodetype);
31520 
31521     page->pagepos = 0;
31522     page->dirty = BT_LOCK;
31523     page->lockfor = 1971;
31524     buf = page->buf;
31525     nodetype = BT_ROOT;
31526     SBT_NODETYPE(buf,nodetype);
31527 
31528     --cache->plevel;
31529 
31530     if(cache->plevel)
31531     {
31532 	/*
31533 	 ** Update the PREV pointers of the new root's children
31534 	 */
31535 	GBT_NKEYS(buf,&nkeys);
31536 	btreeGetKeys(cache,buf,&karray,&parray);
31537 
31538 	for(i=0;i<=nkeys;++i)
31539 	{
31540 	    page = btreePricacheRead(cache,parray[i]);
31541 	    lbuf = page->buf;
31542 	    SBT_PREV(lbuf,prev);
31543 	    page->dirty = BT_DIRTY;
31544 	}
31545     }
31546 
31547     btreeDeallocPriArray(cache,arrays);
31548 
31549     return 0UL;
31550 }
31551 
31552 
31553 
31554 
31555 /* @funcstatic btreeIsSecEmpty ************************************************
31556 **
31557 ** Tests whether a secondary tree is empty
31558 **
31559 ** Deletion software.
31560 **
31561 ** @param [u] cache [AjPBtcache] cache
31562 **
31563 ** @return [AjBool] true if empty, false otherwise
31564 **
31565 ** @release 6.1.0
31566 ** @@
31567 ******************************************************************************/
31568 
btreeIsSecEmpty(AjPBtcache cache)31569 static AjBool btreeIsSecEmpty(AjPBtcache cache)
31570 {
31571     AjPBtpage rootpage  = NULL;
31572     AjPSecbucket bucket = NULL;
31573     unsigned char *buf  = NULL;
31574     AjPStr *karray  = NULL;
31575     ajulong *parray  = NULL;
31576     AjPBtMem arrays = NULL;
31577     ajuint nkeys = 0;
31578     ajuint tkeys = 0;
31579 
31580     if(cache->slevel)
31581         return ajFalse;
31582 
31583     rootpage = btreeSeccacheLocate(cache,cache->secrootblock);
31584 
31585     if(!rootpage)
31586         ajFatal("btreeSecIsEmpty: root page unlocked");
31587 
31588     buf = rootpage->buf;
31589     GBT_NKEYS(buf,&nkeys);
31590 
31591     if(!nkeys)
31592         return ajTrue;
31593 
31594     if(nkeys > 1)
31595         return ajFalse;
31596 
31597     arrays = btreeAllocSecArray(cache);
31598     karray = arrays->karray;
31599     parray = arrays->parray;
31600     btreeGetKeys(cache,buf,&karray,&parray);
31601 
31602     tkeys = 0;
31603 
31604     if(parray[0])
31605     {
31606         bucket = btreeReadSecbucket(cache,parray[0]);
31607         tkeys += bucket->Nentries;
31608     }
31609 
31610     if(parray[1])
31611     {
31612         bucket = btreeReadSecbucket(cache,parray[1]);
31613         tkeys += bucket->Nentries;
31614     }
31615 
31616     btreeDeallocSecArray(cache,arrays);
31617 
31618     if(tkeys)
31619         return ajFalse;
31620 
31621     return ajTrue;
31622 }
31623 
31624 
31625 
31626 
31627 /* @funcstatic btreeNodetype **************************************************
31628 **
31629 ** Returns a name for the nodetype of a buffer
31630 **
31631 ** @param [r] buf [const unsigned char*] buffer
31632 ** @return [const char*] Name of nodetype
31633 **
31634 ** @release 6.4.0
31635 ** @@
31636 ******************************************************************************/
31637 
btreeNodetype(const unsigned char * buf)31638 static const char* btreeNodetype(const unsigned char* buf)
31639 {
31640     static AjBool called = AJFALSE;
31641     static char nodetypebuf[512];
31642     ajuint nodetype;
31643 
31644     if(!called)
31645     {
31646         AJCNEW0(btreeNodetypeNames, BT_MAXTYPE+1);
31647         btreeNodetypeNames[BT_SECFREEPAGE] = "secfree";
31648         btreeNodetypeNames[BT_FREEPAGE]    = "free";
31649         btreeNodetypeNames[BT_ROOT]        = "root";
31650         btreeNodetypeNames[BT_INTERNAL]    = "internal";
31651         btreeNodetypeNames[BT_LEAF]        = "leaf";
31652         btreeNodetypeNames[BT_IDBUCKET]    = "bucket";
31653         btreeNodetypeNames[BT_OVERFLOW]    = "overflow";
31654         btreeNodetypeNames[BT_PRIBUCKET]   = "pribucket";
31655         btreeNodetypeNames[BT_SECBUCKET]   = "secbucket";
31656         btreeNodetypeNames[BT_NUMBUCKET]   = "numbucket";
31657         btreeNodetypeNames[BT_SECROOT]     = "secroot";
31658         btreeNodetypeNames[BT_SECINTERNAL] = "secinternal";
31659         btreeNodetypeNames[BT_SECLEAF]     = "secleaf";
31660         called = ajTrue;
31661     }
31662 
31663     GBT_NODETYPE(buf, &nodetype);
31664 
31665     if(nodetype >= BT_MAXTYPE)
31666     {
31667         sprintf(nodetypebuf, "unknown (%u)", nodetype);
31668         return nodetypebuf;
31669     }
31670 
31671     if(!btreeNodetypeNames[nodetype])
31672     {
31673         sprintf(nodetypebuf, "unknown (%u)", nodetype);
31674         return nodetypebuf;
31675     }
31676 
31677     if(nodetype == BT_OVERFLOW || nodetype == BT_SECOVERFLOW)
31678     {
31679         ajWarn("Overflow page found");
31680     }
31681 
31682 
31683     return btreeNodetypeNames[nodetype];
31684 }
31685 
31686 
31687 
31688 
31689 /* @func ajBtreePageIsPrimary *************************************************
31690 **
31691 ** Returns a true is a page is primary
31692 **
31693 ** @param [r] page [const AjPBtpage] Page
31694 ** @return [AjBool] True if page is primary index or bucket
31695 **
31696 ** @release 6.5.0
31697 ** @@
31698 ******************************************************************************/
31699 
ajBtreePageIsPrimary(const AjPBtpage page)31700 AjBool ajBtreePageIsPrimary(const AjPBtpage page)
31701 {
31702     unsigned char* buf = NULL;
31703     ajuint nodetype;
31704 
31705     buf = page->buf;
31706 
31707     GBT_NODETYPE(buf, &nodetype);
31708 
31709     switch (nodetype)
31710     {
31711         case BT_FREEPAGE:       /* no need to save the page */
31712         case BT_ROOT:
31713         case BT_INTERNAL:
31714         case BT_LEAF:
31715         case BT_IDBUCKET:
31716         case BT_PRIBUCKET:
31717         case BT_OVERFLOW:
31718             return ajTrue;
31719             break;
31720         case BT_SECFREEPAGE:
31721         case BT_SECROOT:
31722         case BT_SECINTERNAL:
31723         case BT_SECLEAF:
31724         case BT_SECOVERFLOW:
31725         case BT_SECBUCKET:
31726         case BT_NUMBUCKET:
31727             return ajFalse;
31728             break;
31729         default:
31730             return ajTrue;
31731     }
31732 
31733     return ajTrue;
31734 }
31735 
31736 
31737 
31738 
31739 
31740 /* @func ajBtreeCacheGetPagecount *********************************************
31741 **
31742 ** Returns the total number of pages in a tree index
31743 **
31744 ** @param [r] cache [const AjPBtcache] cache
31745 **
31746 ** @return [ajulong] number of pages in a tree index
31747 **
31748 ** @release 6.4.0
31749 ** @@
31750 ******************************************************************************/
31751 
ajBtreeCacheGetPagecount(const AjPBtcache cache)31752 ajulong ajBtreeCacheGetPagecount(const AjPBtcache cache)
31753 {
31754     return cache->pripagecount + cache->secpagecount;
31755 }
31756 
31757 
31758 
31759 
31760 /* @func ajBtreeCacheGetPripagesize *******************************************
31761 **
31762 ** Returns the primary page size of a tree index
31763 **
31764 ** @param [r] cache [const AjPBtcache] cache
31765 **
31766 ** @return [ajuint] Page size of a tree index
31767 **
31768 ** @release 6.5.0
31769 ** @@
31770 ******************************************************************************/
31771 
ajBtreeCacheGetPripagesize(const AjPBtcache cache)31772 ajuint ajBtreeCacheGetPripagesize(const AjPBtcache cache)
31773 {
31774     return cache->pripagesize;
31775 }
31776 
31777 
31778 
31779 
31780 /* @func ajBtreeCacheGetSecpagesize *******************************************
31781 **
31782 ** Returns the secondary page size of a tree index
31783 **
31784 ** @param [r] cache [const AjPBtcache] cache
31785 **
31786 ** @return [ajuint] Page size of a tree index
31787 **
31788 ** @release 6.5.0
31789 ** @@
31790 ******************************************************************************/
31791 
ajBtreeCacheGetSecpagesize(const AjPBtcache cache)31792 ajuint ajBtreeCacheGetSecpagesize(const AjPBtcache cache)
31793 {
31794     return cache->secpagesize;
31795 }
31796 
31797 
31798 
31799 
31800 /* @func ajBtreeCacheGetTotsize ***********************************************
31801 **
31802 ** Returns the number of bytes in an uncompressed tree index
31803 **
31804 ** @param [r] cache [const AjPBtcache] cache
31805 **
31806 ** @return [ajulong] number of bytes in a tree index
31807 **
31808 ** @release 6.4.0
31809 ** @@
31810 ******************************************************************************/
31811 
ajBtreeCacheGetTotsize(const AjPBtcache cache)31812 ajulong ajBtreeCacheGetTotsize(const AjPBtcache cache)
31813 {
31814     return (cache->pripagesize * cache->pripagecount) +
31815         (cache->secpagesize * cache->secpagecount);
31816 }
31817 
31818 
31819 
31820 
31821 /* @func ajBtreePageGetTypename ***********************************************
31822 **
31823 ** Returns the node type name of a tree index page
31824 **
31825 ** @param [r] page [const AjPBtpage] Page
31826 **
31827 ** @return [const char*] Name of page type
31828 **
31829 ** @release 6.4.0
31830 ** @@
31831 ******************************************************************************/
31832 
ajBtreePageGetTypename(const AjPBtpage page)31833 const char* ajBtreePageGetTypename(const AjPBtpage page)
31834 {
31835     return btreeNodetype(page->buf);
31836 }
31837 
31838 
31839 
31840 
31841 /* @func ajBtreeStatNode ******************************************************
31842 **
31843 ** Returns number of keys, overflow pages visited and free space unused
31844 ** for a b+ tree node page
31845 **
31846 ** @param [w] cache [AjPBtcache] cache
31847 ** @param [r] page [const AjPBtpage] Page
31848 ** @param [r] full [AjBool] If true, write detailed debug report
31849 ** @param [w] nkeys [ajuint*] Number of keys
31850 ** @param [w] overflows [ajuint*] Number of overflow pages
31851 ** @param [w] freespace [ajuint*] Unused space in this page
31852 ** @param [u] refs [ajulong*] References to pages
31853 ** @param [r] newpostable [const AjPTable] Page numbers for page offsets
31854 ** @return [AjBool] True on success
31855 **
31856 ** @release 6.4.0
31857 ** @@
31858 ******************************************************************************/
31859 
ajBtreeStatNode(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nkeys,ajuint * overflows,ajuint * freespace,ajulong * refs,const AjPTable newpostable)31860 AjBool ajBtreeStatNode(AjPBtcache cache, const AjPBtpage page, AjBool full,
31861                        ajuint* nkeys, ajuint* overflows,
31862                        ajuint* freespace, ajulong* refs,
31863                        const AjPTable newpostable)
31864 {
31865     ajuint i;
31866     ajuint m;
31867 
31868     ajuint klen;
31869     const AjPBtpage bpage;
31870     unsigned char *tbuf;
31871 
31872     ajuint    pagesize = 0U;
31873     ajulong   overflow = 0UL;
31874     ajulong blockno;
31875     ajuint totlen;
31876     ajulong left;
31877     ajulong right;
31878     ajulong prev;
31879     unsigned char *lenptr = NULL;
31880     unsigned char *keyptr = NULL;
31881     ajulong lv;
31882     unsigned char* kp;
31883 
31884 /*    ajuint freebytes;*/
31885 /*    ajuint nodetype;*/
31886 
31887     if(ajBtreePageIsPrimary(page))
31888         pagesize = cache->pripagesize;
31889     else
31890         pagesize = cache->secpagesize;
31891 
31892     bpage = page;
31893     tbuf = bpage->buf;
31894     GBT_TOTLEN(tbuf,&totlen);
31895 
31896     if(!totlen)
31897       return ajBtreeStatNumnode(cache, page, full,
31898                                 nkeys, overflows, freespace, refs,
31899                                 newpostable);
31900 
31901     btreeCheckNode(cache, page);
31902 
31903     GBT_NKEYS(tbuf,&m);
31904     *nkeys = m;
31905     *overflows = 0;
31906     *freespace = 0;
31907 
31908     GBT_OVERFLOW(tbuf,&overflow);
31909 
31910     GBT_LEFT(tbuf,&left);
31911     GBT_RIGHT(tbuf,&right);
31912     GBT_PREV(tbuf,&prev);
31913     GBT_BLOCKNUMBER(tbuf,&blockno);
31914 
31915     if(full)
31916         ajDebug("\n"
31917                 "StatNode      %12Lu totlen:%u '%s'\n"
31918                 "         left:%12Lu  right:%12Lu  prev:%12Lu\n"
31919                 "        block:%12Lu   over:%12Lu\n",
31920                 page->pagepos, totlen, btreeNodetype(tbuf),
31921                 left, right, prev, blockno, overflow);
31922     if(full)
31923         ajDebug("\nStatNode      %12Lu      pointer (%u)\n",
31924                 page->pagepos, m);
31925     if(full)
31926         ajDebug("--------      ------------      -------\n");
31927 
31928     lenptr =  PBT_KEYLEN(tbuf);
31929     keyptr = lenptr + m * sizeof(ajuint);
31930 
31931     for(i=0;i<m;i++)
31932     {
31933 	BT_GETAJUINT(lenptr,&klen);
31934 
31935 /*
31936 //        if((ajuint)((keyptr-tbuf+2) + klen + sizeof(ajulong)) > pagesize)
31937 //	{
31938 //            freebytes = pagesize - (keyptr-tbuf);
31939 //            *freespace += freebytes;
31940 //            (*overflows)++;
31941 //    	    /# ajDebug("btreeStatNode: Overflow\n"); #/
31942 //	    bpage = ajBtreeCacheRead(cache,overflow);
31943 //	    tbuf = bpage->buf;
31944 //	    GBT_NODETYPE(tbuf,&nodetype);
31945 //
31946 //	    if(nodetype != BT_OVERFLOW)
31947 //		ajFatal("StatNode Overflow node %Lu expected but not found "
31948 //                        "cache %S page %Lu "
31949 //                        "pagepos:%Lu key:%u/%u free:%u keyptr:%x lenptr:%x "
31950 //                        "tbuf:%x klen:%u old nodetype '%s' newnodetype '%s' "
31951 //                        "blockno:%Lu totlen:%u left:%Lu right:%Lu prev:%Lu",
31952 //                        overflow, cache->filename,
31953 //                        page->pagepos, page->pagepos/pagesize,
31954 //                        i, m, freebytes,
31955 //                        keyptr, lenptr, tbuf, klen,
31956 //                        btreeNodetype(page->buf), btreeNodetype(tbuf),
31957 //                        blockno,totlen, left, right, prev);
31958 //
31959 //            GBT_BLOCKNUMBER(tbuf,&blockno);
31960 //            GBT_TOTLEN(tbuf,&totlen);
31961 //            GBT_LEFT(tbuf,&left);
31962 //            GBT_RIGHT(tbuf,&right);
31963 //            GBT_PREV(tbuf,&prev);
31964 //
31965 //	    /#
31966 //	     ** The length pointer is restricted to the initial page.
31967 //	     ** The keyptr in overflow pages starts at the Key Lengths
31968 //	     ** position!
31969 //	     #/
31970 //            keyptr = PBT_KEYLEN(tbuf);
31971 //	}
31972 */
31973 
31974         kp = keyptr;
31975         keyptr += klen+1;
31976         BT_GETAJULONG(keyptr,&lv);
31977         ++refs[*ajTableulongFetch(newpostable, &lv)];
31978 	keyptr += sizeof(ajulong);
31979 
31980         if(full)
31981             ajDebug("#StatNode     %12Lu %12Lu '%*s'\n",
31982                     page->pagepos, lv, klen, kp);
31983 
31984 	lenptr += sizeof(ajuint);
31985     }
31986 
31987     BT_GETAJULONG(keyptr,&lv);
31988     keyptr += sizeof(ajulong);
31989     ++refs[*ajTableulongFetch(newpostable, &lv)];
31990 
31991     if(full)
31992         ajDebug("#StatNode     %12Lu %12Lu '<end>'\n",
31993                 page->pagepos, lv);
31994 
31995     *freespace += pagesize - (keyptr-tbuf);
31996 
31997     if(full)
31998         ajDebug("StatNode      %12Lu keys:%6u over:%6u free:%6u\n",
31999                 page->pagepos, *nkeys, *overflows, *freespace);
32000 
32001     return ajTrue;
32002 }
32003 
32004 
32005 
32006 
32007 /* @func ajBtreeStatNumnode ***************************************************
32008 **
32009 ** Returns number of keys, overflow pages visited and free space unused
32010 ** for a b+ tree numeric node page
32011 **
32012 ** @param [w] cache [AjPBtcache] cache
32013 ** @param [r] page [const AjPBtpage] Page
32014 ** @param [r] full [AjBool] If true, write detailed debug report
32015 ** @param [w] nkeys [ajuint*] Number of keys
32016 ** @param [w] overflows [ajuint*] Number of overflow pages
32017 ** @param [w] freespace [ajuint*] Unused space in this page
32018 ** @param [u] refs [ajulong*] References to pages
32019 ** @param [r] newpostable [const AjPTable] Page numbers for page offsets
32020 ** @return [AjBool] True on success
32021 **
32022 ** @release 6.4.0
32023 ** @@
32024 ******************************************************************************/
32025 
ajBtreeStatNumnode(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nkeys,ajuint * overflows,ajuint * freespace,ajulong * refs,const AjPTable newpostable)32026 AjBool ajBtreeStatNumnode(AjPBtcache cache, const AjPBtpage page, AjBool full,
32027                           ajuint* nkeys, ajuint* overflows,
32028                           ajuint* freespace, ajulong *refs,
32029                           const AjPTable newpostable)
32030 {
32031     ajuint i;
32032     ajuint m;
32033     const AjPBtpage bpage;
32034     unsigned char *tbuf;
32035 
32036     ajuint   pagesize = 0U;
32037     ajulong   overflow = 0UL;
32038     ajulong blockno;
32039     ajuint totlen;
32040     ajulong left;
32041     ajulong right;
32042     ajulong prev;
32043     unsigned char *keyptr = NULL;
32044     unsigned char *valptr = NULL;
32045     ajulong lv;
32046     ajulong lk;
32047 
32048     btreeCheckNumnode(cache, page);
32049 
32050     if(ajBtreePageIsPrimary(page))
32051         pagesize = cache->pripagesize;
32052     else
32053         pagesize = cache->secpagesize;
32054 
32055     bpage = page;
32056     tbuf = bpage->buf;
32057     GBT_TOTLEN(tbuf,&totlen);
32058 
32059     GBT_NKEYS(tbuf,&m);
32060     *nkeys = m;
32061     *overflows = 0;
32062     *freespace = 0;
32063 
32064     GBT_LEFT(tbuf,&left);
32065     GBT_RIGHT(tbuf,&right);
32066     GBT_BLOCKNUMBER(tbuf,&blockno);
32067     GBT_PREV(tbuf,&prev);
32068     GBT_OVERFLOW(tbuf,&overflow);
32069 
32070     if(full)
32071         btreeStatNumnode(cache, page);
32072 
32073     keyptr =  PBT_KEYLEN(tbuf);
32074     valptr = keyptr + m * sizeof(ajulong);
32075     for(i=0;i<m;i++)
32076     {
32077         BT_GETAJULONG(valptr,&lv);
32078         ++refs[*ajTableulongFetch(newpostable, &lv)];
32079         BT_GETAJULONG(keyptr,&lk);
32080         keyptr += sizeof(ajulong);
32081 	valptr += sizeof(ajulong);
32082     }
32083     BT_GETAJULONG(valptr,&lv);
32084     valptr += sizeof(ajulong);
32085     ++refs[*ajTableulongFetch(newpostable, &lv)];
32086     *freespace += pagesize - (valptr-tbuf);
32087 
32088     return ajTrue;
32089 }
32090 
32091 
32092 
32093 
32094 /* @funcstatic btreeStatNumnode ***********************************************
32095 **
32096 ** Writes a numeric node page to the debug file
32097 **
32098 ** @param [w] cache [AjPBtcache] cache
32099 ** @param [r] page [const AjPBtpage] Page
32100 ** @return [void]
32101 **
32102 ** @release 6.4.0
32103 ** @@
32104 ******************************************************************************/
32105 
btreeStatNumnode(AjPBtcache cache,const AjPBtpage page)32106 static void btreeStatNumnode(AjPBtcache cache, const AjPBtpage page)
32107 {
32108     ajuint i;
32109     ajuint m;
32110     const AjPBtpage bpage;
32111     unsigned char *tbuf;
32112 
32113     ajulong   overflow = 0UL;
32114     ajulong blockno;
32115     ajuint totlen;
32116     ajulong left;
32117     ajulong right;
32118     ajulong prev;
32119     unsigned char *keyptr = NULL;
32120     unsigned char *valptr = NULL;
32121     ajulong lv;
32122     ajulong lk;
32123 
32124     (void) cache;
32125 
32126     bpage = page;
32127     tbuf = bpage->buf;
32128     GBT_TOTLEN(tbuf,&totlen);
32129 
32130     GBT_NKEYS(tbuf,&m);
32131 
32132     GBT_LEFT(tbuf,&left);
32133     GBT_RIGHT(tbuf,&right);
32134     GBT_BLOCKNUMBER(tbuf,&blockno);
32135     GBT_PREV(tbuf,&prev);
32136     GBT_OVERFLOW(tbuf,&overflow);
32137 
32138     ajDebug("\n"
32139                 "StatNumNode   %12Lu totlen:%u\n"
32140                 "         left:%12Lu  right:%12Lu  prev:%12Lu\n"
32141                 "        block:%12Lu   over:%12Lu\n",
32142                 page->pagepos, totlen, left, right, prev, blockno, overflow);
32143     ajDebug("\nStatNumnode   %12Lu   pageoffset (%u)\n",
32144                 page->pagepos, m);
32145     ajDebug("-----------   ------------ ------------ -------\n");
32146 
32147 
32148     keyptr =  PBT_KEYLEN(tbuf);
32149     valptr = keyptr + m * sizeof(ajulong);
32150 
32151     for(i=0;i<m;i++)
32152     {
32153         BT_GETAJULONG(valptr,&lv);
32154         BT_GETAJULONG(keyptr,&lk);
32155         ajDebug("#StatNumnode  %12Lu %12Lu %Lu\n", page->pagepos, lv, lk);
32156 	keyptr += sizeof(ajulong);
32157 	valptr += sizeof(ajulong);
32158     }
32159 
32160     BT_GETAJULONG(valptr,&lv);
32161     ajDebug("#StatNumnode  %12Lu %12Lu '<end>'\n", page->pagepos, lv);
32162     ajDebug("StatNumnode   %12Lu keys:%6u\n",
32163                 page->pagepos, m);
32164     return;
32165 }
32166 
32167 
32168 
32169 
32170 /* @func ajBtreeStatIdbucket **************************************************
32171 **
32172 ** Returns number of keys, overflow pages visited and free space unused
32173 ** for a b+ tree bucket page
32174 **
32175 ** @param [w] cache [AjPBtcache] cache
32176 ** @param [r] page [const AjPBtpage] Page
32177 ** @param [r] full [AjBool] If true, write detailed debug report
32178 ** @param [w] nentries [ajuint*] Number of entries
32179 ** @param [w] ndups [ajuint*] Number of entries with duplicates
32180 ** @param [w] nextra [ajuint*] Number of extra entries as duplicates
32181 ** @param [w] overflows [ajuint*] Number of overflow pages
32182 ** @param [w] freespace [ajuint*] Unused space in this page
32183 ** @param [u] refs [ajulong*] References to pages
32184 ** @param [r] newpostable [const AjPTable] Page numbers for page offsets
32185 ** @return [AjBool] True on success
32186 **
32187 ** @release 6.4.0
32188 ** @@
32189 ******************************************************************************/
32190 
ajBtreeStatIdbucket(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nentries,ajuint * ndups,ajuint * nextra,ajuint * overflows,ajuint * freespace,ajulong * refs,const AjPTable newpostable)32191 AjBool ajBtreeStatIdbucket(AjPBtcache cache, const AjPBtpage page, AjBool full,
32192                            ajuint* nentries, ajuint *ndups, ajuint *nextra,
32193                            ajuint* overflows, ajuint* freespace, ajulong* refs,
32194                            const AjPTable newpostable)
32195 {
32196     ajuint i;
32197     ajuint iref;
32198     ajuint m;
32199     ajuint klen;
32200     const AjPBtpage bpage;
32201     unsigned char *tbuf;
32202 
32203     ajuint   pagesize = 0U;
32204     ajulong   overflow = 0UL;
32205     unsigned char *idptr = NULL;
32206     unsigned char *keyptr = NULL;
32207     unsigned char *kp = NULL;
32208 
32209     ajuint uv;
32210     ajuint uv2;
32211     ajulong lv;
32212     ajulong lv2;
32213     ajuint idlen;
32214     ajuint keyskip = BT_DDOFF + cache->refcount*BT_EXTRA;
32215 
32216 /*    ajuint freebytes;*/
32217 /*    ajuint nodetype;*/
32218 
32219     if(ajBtreePageIsPrimary(page))
32220         pagesize = cache->pripagesize;
32221     else
32222         pagesize = cache->secpagesize;
32223 
32224     bpage = page;
32225     tbuf = bpage->buf;
32226 
32227     GBT_BUCKNENTRIES(tbuf,&m);
32228     *nentries = m;
32229     *overflows = 0;
32230     *freespace = 0;
32231     *ndups = 0;
32232     *nextra = 0;
32233 
32234     GBT_BUCKOVERFLOW(tbuf,&overflow);
32235 
32236     keyptr = PBT_BUCKKEYLEN(tbuf);
32237     idptr = keyptr + (m * sizeof(ajuint));
32238 
32239     if(full)
32240         ajDebug("\n"
32241                 "StatBucket     %12Lu over:%12Lu\n",
32242                 page->pagepos, overflow);
32243     if(full)
32244         ajDebug("\nStatBucket     %12Lu   dbno   dups       offset",
32245                 page->pagepos);
32246 
32247     if(full && cache->refcount)
32248     {
32249         for(iref=0; iref < cache->refcount; iref++)
32250             ajDebug("    refoffset%d", iref+1);
32251     }
32252 
32253     if(full)
32254         ajDebug(" (%u)\n", m);
32255 
32256     if(full)
32257         ajDebug("----------                    ----   ----");
32258 
32259     if(full && cache->refcount)
32260     {
32261         for(iref=0; iref < cache->refcount; iref++)
32262             ajDebug("       ------");
32263     }
32264 
32265     if(full)
32266         ajDebug("    ---------\n");
32267 
32268     for(i=0;i<m;i++)
32269     {
32270 	BT_GETAJUINT(keyptr,&klen);
32271         idlen = klen - keyskip;
32272 
32273 /*
32274 //	if((idptr-tbuf+2) + klen > cache->pagesize)	/# overflow #/
32275 //	{
32276 //            freebytes = pagesize - (idptr-tbuf);
32277 //            *freespace += freebytes;
32278 //            (*overflows)++;
32279 //    	    /# ajDebug("btreeStatNode: Overflow\n"); #/
32280 //	    GBT_BUCKOVERFLOW(tbuf,&overflow);
32281 //	    bpage = ajBtreeCacheRead(cache,overflow);
32282 //	    tbuf = bpage->buf;
32283 //	    GBT_BUCKNODETYPE(tbuf,&nodetype);
32284 //	    if(nodetype != BT_OVERFLOW)
32285 //		ajFatal("StatBucket Overflow node %Lu expected but not found "
32286 //                        "for cache '%S'",
32287 //                        overflow, cache->filename);
32288 //	    /# overflow bucket ids start at the keylen position #/
32289 //	    idptr = PBT_BUCKKEYLEN(tbuf);
32290 //	}
32291 */
32292 
32293         kp = idptr;
32294 	idptr += idlen;
32295         BT_GETAJUINT(idptr, &uv);
32296         idptr += sizeof(ajuint);
32297 	BT_GETAJUINT(idptr,&uv2);
32298         if(uv2)
32299         {
32300             (*ndups)++;
32301             *nextra += uv2;
32302         }
32303 	idptr += sizeof(ajuint);
32304 	BT_GETAJULONG(idptr,&lv);
32305         if(uv2)
32306             ++refs[*ajTableulongFetch(newpostable, &lv)];
32307 	idptr += sizeof(ajulong);
32308 
32309         if(full)
32310             ajDebug("#StatBucket    %12Lu %6u %6u %12Lu ",
32311                     page->pagepos, uv, uv2, lv);
32312 
32313         if(cache->refcount)
32314         {
32315             for(iref=0; iref < cache->refcount; iref++)
32316             {
32317                 BT_GETAJULONG(idptr,&lv2);
32318                 idptr += sizeof(ajulong);
32319                 if(full)
32320                     ajDebug("%12Lu ", lv2);
32321             }
32322         }
32323 
32324         if(full)
32325             ajDebug("'%*s'\n", idlen-1, kp);
32326 
32327 	keyptr += sizeof(ajuint);
32328     }
32329     *freespace += pagesize - (idptr-tbuf);
32330     *nextra -= *ndups;
32331     if(full)
32332         ajDebug("StatBucket     %12Lu keys:%6u over:%6u free:%6u\n",
32333                 page->pagepos, *nentries, *overflows, *freespace);
32334 
32335     return ajTrue;
32336 }
32337 
32338 
32339 
32340 
32341 /* @func ajBtreeStatNumbucket *************************************************
32342 **
32343 ** Returns number of keys, overflow pages visited and free space unused
32344 ** for a b+ tree secondary bucket page
32345 **
32346 ** @param [w] cache [AjPBtcache] cache
32347 ** @param [r] page [const AjPBtpage] Page
32348 ** @param [r] full [AjBool] If true, write detailed debug report
32349 ** @param [w] nentries [ajuint*] Number of entries
32350 ** @param [w] overflows [ajuint*] Number of overflow pages
32351 ** @param [w] freespace [ajuint*] Unused space in this page
32352 ** @return [AjBool] True on success
32353 **
32354 ** @release 6.4.0
32355 ** @@
32356 ******************************************************************************/
32357 
ajBtreeStatNumbucket(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nentries,ajuint * overflows,ajuint * freespace)32358 AjBool ajBtreeStatNumbucket(AjPBtcache cache, const AjPBtpage page, AjBool full,
32359                             ajuint* nentries, ajuint* overflows,
32360                             ajuint* freespace)
32361 {
32362     ajuint iref;
32363     ajuint i;
32364     ajuint m;
32365     const AjPBtpage bpage;
32366     unsigned char *tbuf;
32367 
32368     ajulong lv;
32369     ajulong lv2;
32370     ajuint v;
32371     ajuint   pagesize = 0U;
32372     ajulong   overflow = 0UL;
32373     unsigned char *keyptr = NULL;
32374 
32375     pagesize = cache->secpagesize;
32376 
32377     bpage = page;
32378     tbuf = bpage->buf;
32379     GBT_BUCKNENTRIES(tbuf,&m);
32380     *nentries = m;
32381     *overflows = 0;
32382     *freespace = 0;
32383 
32384     GBT_BUCKOVERFLOW(tbuf,&overflow);
32385 
32386     keyptr = PBT_BUCKKEYLEN(tbuf);
32387 
32388     if(full)
32389         ajDebug("\n"
32390                 "StatNumbucket  %12Lu over:%12Lu\n",
32391                 page->pagepos, overflow);
32392     if(full)
32393         ajDebug("\nStatNumbucket  %12Lu       offset    ",
32394                 page->pagepos);
32395 
32396     if(full && cache->refcount)
32397     {
32398         for(iref=0; iref < cache->refcount; iref++)
32399             ajDebug("refoffset%d", iref+1);
32400     }
32401 
32402     if (full)
32403         ajDebug("dbno (%u)\n",
32404                 page->pagepos, m);
32405 
32406     if(full)
32407         ajDebug("-------------  ------------      -------");
32408 
32409     if(full && cache->refcount)
32410     {
32411         for(iref=0; iref < cache->refcount; iref++)
32412             ajDebug("   ----------");
32413     }
32414 
32415     if(full)
32416         ajDebug("         ----\n");
32417 
32418     for(i=0;i<m;i++)
32419     {
32420         BT_GETAJULONG(keyptr,&lv);
32421 	keyptr += sizeof(ajulong);
32422 
32423         if(full)
32424             ajDebug("#StatNumbucket              %12Lu", lv);
32425 
32426         if(cache->refcount)
32427         {
32428             for(iref=0; iref < cache->refcount; iref++)
32429             {
32430                 BT_GETAJULONG(keyptr,&lv2);
32431                 keyptr += sizeof(ajulong);
32432                 if(full)
32433                     ajDebug(" %12Lu", lv2);
32434             }
32435 
32436         }
32437 
32438         BT_GETAJUINT(keyptr,&v);
32439 	keyptr += sizeof(ajuint);
32440         if(full)
32441             ajDebug(" %12u\n", v);
32442     }
32443     *freespace += pagesize - (keyptr-tbuf);
32444     if(full)
32445         ajDebug("StatNumbucket  %12Lu keys:%6u over:%6u free:%6u\n",
32446                 page->pagepos, *nentries, *overflows, *freespace);
32447 
32448     return ajTrue;
32449 }
32450 
32451 
32452 
32453 
32454 /* @func ajBtreeStatPribucket *************************************************
32455 **
32456 ** Returns number of keys, overflow pages visited and free space unused
32457 ** for a b+ tree primary bucket page
32458 **
32459 ** @param [w] cache [AjPBtcache] cache
32460 ** @param [r] page [const AjPBtpage] Page
32461 ** @param [r] full [AjBool] If true, write detailed debug report
32462 ** @param [w] nentries [ajuint*] Number of entries
32463 ** @param [w] overflows [ajuint*] Number of overflow pages
32464 ** @param [w] freespace [ajuint*] Unused space in this page
32465 ** @param [u] refs [ajulong*] References to pages
32466 ** @param [r] newpostable [const AjPTable] Page numbers for page offsets
32467 ** @return [AjBool] True on success
32468 **
32469 ** @release 6.4.0
32470 ** @@
32471 ******************************************************************************/
32472 
ajBtreeStatPribucket(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nentries,ajuint * overflows,ajuint * freespace,ajulong * refs,const AjPTable newpostable)32473 AjBool ajBtreeStatPribucket(AjPBtcache cache, const AjPBtpage page, AjBool full,
32474                             ajuint* nentries, ajuint* overflows,
32475                             ajuint* freespace, ajulong *refs,
32476                             const AjPTable newpostable)
32477 {
32478     ajuint i;
32479     ajuint m;
32480     ajuint klen;
32481     const AjPBtpage bpage;
32482     unsigned char *tbuf;
32483 
32484     ajuint   pagesize = 0U;
32485     ajulong   overflow = 0UL;
32486     unsigned char *idptr = NULL;
32487     unsigned char *keyptr = NULL;
32488 
32489     unsigned char* lp;
32490     ajulong lv;
32491 
32492 /*    ajuint nodetype;*/
32493 /*    ajuint freebytes;*/
32494 
32495     pagesize = cache->pripagesize;
32496 
32497     bpage = page;
32498     tbuf = bpage->buf;
32499     GBT_BUCKNENTRIES(tbuf,&m);
32500     *nentries = m;
32501     *overflows = 0;
32502     *freespace = 0;
32503 
32504     GBT_BUCKOVERFLOW(tbuf,&overflow);
32505 
32506     keyptr = PBT_BUCKKEYLEN(tbuf);
32507     idptr = keyptr + (m * sizeof(ajuint));
32508 
32509     if(full)
32510         ajDebug("\n"
32511                 "StatPribucket  %12Lu over:%12Lu\n",
32512                 page->pagepos, overflow);
32513     if(full)
32514         ajDebug("\nStatPribucket  %12Lu (%u)\n",
32515                 page->pagepos, m);
32516     if(full)
32517         ajDebug("-------------  ------------      -------\n");
32518     for(i=0;i<m;i++)
32519     {
32520 	BT_GETAJUINT(keyptr,&klen);
32521 
32522 /*
32523 //	if((idptr-tbuf+2) + klen > pagesize)	/# overflow #/
32524 //	{
32525 //            freebytes = pagesize - (idptr-tbuf);
32526 //            *freespace += freebytes;
32527 //            (*overflows)++;
32528 //    	    /# ajDebug("btreeStatNode: Overflow\n"); #/
32529 //	    GBT_BUCKOVERFLOW(tbuf,&overflow);
32530 //	    bpage = btreePricacheRead(cache,overflow);
32531 //	    tbuf = bpage->buf;
32532 //	    GBT_BUCKNODETYPE(tbuf,&nodetype);
32533 //	    if(nodetype != BT_OVERFLOW)
32534 //		ajFatal("StatPribucket Overflow node %Lu expected but not found",
32535 //                        overflow, cache->filename);
32536 //	    /# overflow bucket ids start at the keylen position #/
32537 //	    idptr = PBT_BUCKKEYLEN(tbuf);
32538 //	}
32539 */
32540 
32541         lp = idptr+ klen-8;
32542         BT_GETAJULONG(lp,&lv);
32543         ++refs[*ajTableulongFetch(newpostable, &lv)];
32544         if(full)
32545             ajDebug("#StatPribucket %12Lu %12Lu '%*s'\n",
32546                     page->pagepos, lv, klen-9, idptr);
32547 	idptr += klen;    /* string */
32548 	keyptr += sizeof(ajuint);
32549     }
32550     *freespace += pagesize - (idptr-tbuf);
32551     if(full)
32552         ajDebug("StatPribucket  %12Lu keys:%6u over:%6u free:%6u\n",
32553                 page->pagepos, *nentries, *overflows, *freespace);
32554     return ajTrue;
32555 }
32556 
32557 
32558 
32559 
32560 /* @func ajBtreeStatSecbucket *************************************************
32561 **
32562 ** Returns number of keys, overflow pages visited and free space unused
32563 ** for a b+ tree secondary bucket page
32564 **
32565 ** @param [w] cache [AjPBtcache] cache
32566 ** @param [r] page [const AjPBtpage] Page
32567 ** @param [r] full [AjBool] If true, write detailed debug report
32568 ** @param [w] nentries [ajuint*] Number of entries
32569 ** @param [w] overflows [ajuint*] Number of overflow pages
32570 ** @param [w] freespace [ajuint*] Unused space in this page
32571 ** @return [AjBool] True on success
32572 **
32573 ** @release 6.4.0
32574 ** @@
32575 ******************************************************************************/
32576 
ajBtreeStatSecbucket(AjPBtcache cache,const AjPBtpage page,AjBool full,ajuint * nentries,ajuint * overflows,ajuint * freespace)32577 AjBool ajBtreeStatSecbucket(AjPBtcache cache, const AjPBtpage page, AjBool full,
32578                             ajuint* nentries, ajuint* overflows,
32579                             ajuint* freespace)
32580 {
32581     ajuint i;
32582     ajuint m;
32583     ajuint klen;
32584     const AjPBtpage bpage;
32585     unsigned char *tbuf;
32586 
32587     ajuint   pagesize = 0U;
32588     ajulong   overflow = 0UL;
32589     unsigned char *idptr = NULL;
32590     unsigned char *keyptr = NULL;
32591 
32592 /*    ajuint nodetype;*/
32593 /*    ajuint freebytes;*/
32594 
32595     pagesize = cache->secpagesize;
32596 
32597     bpage = page;
32598     tbuf = bpage->buf;
32599     GBT_BUCKNENTRIES(tbuf,&m);
32600     *nentries = m;
32601     *overflows = 0;
32602     *freespace = 0;
32603 
32604     GBT_BUCKOVERFLOW(tbuf,&overflow);
32605 
32606     keyptr = PBT_BUCKKEYLEN(tbuf);
32607     idptr = keyptr + (m * sizeof(ajuint));
32608 
32609     if(full)
32610         ajDebug("\n"
32611                 "StatSecbucket  %12Lu over:%12Lu\n",
32612                 page->pagepos, overflow);
32613     ajDebug("\nStatSecbucket  %12Lu (%u)\n",
32614             page->pagepos, m);
32615     if(full)
32616         ajDebug("-------------  ------------      -------\n");
32617     for(i=0;i<m;i++)
32618     {
32619 	BT_GETAJUINT(keyptr,&klen);
32620 
32621 /*
32622 //	if((idptr-tbuf+2) + klen > pagesize)	/# overflow #/
32623 //	{
32624 //            freebytes = pagesize - (idptr-tbuf);
32625 //            *freespace += freebytes;
32626 //            (*overflows)++;
32627 //    	    /# ajDebug("btreeStatNode: Overflow\n"); #/
32628 //	    GBT_BUCKOVERFLOW(tbuf,&overflow);
32629 //	    bpage = btreeSeccacheRead(cache,overflow);
32630 //	    tbuf = bpage->buf;
32631 //            GBT_BUCKNODETYPE(tbuf,&nodetype);
32632 //	    if(nodetype != BT_OVERFLOW)
32633 //		ajFatal("StatSecbucket Overflow node %Lu expected "
32634 //                        "but not found",
32635 //                        overflow, cache->filename);
32636 //	    /# overflow bucket ids start at the keylen position #/
32637 //	    idptr = PBT_BUCKKEYLEN(tbuf);
32638 //	}
32639 */
32640 
32641         if(full)
32642             ajDebug("#StatSecbucket %12Lu '%*s'\n",
32643                     page->pagepos, klen-1, idptr);
32644 	idptr += klen;    /* string */
32645 
32646 	keyptr += sizeof(ajuint);
32647     }
32648     *freespace += pagesize - (idptr-tbuf);
32649     if(full)
32650         ajDebug("StatSecbucket  %12Lu keys:%6u over:%6u free:%6u\n",
32651                 page->pagepos, *nentries, *overflows, *freespace);
32652 
32653     return ajTrue;
32654 }
32655 
32656 
32657 
32658 
32659 #if 0
32660 /* #funcstatic btreeIdbucketSplitCalc *****************************************
32661 **
32662 ** Calculate new bucket sizes to split a node.
32663 ** Make sure all existing buckets are reused.
32664 **
32665 ** #param [r] totalkeys [ajuint] Total number of keys in all buckets
32666 ** #param [r] totalbuckets [ajuint] Total number of current buckets
32667 ** #param [r] maxbucketsize [ajuint] Maximum number of keys in a buckets
32668 ** #param [w] leftbuckets [ajuint*] Number of left node buckets
32669 ** #param [w] leftmax [ajuint*] Maximum number of keys per left node bucket
32670 ** #param [w] leftkeys [ajuint*] Maximum number of keys under left node
32671 ** #param [w] rightbuckets [ajuint*] Number of right node buckets
32672 ** #param [w] rightmax [ajuint*] Maximum number of keys per right node bucket
32673 ** #param [w] rightkeys [ajuint*] Maximum number of keys under right node
32674 ** #return [AjBool] True if calculated value reused all buckets
32675 **                  False if increased to use all buckets
32676 **
32677 ** #release 6.4.0
32678 ** ##
32679 ******************************************************************************/
32680 /*
32681 static AjBool btreeIdbucketSplitCalc(ajuint totalkeys,
32682                                      ajuint totalbuckets,
32683                                      ajuint maxbucketsize,
32684                                      ajuint *leftbuckets, ajuint *leftmax,
32685                                      ajuint *leftkeys,
32686                                      ajuint *rightbuckets, ajuint *rightmax,
32687                                      ajuint *rightkeys)
32688 {
32689     ajuint lbuckets;
32690     ajuint rbuckets;
32691     ajuint lmax;
32692     ajuint rmax;
32693     ajuint lkeys;
32694     ajuint rkeys;
32695 
32696     ajuint lmax2;
32697     ajuint rmax2;
32698     ajuint lbuckets1;
32699     ajuint rbuckets1;
32700     ajuint lbuckets2;
32701     ajuint rbuckets2;
32702 
32703     ajuint bucketn;
32704     ajuint lrest;
32705     ajuint rrest;
32706 
32707     AjBool ret = ajTrue;
32708 
32709 /# we always split the keys to left and right #/
32710 
32711     lkeys = totalkeys/2;
32712     rkeys = totalkeys - lkeys;
32713 
32714 /# values to reuse all the current buckets #/
32715 
32716     lbuckets = totalbuckets/2;
32717     rbuckets = totalbuckets - lbuckets;
32718 
32719     lmax = lkeys/lbuckets;
32720     if(!(lkeys % lbuckets))
32721         lmax--;
32722     if(!lmax) lmax = 1;
32723 
32724     bucketn = lkeys/lmax;
32725     if(lkeys % lmax)
32726         bucketn++;
32727     lbuckets1 = bucketn - 1;
32728 
32729     rmax = rkeys/rbuckets;
32730     if(!(rkeys % rbuckets))
32731         rmax--;
32732     if(!rmax) rmax = 1;
32733 
32734     bucketn = rkeys/rmax;
32735     if(rkeys % rmax)
32736         bucketn++;
32737     rbuckets1 = bucketn - 1;
32738 
32739 /# minimum needed to fit what we need with half-full buckets #/
32740 
32741     lmax2 = maxbucketsize/2;
32742     lmax2++;
32743 
32744     bucketn = lkeys/lmax2;
32745     if(lkeys % lmax2)
32746         bucketn++;
32747     lbuckets2 = bucketn - 1;
32748 
32749     rmax2 = maxbucketsize/2;
32750     rmax2++;
32751 
32752     bucketn = rkeys/rmax2;
32753     if(rkeys % rmax2)
32754         bucketn++;
32755     rbuckets2 = bucketn - 1;
32756 
32757 /# we always use the same keys split #/
32758 
32759     *leftkeys = lkeys;
32760     *rightkeys = rkeys;
32761 
32762     if((rbuckets2 + lbuckets2) < totalbuckets)
32763     {
32764         /# use all the current buckets #/
32765 
32766         *leftmax = lmax;
32767         *leftbuckets = lbuckets1;
32768         *rightmax = rmax;
32769         *rightbuckets = rbuckets1;
32770         ret = ajFalse;
32771     }
32772     else
32773     {
32774         /# split to same or more half-full buckets #/
32775 
32776         *leftmax = lmax2;
32777         *leftbuckets = lbuckets2;
32778         *rightmax = rmax2;
32779         *rightbuckets = rbuckets2;
32780     }
32781 
32782     lrest = (*leftkeys) - (*leftbuckets)*(*leftmax);
32783     rrest = (*rightkeys) - (*rightbuckets)*(*rightmax);
32784 
32785 #if AJINDEX_DEBUG
32786     ajDebug("btreeIdbucketSplitCalc totalkeys:%u  totalbuckets:%u "
32787             "maxbucketsize:%u => %u*%u + %u (%u) "
32788             "%u*%u + %u (%u)\n",
32789             totalkeys, totalbuckets, maxbucketsize,
32790             *leftbuckets, *leftmax, lrest, *leftkeys,
32791             *rightbuckets, *rightmax, rrest, *rightkeys);
32792 #endif
32793 
32794     if(lrest < 1 || lrest > (*leftmax))
32795         ajFatal("Bad btreeIdbucketSplitCalc lrest:%u", lrest);
32796     if(rrest < 1 || rrest > (*rightmax))
32797         ajFatal("Bad btreeIdbucketSplitCalc rrest:%u", rrest);
32798 
32799     return ret;
32800 }
32801 */
32802 #endif
32803 
32804 
32805 
32806 
32807 /* @funcstatic btreeBucketSplitCalc *******************************************
32808 **
32809 ** Calculate new bucket sizes to split a node.
32810 ** Make sure all existing buckets are reused.
32811 **
32812 ** @param [r] totalkeys [ajuint] Total number of keys in all buckets
32813 ** @param [r] totalbuckets [ajuint] Total number of current buckets
32814 ** @param [r] maxbucketsize [ajuint] Maximum number of keys in a buckets
32815 ** @param [w] leftbuckets [ajuint*] Number of left node buckets
32816 ** @param [w] leftmax [ajuint*] Maximum number of keys per left node bucket
32817 ** @param [w] leftkeys [ajuint*] Maximum number of keys under left node
32818 ** @param [w] rightbuckets [ajuint*] Number of right node buckets
32819 ** @param [w] rightmax [ajuint*] Maximum number of keys per right node bucket
32820 ** @param [w] rightkeys [ajuint*] Maximum number of keys under right node
32821 ** @return [AjBool] True if calculated value reused all buckets
32822 **                  False if increased to use all buckets
32823 **
32824 ** @release 6.4.0
32825 ** @@
32826 ******************************************************************************/
32827 
btreeBucketSplitCalc(ajuint totalkeys,ajuint totalbuckets,ajuint maxbucketsize,ajuint * leftbuckets,ajuint * leftmax,ajuint * leftkeys,ajuint * rightbuckets,ajuint * rightmax,ajuint * rightkeys)32828 static AjBool btreeBucketSplitCalc(ajuint totalkeys,
32829                                    ajuint totalbuckets,
32830                                    ajuint maxbucketsize,
32831                                    ajuint *leftbuckets, ajuint *leftmax,
32832                                    ajuint *leftkeys,
32833                                    ajuint *rightbuckets, ajuint *rightmax,
32834                                    ajuint *rightkeys)
32835 {
32836     ajuint lbuckets;
32837     ajuint rbuckets;
32838     ajuint lmax;
32839     ajuint rmax;
32840     ajuint lkeys;
32841     ajuint rkeys;
32842 
32843     ajuint lmax2;
32844     ajuint rmax2;
32845     ajuint lbuckets1;
32846     ajuint rbuckets1;
32847     ajuint lbuckets2;
32848     ajuint rbuckets2;
32849 
32850     ajuint bucketn;
32851     ajuint lrest;
32852     ajuint rrest;
32853 
32854     AjBool ret = ajTrue;
32855 
32856 /* we always split the keys to left and right */
32857 
32858     lkeys = totalkeys/2;
32859     rkeys = totalkeys - lkeys;
32860 
32861 /* values to reuse all the current buckets */
32862 
32863     lbuckets = totalbuckets/2;
32864     rbuckets = totalbuckets - lbuckets;
32865 
32866     lmax = lkeys/lbuckets;
32867     if(!(lkeys % lbuckets))
32868         lmax--;
32869     if(!lmax) lmax = 1;
32870 
32871     bucketn = lkeys/lmax;
32872     if(lkeys % lmax)
32873         bucketn++;
32874     lbuckets1 = bucketn - 1;
32875 
32876     rmax = rkeys/rbuckets;
32877     if(!(rkeys % rbuckets))
32878         rmax--;
32879     if(!rmax) rmax = 1;
32880 
32881     bucketn = rkeys/rmax;
32882     if(rkeys % rmax)
32883         bucketn++;
32884     rbuckets1 = bucketn - 1;
32885 
32886 /* minimum needed to fit what we need with half-full buckets */
32887 
32888     lmax2 = maxbucketsize/2;
32889     lmax2++;
32890 
32891     bucketn = lkeys/lmax2;
32892     if(lkeys % lmax2)
32893         bucketn++;
32894     lbuckets2 = bucketn - 1;
32895 
32896     rmax2 = maxbucketsize/2;
32897     rmax2++;
32898 
32899     bucketn = rkeys/rmax2;
32900     if(rkeys % rmax2)
32901         bucketn++;
32902     rbuckets2 = bucketn - 1;
32903 
32904 /* we always use the same keys split */
32905 
32906     *leftkeys = lkeys;
32907     *rightkeys = rkeys;
32908 
32909     if((rbuckets2 + lbuckets2) < totalbuckets)
32910     {
32911         /* use all the current buckets */
32912 
32913         *leftmax = lmax;
32914         *leftbuckets = lbuckets1;
32915         *rightmax = rmax;
32916         *rightbuckets = rbuckets1;
32917         ret = ajFalse;
32918     }
32919     else
32920     {
32921         /* split to same or more half-full buckets */
32922 
32923         *leftmax = lmax2;
32924         *leftbuckets = lbuckets2;
32925         *rightmax = rmax2;
32926         *rightbuckets = rbuckets2;
32927     }
32928 
32929     lrest = (*leftkeys) - (*leftbuckets)*(*leftmax);
32930     rrest = (*rightkeys) - (*rightbuckets)*(*rightmax);
32931 
32932 #if AJINDEX_DEBUG
32933     ajDebug("btreeBucketSplitCalc totalkeys:%u  totalbuckets:%u "
32934             "maxbucketsize:%u => %u*%u + %u (%u) "
32935             "%u*%u + %u (%u)\n",
32936             totalkeys, totalbuckets, maxbucketsize,
32937             *leftbuckets, *leftmax, lrest, *leftkeys,
32938             *rightbuckets, *rightmax, rrest, *rightkeys);
32939 #endif
32940 
32941     if(lrest < 1 || lrest > (*leftmax))
32942         ajFatal("Bad btreeBucketSplitCalc lrest:%u", lrest);
32943     if(rrest < 1 || rrest > (*rightmax))
32944         ajFatal("Bad btreeBucketSplitCalc rrest:%u", rrest);
32945 
32946     return ret;
32947 }
32948 
32949 
32950 
32951 
32952 #if 0
32953 /* #funcstatic btreeIdbucketCalc **********************************************
32954 **
32955 ** Calculate new bucket sizes to split a node.
32956 ** Make sure all existing buckets are reused.
32957 **
32958 ** #param [r] totalkeys [ajuint] Total number of keys in all buckets
32959 ** #param [r] totalbuckets [ajuint] Total number of current buckets
32960 ** #param [r] maxbucketsize [ajuint] Maximum number of keys in a buckets
32961 ** #param [w] newbuckets [ajuint*] Number of new buckets
32962 ** #param [w] newmax [ajuint*] Maximum number of keys per new bucket
32963 ** #return [AjBool] True if calculated value reused all buckets
32964 **                  False if increased to use all buckets
32965 **
32966 ** #release 6.4.0
32967 ** ##
32968 ******************************************************************************/
32969 
32970 /*
32971 static AjBool btreeIdbucketCalc(ajuint totalkeys, ajuint totalbuckets,
32972                                 ajuint maxbucketsize,
32973                                 ajuint *newbuckets, ajuint *newmax)
32974 {
32975     ajuint bmax;
32976     ajuint bmax2;
32977 
32978     ajuint buckets1;
32979     ajuint buckets2;
32980 
32981     ajuint bucketn;
32982     ajuint rest;
32983 
32984     AjBool ret = ajTrue;
32985 
32986     /# from the original keys and buckets #/
32987 
32988     if(!totalbuckets)
32989         ajFatal("btreeIdbucketCalc zero buckets keys:%u", totalkeys);
32990     bmax = totalkeys/totalbuckets;
32991     if(!(totalkeys % totalbuckets))
32992         bmax--;
32993     if(!bmax) bmax = 1;
32994 
32995     bucketn = totalkeys/bmax;
32996     if(totalkeys % bmax)
32997         bucketn++;
32998     buckets1 = bucketn - 1;
32999 
33000     /# minimum space required #/
33001 
33002     bmax2 = maxbucketsize/2;
33003     if(!bmax2)
33004         ++bmax2;
33005 
33006     bucketn = totalkeys/bmax2;
33007     if(totalkeys % bmax2)
33008         bucketn++;
33009     buckets2 = bucketn - 1;
33010 
33011 
33012     if(buckets2 < totalbuckets)
33013     {
33014         *newmax = bmax;
33015         *newbuckets = buckets1;
33016         ret = ajFalse;
33017     }
33018     else
33019     {
33020         *newmax = bmax2;
33021         *newbuckets = buckets2;
33022     }
33023     rest = (totalkeys) - (*newbuckets)*(*newmax);
33024 
33025 #if AJINDEX_DEBUG
33026     ajDebug("btreeBucketCalc totalkeys:%u totalbuckets:%u "
33027             "maxbucketsize:%u => %u*%u + %u ret:%B\n",
33028             totalkeys, totalbuckets, maxbucketsize,
33029             *newbuckets, *newmax, rest, ret);
33030 #endif
33031 
33032     if(rest < 1 || rest > (*newmax))
33033         ajFatal("Bad ajBtreeBucketCalc rest:%u", rest);
33034 
33035     return ret;
33036 }
33037 */
33038 #endif
33039 
33040 
33041 
33042 
33043 /* @funcstatic btreeBucketCalc ************************************************
33044 **
33045 ** Calculate new bucket sizes to split a node.
33046 ** Make sure all existing buckets are reused.
33047 **
33048 ** @param [r] totalkeys [ajuint] Total number of keys in all buckets
33049 ** @param [r] totalbuckets [ajuint] Total number of current buckets
33050 ** @param [r] maxbucketsize [ajuint] Maximum number of keys in a buckets
33051 ** @param [w] newbuckets [ajuint*] Number of new buckets
33052 ** @param [w] newmax [ajuint*] Maximum number of keys per new bucket
33053 ** @return [AjBool] True if calculated value reused all buckets
33054 **                  False if increased to use all buckets
33055 **
33056 ** @release 6.4.0
33057 ** @@
33058 ******************************************************************************/
33059 
btreeBucketCalc(ajuint totalkeys,ajuint totalbuckets,ajuint maxbucketsize,ajuint * newbuckets,ajuint * newmax)33060 static AjBool btreeBucketCalc(ajuint totalkeys, ajuint totalbuckets,
33061                               ajuint maxbucketsize,
33062                               ajuint *newbuckets, ajuint *newmax)
33063 {
33064     ajuint bmax;
33065     ajuint bmax2;
33066 
33067     ajuint buckets1;
33068     ajuint buckets2;
33069 
33070     ajuint bucketn;
33071     ajuint rest;
33072 
33073     AjBool ret = ajTrue;
33074 
33075     /* from the original keys and buckets */
33076 
33077     if(!totalbuckets)
33078         ajFatal("btreeBucketCalc zero buckets keys:%u", totalkeys);
33079     bmax = totalkeys/totalbuckets;
33080     if(!(totalkeys % totalbuckets))
33081         bmax--;
33082     if(!bmax) bmax = 1;
33083 
33084     bucketn = totalkeys/bmax;
33085     if(totalkeys % bmax)
33086         bucketn++;
33087     buckets1 = bucketn - 1;
33088 
33089     /* minimum space required */
33090 
33091     bmax2 = maxbucketsize/2;
33092     if(!bmax2)
33093         ++bmax2;
33094 
33095     bucketn = totalkeys/bmax2;
33096     if(totalkeys % bmax2)
33097         bucketn++;
33098     buckets2 = bucketn - 1;
33099 
33100 
33101     if(buckets2 < totalbuckets)
33102     {
33103         *newmax = bmax;
33104         *newbuckets = buckets1;
33105         ret = ajFalse;
33106     }
33107     else
33108     {
33109         *newmax = bmax2;
33110         *newbuckets = buckets2;
33111     }
33112     rest = (totalkeys) - (*newbuckets)*(*newmax);
33113 
33114 #if AJINDEX_DEBUG
33115     ajDebug("btreeBucketCalc totalkeys:%u totalbuckets:%u "
33116             "maxbucketsize:%u => %u*%u + %u ret:%B\n",
33117             totalkeys, totalbuckets, maxbucketsize,
33118             *newbuckets, *newmax, rest, ret);
33119 #endif
33120 
33121     if(rest < 1 || rest > (*newmax))
33122         ajFatal("Bad ajBtreeBucketCalc rest:%u", rest);
33123 
33124     return ret;
33125 }
33126 
33127 
33128 
33129 
33130 /* @funcstatic btreePageposCompress *******************************************
33131 **
33132 ** Return the new start position of a compressed page position
33133 **
33134 ** @param [r] oldpos [ajulong] Page position
33135 ** @param [r] newpostable [const AjPTable] Table of new page positions
33136 ** @param [r] where [const char*] Location in index for reporting in
33137 **                                warning message.
33138 ** @return [ajulong] New page start position
33139 **
33140 ** @release 6.4.0
33141 ** @@
33142 ******************************************************************************/
33143 
btreePageposCompress(ajulong oldpos,const AjPTable newpostable,const char * where)33144 static ajulong btreePageposCompress(ajulong oldpos,
33145                                     const AjPTable newpostable,
33146                                     const char* where)
33147 {
33148     const ajulong *newpos;
33149 
33150     ajDebug("btreePageposCompress oldpos %Lu table size:%Lu\n",
33151             oldpos, ajTableGetLength(newpostable));
33152 
33153     newpos =  ajTableulongFetch(newpostable, &oldpos);
33154 
33155     if(!newpos)
33156     {
33157         ajWarn("oldpos %Lu not found (%s)",
33158                oldpos, where);
33159         return 0UL;
33160     }
33161 
33162     ajDebug("btreePageposCompress oldpos %Lu newpos %Lu\n",
33163             oldpos, *newpos);
33164 
33165     return *newpos;
33166 }
33167 
33168 
33169 
33170 
33171 /* @funcstatic btreePageposUncompress *****************************************
33172 **
33173 ** Return the new start position of a compressed page
33174 **
33175 ** @param [r] oldpos [ajulong] Page position
33176 ** @param [r] newpostable [const AjPTable] Array of new page positions
33177 ** @param [r] where [const char*] Location in index for reporting in
33178 **                                warning message.
33179 ** @return [ajulong] new start position
33180 **
33181 ** @release 6.4.0
33182 ** @@
33183 ******************************************************************************/
33184 
btreePageposUncompress(ajulong oldpos,const AjPTable newpostable,const char * where)33185 static ajulong btreePageposUncompress(ajulong oldpos,
33186                                       const AjPTable newpostable,
33187                                       const char* where)
33188 {
33189     const ajulong *newpos;
33190 
33191     ajDebug("btreePageposUncompress oldpos %Lu table size:%Lu\n",
33192             oldpos, ajTableGetLength(newpostable));
33193 
33194     newpos =  ajTableulongFetch(newpostable, &oldpos);
33195 
33196     if(!newpos)
33197     {
33198         ajWarn("oldpos %Lu not found (%s)",
33199                oldpos, where);
33200         return 0UL;
33201     }
33202 
33203     ajDebug("btreePageposUncompress oldpos %Lu newpos %Lu\n",
33204             oldpos, *newpos);
33205 
33206     return *newpos;
33207 }
33208 
33209 
33210 
33211 
33212 /* @funcstatic btreePageUncompress ********************************************
33213 **
33214 ** Uncompress a cache page using a table of new positions for pages
33215 **
33216 ** @param [u] page [AjPBtpage] Page
33217 ** @param [r] newpostable [const AjPTable] Table of new page positions
33218 ** @param [r] refcount [ajuint] Reference offset count
33219 ** @return [AjBool] True on success
33220 **
33221 ** @release 6.4.0
33222 ** @@
33223 ******************************************************************************/
33224 
btreePageUncompress(AjPBtpage page,const AjPTable newpostable,ajuint refcount)33225 static AjBool btreePageUncompress(AjPBtpage page,
33226                                   const AjPTable newpostable,
33227                                   ajuint refcount)
33228 {
33229     const unsigned char* buf = page->buf;
33230     ajuint nodetype;
33231 
33232     GBT_NODETYPE(buf, &nodetype);
33233 
33234     ajDebug("btreePageUncompress %Lu (%s)\n",
33235             page->pagepos, btreeNodetype(buf));
33236 
33237     switch (nodetype)
33238     {
33239         case BT_SECFREEPAGE:    /* no need to save the page */
33240         case BT_FREEPAGE:       /* no need to save the page */
33241             return ajFalse;
33242         case BT_ROOT:
33243         case BT_INTERNAL:
33244         case BT_LEAF:
33245             btreePageUncompressNode(page, newpostable);
33246             return ajTrue;
33247         case BT_SECROOT:
33248         case BT_SECINTERNAL:
33249         case BT_SECLEAF:
33250             btreePageUncompressNode(page, newpostable);
33251             return ajTrue;
33252         case BT_IDBUCKET:
33253             btreePageUncompressIdbucket(page, newpostable,
33254                                         refcount);
33255             return ajTrue;
33256         case BT_OVERFLOW:
33257         case BT_SECOVERFLOW:
33258             ajErr("Overflow page type %u in btreePageUncompress", nodetype);
33259             return ajTrue;
33260         case BT_PRIBUCKET:
33261             btreePageUncompressPribucket(page, newpostable);
33262             return ajTrue;
33263         case BT_SECBUCKET:
33264             /* no index page offsets to adjust */
33265             return ajTrue;
33266         case BT_NUMBUCKET:
33267             /* no index page offsets to adjust */
33268             return ajTrue;
33269         default:
33270             ajErr("Unknown page type %u in btreePageUncompress", nodetype);
33271             break;
33272     }
33273 
33274     return ajTrue;
33275 }
33276 
33277 
33278 
33279 
33280 /* @funcstatic btreePageUncompressIdbucket ************************************
33281 **
33282 ** Uncompress a bucket cache page using an array of new positions for
33283 ** uncompressed pages
33284 **
33285 ** @param [u] page [AjPBtpage] Page
33286 ** @param [r] newpostable [const AjPTable] Table of new page positions
33287 ** @param [r] refcount [ajuint] Number of reference offsets
33288 ** @return [void]
33289 **
33290 ** @release 6.5.0
33291 ** @@
33292 ******************************************************************************/
33293 
btreePageUncompressIdbucket(AjPBtpage page,const AjPTable newpostable,ajuint refcount)33294 static void btreePageUncompressIdbucket(AjPBtpage page,
33295                                         const AjPTable newpostable,
33296                                         ajuint refcount)
33297 {
33298     unsigned char* buf = page->buf;
33299     ajulong   overflow = 0UL;
33300     unsigned char *idptr = NULL;
33301     unsigned char *keyptr = NULL;
33302     ajulong lv;
33303     ajuint klen;
33304     ajuint idlen;
33305     ajuint nentries;
33306     ajuint dups;
33307     ajuint i;
33308     ajuint keyskip = BT_DDOFF + refcount*BT_EXTRA;
33309 
33310     buf = page->buf;
33311 
33312     GBT_BUCKNENTRIES(buf,&nentries);
33313     GBT_BUCKOVERFLOW(buf,&overflow);
33314 /*
33315 //    if(overflow)
33316 //    {
33317 //        overflow = btreePageposUncompress(overflow, newpostable,
33318 //                                          "Bucket overflow");
33319 //        SBT_BUCKOVERFLOW(buf, overflow);
33320 //    }
33321 */
33322     keyptr = PBT_BUCKKEYLEN(buf);
33323     idptr = keyptr + (nentries* sizeof(ajuint));
33324 
33325     /*
33326     ** ajulong values in bucket are offset and refoffset in files for dups=1
33327     ** but for dups > 1 offset is the secondary root page for the duplicates
33328     */
33329 
33330     for(i=0;i<nentries;i++)
33331     {
33332 	BT_GETAJUINT(keyptr,&klen);
33333         idlen = klen - keyskip;
33334 
33335 	idptr += idlen + sizeof(ajuint);
33336 	BT_GETAJUINT(idptr,&dups);
33337         idptr += sizeof(ajuint);
33338 
33339         if(dups)
33340         {
33341             BT_GETAJULONG(idptr,&lv);
33342             lv = btreePageposUncompress(lv, newpostable,
33343                                         "Bucket dup offset (secrootblock)");
33344             BT_SETAJULONG(idptr, lv);
33345         }
33346 
33347         idptr += sizeof(ajulong) + refcount*sizeof(ajulong);
33348 	keyptr += sizeof(ajuint);
33349     }
33350 
33351     return;
33352 }
33353 
33354 
33355 
33356 
33357 /* @funcstatic btreePageUncompressNode ****************************************
33358 **
33359 ** Uncompress a node cache page using a table of new positions
33360 **
33361 ** @param [u] page [AjPBtpage] Page
33362 ** @param [r] newpostable [const AjPTable] Table of new page positions
33363 ** @return [void]
33364 **
33365 ** @release 6.4.0
33366 ** @@
33367 ******************************************************************************/
33368 
btreePageUncompressNode(AjPBtpage page,const AjPTable newpostable)33369 static void btreePageUncompressNode(AjPBtpage page,
33370                                     const AjPTable newpostable)
33371 {
33372     unsigned char* buf = page->buf;
33373     ajuint nkeys = 0U;
33374     ajulong overflow = 0UL;
33375     unsigned char *lenptr = NULL;
33376     unsigned char *keyptr = NULL;
33377 
33378     ajuint klen;
33379     ajuint totlen;
33380     ajuint i;
33381     ajulong blockno;
33382     ajulong left;
33383     ajulong right;
33384     ajulong prev;
33385     ajulong lv;
33386 
33387     buf = page->buf;
33388 
33389     GBT_TOTLEN(buf,&totlen);
33390     if(!totlen)
33391     {
33392         btreePageUncompressNumnode(page, newpostable);
33393         return;
33394     }
33395 
33396     GBT_NKEYS(buf,&nkeys);
33397     GBT_OVERFLOW(buf,&overflow);
33398 
33399 /*
33400 //    if(overflow)
33401 //    {
33402 //        overflow = btreePageposUncompress(overflow, newpostable,
33403 //                                          "Node overflow");
33404 //        SBT_OVERFLOW(buf, overflow);
33405 //    }
33406 */
33407 
33408     GBT_LEFT(buf,&left);
33409 
33410     if(left)
33411     {
33412         left = btreePageposUncompress(left, newpostable,
33413                                       "Node left");
33414         SBT_LEFT(buf, left);
33415     }
33416 
33417     GBT_RIGHT(buf,&right);
33418 
33419     if(right > BT_NODEPREAMBLE)      /* not a level, must be a page position */
33420     {
33421         right = btreePageposUncompress(right, newpostable,
33422                                        "Node right");
33423         SBT_RIGHT(buf, right);
33424     }
33425 
33426     GBT_PREV(buf,&prev);
33427 
33428     if(prev)
33429     {
33430         prev = btreePageposUncompress(prev, newpostable,
33431                                       "Node prev");
33432         SBT_PREV(buf, prev);
33433     }
33434 
33435     GBT_BLOCKNUMBER(buf,&blockno);
33436     if(blockno)
33437     {
33438         blockno = btreePageposUncompress(blockno, newpostable,
33439                                          "Node blockno");
33440         SBT_BLOCKNUMBER(buf, blockno);
33441     }
33442 
33443     lenptr = PBT_KEYLEN(buf);
33444     keyptr = lenptr + nkeys * sizeof(ajuint);
33445 
33446     for(i=0;i<nkeys;i++)
33447     {
33448 	BT_GETAJUINT(lenptr,&klen);
33449         keyptr += klen + 1;
33450         BT_GETAJULONG(keyptr,&lv); /* pagepos for key */
33451 
33452         if(lv)
33453         {
33454             lv = btreePageposUncompress(lv, newpostable,
33455                                         "Node keypos");
33456             BT_SETAJULONG(keyptr, lv);
33457         }
33458 
33459         keyptr += sizeof(ajulong);
33460 	lenptr += sizeof(ajuint);
33461     }
33462 
33463     BT_GETAJULONG(keyptr,&lv); /* pagepos after last key */
33464 
33465     if(lv)
33466     {
33467         lv = btreePageposUncompress(lv, newpostable,
33468                                     "Node last keypos");
33469         BT_SETAJULONG(keyptr, lv);
33470     }
33471 
33472     return;
33473 }
33474 
33475 
33476 
33477 
33478 /* @funcstatic btreePageUncompressNumnode *************************************
33479 **
33480 ** uncompress a numeric node cache page using a table of new positions
33481 **
33482 ** @param [u] page [AjPBtpage] Page
33483 ** @param [r] newpostable [const AjPTable] Table of new page positions
33484 ** @return [void]
33485 **
33486 ** @release 6.4.0
33487 ** @@
33488 ******************************************************************************/
33489 
btreePageUncompressNumnode(AjPBtpage page,const AjPTable newpostable)33490 static void btreePageUncompressNumnode(AjPBtpage page,
33491                                        const AjPTable newpostable)
33492 {
33493     unsigned char* buf = page->buf;
33494     ajuint nkeys;
33495     ajulong   overflow = 0UL;
33496     unsigned char *keyptr = NULL;
33497     unsigned char *valptr = NULL;
33498     ajuint i;
33499     ajulong blockno;
33500     ajulong left;
33501     ajulong right;
33502     ajulong prev;
33503     ajulong lv;
33504 
33505     buf = page->buf;
33506 
33507     GBT_NKEYS(buf,&nkeys);
33508     GBT_OVERFLOW(buf,&overflow);
33509 
33510 /*
33511 //    if(overflow)
33512 //    {
33513 //        overflow = btreePageposUncompress(overflow, newpostable,
33514 //                                          "Numnode overflow");
33515 //        SBT_OVERFLOW(buf, overflow);
33516 //    }
33517 */
33518 
33519     GBT_LEFT(buf,&left);
33520 
33521     if(left)
33522     {
33523         left = btreePageposUncompress(left, newpostable,
33524                                       "Numnode left");
33525         SBT_LEFT(buf, left);
33526     }
33527 
33528     GBT_RIGHT(buf,&right);
33529 
33530     if(right > BT_NODEPREAMBLE)      /* not a level, must be a page position */
33531     {
33532         right = btreePageposUncompress(right, newpostable,
33533                                        "Numnode right");
33534         SBT_RIGHT(buf, right);
33535     }
33536 
33537     GBT_PREV(buf,&prev);
33538 
33539     if(prev)
33540     {
33541         prev = btreePageposUncompress(prev, newpostable,
33542                                       "Numnode prev");
33543         SBT_PREV(buf, prev);
33544     }
33545 
33546     GBT_BLOCKNUMBER(buf,&blockno);
33547     if(blockno)
33548     {
33549         prev = btreePageposUncompress(blockno, newpostable,
33550                                       "Numnode blockno");
33551         SBT_BLOCKNUMBER(buf, blockno);
33552     }
33553 
33554     keyptr =  PBT_KEYLEN(buf);
33555     valptr = keyptr + nkeys * sizeof(ajulong);
33556 
33557     for(i = 0; i < nkeys; i++)
33558     {
33559         BT_GETAJULONG(valptr,&lv); /* pagepos for key */
33560 
33561         if(lv)
33562         {
33563             lv = btreePageposUncompress(lv, newpostable,
33564                                         "Numnode keypos");
33565             BT_SETAJULONG(valptr, lv);
33566         }
33567 
33568         valptr += sizeof(ajulong);
33569     }
33570 
33571     BT_GETAJULONG(valptr,&lv); /* pagepos after last key */
33572 
33573     if(lv)
33574     {
33575         lv = btreePageposUncompress(lv, newpostable,
33576                                     "Numnode last keypos");
33577         BT_SETAJULONG(valptr, lv);
33578     }
33579 
33580     return;
33581 }
33582 
33583 
33584 
33585 
33586 /* @funcstatic btreePageUncompressPribucket ***********************************
33587 **
33588 ** Uncompress a primary bucket cache page using a table of new positions
33589 **
33590 ** @param [u] page [AjPBtpage] Page
33591 ** @param [r] newpostable [const AjPTable] Table of new page positions
33592 ** @return [void]
33593 **
33594 ** @release 6.4.0
33595 ** @@
33596 ******************************************************************************/
33597 
btreePageUncompressPribucket(AjPBtpage page,const AjPTable newpostable)33598 static void btreePageUncompressPribucket(AjPBtpage page,
33599                                          const AjPTable newpostable)
33600 {
33601     unsigned char* buf = page->buf;
33602     ajuint nentries;
33603     ajulong overflow = 0UL;
33604     unsigned char *idptr = NULL;
33605     unsigned char *keyptr = NULL;
33606     ajuint klen;
33607     ajuint i;
33608     ajulong lv;
33609     unsigned char * lp = NULL;
33610 
33611     buf = page->buf;
33612 
33613     GBT_BUCKNENTRIES(buf,&nentries);
33614     GBT_BUCKOVERFLOW(buf,&overflow);
33615 
33616 /*
33617 //    if(overflow)
33618 //    {
33619 //        overflow = btreePageposUncompress(overflow, newpostable,
33620 //                                          "Pribucket overflow");
33621 //        SBT_BUCKOVERFLOW(buf, overflow);
33622 //    }
33623 */
33624 
33625     keyptr = PBT_BUCKKEYLEN(buf);
33626     idptr = keyptr + (nentries * sizeof(ajuint));
33627 
33628     for(i=0;i<nentries;i++)
33629     {
33630 	BT_GETAJUINT(keyptr,&klen);
33631 	idptr += klen;
33632         lp = idptr - 8;
33633         BT_GETAJULONG(lp,&lv);
33634 
33635         if(lv)
33636         {
33637             lv = btreePageposUncompress(lv, newpostable,
33638                                         "Pribucket keypos");
33639             BT_SETAJULONG(lp, lv);
33640         }
33641 
33642 	keyptr += sizeof(ajuint);
33643     }
33644 
33645     return;
33646 }
33647 
33648 
33649 
33650 
33651 /* @funcstatic btreePageCompress **********************************************
33652 **
33653 ** Compress a cache page using an array of new positions for uncompressed pages
33654 **
33655 ** @param [u] page [AjPBtpage] Page
33656 ** @param [r] newpostable [const AjPTable] Table of new page positions
33657 ** @param [r] refcount [ajuint] Number of reference offsets
33658 ** @return [AjBool] True on success
33659 **
33660 ** @release 6.4.0
33661 ** @@
33662 ******************************************************************************/
33663 
btreePageCompress(AjPBtpage page,const AjPTable newpostable,ajuint refcount)33664 static AjBool btreePageCompress(AjPBtpage page,
33665                                 const AjPTable newpostable,
33666                                 ajuint refcount)
33667 {
33668     const unsigned char* buf = page->buf;
33669     ajuint nodetype;
33670 
33671     GBT_NODETYPE(buf, &nodetype);
33672 
33673     ajDebug("btreePageCompress %Lu (%s)\n",
33674             page->pagepos, btreeNodetype(buf));
33675 
33676     switch (nodetype)
33677     {
33678         case BT_SECFREEPAGE:    /* no need to save the page */
33679         case BT_FREEPAGE:       /* no need to save the page */
33680             return ajFalse;
33681         case BT_ROOT:
33682         case BT_INTERNAL:
33683         case BT_LEAF:
33684             btreePageCompressNode(page, newpostable);
33685             return ajTrue;
33686         case BT_SECROOT:
33687         case BT_SECINTERNAL:
33688         case BT_SECLEAF:
33689             btreePageCompressNode(page, newpostable);
33690             return ajTrue;
33691         case BT_IDBUCKET:
33692             btreePageCompressIdbucket(page, newpostable, refcount);
33693             return ajTrue;
33694         case BT_OVERFLOW:
33695         case BT_SECOVERFLOW:
33696             ajErr("Overflow page type %u in btreePageCompress", nodetype);
33697             return ajTrue;
33698         case BT_PRIBUCKET:
33699             btreePageCompressPribucket(page, newpostable);
33700             return ajTrue;
33701         case BT_SECBUCKET:
33702             /* no index page offsets to adjust */
33703             return ajTrue;
33704         case BT_NUMBUCKET:
33705             /* no index page offsets to adjust */
33706             return ajTrue;
33707         default:
33708             ajErr("Unknown page type %u in btreePageCompress", nodetype);
33709             break;
33710     }
33711     return ajTrue;
33712 }
33713 
33714 
33715 
33716 
33717 /* @funcstatic btreePageCompressIdbucket **************************************
33718 **
33719 ** Compress a bucket cache page using an array of new positions for
33720 ** uncompressed pages
33721 **
33722 ** @param [u] page [AjPBtpage] Page
33723 ** @param [r] newpostable [const AjPTable] Table of new page positions
33724 ** @param [r] refcount [ajuint] Number of reference offsets
33725 ** @return [void]
33726 **
33727 ** @release 6.4.0
33728 ** @@
33729 ******************************************************************************/
33730 
btreePageCompressIdbucket(AjPBtpage page,const AjPTable newpostable,ajuint refcount)33731 static void btreePageCompressIdbucket(AjPBtpage page,
33732                                       const AjPTable newpostable,
33733                                       ajuint refcount)
33734 {
33735     unsigned char* buf = page->buf;
33736     ajulong   overflow = 0UL;
33737     unsigned char *idptr = NULL;
33738     unsigned char *keyptr = NULL;
33739     ajulong lv;
33740     ajuint klen;
33741     ajuint idlen;
33742     ajuint nentries;
33743     ajuint dups;
33744     ajuint i;
33745     ajuint keyskip = BT_DDOFF + refcount*BT_EXTRA;
33746 
33747     buf = page->buf;
33748 
33749     GBT_BUCKNENTRIES(buf,&nentries);
33750     GBT_BUCKOVERFLOW(buf,&overflow);
33751 
33752 /*
33753 //    if(overflow)
33754 //    {
33755 //        overflow = btreePageposCompress(overflow, newpostable,
33756 //                                        "Bucket overflow");
33757 //        SBT_BUCKOVERFLOW(buf, overflow);
33758 //    }
33759 */
33760 
33761     keyptr = PBT_BUCKKEYLEN(buf);
33762     idptr = keyptr + (nentries* sizeof(ajuint));
33763 
33764     /*
33765     ** ajulong values in bucket are offset and refoffset in files for dups=1
33766     ** but for dups > 1 offset is the secondary root page for the duplicates
33767     */
33768 
33769     for(i=0;i<nentries;i++)
33770     {
33771 	BT_GETAJUINT(keyptr,&klen);
33772         idlen = klen - keyskip;
33773 
33774 	idptr += idlen + sizeof(ajuint);
33775 	BT_GETAJUINT(idptr,&dups);
33776         idptr += sizeof(ajuint);
33777 
33778         if(dups)
33779         {
33780             BT_GETAJULONG(idptr,&lv);
33781             lv = btreePageposCompress(lv, newpostable,
33782                                       "Bucket dup offset (secrootblock)");
33783             BT_SETAJULONG(idptr, lv);
33784         }
33785 
33786         idptr += sizeof(ajulong) + refcount*sizeof(ajulong);
33787 	keyptr += sizeof(ajuint);
33788     }
33789 
33790     return;
33791 }
33792 
33793 
33794 
33795 
33796 /* @funcstatic btreePageCompressNode ******************************************
33797 **
33798 ** Compress a node cache page using an array of new positions for
33799 ** uncompressed pages
33800 **
33801 ** @param [u] page [AjPBtpage] Page
33802 ** @param [r] newpostable [const AjPTable] Table of new page positions
33803 ** @return [void]
33804 **
33805 ** @release 6.5.0
33806 ** @@
33807 ******************************************************************************/
33808 
btreePageCompressNode(AjPBtpage page,const AjPTable newpostable)33809 static void btreePageCompressNode(AjPBtpage page,
33810                                  const AjPTable newpostable)
33811 {
33812     unsigned char* buf = page->buf;
33813     ajuint nkeys = 0U;
33814     ajulong overflow = 0UL;
33815     unsigned char *lenptr = NULL;
33816     unsigned char *keyptr = NULL;
33817 
33818     ajuint klen;
33819     ajuint totlen;
33820     ajuint i;
33821     ajulong blockno;
33822     ajulong left;
33823     ajulong right;
33824     ajulong prev;
33825     ajulong lv;
33826 
33827     buf = page->buf;
33828 
33829     GBT_TOTLEN(buf,&totlen);
33830 
33831     if(!totlen)
33832     {
33833         btreePageCompressNumnode(page, newpostable);
33834         return;
33835     }
33836 
33837     GBT_NKEYS(buf,&nkeys);
33838     GBT_OVERFLOW(buf,&overflow);
33839 
33840 /*
33841 //    if(overflow)
33842 //    {
33843 //        overflow = btreePageposCompress(overflow, newpostable,
33844 //                                        "Node overflow");
33845 //        SBT_OVERFLOW(buf, overflow);
33846 //    }
33847 */
33848 
33849     GBT_LEFT(buf,&left);
33850 
33851     if(left)
33852     {
33853         left = btreePageposCompress(left, newpostable,
33854                                     "Node left");
33855         SBT_LEFT(buf, left);
33856     }
33857 
33858     GBT_RIGHT(buf,&right);
33859 
33860     if(right > BT_NODEPREAMBLE)      /* not a level, must be a page position */
33861     {
33862         right = btreePageposCompress(right, newpostable,
33863                                      "Node right");
33864         SBT_RIGHT(buf, right);
33865     }
33866 
33867     GBT_PREV(buf,&prev);
33868 
33869     if(prev)
33870     {
33871         prev = btreePageposCompress(prev, newpostable,
33872                                     "Node prev");
33873         SBT_PREV(buf, prev);
33874     }
33875 
33876     GBT_BLOCKNUMBER(buf,&blockno);
33877     if(blockno)
33878     {
33879         blockno = btreePageposCompress(blockno, newpostable,
33880                                        "Node blockno");
33881         SBT_BLOCKNUMBER(buf, blockno);
33882     }
33883 
33884     lenptr = PBT_KEYLEN(buf);
33885     keyptr = lenptr + nkeys * sizeof(ajuint);
33886 
33887     for(i=0;i<nkeys;i++)
33888     {
33889 	BT_GETAJUINT(lenptr,&klen);
33890         keyptr += klen + 1;
33891         BT_GETAJULONG(keyptr,&lv); /* pagepos for key */
33892 
33893         if(lv)
33894         {
33895             lv = btreePageposCompress(lv, newpostable,
33896                                       "Node keypos");
33897             BT_SETAJULONG(keyptr, lv);
33898         }
33899 
33900         keyptr += sizeof(ajulong);
33901 	lenptr += sizeof(ajuint);
33902     }
33903 
33904     BT_GETAJULONG(keyptr,&lv); /* pagepos after last key */
33905 
33906     if(lv)
33907     {
33908         lv = btreePageposCompress(lv, newpostable,
33909                                   "Node last keypos");
33910         BT_SETAJULONG(keyptr, lv);
33911     }
33912 
33913     return;
33914 }
33915 
33916 
33917 
33918 
33919 /* @funcstatic btreePageCompressNumnode ***************************************
33920 **
33921 ** Compress a numeric node cache page using an array of new
33922 ** positions for uncompressed pages
33923 **
33924 ** @param [u] page [AjPBtpage] Page
33925 ** @param [r] newpostable [const AjPTable] Table of new page positions
33926 ** @return [void]
33927 **
33928 ** @release 6.5.0
33929 ** @@
33930 ******************************************************************************/
33931 
btreePageCompressNumnode(AjPBtpage page,const AjPTable newpostable)33932 static void btreePageCompressNumnode(AjPBtpage page,
33933                                      const AjPTable newpostable)
33934 {
33935     unsigned char* buf = page->buf;
33936     ajuint nkeys;
33937     ajulong   overflow = 0UL;
33938     unsigned char *keyptr = NULL;
33939     unsigned char *valptr = NULL;
33940     ajuint i;
33941     ajulong blockno;
33942     ajulong left;
33943     ajulong right;
33944     ajulong prev;
33945     ajulong lv;
33946 
33947     buf = page->buf;
33948 
33949     GBT_NKEYS(buf,&nkeys);
33950     GBT_OVERFLOW(buf,&overflow);
33951 
33952 /*
33953 //    if(overflow)
33954 //    {
33955 //        overflow = btreePageposCompress(overflow, newpostable,
33956 //                                        "Numnode overflow");
33957 //        SBT_OVERFLOW(buf, overflow);
33958 //    }
33959 */
33960 
33961     GBT_LEFT(buf,&left);
33962 
33963     if(left)
33964     {
33965         left = btreePageposCompress(left, newpostable,
33966                                     "Numnode left");
33967         SBT_LEFT(buf, left);
33968     }
33969 
33970     GBT_RIGHT(buf,&right);
33971 
33972     if(right > BT_NODEPREAMBLE)      /* not a level, must be a page position */
33973     {
33974         right = btreePageposCompress(right, newpostable,
33975                                      "Numnode right");
33976         SBT_RIGHT(buf, right);
33977     }
33978 
33979     GBT_PREV(buf,&prev);
33980 
33981     if(prev)
33982     {
33983         prev = btreePageposCompress(prev, newpostable,
33984                                     "Numnode prev");
33985         SBT_PREV(buf, prev);
33986     }
33987 
33988     GBT_BLOCKNUMBER(buf,&blockno);
33989     if(blockno)
33990     {
33991         blockno = btreePageposCompress(blockno, newpostable,
33992                                        "Numnode blockno");
33993         SBT_BLOCKNUMBER(buf, blockno);
33994     }
33995 
33996     keyptr =  PBT_KEYLEN(buf);
33997     valptr = keyptr + nkeys * sizeof(ajulong);
33998 
33999     for(i = 0; i < nkeys; i++)
34000     {
34001         BT_GETAJULONG(valptr,&lv); /* pagepos for key */
34002 
34003         if(lv)
34004         {
34005             lv = btreePageposCompress(lv, newpostable,
34006                                       "Numnode keypos");
34007             BT_SETAJULONG(valptr, lv);
34008         }
34009 
34010         valptr += sizeof(ajulong);
34011     }
34012 
34013     BT_GETAJULONG(valptr,&lv); /* pagepos after last key */
34014 
34015     if(lv)
34016     {
34017         lv = btreePageposCompress(lv, newpostable,
34018                                   "Numnode last keypos");
34019         BT_SETAJULONG(valptr, lv);
34020     }
34021 
34022     return;
34023 }
34024 
34025 
34026 
34027 
34028 /* @funcstatic btreePageCompressPribucket *************************************
34029 **
34030 ** Compress a primary bucket cache page using an array of new
34031 ** positions for uncompressed pages
34032 **
34033 ** @param [u] page [AjPBtpage] Page
34034 ** @param [r] newpostable [const AjPTable] Table of new page positions
34035 ** @return [void]
34036 **
34037 ** @release 6.5.0
34038 ** @@
34039 ******************************************************************************/
34040 
btreePageCompressPribucket(AjPBtpage page,const AjPTable newpostable)34041 static void btreePageCompressPribucket(AjPBtpage page,
34042                                        const AjPTable newpostable)
34043 {
34044     unsigned char* buf = page->buf;
34045     ajuint nentries;
34046     ajulong overflow = 0UL;
34047     unsigned char *idptr = NULL;
34048     unsigned char *keyptr = NULL;
34049     ajuint klen;
34050     ajuint i;
34051     ajulong lv;
34052     unsigned char * lp = NULL;
34053 
34054     buf = page->buf;
34055 
34056     GBT_BUCKNENTRIES(buf,&nentries);
34057     GBT_BUCKOVERFLOW(buf,&overflow);
34058 
34059 /*
34060 //    if(overflow)
34061 //    {
34062 //        overflow = btreePageposCompress(overflow, newpostable,
34063 //                                        "Pribucket overflow");
34064 //        SBT_BUCKOVERFLOW(buf, overflow);
34065 //    }
34066 */
34067 
34068     keyptr = PBT_BUCKKEYLEN(buf);
34069     idptr = keyptr + (nentries * sizeof(ajuint));
34070 
34071     for(i=0;i<nentries;i++)
34072     {
34073 	BT_GETAJUINT(keyptr,&klen);
34074 	idptr += klen;
34075         lp = idptr - 8;
34076         BT_GETAJULONG(lp,&lv);
34077 
34078         if(lv)
34079         {
34080             lv = btreePageposCompress(lv, newpostable,
34081                                       "Pribucket keypos");
34082             BT_SETAJULONG(lp, lv);
34083         }
34084 
34085 	keyptr += sizeof(ajuint);
34086     }
34087 
34088     return;
34089 }
34090 
34091 
34092 
34093 
34094 /* @func ajBtreePageGetSize ***************************************************
34095 **
34096 ** Return the used size of a cache page
34097 **
34098 ** @param [r] page [const AjPBtpage] Page
34099 *8 @param [r] refcount [ajuint] Number of reference offsets
34100 ** @return [ajuint] Used page size
34101 **
34102 ** @release 6.4.0
34103 ** @@
34104 ******************************************************************************/
34105 
ajBtreePageGetSize(const AjPBtpage page,ajuint refcount)34106 ajuint ajBtreePageGetSize(const AjPBtpage page, ajuint refcount)
34107 {
34108     const unsigned char* buf = page->buf;
34109     ajuint nodetype;
34110 
34111     GBT_NODETYPE(buf, &nodetype);
34112     switch (nodetype)
34113     {
34114         case BT_SECFREEPAGE:    /* no need to save the page */
34115         case BT_FREEPAGE:       /* no need to save the page */
34116             return 0;
34117         case BT_ROOT:
34118         case BT_INTERNAL:
34119         case BT_LEAF:
34120         case BT_SECROOT:
34121         case BT_SECINTERNAL:
34122         case BT_SECLEAF:
34123             return btreePageGetSizeNode(page);
34124         case BT_IDBUCKET:
34125             return btreePageGetSizeIdbucket(page, refcount);
34126         case BT_OVERFLOW:
34127         case BT_SECOVERFLOW:
34128             ajErr("Overflow page type %u in btreePageGetSize at %Lu",
34129                   nodetype, page->pagepos);
34130             return 0;
34131         case BT_PRIBUCKET:
34132             return btreePageGetSizePribucket(page);
34133         case BT_SECBUCKET:
34134             return btreePageGetSizeSecbucket(page);
34135         case BT_NUMBUCKET:
34136             return btreePageGetSizeNumbucket(page, refcount);
34137         default:
34138             ajErr("Unknown page type %u in btreePageGetSize at %Lu",
34139                   nodetype, page->pagepos);
34140             break;
34141     }
34142     return 0;
34143 }
34144 
34145 
34146 
34147 
34148 /* @funcstatic btreePageGetSizeIdbucket ***************************************
34149 **
34150 ** Return the used size of a bucket cache page
34151 **
34152 ** @param [r] page [const AjPBtpage] Page
34153 ** @param [r] refcount [ajuint] Number of reference offsets
34154 ** @return [ajuint] Used page size
34155 **
34156 ** @release 6.4.0
34157 ** @@
34158 ******************************************************************************/
34159 
btreePageGetSizeIdbucket(const AjPBtpage page,ajuint refcount)34160 static ajuint btreePageGetSizeIdbucket(const AjPBtpage page, ajuint refcount)
34161 {
34162     const unsigned char* buf = page->buf;
34163     ajuint nentries;
34164     ajulong   overflow = 0UL;
34165     const unsigned char *idptr = NULL;
34166     const unsigned char *keyptr = NULL;
34167     ajuint klen;
34168     ajuint idlen;
34169     ajuint i;
34170     ajuint keyskip = BT_DDOFF + refcount*BT_EXTRA;
34171 
34172     buf = page->buf;
34173 
34174     GBT_BUCKNENTRIES(buf,&nentries);
34175     GBT_BUCKOVERFLOW(buf,&overflow);
34176 
34177     keyptr = PBT_BUCKKEYLEN(buf);
34178     idptr = keyptr + (nentries * sizeof(ajuint));
34179 
34180     for(i=0;i<nentries;i++)
34181     {
34182 	BT_GETAJUINT(keyptr,&klen);
34183         idlen = klen - keyskip;
34184 	idptr += (idlen + sizeof(ajuint) + sizeof(ajuint) +
34185                   sizeof(ajulong) + refcount*sizeof(ajulong));
34186 	keyptr += sizeof(ajuint);
34187     }
34188 
34189     return (idptr-buf);
34190 }
34191 
34192 
34193 
34194 
34195 /* @funcstatic btreePageGetSizeNode *******************************************
34196 **
34197 ** Return the used size of a node cache page
34198 **
34199 ** @param [r] page [const AjPBtpage] Page
34200 ** @return [ajuint] Used page size
34201 **
34202 ** @release 6.4.0
34203 ** @@
34204 ******************************************************************************/
34205 
btreePageGetSizeNode(const AjPBtpage page)34206 static ajuint btreePageGetSizeNode(const AjPBtpage page)
34207 {
34208     const unsigned char* buf = page->buf;
34209     ajuint nkeys = 0U;
34210     ajulong overflow = 0UL;
34211     const unsigned char *lenptr = NULL;
34212     const unsigned char *keyptr = NULL;
34213 
34214     ajuint klen;
34215     ajuint totlen;
34216     ajuint i;
34217 
34218     buf = page->buf;
34219 
34220     GBT_TOTLEN(buf,&totlen);
34221     if(!totlen)
34222       return btreePageGetSizeNumnode(page);
34223 
34224     GBT_NKEYS(buf,&nkeys);
34225     GBT_OVERFLOW(buf,&overflow);
34226 
34227     lenptr = PBT_KEYLEN(buf);
34228     keyptr = lenptr + nkeys * sizeof(ajuint);
34229 
34230     for(i=0;i<nkeys;i++)
34231     {
34232 	BT_GETAJUINT(lenptr,&klen);
34233         keyptr += klen + 1 + sizeof(ajulong);
34234 	lenptr += sizeof(ajuint);
34235     }
34236 
34237     keyptr += sizeof(ajulong);
34238 
34239     return (keyptr-buf);
34240 }
34241 
34242 
34243 
34244 
34245 /* @funcstatic btreePageGetSizeNumbucket **************************************
34246 **
34247 ** Return the used size of a numeric bucket cache page
34248 **
34249 ** @param [r] page [const AjPBtpage] Page
34250 ** @param [r] refcount [ajuint] Number of reference offsets
34251 ** @return [ajuint] Used page size
34252 **
34253 ** @release 6.4.0
34254 ** @@
34255 ******************************************************************************/
34256 
btreePageGetSizeNumbucket(const AjPBtpage page,ajuint refcount)34257 static ajuint btreePageGetSizeNumbucket(const AjPBtpage page,
34258                                         ajuint refcount)
34259 {
34260     const unsigned char* buf = page->buf;
34261     ajuint nentries;
34262     ajulong   overflow = 0UL;
34263     const unsigned char *keyptr = NULL;
34264 
34265     buf = page->buf;
34266 
34267     GBT_BUCKNENTRIES(buf,&nentries);
34268     GBT_BUCKOVERFLOW(buf,&overflow);
34269 
34270     keyptr = PBT_BUCKKEYLEN(buf);
34271 
34272     keyptr += nentries * (sizeof(ajlong) + refcount*sizeof(ajlong) +
34273                           sizeof(ajuint));
34274 
34275     return (keyptr-buf);
34276 }
34277 
34278 
34279 
34280 
34281 /* @funcstatic btreePageGetSizeNumnode ****************************************
34282 **
34283 ** Return the used size of a numeric node cache page
34284 **
34285 ** @param [r] page [const AjPBtpage] Page
34286 ** @return [ajuint] Used page size
34287 **
34288 ** @release 6.4.0
34289 ** @@
34290 ******************************************************************************/
34291 
btreePageGetSizeNumnode(const AjPBtpage page)34292 static ajuint btreePageGetSizeNumnode(const AjPBtpage page)
34293 {
34294     const unsigned char* buf = page->buf;
34295     ajuint nkeys;
34296     ajulong   overflow = 0UL;
34297     const unsigned char *keyptr = NULL;
34298     const unsigned char *valptr = NULL;
34299 
34300     buf = page->buf;
34301 
34302     GBT_NKEYS(buf,&nkeys);
34303     GBT_OVERFLOW(buf,&overflow);
34304 
34305     keyptr =  PBT_KEYLEN(buf);
34306     valptr = keyptr + nkeys * sizeof(ajulong) +
34307         nkeys * sizeof(ajulong) + sizeof(ajulong);
34308 
34309     return (valptr-buf);
34310 }
34311 
34312 
34313 
34314 
34315 /* @funcstatic btreePageGetSizePribucket **************************************
34316 **
34317 ** Return the used size of a primary bucket cache page
34318 **
34319 ** @param [r] page [const AjPBtpage] Page
34320 ** @return [ajuint] Used page size
34321 **
34322 ** @release 6.4.0
34323 ** @@
34324 ******************************************************************************/
34325 
btreePageGetSizePribucket(const AjPBtpage page)34326 static ajuint btreePageGetSizePribucket(const AjPBtpage page)
34327 {
34328     const unsigned char* buf = page->buf;
34329     ajuint nentries;
34330     ajulong overflow = 0UL;
34331     const unsigned char *idptr = NULL;
34332     const unsigned char *keyptr = NULL;
34333     ajuint klen;
34334     ajuint i;
34335 
34336     buf = page->buf;
34337 
34338     GBT_BUCKNENTRIES(buf,&nentries);
34339     GBT_BUCKOVERFLOW(buf,&overflow);
34340 
34341     keyptr = PBT_BUCKKEYLEN(buf);
34342     idptr = keyptr + (nentries * sizeof(ajuint));
34343 
34344     for(i=0;i<nentries;i++)
34345     {
34346 	BT_GETAJUINT(keyptr,&klen);
34347 	idptr += klen;
34348 	keyptr += sizeof(ajuint);
34349     }
34350 
34351     return (idptr-buf);
34352 }
34353 
34354 
34355 
34356 
34357 /* @funcstatic btreePageGetSizeSecbucket **************************************
34358 **
34359 ** Return the used size of a secondary bucket cache page
34360 **
34361 ** @param [r] page [const AjPBtpage] Page
34362 ** @return [ajuint] Used page size
34363 **
34364 ** @release 6.4.0
34365 ** @@
34366 ******************************************************************************/
34367 
btreePageGetSizeSecbucket(const AjPBtpage page)34368 static ajuint btreePageGetSizeSecbucket(const AjPBtpage page)
34369 {
34370     const unsigned char* buf = page->buf;
34371     ajuint nentries;
34372     ajulong   overflow = 0UL;
34373     const unsigned char *idptr = NULL;
34374     const unsigned char *keyptr = NULL;
34375     ajuint klen;
34376     ajuint i;
34377 
34378     buf = page->buf;
34379 
34380     GBT_BUCKNENTRIES(buf,&nentries);
34381     GBT_BUCKOVERFLOW(buf,&overflow);
34382 
34383     keyptr = PBT_BUCKKEYLEN(buf);
34384     idptr = keyptr + (nentries * sizeof(ajuint));
34385 
34386     for(i=0;i<nentries;i++)
34387     {
34388 	BT_GETAJUINT(keyptr,&klen);
34389 	idptr += klen;
34390 	keyptr += sizeof(ajuint);
34391     }
34392 
34393     return (idptr-buf);
34394 }
34395 
34396 
34397 
34398 
34399 /* @funcstatic btreeCheckNode *************************************************
34400 **
34401 ** Checks consistency of a node page
34402 **
34403 ** @param [w] cache [AjPBtcache] cache
34404 ** @param [r] page [const AjPBtpage] Page
34405 ** @return [AjBool] True on success
34406 **
34407 ** @release 6.4.0
34408 ** @@
34409 ******************************************************************************/
34410 
btreeCheckNode(AjPBtcache cache,const AjPBtpage page)34411 static AjBool btreeCheckNode(AjPBtcache cache, const AjPBtpage page)
34412 {
34413     ajuint i;
34414     ajuint m;
34415     ajuint klen;
34416     const AjPBtpage bpage;
34417     unsigned char *tbuf;
34418 
34419 /*    ajuint   pagesize = 0U;*/
34420     ajulong   overflow = 0UL;
34421     ajulong blockno;
34422     ajuint totlen;
34423     ajulong left;
34424     ajulong right;
34425     ajulong prev;
34426     unsigned char *lenptr = NULL;
34427     unsigned char *keyptr = NULL;
34428     ajulong lv;
34429     AjBool ret = ajTrue;
34430     AjPBtpage ncpage = NULL;
34431 
34432 /*    ajuint nodetype;*/
34433 /*    ajuint freebytes;*/
34434 
34435     if(!btreeCheckNodeHeader(cache, page, "node"))
34436         ret = ajFalse;
34437 
34438 /*    pagesize = cache->pagesize;*/
34439 
34440     bpage = page;
34441     tbuf = bpage->buf;
34442     GBT_TOTLEN(tbuf,&totlen);
34443 
34444     GBT_NKEYS(tbuf,&m);
34445 
34446     GBT_LEFT(tbuf,&left);
34447     GBT_RIGHT(tbuf,&right);
34448     GBT_BLOCKNUMBER(tbuf,&blockno);
34449     GBT_PREV(tbuf,&prev);
34450     GBT_OVERFLOW(tbuf,&overflow);
34451 
34452 
34453     lenptr =  PBT_KEYLEN(tbuf);
34454     keyptr = lenptr + m * sizeof(ajuint);
34455 
34456     AJNEW0(ncpage);
34457 
34458     for(i=0;i<m;i++)
34459     {
34460 	BT_GETAJUINT(lenptr,&klen);
34461 
34462 /*
34463 //	if((ajuint)((keyptr-tbuf+2) + klen + sizeof(ajulong)) > pagesize)
34464 //	{
34465 //            freebytes = pagesize - (keyptr-tbuf);
34466 //    	    /# ajDebug("btreeStatNode: Overflow\n"); #/
34467 //	    btreeNocacheFetch(cache, ncpage, overflow);
34468 //	    tbuf = bpage->buf;
34469 //	    GBT_NODETYPE(tbuf,&nodetype);
34470 //	    if(nodetype != BT_OVERFLOW)
34471 //		ajFatal("btreeCheckNode "
34472 //                        "Overflow node %Lu expected but not found "
34473 //                        "cache %S page %Lu "
34474 //                        "pagepos:%Lu key:%u/%u free:%u keyptr:%x lenptr:%x "
34475 //                        "tbuf:%x klen:%u old nodetype '%s' newnodetype '%s' "
34476 //                        "blockno:%Lu totlen:%u left:%Lu right:%Lu prev:%Lu",
34477 //                        overflow, cache->filename,
34478 //                        page->pagepos, page->pagepos/pagesize,
34479 //                        i, m, freebytes,
34480 //                        keyptr, lenptr, tbuf, klen,
34481 //                        btreeNodetype(page->buf), btreeNodetype(tbuf),
34482 //                        blockno,totlen, left, right, prev);
34483 //
34484 //            GBT_BLOCKNUMBER(tbuf,&blockno);
34485 //            GBT_TOTLEN(tbuf,&totlen);
34486 //            GBT_LEFT(tbuf,&left);
34487 //            GBT_RIGHT(tbuf,&right);
34488 //            GBT_PREV(tbuf,&prev);
34489 //
34490 //	    /#
34491 //	     ** The length pointer is restricted to the initial page.
34492 //	     ** The keyptr in overflow pages starts at the Key Lengths
34493 //	     ** position!
34494 //	     #/
34495 //            keyptr = PBT_KEYLEN(tbuf);
34496 //	}
34497 */
34498 
34499         keyptr += klen+1;
34500         BT_GETAJULONG(keyptr,&lv);
34501 	keyptr += sizeof(ajulong);
34502         lenptr += sizeof(ajuint);
34503     }
34504 
34505     BT_GETAJULONG(keyptr,&lv);
34506     AJFREE(ncpage);
34507 
34508     return ret;
34509 }
34510 
34511 
34512 
34513 
34514 /* @funcstatic btreeCheckNumnode **********************************************
34515 **
34516 ** Checks consistency of a numeric node page
34517 **
34518 ** @param [w] cache [AjPBtcache] cache
34519 ** @param [r] page [const AjPBtpage] Page
34520 ** @return [AjBool] True on success
34521 **
34522 ** @release 6.4.0
34523 ** @@
34524 ******************************************************************************/
34525 
btreeCheckNumnode(AjPBtcache cache,const AjPBtpage page)34526 static AjBool btreeCheckNumnode(AjPBtcache cache, const AjPBtpage page)
34527 {
34528     ajuint i;
34529     ajuint m;
34530     const AjPBtpage bpage;
34531     unsigned char *tbuf;
34532 
34533     ajulong   overflow = 0UL;
34534     ajulong blockno;
34535     ajuint totlen;
34536     ajulong left;
34537     ajulong right;
34538     ajulong prev;
34539     unsigned char *keyptr = NULL;
34540     unsigned char *valptr = NULL;
34541     ajulong lv;
34542     ajulong lk;
34543     ajulong lastk = 0UL;
34544 
34545     AjBool ok = ajTrue;
34546     AjBool ret = ajTrue;
34547 
34548     if(!btreeDoExtra)
34549         return ret;
34550 
34551     bpage = page;
34552     tbuf = bpage->buf;
34553     GBT_TOTLEN(tbuf,&totlen);
34554 
34555     GBT_NKEYS(tbuf,&m);
34556 
34557     GBT_LEFT(tbuf,&left);
34558     GBT_RIGHT(tbuf,&right);
34559     GBT_BLOCKNUMBER(tbuf,&blockno);
34560     GBT_PREV(tbuf,&prev);
34561     GBT_OVERFLOW(tbuf,&overflow);
34562 
34563     if(!btreeCheckNodeHeader(cache, page, "numnode"))
34564         ret = ajFalse;
34565 
34566     keyptr =  PBT_KEYLEN(tbuf);
34567     valptr = keyptr + m*sizeof(ajulong);
34568     for(i=0;i<m;i++)
34569     {
34570         BT_GETAJULONG(valptr,&lv);
34571         BT_GETAJULONG(keyptr,&lk);
34572 
34573         if(lk <= lastk)
34574         {
34575             ok = ajFalse;
34576             ajWarn("numnode:%Lu lk[%u] %12Lu <= %12Lu in %S",
34577                    page->pagepos, i, lk, lastk,
34578                    cache->filename);
34579         }
34580         lastk = lk;
34581 
34582 	keyptr += sizeof(ajulong);
34583 	valptr += sizeof(ajulong);
34584     }
34585 
34586     BT_GETAJULONG(valptr,&lv);
34587 
34588     if(!ok)
34589     {
34590         ret = ajFalse;
34591         btreeStatNumnode(cache, page);
34592     }
34593 
34594     return ret;
34595 }
34596 
34597 
34598 
34599 
34600 /* @funcstatic btreeCheckNodeHeader *******************************************
34601 **
34602 ** Checks consistency of any node header
34603 **
34604 ** @param [w] cache [AjPBtcache] cache
34605 ** @param [r] page [const AjPBtpage] Page
34606 ** @param [r] type [const char*] Node type
34607 ** @return [AjBool] True on success
34608 **
34609 ** @release 6.4.0
34610 ** @@
34611 ******************************************************************************/
34612 
btreeCheckNodeHeader(AjPBtcache cache,const AjPBtpage page,const char * type)34613 static AjBool btreeCheckNodeHeader(AjPBtcache cache, const AjPBtpage page,
34614                                    const char* type)
34615 {
34616     ajuint nkeys;
34617     unsigned char *tbuf;
34618 
34619     ajulong   overflow = 0UL;
34620     ajulong blockno;
34621     ajuint totlen;
34622     ajulong left;
34623     ajulong right;
34624     ajulong prev;
34625     const char* nodetype;
34626 
34627     AjBool ret = ajTrue;
34628 
34629     tbuf = page->buf;
34630     nodetype = btreeNodetype(tbuf);
34631 
34632     GBT_TOTLEN(tbuf,&totlen);
34633 
34634     GBT_NKEYS(tbuf,&nkeys);
34635 
34636     GBT_LEFT(tbuf,&left);
34637     GBT_RIGHT(tbuf,&right);
34638     GBT_BLOCKNUMBER(tbuf,&blockno);
34639     GBT_PREV(tbuf,&prev);
34640     GBT_OVERFLOW(tbuf,&overflow);
34641 
34642     if(!strcmp(type, "node"))
34643     {
34644         if(!strcmp(nodetype, "numbucket"))
34645         {
34646             ret = ajFalse;
34647             ajWarn("%s:%Lu expected non-numeric node in %S",
34648                    nodetype,
34649                    cache->filename);
34650         }
34651     }
34652     else if(!strcmp(type, "numnode"))
34653     {
34654         if(strcmp(nodetype, "numbucket"))
34655         {
34656             ret = ajFalse;
34657             ajWarn("%s:%Lu expected numeric node in %S",
34658                    nodetype,
34659                    cache->filename);
34660         }
34661     }
34662     else
34663     {
34664         ret = ajFalse;
34665         ajWarn("%s:%Lu expected '%s' in %S",
34666                nodetype, type,
34667                cache->filename);
34668     }
34669 
34670     if(ajBtreePageIsPrimary(page))
34671     {
34672         if(nkeys >= cache->porder)
34673         {
34674             ret = ajFalse;
34675             ajWarn("%s:%Lu nkeys: %u porder:%u in %S",
34676                    nodetype, page->pagepos, nkeys,
34677                    cache->porder,
34678                    cache->filename);
34679         }
34680     }
34681     else
34682     {
34683         if(nkeys > cache->sorder)
34684         {
34685             ret = ajFalse;
34686             ajWarn("%s:%Lu nkeys: %u sorder:%u in %S",
34687                    nodetype, page->pagepos, nkeys,
34688                    cache->sorder,
34689                    cache->filename);
34690         }
34691     }
34692     if((blockno != page->pagepos))
34693     {
34694         ret = ajFalse;
34695         ajWarn("%s:%Lu blockno: %Lu in %S",
34696                nodetype, page->pagepos, blockno,
34697                cache->filename);
34698     }
34699 
34700     return ret;
34701 }
34702 
34703 
34704 
34705 
34706 /* @funcstatic btreeNocacheFetch **********************************************
34707 **
34708 ** Fetch a cache page from disc
34709 **
34710 ** @param [r] cache [const AjPBtcache] cache
34711 ** @param [w] cpage [AjPBtpage] cache page
34712 ** @param [r] pagepos [ajulong] page number
34713 **
34714 ** @return [void]
34715 **
34716 ** @release 6.4.0
34717 ** @@
34718 ******************************************************************************/
34719 
btreeNocacheFetch(const AjPBtcache cache,AjPBtpage cpage,ajulong pagepos)34720 static void btreeNocacheFetch(const AjPBtcache cache, AjPBtpage cpage,
34721                               ajulong pagepos)
34722 {
34723     ajuint sum = 0;
34724     ajuint retries = 0;
34725     ajuint pagesize;
34726 
34727     /* ajDebug("In btreeNoCacheFetch\n"); */
34728 
34729     if(cache->secpagesize > cache->pripagesize)
34730         pagesize = cache->pripagesize;
34731     else
34732         pagesize = cache->secpagesize;
34733 
34734     if(fseek(cache->fp,pagepos,SEEK_SET))
34735 	ajFatal("Seek error '%s' in btreeNocacheFetch file %S",
34736                 strerror(ferror(cache->fp)), cache->filename);
34737 
34738     while(sum != pagesize && retries != BT_MAXRETRIES)
34739     {
34740 	sum += fread((void *)(cpage->buf+sum),1,pagesize-sum,
34741 		     cache->fp);
34742 	++retries;
34743     }
34744 
34745     if(retries == BT_MAXRETRIES)
34746 	ajFatal("Maximum retries (%u) reached in btreeNocacheFetch "
34747                 "for page %Lu",
34748 		BT_MAXRETRIES,pagepos);
34749 
34750     if(ajBtreePageIsPrimary(cpage))
34751     {
34752         retries = 0;
34753         while(sum < cache->pripagesize && retries != BT_MAXRETRIES)
34754         {
34755             sum += fread((void *)(cpage->buf+sum),1,cache->pripagesize-sum,
34756                          cache->fp);
34757             ++retries;
34758         }
34759         if(retries == BT_MAXRETRIES)
34760             ajFatal("Maximum retries (%u) reached in btreeNocacheFetch "
34761                     "for page %Lu",
34762                     BT_MAXRETRIES,pagepos);
34763     }
34764     else
34765     {
34766         retries = 0;
34767         while(sum < cache->secpagesize && retries != BT_MAXRETRIES)
34768         {
34769             sum += fread((void *)(cpage->buf+sum),1,cache->secpagesize-sum,
34770                          cache->fp);
34771             ++retries;
34772         }
34773         if(retries == BT_MAXRETRIES)
34774             ajFatal("Maximum retries (%u) reached in btreeNocacheFetch "
34775                     "for page %Lu",
34776                     BT_MAXRETRIES,pagepos);
34777     }
34778 
34779     cpage->pagepos = pagepos;
34780 
34781     return;
34782 }
34783 
34784 
34785 
34786 
34787 /* @funcstatic btreePripageSetfree ********************************************
34788 **
34789 ** Clear a primary cache page and set as free
34790 **
34791 ** @param [u] cache [AjPBtcache] cache
34792 ** @param [r] pagepos [ajulong] Pahge number
34793 **
34794 ** @return [void]
34795 **
34796 ** @release 6.5.0
34797 ** @@
34798 ******************************************************************************/
34799 
btreePripageSetfree(AjPBtcache cache,ajulong pagepos)34800 static void btreePripageSetfree(AjPBtcache cache, ajulong pagepos)
34801 {
34802     AjPBtpage page = NULL;
34803     unsigned char *p;
34804     ajuint nodetype;
34805 
34806     page = btreePricacheRead(cache,pagepos);
34807 
34808     if(!ajBtreePageIsPrimary(page))
34809         ajWarn("btreePripageSetfree secondary page %Lu '%s'",
34810                pagepos, btreeNodetype(page->buf));
34811 
34812     p = page->buf;
34813     AJCSET0(p, cache->pripagesize); /* clear buffer to zeros */
34814 
34815     page->next = NULL;
34816     page->prev = NULL;
34817 
34818     nodetype = BT_FREEPAGE;
34819     SBT_NODETYPE(p,nodetype);
34820 
34821     page->dirty = BT_DIRTY;
34822 
34823     return;
34824 }
34825 
34826 
34827 
34828 
34829 /* @funcstatic btreeSecpageSetfree *********************************************
34830 **
34831 ** Clear a secondary cache page and set as free
34832 **
34833 ** @param [u] cache [AjPBtcache] cache
34834 ** @param [r] pagepos [ajulong] Pahge number
34835 **
34836 ** @return [void]
34837 **
34838 ** @release 6.4.0
34839 ** @@
34840 ******************************************************************************/
34841 
btreeSecpageSetfree(AjPBtcache cache,ajulong pagepos)34842 static void btreeSecpageSetfree(AjPBtcache cache, ajulong pagepos)
34843 {
34844     AjPBtpage page = NULL;
34845     unsigned char *p;
34846     ajuint nodetype;
34847 
34848     page = btreeSeccacheRead(cache,pagepos);
34849 
34850     if(ajBtreePageIsPrimary(page))
34851         ajWarn("btreeSecpageSetfree primary page %Lu '%s'",
34852                pagepos, btreeNodetype(page->buf));
34853 
34854     p = page->buf;
34855     AJCSET0(p, cache->secpagesize); /* clear buffer to zeros */
34856 
34857     page->next = NULL;
34858     page->prev = NULL;
34859 
34860     nodetype = BT_SECFREEPAGE;
34861     SBT_NODETYPE(p,nodetype);
34862 
34863     page->dirty = BT_DIRTY;
34864 
34865     return;
34866 }
34867 
34868 
34869 
34870 
34871 /* @funcstatic btreePripageClear **********************************************
34872 **
34873 ** Clear a primary cache page
34874 **
34875 ** @param [u] cache [AjPBtcache] cache
34876 ** @param [u] page [AjPBtpage] Page
34877 **
34878 ** @return [void]
34879 **
34880 ** @release 6.4.0
34881 ** @@
34882 ******************************************************************************/
34883 
btreePripageClear(AjPBtcache cache,AjPBtpage page)34884 static void btreePripageClear(AjPBtcache cache, AjPBtpage page)
34885 {
34886     unsigned char *p;
34887     ajuint nodetype;
34888 
34889     p = page->buf;
34890 
34891     if(!ajBtreePageIsPrimary(page))
34892         ajWarn("btreePripageClear secondary page %Lu '%s'",
34893                page->pagepos, btreeNodetype(page->buf));
34894 
34895     AJCSET0(p, cache->pripagesize); /* clear buffer to zeros */
34896 
34897     nodetype     = BT_FREEPAGE;
34898     SBT_NODETYPE(p,nodetype);
34899 
34900     page->next = NULL;
34901     page->prev = NULL;
34902 
34903     page->dirty = BT_CLEAN;
34904 
34905     return;
34906 }
34907 
34908 
34909 
34910 
34911 /* @funcstatic btreeSecpageClear **********************************************
34912 **
34913 ** Clear a secondary cache page
34914 **
34915 ** @param [u] cache [AjPBtcache] cache
34916 ** @param [u] page [AjPBtpage] Page
34917 **
34918 ** @return [void]
34919 **
34920 ** @release 6.4.0
34921 ** @@
34922 ******************************************************************************/
34923 
btreeSecpageClear(AjPBtcache cache,AjPBtpage page)34924 static void btreeSecpageClear(AjPBtcache cache, AjPBtpage page)
34925 {
34926     unsigned char *p;
34927     ajuint nodetype;
34928 
34929     p = page->buf;
34930 
34931     if(ajBtreePageIsPrimary(page))
34932         ajWarn("btreeSecpageClear primary page %Lu '%s'",
34933                page->pagepos, btreeNodetype(page->buf));
34934 
34935     AJCSET0(p, cache->secpagesize); /* clear buffer to zeros */
34936 
34937     nodetype     = BT_SECFREEPAGE;
34938     SBT_NODETYPE(p,nodetype);
34939 
34940     page->next = NULL;
34941     page->prev = NULL;
34942 
34943     page->dirty = BT_CLEAN;
34944 
34945     return;
34946 }
34947 
34948 
34949 
34950 
34951 /* @func ajBtreeStatsOut ******************************************************
34952 **
34953 ** Reports overall statistics from B+tree indexing since the last call
34954 **
34955 ** @param [u] outf [AjPFile] output file file
34956 ** @param [u] Psplitrootid [ajulong*] Number of hyb splitroot calls to date
34957 ** @param [u] Psplitrootnum [ajulong*] Number of num splitroot calls to date
34958 ** @param [u] Psplitrootkey [ajulong*] Number of pri splitroot calls to date
34959 ** @param [u] Psplitrootsec [ajulong*] Number of sec splitroot calls to date
34960 ** @param [u] Psplitleafid  [ajulong*] Number of id splitleaf calls to date
34961 ** @param [u] Psplitleafnum [ajulong*] Number of num splitleaf calls to date
34962 ** @param [u] Psplitleafkey [ajulong*] Number of pri splitleaf calls to date
34963 ** @param [u] Psplitleafsec [ajulong*] Number of sec splitleaf calls to date
34964 ** @param [u] Preorderid  [ajulong*] Number of id reorderings to date
34965 ** @param [u] Preordernum [ajulong*] Number of num reorderings to date
34966 ** @param [u] Preorderkey [ajulong*] Number of pri reorderings to date
34967 ** @param [u] Preordersec [ajulong*] Number of sec reorderings to date
34968 ** @return [void]
34969 **
34970 ** @release 6.5.0
34971 ** @@
34972 ******************************************************************************/
34973 
ajBtreeStatsOut(AjPFile outf,ajulong * Psplitrootid,ajulong * Psplitrootnum,ajulong * Psplitrootkey,ajulong * Psplitrootsec,ajulong * Psplitleafid,ajulong * Psplitleafnum,ajulong * Psplitleafkey,ajulong * Psplitleafsec,ajulong * Preorderid,ajulong * Preordernum,ajulong * Preorderkey,ajulong * Preordersec)34974 void ajBtreeStatsOut(AjPFile outf,
34975                      ajulong* Psplitrootid,  ajulong* Psplitrootnum,
34976                      ajulong* Psplitrootkey, ajulong* Psplitrootsec,
34977                      ajulong* Psplitleafid,  ajulong* Psplitleafnum,
34978                      ajulong* Psplitleafkey, ajulong* Psplitleafsec,
34979                      ajulong* Preorderid,    ajulong* Preordernum,
34980                      ajulong* Preorderkey,   ajulong* Preordersec)
34981 {
34982     ajulong splitrootid;
34983     ajulong splitrootnum;
34984     ajulong splitrootkey;
34985     ajulong splitrootsec;
34986     ajulong splitleafid;
34987     ajulong splitleafnum;
34988     ajulong splitleafkey;
34989     ajulong splitleafsec;
34990     ajulong reorderid;
34991     ajulong reordernum;
34992     ajulong reorderkey;
34993     ajulong reordersec;
34994 
34995     splitrootid  = statCallIdSplitroot  - *Psplitrootid;
34996     splitrootnum = statCallNumSplitroot - *Psplitrootnum;
34997     splitrootkey = statCallPriSplitroot - *Psplitrootkey;
34998     splitrootsec = statCallSecSplitroot - *Psplitrootsec;
34999     splitleafid  = statCallIdSplitleaf  - *Psplitleafid;
35000     splitleafnum = statCallNumSplitleaf - *Psplitleafnum;
35001     splitleafkey = statCallKeySplitleaf - *Psplitleafkey;
35002     splitleafsec = statCallSecSplitleaf - *Psplitleafsec;
35003     reorderid  = statCallIdbucketsReorder - *Preorderid;
35004     reordernum = statCallNumbucketsReorder - *Preordernum;
35005     reorderkey = statCallPribucketsReorder - *Preorderkey;
35006     reordersec = statCallSecbucketsReorder - *Preordersec;
35007 
35008     ajFmtPrintF(outf,
35009                 "ajBtreeStatsOut splitroot id: %8Lu "
35010                 "num: %8Lu key: %8Lu sec: %8Lu\n",
35011                 splitrootid, splitrootnum,
35012                 splitrootkey, splitrootsec);
35013 
35014     ajFmtPrintF(outf,
35015                 "ajBtreeStatsOut splitleaf id: %8Lu "
35016                 "num: %8Lu key: %8Lu sec: %8Lu\n",
35017                 splitleafid, splitleafnum,
35018                 splitleafkey, splitleafsec);
35019 
35020     ajFmtPrintF(outf,
35021                 "ajBtreeStatsOut reorder   id: %8Lu "
35022                 "num: %8Lu key: %8Lu sec: %8Lu\n",
35023                 reorderid, reordernum,
35024                 reorderkey, reordersec);
35025 
35026     *Psplitrootid  = statCallIdSplitroot;
35027     *Psplitrootnum = statCallNumSplitroot;
35028     *Psplitrootkey = statCallPriSplitroot;
35029     *Psplitrootsec = statCallSecSplitroot;
35030 
35031     *Psplitleafid  = statCallIdSplitleaf;
35032     *Psplitleafnum = statCallNumSplitleaf;
35033     *Psplitleafkey = statCallKeySplitleaf;
35034     *Psplitleafsec = statCallSecSplitleaf;
35035 
35036     *Preorderid    = statCallIdbucketsReorder;
35037     *Preordernum   = statCallNumbucketsReorder;
35038     *Preorderkey   = statCallPribucketsReorder;
35039     *Preordersec   = statCallSecbucketsReorder;
35040 
35041     return;
35042 }
35043 
35044 
35045 
35046 
35047 /* @func ajBtreeExit **********************************************************
35048 **
35049 ** Cleans up B+tree indexing internal memory
35050 **
35051 ** @return [void]
35052 **
35053 ** @release 6.4.0
35054 ** @@
35055 ******************************************************************************/
35056 
ajBtreeExit(void)35057 void ajBtreeExit(void)
35058 {
35059     void *tmpfree = NULL;
35060 
35061     ajDebug("ajBtreeExit\n");
35062 
35063     ajDebug("sync calls: %Lu Lock:%Lu Write: %Lu\n",
35064             statCallSync, statSyncLocked, statSyncWrite);
35065 
35066     ajDebug("Rootsync calls: %Lu Lock:%Lu (max %u) Unlock: %Lu (max %u)\n",
35067             statCallRootSync, statRootSyncLocked, statRootSyncMaxLocked,
35068             statRootSyncUnlocked, statRootSyncMaxUnlocked);
35069 
35070     ajDebug("   statSaveIdbucketNext: %u\n", statSaveIdbucketNext);
35071     while(statSaveIdbucketNext)
35072         btreeIdbucketFree(&statSaveIdbucket[--statSaveIdbucketNext]);
35073     AJFREE(statSaveIdbucket);
35074     statSaveIdbucketMax=0;
35075     statSaveIdbucketNext=0;
35076 
35077     ajDebug("   statSaveIdbucketEmptyNext: %u\n", statSaveIdbucketEmptyNext);
35078     while(statSaveIdbucketEmptyNext)
35079         btreeIdbucketFree(&statSaveIdbucketEmpty[--statSaveIdbucketEmptyNext]);
35080     AJFREE(statSaveIdbucketEmpty);
35081     statSaveIdbucketEmptyMax=0;
35082     statSaveIdbucketEmptyNext=0;
35083 
35084     ajDebug("   statSavePribucketNext: %u\n", statSavePribucketNext);
35085     while(statSavePribucketNext)
35086         btreePribucketFree(&statSavePribucket[--statSavePribucketNext]);
35087     AJFREE(statSavePribucket);
35088     statSavePribucketMax=0;
35089     statSavePribucketNext=0;
35090 
35091     ajDebug("   statSavePribucketEmptyNext: %u\n", statSavePribucketEmptyNext);
35092     while(statSavePribucketEmptyNext)
35093         btreePribucketFree(&statSavePribucketEmpty[--statSavePribucketEmptyNext]);
35094     AJFREE(statSavePribucketEmpty);
35095     statSavePribucketEmptyMax=0;
35096     statSavePribucketEmptyNext=0;
35097 
35098     ajDebug("   statSaveSecbucketNext: %u\n", statSaveSecbucketNext);
35099     while(statSaveSecbucketNext)
35100         btreeSecbucketFree(&statSaveSecbucket[--statSaveSecbucketNext]);
35101     AJFREE(statSaveSecbucket);
35102     statSaveSecbucketMax=0;
35103     statSaveSecbucketNext=0;
35104 
35105     ajDebug("   statSaveSecbucketEmptyNext: %u\n", statSaveSecbucketEmptyNext);
35106     while(statSaveSecbucketEmptyNext)
35107         btreeSecbucketFree(&statSaveSecbucketEmpty[--statSaveSecbucketEmptyNext]);
35108     AJFREE(statSaveSecbucketEmpty);
35109     statSaveSecbucketEmptyMax=0;
35110     statSaveSecbucketEmptyNext=0;
35111 
35112     ajDebug("   statSaveNumbucketNext: %u\n", statSaveNumbucketNext);
35113     while(statSaveNumbucketNext)
35114         btreeNumbucketFree(&statSaveNumbucket[--statSaveNumbucketNext]);
35115     AJFREE(statSaveNumbucket);
35116     statSaveNumbucketMax=0;
35117     statSaveNumbucketNext=0;
35118 
35119     ajDebug("   statSaveSecIdNext: %u\n", statSaveSecIdNext);
35120     while(statSaveSecIdNext)
35121         ajStrDel(&statSaveSecId[--statSaveSecIdNext]);
35122     AJFREE(statSaveSecId);
35123     statSaveSecIdMax=0;
35124     statSaveSecIdNext=0;
35125 
35126     ajDebug("   statSaveBtreeIdNext: %u\n", statSaveBtreeIdNext);
35127     while(statSaveBtreeIdNext)
35128         btreeIdFree(&statSaveBtreeId[--statSaveBtreeIdNext]);
35129     AJFREE(statSaveBtreeId);
35130     statSaveBtreeIdMax=0;
35131     statSaveBtreeIdNext=0;
35132 
35133     ajDebug("   statSaveBtreeHitrefNext: %u\n", statSaveBtreeHitrefNext);
35134     while(statSaveBtreeHitrefNext)
35135         btreeHitrefFree(&statSaveBtreeHitref[--statSaveBtreeHitrefNext]);
35136     AJFREE(statSaveBtreeHitref);
35137     statSaveBtreeHitrefMax=0;
35138     statSaveBtreeHitrefNext=0;
35139 
35140      ajDebug("   statSaveBtreeHitNext: %u\n", statSaveBtreeHitNext);
35141     while(statSaveBtreeHitNext)
35142         btreeHitFree(&statSaveBtreeHit[--statSaveBtreeHitNext]);
35143     AJFREE(statSaveBtreeHit);
35144     statSaveBtreeHitMax=0;
35145     statSaveBtreeHitNext=0;
35146 
35147    ajDebug("   statSaveBtreePriNext: %u\n", statSaveBtreePriNext);
35148     while(statSaveBtreePriNext)
35149         btreePriFree(&statSaveBtreePri[--statSaveBtreePriNext]);
35150     AJFREE(statSaveBtreePri);
35151     statSaveBtreePriMax=0;
35152     statSaveBtreePriNext=0;
35153 
35154     ajDebug("Primary Arrays: New:%Lu Reuse:%Lu Delete:%Lu Free:%Lu\n",
35155                 statCountAllocPriArrayNew, statCountAllocPriArrayReuse,
35156                 statCountAllocPriArrayDel, statCountAllocPriArrayFree);
35157 
35158     ajDebug("Secondary Arrays: New:%Lu Reuse:%Lu Delete:%Lu Free:%Lu\n",
35159                 statCountAllocSecArrayNew, statCountAllocSecArrayReuse,
35160                 statCountAllocSecArrayDel, statCountAllocSecArrayFree);
35161 
35162     ajDebug("Split root:%Lu Sec:%Lu Hyb:%Lu Num:%Lu\n",
35163             statCallPriSplitroot,statCallSecSplitroot,
35164             statCallIdSplitroot,statCallNumSplitroot);
35165 
35166     ajDebug("Split leaf: Pri:%Lu Sec:%Lu Id:%Lu Num:%Lu\n",
35167             statCallKeySplitleaf,statCallSecSplitleaf,
35168             statCallIdSplitleaf,statCallNumSplitleaf);
35169 
35170     ajDebug("Split reorder:%Lu Pri:%Lu Sec:%Lu  Id:%Lu Num:%Lu\n",
35171             statCallIdbucketsReorder,
35172             statCallPribucketsReorder,statCallSecbucketsReorder,
35173             statCallIdbucketsReorder,statCallNumbucketsReorder);
35174 
35175     ajStrDel(&btreeFieldnameTmp);
35176     ajTableMapDel(btreeFieldsTable, &btreeFieldMapDel, NULL);
35177     ajTableFree(&btreeFieldsTable);
35178 
35179     tmpfree = (void *) btreeNodetypeNames;
35180     AJFREE(tmpfree);
35181     btreeNodetypeNames = NULL;
35182 
35183     if(btreeTestpage)
35184     {
35185         AJFREE(btreeTestpage->buf);
35186         AJFREE(btreeTestpage);
35187     }
35188 
35189     ajStrDel(&indexKeyword);
35190     ajStrDel(&indexId);
35191 
35192     return;
35193 }
35194 
35195 
35196 
35197 
35198 /* @func ajBtreeIdCmp *********************************************************
35199 **
35200 ** Comparison function for sorting B+tree ID objects by database file number
35201 ** and offset.
35202 **
35203 ** Using the file and position allows unique identifiers to be
35204 ** compared with other primary keys such bas accession number.
35205 **
35206 ** @param [r] x [const void*] Standard argument. Item value.
35207 ** @param [r] y [const void*] Standard argument. Comparison item value.
35208 ** @return [ajint] Comparison result. Zero if equal, non-zero if different.
35209 **
35210 ** @release 6.4.0
35211 ** @@
35212 ******************************************************************************/
35213 
ajBtreeIdCmp(const void * x,const void * y)35214 ajint ajBtreeIdCmp(const void* x, const void* y)
35215 {
35216     const AjPBtId idx;
35217     const AjPBtId idy;
35218 
35219     idx = (const AjPBtId) x;
35220     idy = (const AjPBtId) y;
35221 
35222     if(idx->dbno > idy->dbno)
35223         return 1;
35224 
35225     if(idx->dbno < idy->dbno)
35226         return -1;
35227 
35228     if(idx->offset > idy->offset)
35229         return 1;
35230 
35231     if(idx->offset < idy->offset)
35232         return -1;
35233 
35234     return 0;
35235 }
35236 
35237 
35238 
35239 
35240 /* @func ajBtreeIdHash ********************************************************
35241 **
35242 ** Hash function for a table with a B+tree primary key
35243 **
35244 ** @param [r] key [const void*] Standard argument. Table key.
35245 ** @param [r] hashsize [ajulong] Standard argument. Estimated Hash size.
35246 ** @return [ajulong] Hash value in range 0 to hashsize-1
35247 **
35248 ** @release 6.4.0
35249 ** @@
35250 ******************************************************************************/
35251 
ajBtreeIdHash(const void * key,ajulong hashsize)35252 ajulong ajBtreeIdHash(const void* key, ajulong hashsize)
35253 {
35254     const AjPBtId id;
35255     ajulong hash;
35256     ajulong ia;
35257 
35258     id = (const AjPBtId) key;
35259 
35260      if(!key)
35261         return 0;
35262 
35263     if(!hashsize)
35264         return 0;
35265 
35266     ia = id->offset + (ajulong) id->dbno;
35267 
35268     hash = (ia >> 2) % hashsize;
35269 
35270     return hash;
35271 }
35272 
35273 
35274 
35275 
35276 /* @func ajBtreeHitCmp ********************************************************
35277 **
35278 ** Comparison function for sorting B+tree hit objects by database file number
35279 ** and offset.
35280 **
35281 ** Using the file and position allows unique identifiers to be
35282 ** compared with other primary keys such bas accession number.
35283 **
35284 ** @param [r] x [const void*] Standard argument. Item value.
35285 ** @param [r] y [const void*] Standard argument. Comparison item value.
35286 ** @return [ajint] Comparison result. Zero if equal, non-zero if different.
35287 **
35288 ** @release 6.5.0
35289 ** @@
35290 ******************************************************************************/
35291 
ajBtreeHitCmp(const void * x,const void * y)35292 ajint ajBtreeHitCmp(const void* x, const void* y)
35293 {
35294     const AjPBtHit hitx;
35295     const AjPBtHit hity;
35296 
35297     hitx = (const AjPBtHit) x;
35298     hity = (const AjPBtHit) y;
35299 
35300     if(hitx->dbno > hity->dbno)
35301         return 1;
35302 
35303     if(hitx->dbno < hity->dbno)
35304         return -1;
35305 
35306     if(hitx->offset > hity->offset)
35307         return 1;
35308 
35309     if(hitx->offset < hity->offset)
35310         return -1;
35311 
35312     return 0;
35313 }
35314 
35315 
35316 
35317 
35318 /* @func ajBtreeHitHash *******************************************************
35319 **
35320 ** Hash function for a table with a B+tree hit primary key
35321 **
35322 ** @param [r] key [const void*] Standard argument. Table key.
35323 ** @param [r] hashsize [ajulong] Standard argument. Estimated Hash size.
35324 ** @return [ajulong] Hash value in range 0 to hashsize-1
35325 **
35326 ** @release 6.5.0
35327 ** @@
35328 ******************************************************************************/
35329 
ajBtreeHitHash(const void * key,ajulong hashsize)35330 ajulong ajBtreeHitHash(const void* key, ajulong hashsize)
35331 {
35332     const AjPBtHit hit;
35333     ajulong hash;
35334     ajulong ia;
35335 
35336     hit = (const AjPBtHit) key;
35337 
35338      if(!key)
35339         return 0;
35340 
35341     if(!hashsize)
35342         return 0;
35343 
35344     ia = hit->offset + (ajulong) hit->dbno;
35345 
35346     hash = (ia >> 2) % hashsize;
35347 
35348     return hash;
35349 }
35350 
35351 
35352 
35353 
35354 /* @func ajBtreeHitrefCmp *****************************************************
35355 **
35356 ** Comparison function for sorting B+tree reference hit objects by
35357 ** database file number and offset.
35358 **
35359 ** Using the file and position allows unique identifiers to be
35360 ** compared with other primary keys such bas accession number.
35361 **
35362 ** @param [r] x [const void*] Standard argument. Item value.
35363 ** @param [r] y [const void*] Standard argument. Comparison item value.
35364 ** @return [ajint] Comparison result. Zero if equal, non-zero if different.
35365 **
35366 ** @release 6.5.0
35367 ** @@
35368 ******************************************************************************/
35369 
ajBtreeHitrefCmp(const void * x,const void * y)35370 ajint ajBtreeHitrefCmp(const void* x, const void* y)
35371 {
35372     const AjPBtHitref hitrefx;
35373     const AjPBtHitref hitrefy;
35374 
35375     hitrefx = (const AjPBtHitref) x;
35376     hitrefy = (const AjPBtHitref) y;
35377 
35378     if(hitrefx->dbno > hitrefy->dbno)
35379         return 1;
35380 
35381     if(hitrefx->dbno < hitrefy->dbno)
35382         return -1;
35383 
35384     if(hitrefx->offset > hitrefy->offset)
35385         return 1;
35386 
35387     if(hitrefx->offset < hitrefy->offset)
35388         return -1;
35389 
35390     return 0;
35391 }
35392 
35393 
35394 
35395 
35396 /* @func ajBtreeHitrefHash ****************************************************
35397 **
35398 ** Hash function for a table with a B+tree reference hit primary key
35399 **
35400 ** @param [r] key [const void*] Standard argument. Table key.
35401 ** @param [r] hashsize [ajulong] Standard argument. Estimated Hash size.
35402 ** @return [ajulong] Hash value in range 0 to hashsize-1
35403 **
35404 ** @release 6.5.0
35405 ** @@
35406 ******************************************************************************/
35407 
ajBtreeHitrefHash(const void * key,ajulong hashsize)35408 ajulong ajBtreeHitrefHash(const void* key, ajulong hashsize)
35409 {
35410     const AjPBtHitref hitref;
35411     ajulong hash;
35412     ajulong ia;
35413 
35414     hitref = (const AjPBtHitref) key;
35415 
35416      if(!key)
35417         return 0;
35418 
35419     if(!hashsize)
35420         return 0;
35421 
35422     ia = hitref->offset + (ajulong) hitref->dbno;
35423 
35424     hash = (ia >> 2) % hashsize;
35425 
35426     return hash;
35427 }
35428 
35429 
35430 
35431 
35432 #if AJINDEX_STATIC
35433 /* @funcstatic btreePageDump **************************************************
35434 **
35435 ** Dump a page
35436 **
35437 ** @param [r] cache [const AjPBtcache] Page
35438 ** @param [r] page [const AjPBtpage] Page
35439 ** @return [void]
35440 **
35441 ** @release 6.5.0
35442 ** @@
35443 ******************************************************************************/
35444 
btreePageDump(const AjPBtcache cache,const AjPBtpage page)35445 static void btreePageDump(const AjPBtcache cache, const AjPBtpage page)
35446 {
35447     unsigned char *buf = page->buf;
35448     AjBool isprimary;
35449     ajuint i=0;
35450     ajuint pagesize;
35451 
35452     isprimary =  ajBtreePageIsPrimary(page);
35453     if(isprimary)
35454         pagesize = cache->pripagesize;
35455     else
35456         pagesize = cache->secpagesize;
35457 
35458     ajUser("PageDump page %Lu '%s' primary: %B",
35459            page->pagepos, btreeNodetype(buf), isprimary);
35460 
35461     i = 0;
35462     while(i < pagesize)
35463     {
35464         ajUser("%4x %2x%2x%2x%2x %2x%2x%2x%2x %2x%2x%2x%2x %2x%2x%2x%2x",
35465                i,
35466                buf[i],buf[i+1],buf[i+2],buf[i+3],
35467                buf[i+4],buf[i+5],buf[i+6],buf[i+7],
35468                buf[i+8],buf[i+9],buf[i+10],buf[i+11],
35469                buf[i+12],buf[i+13],buf[i+14],buf[i+15] );
35470 
35471         i += 16;
35472     }
35473 
35474     ajUser("pagesize: %u", pagesize);
35475 
35476     return;
35477 }
35478 #endif
35479 
35480 
35481 
35482 #ifdef AJ_COMPILE_DEPRECATED_BOOK
35483 #endif
35484 
35485 
35486 
35487 
35488 #ifdef AJ_COMPILE_DEPRECATED
35489 /* @obsolete ajBtreeWriteParams
35490 ** @rename ajBtreeWriteParamsC
35491 */
ajBtreeWriteParams(const AjPBtcache cache,const char * fn,const char * ext,const char * idir)35492 __deprecated void ajBtreeWriteParams(const AjPBtcache cache, const char *fn,
35493 			const char *ext, const char *idir)
35494 {
35495     ajBtreeWriteParamsC(cache, fn, ext, idir);
35496     return;
35497 }
35498 #endif
35499