1 /*************************************************************************************************
2  * The table database API of Tokyo Cabinet
3  *                                                               Copyright (C) 2006-2012 FAL Labs
4  * This file is part of Tokyo Cabinet.
5  * Tokyo Cabinet is free software; you can redistribute it and/or modify it under the terms of
6  * the GNU Lesser General Public License as published by the Free Software Foundation; either
7  * version 2.1 of the License or any later version.  Tokyo Cabinet is distributed in the hope
8  * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
10  * License for more details.
11  * You should have received a copy of the GNU Lesser General Public License along with Tokyo
12  * Cabinet; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
13  * Boston, MA 02111-1307 USA.
14  *************************************************************************************************/
15 
16 
17 #include "tcutil.h"
18 #include "tchdb.h"
19 #include "tcbdb.h"
20 #include "tctdb.h"
21 #include "myconf.h"
22 
23 #define TDBOPAQUESIZ   64                // size of using opaque field
24 #define TDBLEFTOPQSIZ  64                // size of left opaque field
25 #define TDBPAGEBUFSIZ  32768             // size of a buffer to read each page
26 
27 #define TDBDEFBNUM     131071            // default bucket number
28 #define TDBDEFAPOW     4                 // default alignment power
29 #define TDBDEFFPOW     10                // default free block pool power
30 #define TDBDEFLCNUM    4096              // default number of leaf cache
31 #define TDBDEFNCNUM    512               // default number of node cache
32 #define TDBDEFXMSIZ    (64LL<<20)        // default size of the extra mapped memory
33 #define TDBIDXSUFFIX   "idx"             // suffix of column index file
34 #define TDBIDXLMEMB    64                // number of members in each leaf of the index
35 #define TDBIDXNMEMB    256               // number of members in each node of the index
36 #define TDBIDXLSMAX    4096              // maximum size of each leaf of the index
37 #define TDBIDXICCBNUM  262139            // bucket number of the index cache
38 #define TDBIDXICCMAX   (64LL<<20)        // maximum size of the index cache
39 #define TDBIDXICCSYNC  0.01              // ratio of cache synchronization
40 #define TDBIDXQGUNIT   3                 // unit number of the q-gram index
41 #define TDBFTSUNITMAX  32                // maximum number of full-text search units
42 #define TDBFTSOCRUNIT  8192              // maximum number of full-text search units
43 #define TDBFTSBMNUM    524287            // number of elements of full-text search bitmap
44 #define TDBNUMCNTCOL   "_num"            // column name of number counting
45 #define TDBCOLBUFSIZ   1024              // size of a buffer for a column value
46 #define TDBNUMCOLMAX   16                // maximum number of columns of the long double
47 #define TDBHINTUSIZ    256               // unit size of the hint string
48 #define TDBORDRATIO    0.2               // ratio of records to use the order index
49 
50 enum {                                   // enumeration for duplication behavior
51   TDBPDOVER,                             // overwrite an existing value
52   TDBPDKEEP,                             // keep the existing value
53   TDBPDCAT                               // concatenate values
54 };
55 
56 typedef struct {                         // type of structure for a sort record
57   const char *kbuf;                      // pointer to the primary key
58   int ksiz;                              // size of the primary key
59   char *vbuf;                            // pointer to the value
60   int vsiz;                              // size of the value
61 } TDBSORTREC;
62 
63 typedef struct {                         // type of structure for a full-text search unit
64   TCLIST *tokens;                        // q-gram tokens
65   bool sign;                             // positive sign
66 } TDBFTSUNIT;
67 
68 typedef struct {                         // type of structure for a full-text string occurrence
69   const char *pkbuf;                     // primary key string
70   int32_t pksiz;                         // size of the primary key
71   int32_t off;                           // offset of the token
72   uint16_t seq;                          // sequence number
73   uint16_t hash;                         // hash value for counting sort
74 } TDBFTSSTROCR;
75 
76 typedef struct {                         // type of structure for a full-text number occurrence
77   int64_t pkid;                          // primery key number
78   int32_t off;                           // offset of the token
79   uint16_t seq;                          // sequence number
80   uint16_t hash;                         // hash value for counting sort
81 } TDBFTSNUMOCR;
82 
83 
84 /* private macros */
85 #define TDBLOCKMETHOD(TC_tdb, TC_wr)                            \
86   ((TC_tdb)->mmtx ? tctdblockmethod((TC_tdb), (TC_wr)) : true)
87 #define TDBUNLOCKMETHOD(TC_tdb)                         \
88   ((TC_tdb)->mmtx ? tctdbunlockmethod(TC_tdb) : true)
89 #define TDBTHREADYIELD(TC_tdb)                          \
90   do { if((TC_tdb)->mmtx) sched_yield(); } while(false)
91 
92 
93 /* private function prototypes */
94 static void tctdbclear(TCTDB *tdb);
95 static bool tctdbopenimpl(TCTDB *tdb, const char *path, int omode);
96 static bool tctdbcloseimpl(TCTDB *tdb);
97 static bool tctdbputimpl(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols, int dmode);
98 static bool tctdboutimpl(TCTDB *tdb, const char *pkbuf, int pksiz);
99 static TCMAP *tctdbgetimpl(TCTDB *tdb, const void *pkbuf, int pksiz);
100 static char *tctdbgetonecol(TCTDB *tdb, const void *pkbuf, int pksiz,
101                             const void *nbuf, int nsiz, int *sp);
102 static double tctdbaddnumber(TCTDB *tdb, const void *pkbuf, int pksiz, double num);
103 static bool tctdboptimizeimpl(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts);
104 static bool tctdbvanishimpl(TCTDB *tdb);
105 static bool tctdbcopyimpl(TCTDB *tdb, const char *path);
106 static bool tctdbtranbeginimpl(TCTDB *tdb);
107 static bool tctdbtrancommitimpl(TCTDB *tdb);
108 static bool tctdbtranabortimpl(TCTDB *tdb);
109 static bool tctdbsetindeximpl(TCTDB *tdb, const char *name, int type);
110 static int64_t tctdbgenuidimpl(TCTDB *tdb, int64_t inc);
111 static TCLIST *tctdbqrysearchimpl(TDBQRY *qry);
112 static TCMAP *tctdbqryidxfetch(TDBQRY *qry, TDBCOND *cond, TDBIDX *idx);
113 static long double tctdbatof(const char *str);
114 static bool tctdbqryidxcurjumpnum(BDBCUR *cur, const char *kbuf, int ksiz, bool first);
115 static bool tctdbqryonecondmatch(TDBQRY *qry, TDBCOND *cond, const char *pkbuf, int pksiz);
116 static bool tctdbqryallcondmatch(TDBQRY *qry, const char *pkbuf, int pksiz);
117 static bool tctdbqrycondmatch(TDBCOND *cond, const char *vbuf, int vsiz);
118 static bool tctdbqrycondcheckstrand(const char *tval, const char *oval);
119 static bool tctdbqrycondcheckstror(const char *tval, const char *oval);
120 static bool tctdbqrycondcheckstroreq(const char *vbuf, const char *expr);
121 static bool tctdbqrycondchecknumbt(const char *vbuf, const char *expr);
122 static bool tctdbqrycondchecknumoreq(const char *vbuf, const char *expr);
123 static bool tctdbqrycondcheckfts(const char *vbuf, int vsiz, TDBCOND *cond);
124 static int tdbcmppkeynumasc(const TCLISTDATUM *a, const TCLISTDATUM *b);
125 static int tdbcmppkeynumdesc(const TCLISTDATUM *a, const TCLISTDATUM *b);
126 static int tdbcmpsortrecstrasc(const TDBSORTREC *a, const TDBSORTREC *b);
127 static int tdbcmpsortrecstrdesc(const TDBSORTREC *a, const TDBSORTREC *b);
128 static int tdbcmpsortrecnumasc(const TDBSORTREC *a, const TDBSORTREC *b);
129 static int tdbcmpsortrecnumdesc(const TDBSORTREC *a, const TDBSORTREC *b);
130 static uint16_t tctdbidxhash(const char *pkbuf, int pksiz);
131 static bool tctdbidxput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols);
132 static bool tctdbidxputone(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz, uint16_t hash,
133                            const char *vbuf, int vsiz);
134 static bool tctdbidxputtoken(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
135                              const char *vbuf, int vsiz);
136 static bool tctdbidxputqgram(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
137                              const char *vbuf, int vsiz);
138 static bool tctdbidxout(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols);
139 static bool tctdbidxoutone(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz, uint16_t hash,
140                            const char *vbuf, int vsiz);
141 static bool tctdbidxouttoken(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
142                              const char *vbuf, int vsiz);
143 static bool tctdbidxoutqgram(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
144                              const char *vbuf, int vsiz);
145 static bool tctdbidxsyncicc(TCTDB *tdb, TDBIDX *idx, bool all);
146 static int tctdbidxcmpkey(const char **a, const char **b);
147 static TCMAP *tctdbidxgetbytokens(TCTDB *tdb, TDBIDX *idx, const TCLIST *tokens, int op,
148                                   TCXSTR *hint);
149 static TCMAP *tctdbidxgetbyfts(TCTDB *tdb, TDBIDX *idx, TDBCOND *cond, TCXSTR *hint);
150 static void tctdbidxgetbyftsunion(TDBIDX *idx, const TCLIST *tokens, bool sign,
151                                   TCMAP *ores, TCMAP *nres, TCXSTR *hint);
152 static int tctdbidxftscmpstrocr(TDBFTSSTROCR *a, TDBFTSSTROCR *b);
153 static int tctdbidxftscmpnumocr(TDBFTSNUMOCR *a, TDBFTSNUMOCR *b);
154 static TDBFTSUNIT *tctdbftsparseexpr(const char *expr, int esiz, int op, int *np);
155 static bool tctdbdefragimpl(TCTDB *tdb, int64_t step);
156 static bool tctdbcacheclearimpl(TCTDB *tdb);
157 static bool tctdbforeachimpl(TCTDB *tdb, TCITER iter, void *op);
158 static int tctdbqryprocoutcb(const void *pkbuf, int pksiz, TCMAP *cols, void *op);
159 static bool tctdblockmethod(TCTDB *tdb, bool wr);
160 static bool tctdbunlockmethod(TCTDB *tdb);
161 
162 
163 /* debugging function prototypes */
164 void tctdbprintmeta(TCTDB *tdb);
165 
166 
167 
168 /*************************************************************************************************
169  * API
170  *************************************************************************************************/
171 
172 
173 /* Get the message string corresponding to an error code. */
tctdberrmsg(int ecode)174 const char *tctdberrmsg(int ecode){
175   return tcerrmsg(ecode);
176 }
177 
178 
179 /* Create a table database object. */
tctdbnew(void)180 TCTDB *tctdbnew(void){
181   TCTDB *tdb;
182   TCMALLOC(tdb, sizeof(*tdb));
183   tctdbclear(tdb);
184   tdb->hdb = tchdbnew();
185   tchdbtune(tdb->hdb, TDBDEFBNUM, TDBDEFAPOW, TDBDEFFPOW, 0);
186   tchdbsetxmsiz(tdb->hdb, TDBDEFXMSIZ);
187   return tdb;
188 }
189 
190 
191 /* Delete a table database object. */
tctdbdel(TCTDB * tdb)192 void tctdbdel(TCTDB *tdb){
193   assert(tdb);
194   if(tdb->open) tctdbclose(tdb);
195   tchdbdel(tdb->hdb);
196   if(tdb->mmtx){
197     pthread_rwlock_destroy(tdb->mmtx);
198     TCFREE(tdb->mmtx);
199   }
200   TCFREE(tdb);
201 }
202 
203 
204 /* Get the last happened error code of a table database object. */
tctdbecode(TCTDB * tdb)205 int tctdbecode(TCTDB *tdb){
206   assert(tdb);
207   return tchdbecode(tdb->hdb);
208 }
209 
210 
211 /* Set mutual exclusion control of a table database object for threading. */
tctdbsetmutex(TCTDB * tdb)212 bool tctdbsetmutex(TCTDB *tdb){
213   assert(tdb);
214   if(!TCUSEPTHREAD) return true;
215   if(tdb->mmtx || tdb->open){
216     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
217     return false;
218   }
219   TCMALLOC(tdb->mmtx, sizeof(pthread_rwlock_t));
220   bool err = false;
221   if(pthread_rwlock_init(tdb->mmtx, NULL) != 0) err = true;
222   if(err){
223     TCFREE(tdb->mmtx);
224     tdb->mmtx = NULL;
225     return false;
226   }
227   return tchdbsetmutex(tdb->hdb);
228 }
229 
230 
231 /* Set the tuning parameters of a table database object. */
tctdbtune(TCTDB * tdb,int64_t bnum,int8_t apow,int8_t fpow,uint8_t opts)232 bool tctdbtune(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
233   assert(tdb);
234   if(tdb->open){
235     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
236     return false;
237   }
238   tdb->opts = opts;
239   uint8_t hopts = 0;
240   if(opts & TDBTLARGE) hopts |= HDBTLARGE;
241   if(opts & TDBTDEFLATE) hopts |= HDBTDEFLATE;
242   if(opts & TDBTBZIP) hopts |= HDBTBZIP;
243   if(opts & TDBTTCBS) hopts |= HDBTTCBS;
244   if(opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
245   bnum = (bnum > 0) ? bnum : TDBDEFBNUM;
246   apow = (apow >= 0) ? apow : TDBDEFAPOW;
247   fpow = (fpow >= 0) ? fpow : TDBDEFFPOW;
248   return tchdbtune(tdb->hdb, bnum, apow, fpow, hopts);
249 }
250 
251 
252 /* Set the caching parameters of a table database object. */
tctdbsetcache(TCTDB * tdb,int32_t rcnum,int32_t lcnum,int32_t ncnum)253 bool tctdbsetcache(TCTDB *tdb, int32_t rcnum, int32_t lcnum, int32_t ncnum){
254   assert(tdb);
255   if(tdb->open){
256     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
257     return false;
258   }
259   if(lcnum > 0) tdb->lcnum = lcnum;
260   if(ncnum > 0) tdb->ncnum = ncnum;
261   return tchdbsetcache(tdb->hdb, rcnum);
262 }
263 
264 
265 /* Set the size of the extra mapped memory of a table database object. */
tctdbsetxmsiz(TCTDB * tdb,int64_t xmsiz)266 bool tctdbsetxmsiz(TCTDB *tdb, int64_t xmsiz){
267   assert(tdb);
268   if(tdb->open){
269     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
270     return false;
271   }
272   return tchdbsetxmsiz(tdb->hdb, xmsiz);
273 }
274 
275 
276 /* Set the unit step number of auto defragmentation of a table database object. */
tctdbsetdfunit(TCTDB * tdb,int32_t dfunit)277 bool tctdbsetdfunit(TCTDB *tdb, int32_t dfunit){
278   assert(tdb);
279   if(tdb->open){
280     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
281     return false;
282   }
283   return tchdbsetdfunit(tdb->hdb, dfunit);
284 }
285 
286 
287 /* Open a database file and connect a table database object. */
tctdbopen(TCTDB * tdb,const char * path,int omode)288 bool tctdbopen(TCTDB *tdb, const char *path, int omode){
289   assert(tdb && path);
290   if(!TDBLOCKMETHOD(tdb, true)) return false;
291   if(tdb->open){
292     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
293     TDBUNLOCKMETHOD(tdb);
294     return false;
295   }
296   bool rv = tctdbopenimpl(tdb, path, omode);
297   TDBUNLOCKMETHOD(tdb);
298   return rv;
299 }
300 
301 
302 /* Close a table database object. */
tctdbclose(TCTDB * tdb)303 bool tctdbclose(TCTDB *tdb){
304   assert(tdb);
305   if(!TDBLOCKMETHOD(tdb, true)) return false;
306   if(!tdb->open){
307     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
308     TDBUNLOCKMETHOD(tdb);
309     return false;
310   }
311   bool rv = tctdbcloseimpl(tdb);
312   TDBUNLOCKMETHOD(tdb);
313   return rv;
314 }
315 
316 
317 /* Store a record into a table database object. */
tctdbput(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols)318 bool tctdbput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
319   assert(tdb && pkbuf && pksiz >= 0 && cols);
320   int vsiz;
321   if(tcmapget(cols, "", 0, &vsiz)){
322     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
323     return false;
324   }
325   if(!TDBLOCKMETHOD(tdb, true)) return false;
326   if(!tdb->open || !tdb->wmode){
327     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
328     TDBUNLOCKMETHOD(tdb);
329     return false;
330   }
331   bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER);
332   TDBUNLOCKMETHOD(tdb);
333   return rv;
334 }
335 
336 
337 /* Store a string record into a table database object with a zero separated column string. */
tctdbput2(TCTDB * tdb,const void * pkbuf,int pksiz,const void * cbuf,int csiz)338 bool tctdbput2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
339   assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
340   TCMAP *cols = tcstrsplit4(cbuf, csiz);
341   bool rv = tctdbput(tdb, pkbuf, pksiz, cols);
342   tcmapdel(cols);
343   return rv;
344 }
345 
346 
347 /* Store a string record into a table database object with a tab separated column string. */
tctdbput3(TCTDB * tdb,const char * pkstr,const char * cstr)348 bool tctdbput3(TCTDB *tdb, const char *pkstr, const char *cstr){
349   assert(tdb && pkstr && cstr);
350   TCMAP *cols = tcstrsplit3(cstr, "\t");
351   bool rv = tctdbput(tdb, pkstr, strlen(pkstr), cols);
352   tcmapdel(cols);
353   return rv;
354 }
355 
356 
357 /* Store a new record into a table database object. */
tctdbputkeep(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols)358 bool tctdbputkeep(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
359   assert(tdb && pkbuf && pksiz >= 0 && cols);
360   int vsiz;
361   if(tcmapget(cols, "", 0, &vsiz)){
362     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
363     return false;
364   }
365   if(!TDBLOCKMETHOD(tdb, true)) return false;
366   if(!tdb->open || !tdb->wmode){
367     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
368     TDBUNLOCKMETHOD(tdb);
369     return false;
370   }
371   bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDKEEP);
372   TDBUNLOCKMETHOD(tdb);
373   return rv;
374 }
375 
376 
377 /* Store a new string record into a table database object with a zero separated column string. */
tctdbputkeep2(TCTDB * tdb,const void * pkbuf,int pksiz,const void * cbuf,int csiz)378 bool tctdbputkeep2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
379   assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
380   TCMAP *cols = tcstrsplit4(cbuf, csiz);
381   bool rv = tctdbputkeep(tdb, pkbuf, pksiz, cols);
382   tcmapdel(cols);
383   return rv;
384 }
385 
386 
387 /* Store a new string record into a table database object with a tab separated column string. */
tctdbputkeep3(TCTDB * tdb,const char * pkstr,const char * cstr)388 bool tctdbputkeep3(TCTDB *tdb, const char *pkstr, const char *cstr){
389   assert(tdb && pkstr && cstr);
390   TCMAP *cols = tcstrsplit3(cstr, "\t");
391   bool rv = tctdbputkeep(tdb, pkstr, strlen(pkstr), cols);
392   tcmapdel(cols);
393   return rv;
394 }
395 
396 
397 /* Concatenate columns of the existing record in a table database object. */
tctdbputcat(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols)398 bool tctdbputcat(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
399   assert(tdb && pkbuf && pksiz >= 0 && cols);
400   int vsiz;
401   if(tcmapget(cols, "", 0, &vsiz)){
402     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
403     return false;
404   }
405   if(!TDBLOCKMETHOD(tdb, true)) return false;
406   if(!tdb->open || !tdb->wmode){
407     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
408     TDBUNLOCKMETHOD(tdb);
409     return false;
410   }
411   bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDCAT);
412   TDBUNLOCKMETHOD(tdb);
413   return rv;
414 }
415 
416 
417 /* Concatenate columns in a table database object with a zero separated column string. */
tctdbputcat2(TCTDB * tdb,const void * pkbuf,int pksiz,const void * cbuf,int csiz)418 bool tctdbputcat2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
419   assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
420   TCMAP *cols = tcstrsplit4(cbuf, csiz);
421   bool rv = tctdbputcat(tdb, pkbuf, pksiz, cols);
422   tcmapdel(cols);
423   return rv;
424 }
425 
426 
427 /* Concatenate columns in a table database object with with a tab separated column string. */
tctdbputcat3(TCTDB * tdb,const char * pkstr,const char * cstr)428 bool tctdbputcat3(TCTDB *tdb, const char *pkstr, const char *cstr){
429   assert(tdb && pkstr && cstr);
430   TCMAP *cols = tcstrsplit3(cstr, "\t");
431   bool rv = tctdbputcat(tdb, pkstr, strlen(pkstr), cols);
432   tcmapdel(cols);
433   return rv;
434 }
435 
436 
437 /* Remove a record of a table database object. */
tctdbout(TCTDB * tdb,const void * pkbuf,int pksiz)438 bool tctdbout(TCTDB *tdb, const void *pkbuf, int pksiz){
439   assert(tdb && pkbuf && pksiz >= 0);
440   if(!TDBLOCKMETHOD(tdb, true)) return false;
441   if(!tdb->open || !tdb->wmode){
442     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
443     TDBUNLOCKMETHOD(tdb);
444     return false;
445   }
446   bool rv = tctdboutimpl(tdb, pkbuf, pksiz);
447   TDBUNLOCKMETHOD(tdb);
448   return rv;
449 }
450 
451 
452 /* Remove a string record of a table database object. */
tctdbout2(TCTDB * tdb,const char * pkstr)453 bool tctdbout2(TCTDB *tdb, const char *pkstr){
454   assert(tdb && pkstr);
455   return tctdbout(tdb, pkstr, strlen(pkstr));
456 }
457 
458 
459 /* Retrieve a record in a table database object. */
tctdbget(TCTDB * tdb,const void * pkbuf,int pksiz)460 TCMAP *tctdbget(TCTDB *tdb, const void *pkbuf, int pksiz){
461   assert(tdb && pkbuf && pksiz >= 0);
462   if(!TDBLOCKMETHOD(tdb, false)) return NULL;
463   if(!tdb->open){
464     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
465     TDBUNLOCKMETHOD(tdb);
466     return NULL;
467   }
468   TCMAP *rv = tctdbgetimpl(tdb, pkbuf, pksiz);
469   TDBUNLOCKMETHOD(tdb);
470   return rv;
471 }
472 
473 
474 /* Retrieve a record in a table database object as a zero separated column string. */
tctdbget2(TCTDB * tdb,const void * pkbuf,int pksiz,int * sp)475 char *tctdbget2(TCTDB *tdb, const void *pkbuf, int pksiz, int *sp){
476   assert(tdb && pkbuf && pksiz >= 0 && sp);
477   TCMAP *cols = tctdbget(tdb, pkbuf, pksiz);
478   if(!cols) return NULL;
479   char *cbuf = tcstrjoin4(cols, sp);
480   tcmapdel(cols);
481   return cbuf;
482 }
483 
484 
485 /* Retrieve a string record in a table database object as a tab separated column string. */
tctdbget3(TCTDB * tdb,const char * pkstr)486 char *tctdbget3(TCTDB *tdb, const char *pkstr){
487   assert(tdb && pkstr);
488   TCMAP *cols = tctdbget(tdb, pkstr, strlen(pkstr));
489   if(!cols) return NULL;
490   char *cstr = tcstrjoin3(cols, '\t');
491   tcmapdel(cols);
492   return cstr;
493 }
494 
495 
496 /* Get the size of the value of a record in a table database object. */
tctdbvsiz(TCTDB * tdb,const void * pkbuf,int pksiz)497 int tctdbvsiz(TCTDB *tdb, const void *pkbuf, int pksiz){
498   assert(tdb && pkbuf && pksiz >= 0);
499   if(!TDBLOCKMETHOD(tdb, false)) return -1;
500   if(!tdb->open){
501     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
502     TDBUNLOCKMETHOD(tdb);
503     return -1;
504   }
505   int rv = tchdbvsiz(tdb->hdb, pkbuf, pksiz);
506   TDBUNLOCKMETHOD(tdb);
507   return rv;
508 }
509 
510 
511 /* Get the size of the value of a string record in a table database object. */
tctdbvsiz2(TCTDB * tdb,const char * pkstr)512 int tctdbvsiz2(TCTDB *tdb, const char *pkstr){
513   assert(tdb && pkstr);
514   return tctdbvsiz(tdb, pkstr, strlen(pkstr));
515 }
516 
517 
518 /* Initialize the iterator of a table database object. */
tctdbiterinit(TCTDB * tdb)519 bool tctdbiterinit(TCTDB *tdb){
520   assert(tdb);
521   if(!TDBLOCKMETHOD(tdb, true)) return false;
522   if(!tdb->open){
523     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
524     TDBUNLOCKMETHOD(tdb);
525     return false;
526   }
527   bool rv = tchdbiterinit(tdb->hdb);
528   TDBUNLOCKMETHOD(tdb);
529   return rv;
530 }
531 
532 
533 /* Get the next primary key of the iterator of a table database object. */
tctdbiternext(TCTDB * tdb,int * sp)534 void *tctdbiternext(TCTDB *tdb, int *sp){
535   assert(tdb && sp);
536   if(!TDBLOCKMETHOD(tdb, true)) return NULL;
537   if(!tdb->open){
538     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
539     TDBUNLOCKMETHOD(tdb);
540     return NULL;
541   }
542   char *rv = tchdbiternext(tdb->hdb, sp);
543   TDBUNLOCKMETHOD(tdb);
544   return rv;
545 }
546 
547 
548 /* Get the next primary key string of the iterator of a table database object. */
tctdbiternext2(TCTDB * tdb)549 char *tctdbiternext2(TCTDB *tdb){
550   assert(tdb);
551   int pksiz;
552   return tctdbiternext(tdb, &pksiz);
553 }
554 
555 
556 /* Get the columns of the next record of the iterator of a table database object. */
tctdbiternext3(TCTDB * tdb)557 TCMAP *tctdbiternext3(TCTDB *tdb){
558   assert(tdb);
559   TCXSTR *kstr = tcxstrnew();
560   TCXSTR *vstr = tcxstrnew();
561   TCMAP *cols = NULL;
562   if(tchdbiternext3(tdb->hdb, kstr, vstr)){
563     cols = tcmapload(TCXSTRPTR(vstr), TCXSTRSIZE(vstr));
564     tcmapput(cols, "", 0, TCXSTRPTR(kstr), TCXSTRSIZE(kstr));
565   }
566   tcxstrdel(vstr);
567   tcxstrdel(kstr);
568   return cols;
569 }
570 
571 
572 /* Get forward matching primary keys in a table database object. */
tctdbfwmkeys(TCTDB * tdb,const void * pbuf,int psiz,int max)573 TCLIST *tctdbfwmkeys(TCTDB *tdb, const void *pbuf, int psiz, int max){
574   assert(tdb && pbuf && psiz >= 0);
575   if(!TDBLOCKMETHOD(tdb, true)) return tclistnew();
576   if(!tdb->open){
577     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
578     TDBUNLOCKMETHOD(tdb);
579     return tclistnew();
580   }
581   TCLIST *rv = tchdbfwmkeys(tdb->hdb, pbuf, psiz, max);
582   TDBUNLOCKMETHOD(tdb);
583   return rv;
584 }
585 
586 
587 /* Get forward matching string primary keys in a table database object. */
tctdbfwmkeys2(TCTDB * tdb,const char * pstr,int max)588 TCLIST *tctdbfwmkeys2(TCTDB *tdb, const char *pstr, int max){
589   assert(tdb && pstr);
590   return tctdbfwmkeys(tdb, pstr, strlen(pstr), max);
591 }
592 
593 
594 /* Add an integer to a column of a record in a table database object. */
tctdbaddint(TCTDB * tdb,const void * pkbuf,int pksiz,int num)595 int tctdbaddint(TCTDB *tdb, const void *pkbuf, int pksiz, int num){
596   assert(tdb && pkbuf && pksiz >= 0);
597   if(!TDBLOCKMETHOD(tdb, true)) return INT_MIN;
598   if(!tdb->open){
599     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
600     TDBUNLOCKMETHOD(tdb);
601     return INT_MIN;
602   }
603   double rv = tctdbaddnumber(tdb, pkbuf, pksiz, num);
604   TDBUNLOCKMETHOD(tdb);
605   return isnan(rv) ? INT_MIN : (int)rv;
606 }
607 
608 
609 /* Add a real number to a column of a record in a table database object. */
tctdbadddouble(TCTDB * tdb,const void * pkbuf,int pksiz,double num)610 double tctdbadddouble(TCTDB *tdb, const void *pkbuf, int pksiz, double num){
611   assert(tdb && pkbuf && pksiz >= 0);
612   if(!TDBLOCKMETHOD(tdb, true)) return INT_MIN;
613   if(!tdb->open){
614     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
615     TDBUNLOCKMETHOD(tdb);
616     return INT_MIN;
617   }
618   double rv = tctdbaddnumber(tdb, pkbuf, pksiz, num);
619   TDBUNLOCKMETHOD(tdb);
620   return rv;
621 }
622 
623 
624 /* Synchronize updated contents of a table database object with the file and the device. */
tctdbsync(TCTDB * tdb)625 bool tctdbsync(TCTDB *tdb){
626   assert(tdb);
627   if(!TDBLOCKMETHOD(tdb, true)) return false;
628   if(!tdb->open || !tdb->wmode || tdb->tran){
629     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
630     TDBUNLOCKMETHOD(tdb);
631     return false;
632   }
633   bool rv = tctdbmemsync(tdb, true);
634   TDBUNLOCKMETHOD(tdb);
635   return rv;
636 }
637 
638 
639 /* Optimize the file of a table database object. */
tctdboptimize(TCTDB * tdb,int64_t bnum,int8_t apow,int8_t fpow,uint8_t opts)640 bool tctdboptimize(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
641   assert(tdb);
642   if(!TDBLOCKMETHOD(tdb, true)) return false;
643   if(!tdb->open || !tdb->wmode || tdb->tran){
644     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
645     TDBUNLOCKMETHOD(tdb);
646     return false;
647   }
648   TDBTHREADYIELD(tdb);
649   bool rv = tctdboptimizeimpl(tdb, bnum, apow, fpow, opts);
650   TDBUNLOCKMETHOD(tdb);
651   return rv;
652 }
653 
654 
655 /* Remove all records of a table database object. */
tctdbvanish(TCTDB * tdb)656 bool tctdbvanish(TCTDB *tdb){
657   assert(tdb);
658   if(!TDBLOCKMETHOD(tdb, true)) return false;
659   if(!tdb->open || !tdb->wmode || tdb->tran){
660     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
661     TDBUNLOCKMETHOD(tdb);
662     return false;
663   }
664   TDBTHREADYIELD(tdb);
665   bool rv = tctdbvanishimpl(tdb);
666   TDBUNLOCKMETHOD(tdb);
667   return rv;
668 }
669 
670 
671 /* Copy the database file of a table database object. */
tctdbcopy(TCTDB * tdb,const char * path)672 bool tctdbcopy(TCTDB *tdb, const char *path){
673   assert(tdb && path);
674   if(!TDBLOCKMETHOD(tdb, false)) return false;
675   if(!tdb->open){
676     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
677     TDBUNLOCKMETHOD(tdb);
678     return false;
679   }
680   TDBTHREADYIELD(tdb);
681   bool rv = tctdbcopyimpl(tdb, path);
682   TDBUNLOCKMETHOD(tdb);
683   return rv;
684 }
685 
686 
687 /* Begin the transaction of a table database object. */
tctdbtranbegin(TCTDB * tdb)688 bool tctdbtranbegin(TCTDB *tdb){
689   assert(tdb);
690   for(double wsec = 1.0 / sysconf(_SC_CLK_TCK); true; wsec *= 2){
691     if(!TDBLOCKMETHOD(tdb, true)) return false;
692     if(!tdb->open || !tdb->wmode){
693       tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
694       TDBUNLOCKMETHOD(tdb);
695       return false;
696     }
697     if(!tdb->tran) break;
698     TDBUNLOCKMETHOD(tdb);
699     if(wsec > 1.0) wsec = 1.0;
700     tcsleep(wsec);
701   }
702   if(!tctdbtranbeginimpl(tdb)){
703     TDBUNLOCKMETHOD(tdb);
704     return false;
705   }
706   tdb->tran = true;
707   TDBUNLOCKMETHOD(tdb);
708   return true;
709 }
710 
711 
712 /* Commit the transaction of a table database object. */
tctdbtrancommit(TCTDB * tdb)713 bool tctdbtrancommit(TCTDB *tdb){
714   assert(tdb);
715   if(!TDBLOCKMETHOD(tdb, true)) return false;
716   if(!tdb->open || !tdb->wmode || !tdb->tran){
717     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
718     TDBUNLOCKMETHOD(tdb);
719     return false;
720   }
721   tdb->tran = false;
722   bool err = false;
723   if(!tctdbtrancommitimpl(tdb)) err = true;
724   TDBUNLOCKMETHOD(tdb);
725   return !err;
726 }
727 
728 
729 /* Abort the transaction of a table database object. */
tctdbtranabort(TCTDB * tdb)730 bool tctdbtranabort(TCTDB *tdb){
731   assert(tdb);
732   if(!TDBLOCKMETHOD(tdb, true)) return false;
733   if(!tdb->open || !tdb->wmode || !tdb->tran){
734     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
735     TDBUNLOCKMETHOD(tdb);
736     return false;
737   }
738   tdb->tran = false;
739   bool err = false;
740   if(!tctdbtranabortimpl(tdb)) err = true;
741   TDBUNLOCKMETHOD(tdb);
742   return !err;
743 }
744 
745 
746 /* Get the file path of a table database object. */
tctdbpath(TCTDB * tdb)747 const char *tctdbpath(TCTDB *tdb){
748   assert(tdb);
749   if(!TDBLOCKMETHOD(tdb, false)) return NULL;
750   if(!tdb->open){
751     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
752     TDBUNLOCKMETHOD(tdb);
753     return NULL;
754   }
755   const char *rv = tchdbpath(tdb->hdb);
756   TDBUNLOCKMETHOD(tdb);
757   return rv;
758 }
759 
760 
761 /* Get the number of records of a table database object. */
tctdbrnum(TCTDB * tdb)762 uint64_t tctdbrnum(TCTDB *tdb){
763   assert(tdb);
764   if(!TDBLOCKMETHOD(tdb, false)) return 0;
765   if(!tdb->open){
766     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
767     TDBUNLOCKMETHOD(tdb);
768     return 0;
769   }
770   uint64_t rv = tchdbrnum(tdb->hdb);
771   TDBUNLOCKMETHOD(tdb);
772   return rv;
773 }
774 
775 
776 /* Get the size of the database file of a table database object. */
tctdbfsiz(TCTDB * tdb)777 uint64_t tctdbfsiz(TCTDB *tdb){
778   assert(tdb);
779   if(!TDBLOCKMETHOD(tdb, false)) return 0;
780   if(!tdb->open){
781     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
782     TDBUNLOCKMETHOD(tdb);
783     return 0;
784   }
785   uint64_t rv = tchdbfsiz(tdb->hdb);
786   TDBIDX *idxs = tdb->idxs;
787   int inum = tdb->inum;
788   for(int i = 0; i < inum; i++){
789     TDBIDX *idx = idxs + i;
790     switch(idx->type){
791       case TDBITLEXICAL:
792       case TDBITDECIMAL:
793       case TDBITTOKEN:
794       case TDBITQGRAM:
795         rv += tcbdbfsiz(idx->db);
796         break;
797     }
798   }
799   TDBUNLOCKMETHOD(tdb);
800   return rv;
801 }
802 
803 
804 /* Set a column index to a table database object. */
tctdbsetindex(TCTDB * tdb,const char * name,int type)805 bool tctdbsetindex(TCTDB *tdb, const char *name, int type){
806   assert(tdb && name);
807   if(!TDBLOCKMETHOD(tdb, true)) return false;
808   if(!tdb->open || !tdb->wmode || tdb->tran){
809     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
810     TDBUNLOCKMETHOD(tdb);
811     return false;
812   }
813   bool err = false;
814   double iccsync = tdb->iccsync;
815   tdb->iccsync = 1.0;
816   if(!tctdbsetindeximpl(tdb, name, type)) err = true;
817   if(!tctdbmemsync(tdb, false)) err = true;
818   tdb->iccsync = iccsync;
819   TDBUNLOCKMETHOD(tdb);
820   return !err;
821 }
822 
823 
824 /* Generate a unique ID number of a table database object. */
tctdbgenuid(TCTDB * tdb)825 int64_t tctdbgenuid(TCTDB *tdb){
826   assert(tdb);
827   if(!TDBLOCKMETHOD(tdb, true)) return -1;
828   if(!tdb->open || !tdb->wmode){
829     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
830     TDBUNLOCKMETHOD(tdb);
831     return -1;
832   }
833   int64_t rv = tctdbgenuidimpl(tdb, 1);
834   TDBUNLOCKMETHOD(tdb);
835   return rv;
836 }
837 
838 
839 /* Create a query object. */
tctdbqrynew(TCTDB * tdb)840 TDBQRY *tctdbqrynew(TCTDB *tdb){
841   assert(tdb);
842   TDBQRY *qry;
843   TCMALLOC(qry, sizeof(*qry));
844   qry->tdb = tdb;
845   TCMALLOC(qry->conds, sizeof(qry->conds[0]) * 2);
846   qry->cnum = 0;
847   qry->oname = NULL;
848   qry->otype = TDBQOSTRASC;
849   qry->max = INT_MAX;
850   qry->skip = 0;
851   qry->hint = tcxstrnew3(TDBHINTUSIZ);
852   qry->count = 0;
853   return qry;
854 }
855 
856 
857 /* Delete a query object. */
tctdbqrydel(TDBQRY * qry)858 void tctdbqrydel(TDBQRY *qry){
859   assert(qry);
860   tcxstrdel(qry->hint);
861   TCFREE(qry->oname);
862   TDBCOND *conds = qry->conds;
863   int cnum = qry->cnum;
864   for(int i = 0; i < cnum; i++){
865     TDBCOND *cond = conds + i;
866     if(cond->ftsunits){
867       TDBFTSUNIT *ftsunits = cond->ftsunits;
868       int ftsnum = cond->ftsnum;
869       for(int j = 0; j < ftsnum; j++){
870         TDBFTSUNIT *ftsunit = ftsunits + j;
871         tclistdel(ftsunit->tokens);
872       }
873       TCFREE(ftsunits);
874     }
875     if(cond->regex){
876       regfree(cond->regex);
877       TCFREE(cond->regex);
878     }
879     TCFREE(cond->expr);
880     TCFREE(cond->name);
881   }
882   TCFREE(conds);
883   TCFREE(qry);
884 }
885 
886 
887 /* Add a narrowing condition to a query object. */
tctdbqryaddcond(TDBQRY * qry,const char * name,int op,const char * expr)888 void tctdbqryaddcond(TDBQRY *qry, const char *name, int op, const char *expr){
889   assert(qry && name && expr);
890   int cnum = qry->cnum;
891   TCREALLOC(qry->conds, qry->conds, sizeof(qry->conds[0]) * (cnum + 1));
892   TDBCOND *cond = qry->conds + cnum;
893   int nsiz = strlen(name);
894   int esiz = strlen(expr);
895   TCMEMDUP(cond->name, name, nsiz);
896   cond->nsiz = nsiz;
897   bool sign = true;
898   if(op & TDBQCNEGATE){
899     op &= ~TDBQCNEGATE;
900     sign = false;
901   }
902   bool noidx = false;
903   if(op & TDBQCNOIDX){
904     op &= ~TDBQCNOIDX;
905     noidx = true;
906   }
907   cond->op = op;
908   cond->sign = sign;
909   cond->noidx = noidx;
910   TCMEMDUP(cond->expr, expr, esiz);
911   cond->esiz = esiz;
912   cond->regex = NULL;
913   if(op == TDBQCSTRRX){
914     const char *rxstr = expr;
915     int rxopt = REG_EXTENDED | REG_NOSUB;
916     if(*rxstr == '*'){
917       rxopt |= REG_ICASE;
918       rxstr++;
919     }
920     regex_t rxbuf;
921     if(regcomp(&rxbuf, rxstr, rxopt) == 0){
922       TCMALLOC(cond->regex, sizeof(rxbuf));
923       memcpy(cond->regex, &rxbuf, sizeof(rxbuf));
924     }
925   }
926   cond->ftsunits = NULL;
927   cond->ftsnum = 0;
928   if(op >= TDBQCFTSPH && op <= TDBQCFTSEX){
929     cond->op = TDBQCFTSPH;
930     cond->ftsunits = tctdbftsparseexpr(expr, esiz, op, &(cond->ftsnum));
931   }
932   qry->cnum++;
933 }
934 
935 
936 /* Set the order of a query object. */
tctdbqrysetorder(TDBQRY * qry,const char * name,int type)937 void tctdbqrysetorder(TDBQRY *qry, const char *name, int type){
938   assert(qry && name);
939   if(qry->oname) TCFREE(qry->oname);
940   qry->oname = tcstrdup(name);
941   qry->otype = type;
942 }
943 
944 
945 /* Set the limit number of records of the result of a query object. */
tctdbqrysetlimit(TDBQRY * qry,int max,int skip)946 void tctdbqrysetlimit(TDBQRY *qry, int max, int skip){
947   assert(qry);
948   qry->max = (max >= 0) ? max : INT_MAX;
949   qry->skip = (skip > 0) ? skip : 0;
950 }
951 
952 
953 /* Execute the search of a query object. */
tctdbqrysearch(TDBQRY * qry)954 TCLIST *tctdbqrysearch(TDBQRY *qry){
955   assert(qry);
956   TCTDB *tdb = qry->tdb;
957   if(!TDBLOCKMETHOD(tdb, false)) return tclistnew();
958   if(!tdb->open){
959     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
960     TDBUNLOCKMETHOD(tdb);
961     return tclistnew();
962   }
963   TCLIST *rv = tctdbqrysearchimpl(qry);
964   TDBUNLOCKMETHOD(tdb);
965   return rv;
966 }
967 
968 
969 /* Remove each record corresponding to a query object. */
tctdbqrysearchout(TDBQRY * qry)970 bool tctdbqrysearchout(TDBQRY *qry){
971   assert(qry);
972   return tctdbqryproc(qry, tctdbqryprocoutcb, NULL);
973 }
974 
975 
976 /* Process each record corresponding to a query object. */
tctdbqryproc(TDBQRY * qry,TDBQRYPROC proc,void * op)977 bool tctdbqryproc(TDBQRY *qry, TDBQRYPROC proc, void *op){
978   assert(qry && proc);
979   TCTDB *tdb = qry->tdb;
980   if(!TDBLOCKMETHOD(tdb, true)) return false;
981   if(!tdb->open || !tdb->wmode){
982     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
983     TDBUNLOCKMETHOD(tdb);
984     return false;
985   }
986   bool err = false;
987   int64_t getnum = 0;
988   int64_t putnum = 0;
989   int64_t outnum = 0;
990   TCLIST *res = tctdbqrysearchimpl(qry);
991   int rnum = TCLISTNUM(res);
992   for(int i = 0; i < rnum; i++){
993     const char *pkbuf;
994     int pksiz;
995     TCLISTVAL(pkbuf, res, i, pksiz);
996     TCMAP *cols = tctdbgetimpl(tdb, pkbuf, pksiz);
997     if(!cols){
998       err = true;
999       continue;
1000     }
1001     getnum++;
1002     int flags = proc(pkbuf, pksiz, cols, op);
1003     if(flags & TDBQPPUT){
1004       if(tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER)){
1005         putnum++;
1006       } else {
1007         err = true;
1008       }
1009     } else if(flags & TDBQPOUT){
1010       if(tctdboutimpl(tdb, pkbuf, pksiz)){
1011         outnum++;
1012       } else {
1013         err = true;
1014       }
1015     }
1016     tcmapdel(cols);
1017     if(flags & TDBQPSTOP) break;
1018   }
1019   tclistdel(res);
1020   tcxstrprintf(qry->hint, "post treatment: get=%lld, put=%lld, out=%lld\n",
1021                (long long)getnum, (long long)putnum, (long long)outnum);
1022   TDBUNLOCKMETHOD(tdb);
1023   return !err;
1024 }
1025 
1026 
1027 /* Get the hint string of a query object. */
tctdbqryhint(TDBQRY * qry)1028 const char *tctdbqryhint(TDBQRY *qry){
1029   assert(qry);
1030   return tcxstrptr(qry->hint);
1031 }
1032 
1033 
1034 /* Retrieve records with multiple query objects and get the set of the result. */
tctdbmetasearch(TDBQRY ** qrys,int num,int type)1035 TCLIST *tctdbmetasearch(TDBQRY **qrys, int num, int type){
1036   assert(qrys && num >= 0);
1037   if(num < 1) return tclistnew2(1);
1038   if(num < 2) return tctdbqrysearch(qrys[0]);
1039   const char *oname = qrys[0]->oname;
1040   int onsiz = oname ? strlen(oname) : 0;
1041   int otype = qrys[0]->otype;
1042   TCMAP *uset = tcmapnew();
1043   if(type == TDBMSUNION){
1044     for(int i = 0; i < num; i++){
1045       TDBQRY *qry = qrys[i];
1046       TCTDB *tdb = qry->tdb;
1047       int omax = qry->max;
1048       int oskip = qry->skip;
1049       if(qry->max < INT_MAX - qry->skip) qry->max += qry->skip;
1050       qry->skip = 0;
1051       TCLIST *res = tctdbqrysearch(qry);
1052       qry->max = omax;
1053       qry->skip = oskip;
1054       int rnum = TCLISTNUM(res);
1055       for(int j = 0; j < rnum; j++){
1056         const char *pkbuf;
1057         int pksiz;
1058         TCLISTVAL(pkbuf, res, j, pksiz);
1059         if(oname){
1060           int csiz;
1061           char *cbuf = tctdbget4(tdb, pkbuf, pksiz, oname, onsiz, &csiz);
1062           if(cbuf){
1063             tcmapput4(uset, pkbuf, pksiz, "=", 1, cbuf, csiz);
1064             TCFREE(cbuf);
1065           } else {
1066             tcmapput(uset, pkbuf, pksiz, "*", 1);
1067           }
1068         } else {
1069           tcmapputkeep(uset, pkbuf, pksiz, "", 0);
1070         }
1071       }
1072       tclistdel(res);
1073     }
1074   } else if(type == TDBMSISECT){
1075     int omax = qrys[0]->max;
1076     int oskip = qrys[0]->skip;
1077     qrys[0]->max = INT_MAX;
1078     qrys[0]->skip = 0;
1079     TCLIST *res = tctdbqrysearch(qrys[0]);
1080     qrys[0]->max = omax;
1081     qrys[0]->skip = oskip;
1082     int rnum = TCLISTNUM(res);
1083     for(int i = 0; i < rnum; i++){
1084       const char *pkbuf;
1085       int pksiz;
1086       TCLISTVAL(pkbuf, res, i, pksiz);
1087       tcmapputkeep(uset, pkbuf, pksiz, "", 0);
1088     }
1089     tclistdel(res);
1090     for(int i = 1; i < num; i++){
1091       TDBQRY *qry = qrys[i];
1092       if(TCMAPRNUM(uset) < 1){
1093         tcxstrclear(qry->hint);
1094         tcxstrprintf(qry->hint, "omitted\n");
1095         continue;
1096       }
1097       omax = qry->max;
1098       oskip = qry->skip;
1099       qry->max = INT_MAX;
1100       qry->skip = 0;
1101       res = tctdbqrysearch(qry);
1102       qry->max = omax;
1103       qry->skip = oskip;
1104       rnum = TCLISTNUM(res);
1105       TCMAP *nset = tcmapnew2(tclmin(TCMAPRNUM(uset), rnum) + 1);
1106       for(int j = 0; j < rnum; j++){
1107         const char *pkbuf;
1108         int pksiz;
1109         TCLISTVAL(pkbuf, res, j, pksiz);
1110         int vsiz;
1111         if(tcmapget(uset, pkbuf, pksiz, &vsiz)) tcmapputkeep(nset, pkbuf, pksiz, "", 0);
1112       }
1113       tcmapdel(uset);
1114       uset = nset;
1115       tclistdel(res);
1116     }
1117   } else if(type == TDBMSDIFF){
1118     int omax = qrys[0]->max;
1119     int oskip = qrys[0]->skip;
1120     qrys[0]->max = INT_MAX;
1121     qrys[0]->skip = 0;
1122     TCLIST *res = tctdbqrysearch(qrys[0]);
1123     qrys[0]->max = omax;
1124     qrys[0]->skip = oskip;
1125     int rnum = TCLISTNUM(res);
1126     for(int i = 0; i < rnum; i++){
1127       const char *pkbuf;
1128       int pksiz;
1129       TCLISTVAL(pkbuf, res, i, pksiz);
1130       tcmapputkeep(uset, pkbuf, pksiz, "", 0);
1131     }
1132     tclistdel(res);
1133     for(int i = 1; i < num; i++){
1134       TDBQRY *qry = qrys[i];
1135       if(TCMAPRNUM(uset) < 1){
1136         tcxstrclear(qry->hint);
1137         tcxstrprintf(qry->hint, "omitted\n");
1138         continue;
1139       }
1140       omax = qry->max;
1141       oskip = qry->skip;
1142       qry->max = INT_MAX;
1143       qry->skip = 0;
1144       res = tctdbqrysearch(qry);
1145       qry->max = omax;
1146       qry->skip = oskip;
1147       rnum = TCLISTNUM(res);
1148       for(int j = 0; j < rnum; j++){
1149         const char *pkbuf;
1150         int pksiz;
1151         TCLISTVAL(pkbuf, res, j, pksiz);
1152         tcmapout(uset, pkbuf, pksiz);
1153       }
1154       tclistdel(res);
1155     }
1156   }
1157   int max = qrys[0]->max;
1158   int skip = qrys[0]->skip;
1159   TCLIST *res;
1160   if(oname && type == TDBMSUNION){
1161     int rnum = TCMAPRNUM(uset);
1162     TDBSORTREC *keys;
1163     TCMALLOC(keys, sizeof(*keys) * rnum + 1);
1164     tcmapiterinit(uset);
1165     const char *pkbuf;
1166     int pksiz;
1167     TDBSORTREC *key = keys;
1168     while((pkbuf = tcmapiternext(uset, &pksiz)) != NULL){
1169       int vsiz;
1170       const char *vbuf = tcmapiterval(pkbuf, &vsiz);
1171       key->kbuf = pkbuf;
1172       key->ksiz = pksiz;
1173       if(*vbuf == '='){
1174         key->vbuf = (char *)vbuf + 1;
1175         key->vsiz = vsiz - 1;
1176       } else {
1177         key->vbuf = NULL;
1178         key->vsiz = 0;
1179       }
1180       key++;
1181     }
1182     int (*compar)(const TDBSORTREC *a, const TDBSORTREC *b) = NULL;
1183     switch(otype){
1184       case TDBQOSTRASC:
1185         compar = tdbcmpsortrecstrasc;
1186         break;
1187       case TDBQOSTRDESC:
1188         compar = tdbcmpsortrecstrdesc;
1189         break;
1190       case TDBQONUMASC:
1191         compar = tdbcmpsortrecnumasc;
1192         break;
1193       case TDBQONUMDESC:
1194         compar = tdbcmpsortrecnumdesc;
1195         break;
1196     }
1197     if(compar) qsort(keys, rnum, sizeof(*keys), (int (*)(const void *, const void *))compar);
1198     res = tclistnew2(tclmin(rnum, max));
1199     for(int i = skip; max > 0 && i < rnum; i++){
1200       key = keys + i;
1201       TCLISTPUSH(res, key->kbuf, key->ksiz);
1202       max--;
1203     }
1204     TCFREE(keys);
1205   } else {
1206     res = tclistnew2(tclmin(tcmaprnum(uset), max));
1207     tcmapiterinit(uset);
1208     const char *pkbuf;
1209     int pksiz;
1210     while(max > 0 && (pkbuf = tcmapiternext(uset, &pksiz)) != NULL){
1211       if(skip > 0){
1212         skip--;
1213       } else {
1214         TCLISTPUSH(res, pkbuf, pksiz);
1215         max--;
1216       }
1217     }
1218   }
1219   tcmapdel(uset);
1220   TCXSTR *hint = tcxstrnew();
1221   for(int i = 0; i < num; i++){
1222     TDBQRY *qry = qrys[i];
1223     TCLIST *lines = tcstrsplit(tctdbqryhint(qry), "\n");
1224     int lnum = TCLISTNUM(lines);
1225     for(int j = 0; j < lnum; j++){
1226       const char *line = TCLISTVALPTR(lines, j);
1227       if(*line != 0) tcxstrprintf(hint, "[%d] %s\n", i, line);
1228     }
1229     tclistdel(lines);
1230     tcxstrclear(qry->hint);
1231     qry->count = 0;
1232   }
1233   TCXSTRCAT(qrys[0]->hint, TCXSTRPTR(hint), TCXSTRSIZE(hint));
1234   qrys[0]->count = TCLISTNUM(res);
1235   tcxstrdel(hint);
1236   return res;
1237 }
1238 
1239 
1240 
1241 /*************************************************************************************************
1242  * features for experts
1243  *************************************************************************************************/
1244 
1245 
1246 /* Set the error code of a table database object. */
tctdbsetecode(TCTDB * tdb,int ecode,const char * filename,int line,const char * func)1247 void tctdbsetecode(TCTDB *tdb, int ecode, const char *filename, int line, const char *func){
1248   assert(tdb && filename && line >= 1 && func);
1249   tchdbsetecode(tdb->hdb, ecode, filename, line, func);
1250 }
1251 
1252 
1253 /* Set the file descriptor for debugging output. */
tctdbsetdbgfd(TCTDB * tdb,int fd)1254 void tctdbsetdbgfd(TCTDB *tdb, int fd){
1255   assert(tdb && fd >= 0);
1256   tchdbsetdbgfd(tdb->hdb, fd);
1257 }
1258 
1259 
1260 /* Get the file descriptor for debugging output. */
tctdbdbgfd(TCTDB * tdb)1261 int tctdbdbgfd(TCTDB *tdb){
1262   assert(tdb);
1263   return tchdbdbgfd(tdb->hdb);
1264 }
1265 
1266 
1267 /* Check whether mutual exclusion control is set to a table database object. */
tctdbhasmutex(TCTDB * tdb)1268 bool tctdbhasmutex(TCTDB *tdb){
1269   assert(tdb);
1270   return tdb->mmtx != NULL;
1271 }
1272 
1273 
1274 /* Synchronize updating contents on memory of a table database object. */
tctdbmemsync(TCTDB * tdb,bool phys)1275 bool tctdbmemsync(TCTDB *tdb, bool phys){
1276   assert(tdb);
1277   if(!tdb->open || !tdb->wmode){
1278     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1279     return false;
1280   }
1281   bool err = false;
1282   if(!tchdbmemsync(tdb->hdb, phys)) err = true;
1283   TDBIDX *idxs = tdb->idxs;
1284   int inum = tdb->inum;
1285   for(int i = 0; i < inum; i++){
1286     TDBIDX *idx = idxs + i;
1287     switch(idx->type){
1288       case TDBITTOKEN:
1289       case TDBITQGRAM:
1290         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
1291         break;
1292     }
1293   }
1294   for(int i = 0; i < inum; i++){
1295     TDBIDX *idx = idxs + i;
1296     switch(idx->type){
1297       case TDBITLEXICAL:
1298       case TDBITDECIMAL:
1299       case TDBITTOKEN:
1300       case TDBITQGRAM:
1301         if(!tcbdbmemsync(idx->db, phys)){
1302           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
1303           err = true;
1304         }
1305         break;
1306     }
1307   }
1308   return !err;
1309 }
1310 
1311 
1312 /* Get the number of elements of the bucket array of a table database object. */
tctdbbnum(TCTDB * tdb)1313 uint64_t tctdbbnum(TCTDB *tdb){
1314   assert(tdb);
1315   if(!tdb->open){
1316     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1317     return 0;
1318   }
1319   return tchdbbnum(tdb->hdb);
1320 }
1321 
1322 
1323 /* Get the record alignment of a table database object. */
tctdbalign(TCTDB * tdb)1324 uint32_t tctdbalign(TCTDB *tdb){
1325   assert(tdb);
1326   if(!tdb->open){
1327     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1328     return 0;
1329   }
1330   return tchdbalign(tdb->hdb);
1331 }
1332 
1333 
1334 /* Get the maximum number of the free block pool of a table database object. */
tctdbfbpmax(TCTDB * tdb)1335 uint32_t tctdbfbpmax(TCTDB *tdb){
1336   assert(tdb);
1337   if(!tdb->open){
1338     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1339     return 0;
1340   }
1341   return tchdbfbpmax(tdb->hdb);
1342 }
1343 
1344 
1345 /* Get the inode number of the database file of a table database object. */
tctdbinode(TCTDB * tdb)1346 uint64_t tctdbinode(TCTDB *tdb){
1347   assert(tdb);
1348   if(!tdb->open){
1349     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1350     return 0;
1351   }
1352   return tchdbinode(tdb->hdb);
1353 }
1354 
1355 
1356 /* Get the modification time of the database file of a table database object. */
tctdbmtime(TCTDB * tdb)1357 time_t tctdbmtime(TCTDB *tdb){
1358   assert(tdb);
1359   if(!tdb->open){
1360     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1361     return 0;
1362   }
1363   return tchdbmtime(tdb->hdb);
1364 }
1365 
1366 
1367 /* Get the additional flags of a table database object. */
tctdbflags(TCTDB * tdb)1368 uint8_t tctdbflags(TCTDB *tdb){
1369   assert(tdb);
1370   if(!tdb->open){
1371     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1372     return 0;
1373   }
1374   return tchdbflags(tdb->hdb);
1375 }
1376 
1377 
1378 /* Get the options of a table database object. */
tctdbopts(TCTDB * tdb)1379 uint8_t tctdbopts(TCTDB *tdb){
1380   assert(tdb);
1381   if(!tdb->open){
1382     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1383     return 0;
1384   }
1385   return tdb->opts;
1386 }
1387 
1388 
1389 /* Get the pointer to the opaque field of a table database object. */
tctdbopaque(TCTDB * tdb)1390 char *tctdbopaque(TCTDB *tdb){
1391   assert(tdb);
1392   if(!tdb->open){
1393     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1394     return NULL;
1395   }
1396   return tchdbopaque(tdb->hdb) + TDBOPAQUESIZ;
1397 }
1398 
1399 
1400 /* Get the number of used elements of the bucket array of a table database object. */
tctdbbnumused(TCTDB * tdb)1401 uint64_t tctdbbnumused(TCTDB *tdb){
1402   assert(tdb);
1403   if(!tdb->open){
1404     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1405     return 0;
1406   }
1407   return tchdbbnumused(tdb->hdb);
1408 }
1409 
1410 
1411 /* Get the number of column indices of a table database object. */
tctdbinum(TCTDB * tdb)1412 int tctdbinum(TCTDB *tdb){
1413   assert(tdb);
1414   if(!tdb->open){
1415     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1416     return 0;
1417   }
1418   return tdb->inum;
1419 }
1420 
1421 
1422 /* Get the seed of unique ID unumbers of a table database object. */
tctdbuidseed(TCTDB * tdb)1423 int64_t tctdbuidseed(TCTDB *tdb){
1424   assert(tdb);
1425   if(!TDBLOCKMETHOD(tdb, false)) return -1;
1426   if(!tdb->open){
1427     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1428     TDBUNLOCKMETHOD(tdb);
1429     return -1;
1430   }
1431   int64_t rv = tctdbgenuidimpl(tdb, 0);
1432   TDBUNLOCKMETHOD(tdb);
1433   return rv;
1434 }
1435 
1436 
1437 /* Set the parameters of the inverted cache of a table database object. */
tctdbsetinvcache(TCTDB * tdb,int64_t iccmax,double iccsync)1438 bool tctdbsetinvcache(TCTDB *tdb, int64_t iccmax, double iccsync){
1439   assert(tdb);
1440   if(tdb->open){
1441     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1442     return false;
1443   }
1444   tdb->iccmax = (iccmax > 0) ? iccmax : TDBIDXICCMAX;
1445   tdb->iccsync = (iccsync > 0) ? iccsync : TDBIDXICCSYNC;
1446   return true;
1447 }
1448 
1449 
1450 /* Set the seed of unique ID unumbers of a table database object. */
tctdbsetuidseed(TCTDB * tdb,int64_t seed)1451 bool tctdbsetuidseed(TCTDB *tdb, int64_t seed){
1452   assert(tdb && seed >= 0);
1453   if(!TDBLOCKMETHOD(tdb, true)) return -1;
1454   if(!tdb->open || !tdb->wmode){
1455     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1456     TDBUNLOCKMETHOD(tdb);
1457     return false;
1458   }
1459   tctdbgenuidimpl(tdb, -seed - 1);
1460   TDBUNLOCKMETHOD(tdb);
1461   return true;
1462 }
1463 
1464 
1465 /* Set the custom codec functions of a table database object. */
tctdbsetcodecfunc(TCTDB * tdb,TCCODEC enc,void * encop,TCCODEC dec,void * decop)1466 bool tctdbsetcodecfunc(TCTDB *tdb, TCCODEC enc, void *encop, TCCODEC dec, void *decop){
1467   assert(tdb && enc && dec);
1468   if(!TDBLOCKMETHOD(tdb, true)) return false;
1469   if(tdb->open){
1470     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1471     TDBUNLOCKMETHOD(tdb);
1472     return false;
1473   }
1474   bool rv = tchdbsetcodecfunc(tdb->hdb, enc, encop, dec, decop);
1475   TDBUNLOCKMETHOD(tdb);
1476   return rv;
1477 }
1478 
1479 
1480 /* Get the unit step number of auto defragmentation of a table database object. */
tctdbdfunit(TCTDB * tdb)1481 uint32_t tctdbdfunit(TCTDB *tdb){
1482   assert(tdb);
1483   return tchdbdfunit(tdb->hdb);
1484 }
1485 
1486 
1487 /* Perform dynamic defragmentation of a table database object. */
tctdbdefrag(TCTDB * tdb,int64_t step)1488 bool tctdbdefrag(TCTDB *tdb, int64_t step){
1489   assert(tdb);
1490   if(!TDBLOCKMETHOD(tdb, false)) return false;
1491   if(!tdb->open || !tdb->wmode){
1492     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1493     TDBUNLOCKMETHOD(tdb);
1494     return false;
1495   }
1496   bool rv = tctdbdefragimpl(tdb, step);
1497   TDBUNLOCKMETHOD(tdb);
1498   return rv;
1499 }
1500 
1501 
1502 /* Clear the cache of a table tree database object. */
tctdbcacheclear(TCTDB * tdb)1503 bool tctdbcacheclear(TCTDB *tdb){
1504   assert(tdb);
1505   if(!TDBLOCKMETHOD(tdb, true)) return false;
1506   if(!tdb->open){
1507     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1508     TDBUNLOCKMETHOD(tdb);
1509     return false;
1510   }
1511   bool rv = tctdbcacheclearimpl(tdb);
1512   TDBUNLOCKMETHOD(tdb);
1513   return rv;
1514 }
1515 
1516 
1517 /* Store a record into a table database object with a duplication handler. */
tctdbputproc(TCTDB * tdb,const void * pkbuf,int pksiz,const void * cbuf,int csiz,TCPDPROC proc,void * op)1518 bool tctdbputproc(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz,
1519                   TCPDPROC proc, void *op){
1520   assert(tdb && pkbuf && pksiz >= 0 && proc);
1521   if(!TDBLOCKMETHOD(tdb, true)) return false;
1522   if(!tdb->open || !tdb->wmode){
1523     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1524     TDBUNLOCKMETHOD(tdb);
1525     return false;
1526   }
1527   bool err = false;
1528   TCMAP *cols = tctdbgetimpl(tdb, pkbuf, pksiz);
1529   if(cols){
1530     int zsiz;
1531     char *zbuf = tcstrjoin4(cols, &zsiz);
1532     int ncsiz;
1533     void *ncbuf = proc(zbuf, zsiz, &ncsiz, op);
1534     if(ncbuf == (void *)-1){
1535       if(!tctdboutimpl(tdb, pkbuf, pksiz)) err = true;
1536     } else if(ncbuf){
1537       TCMAP *ncols = tcstrsplit4(ncbuf, ncsiz);
1538       if(!tctdbputimpl(tdb, pkbuf, pksiz, ncols, TDBPDOVER)) err = true;
1539       tcmapdel(ncols);
1540       TCFREE(ncbuf);
1541     } else {
1542       tctdbsetecode(tdb, TCEKEEP, __FILE__, __LINE__, __func__);
1543       err = true;
1544     }
1545     TCFREE(zbuf);
1546     tcmapdel(cols);
1547   } else {
1548     if(cbuf){
1549       cols = tcstrsplit4(cbuf, csiz);
1550       if(!tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER)) err = true;
1551       tcmapdel(cols);
1552     } else {
1553       tctdbsetecode(tdb, TCENOREC, __FILE__, __LINE__, __func__);
1554       err = true;
1555     }
1556   }
1557   TDBUNLOCKMETHOD(tdb);
1558   return !err;
1559 }
1560 
1561 
1562 /* Retrieve the value of a column of a record in a table database object. */
tctdbget4(TCTDB * tdb,const void * pkbuf,int pksiz,const void * nbuf,int nsiz,int * sp)1563 char *tctdbget4(TCTDB *tdb, const void *pkbuf, int pksiz, const void *nbuf, int nsiz, int *sp){
1564   assert(tdb && pkbuf && pksiz >= 0 && nbuf && nsiz >= 0 && sp);
1565   if(!TDBLOCKMETHOD(tdb, false)) return NULL;
1566   if(!tdb->open){
1567     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1568     TDBUNLOCKMETHOD(tdb);
1569     return NULL;
1570   }
1571   char *rv = tctdbgetonecol(tdb, pkbuf, pksiz, nbuf, nsiz, sp);
1572   TDBUNLOCKMETHOD(tdb);
1573   return rv;
1574 }
1575 
1576 
1577 /* Move the iterator to the record corresponding a key of a table database object. */
tctdbiterinit2(TCTDB * tdb,const void * pkbuf,int pksiz)1578 bool tctdbiterinit2(TCTDB *tdb, const void *pkbuf, int pksiz){
1579   assert(tdb && pkbuf && pksiz >= 0);
1580   if(!TDBLOCKMETHOD(tdb, true)) return false;
1581   if(!tdb->open){
1582     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1583     TDBUNLOCKMETHOD(tdb);
1584     return false;
1585   }
1586   bool rv = tchdbiterinit2(tdb->hdb, pkbuf, pksiz);
1587   TDBUNLOCKMETHOD(tdb);
1588   return rv;
1589 }
1590 
1591 
1592 /* Move the iterator to the record corresponding a key string of a table database object. */
tctdbiterinit3(TCTDB * tdb,const char * kstr)1593 bool tctdbiterinit3(TCTDB *tdb, const char *kstr){
1594   assert(tdb && kstr);
1595   return tctdbiterinit2(tdb, kstr, strlen(kstr));
1596 }
1597 
1598 
1599 /* Process each record atomically of a table database object. */
tctdbforeach(TCTDB * tdb,TCITER iter,void * op)1600 bool tctdbforeach(TCTDB *tdb, TCITER iter, void *op){
1601   assert(tdb && iter);
1602   if(!TDBLOCKMETHOD(tdb, false)) return false;
1603   if(!tdb->open){
1604     tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1605     TDBUNLOCKMETHOD(tdb);
1606     return false;
1607   }
1608   TDBTHREADYIELD(tdb);
1609   bool rv = tctdbforeachimpl(tdb, iter, op);
1610   TDBUNLOCKMETHOD(tdb);
1611   return rv;
1612 }
1613 
1614 
1615 /* Process each record corresponding to a query object with non-atomic fashion. */
tctdbqryproc2(TDBQRY * qry,TDBQRYPROC proc,void * op)1616 bool tctdbqryproc2(TDBQRY *qry, TDBQRYPROC proc, void *op){
1617   assert(qry && proc);
1618   TCTDB *tdb = qry->tdb;
1619   TDBCOND *conds = qry->conds;
1620   int cnum = qry->cnum;
1621   bool err = false;
1622   int64_t getnum = 0;
1623   int64_t putnum = 0;
1624   int64_t outnum = 0;
1625   TCLIST *res = tctdbqrysearch(qry);
1626   int rnum = TCLISTNUM(res);
1627   for(int i = 0; i < rnum; i++){
1628     if(!TDBLOCKMETHOD(tdb, true)){
1629       err = true;
1630       break;
1631     }
1632     if(!tdb->open || !tdb->wmode){
1633       tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
1634       TDBUNLOCKMETHOD(tdb);
1635       err = true;
1636       break;
1637     }
1638     int pksiz;
1639     const char *pkbuf;
1640     TCLISTVAL(pkbuf, res, i, pksiz);
1641     TCMAP *cols = tctdbgetimpl(tdb, pkbuf, pksiz);
1642     if(cols){
1643       getnum++;
1644       bool ok = true;
1645       for(int j = 0; j < cnum; j++){
1646         TDBCOND *cond = conds + j;
1647         if(cond->nsiz < 1){
1648           if(tctdbqrycondmatch(cond, pkbuf, pksiz) != cond->sign){
1649             ok = false;
1650             break;
1651           }
1652         } else {
1653           int vsiz;
1654           const char *vbuf = tcmapget(cols, cond->name, cond->nsiz, &vsiz);
1655           if(vbuf){
1656             if(tctdbqrycondmatch(cond, vbuf, vsiz) != cond->sign){
1657               ok = false;
1658               break;
1659             }
1660           } else {
1661             if(cond->sign){
1662               ok = false;
1663               break;
1664             }
1665           }
1666         }
1667       }
1668       if(ok){
1669         int flags = proc(pkbuf, pksiz, cols, op);
1670         if(flags & TDBQPPUT){
1671           if(tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER)){
1672             putnum++;
1673           } else {
1674             err = true;
1675           }
1676         } else if(flags & TDBQPOUT){
1677           if(tctdboutimpl(tdb, pkbuf, pksiz)){
1678             outnum++;
1679           } else {
1680             err = true;
1681           }
1682         }
1683         if(flags & TDBQPSTOP) i = rnum;
1684       }
1685       tcmapdel(cols);
1686     }
1687     TDBUNLOCKMETHOD(tdb);
1688   }
1689   tclistdel(res);
1690   tcxstrprintf(qry->hint, "post treatment: get=%lld, put=%lld, out=%lld\n",
1691                (long long)getnum, (long long)putnum, (long long)outnum);
1692   return !err;
1693 }
1694 
1695 
1696 /* Remove each record corresponding to a query object with non-atomic fashion. */
tctdbqrysearchout2(TDBQRY * qry)1697 bool tctdbqrysearchout2(TDBQRY *qry){
1698   assert(qry);
1699   return tctdbqryproc2(qry, tctdbqryprocoutcb, NULL);
1700 }
1701 
1702 
1703 /* Convert a string into the index type number. */
tctdbstrtoindextype(const char * str)1704 int tctdbstrtoindextype(const char *str){
1705   assert(str);
1706   int type = -1;
1707   int flags = 0;
1708   if(*str == '+'){
1709     flags |= TDBITKEEP;
1710     str++;
1711   }
1712   if(!tcstricmp(str, "LEX") || !tcstricmp(str, "LEXICAL") || !tcstricmp(str, "STR")){
1713     type = TDBITLEXICAL;
1714   } else if(!tcstricmp(str, "DEC") || !tcstricmp(str, "DECIMAL") || !tcstricmp(str, "NUM")){
1715     type = TDBITDECIMAL;
1716   } else if(!tcstricmp(str, "TOK") || !tcstricmp(str, "TOKEN")){
1717     type = TDBITTOKEN;
1718   } else if(!tcstricmp(str, "QGR") || !tcstricmp(str, "QGRAM") || !tcstricmp(str, "FTS")){
1719     type = TDBITQGRAM;
1720   } else if(!tcstricmp(str, "OPT") || !tcstricmp(str, "OPTIMIZE")){
1721     type = TDBITOPT;
1722   } else if(!tcstricmp(str, "VOID") || !tcstricmp(str, "NULL")){
1723     type = TDBITVOID;
1724   } else if(tcstrisnum(str)){
1725     type = tcatoi(str);
1726   }
1727   return type | flags;
1728 }
1729 
1730 
1731 /* Convert a string into the meta search type number. */
tctdbstrtometasearcytype(const char * str)1732 int tctdbstrtometasearcytype(const char *str){
1733   assert(str);
1734   int type = -1;
1735   if(!tcstricmp(str, "UNION") || !tcstricmp(str, "OR")){
1736     type = TDBMSUNION;
1737   } else if(!tcstricmp(str, "ISECT") || !tcstricmp(str, "INTERSECTION") ||
1738             !tcstricmp(str, "AND")){
1739     type = TDBMSISECT;
1740   } else if(!tcstricmp(str, "DIFF") || !tcstricmp(str, "DIFFERENCE") ||
1741             !tcstricmp(str, "ANDNOT") || !tcstricmp(str, "NOT")){
1742     type = TDBMSDIFF;
1743   } else if(tcstrisnum(str)){
1744     type = tcatoi(str);
1745   }
1746   return type;
1747 }
1748 
1749 
1750 /* Get the count of corresponding records of a query object. */
tctdbqrycount(TDBQRY * qry)1751 int tctdbqrycount(TDBQRY *qry){
1752   assert(qry);
1753   return qry->count;
1754 }
1755 
1756 
1757 /* Generate keyword-in-context strings from a query object. */
tctdbqrykwic(TDBQRY * qry,TCMAP * cols,const char * name,int width,int opts)1758 TCLIST *tctdbqrykwic(TDBQRY *qry, TCMAP *cols, const char *name, int width, int opts){
1759   assert(qry && cols && width >= 0);
1760   TDBCOND *conds = qry->conds;
1761   int cnum = qry->cnum;
1762   TDBCOND *cond = NULL;
1763   if(name){
1764     for(int i = 0; i < cnum; i++){
1765       if(!strcmp(conds[i].name, name)){
1766         cond = conds + i;
1767         break;
1768       }
1769     }
1770   } else if(cnum > 0){
1771     cond = conds;
1772     name = cond->name;
1773   }
1774   if(!cond) return tclistnew2(1);
1775   const char *str = tcmapget2(cols, name);
1776   if(!str) return tclistnew2(1);
1777   TCLIST *words;
1778   if(cond->op == TDBQCSTRAND || cond->op == TDBQCSTROR ||
1779      cond->op == TDBQCSTROREQ || cond->op == TDBQCNUMOREQ){
1780     words = tcstrsplit(cond->expr, " ,");
1781   } else if(cond->op == TDBQCFTSPH){
1782     TDBFTSUNIT *ftsunits = cond->ftsunits;
1783     int ftsnum = cond->ftsnum;
1784     if(ftsnum > 0){
1785       words = tclistnew2(ftsnum * 2 + 1);
1786       for(int i = 0; i < ftsnum; i++){
1787         if(!ftsunits[i].sign) continue;
1788         TCLIST *tokens = ftsunits[i].tokens;
1789         int tnum = TCLISTNUM(tokens);
1790         for(int j = 0; j < tnum; j++){
1791           const char *token;
1792           int tsiz;
1793           TCLISTVAL(token, tokens, j, tsiz);
1794           TCLISTPUSH(words, token, tsiz);
1795         }
1796       }
1797     } else {
1798       words = tclistnew2(1);
1799     }
1800   } else {
1801     words = tclistnew3(cond->expr, NULL);
1802   }
1803   TCLIST *texts = tcstrkwic(str, words, width, opts);
1804   tclistdel(words);
1805   return texts;
1806 }
1807 
1808 
1809 /* Convert a string into the query operation number. */
tctdbqrystrtocondop(const char * str)1810 int tctdbqrystrtocondop(const char *str){
1811   assert(str);
1812   int op = -1;
1813   int flags = 0;
1814   if(*str == '~' || *str == '!'){
1815     flags |= TDBQCNEGATE;
1816     str++;
1817   }
1818   if(*str == '+'){
1819     flags |= TDBQCNOIDX;
1820     str++;
1821   }
1822   if(!tcstricmp(str, "STREQ") || !tcstricmp(str, "STR") || !tcstricmp(str, "EQ")){
1823     op = TDBQCSTREQ;
1824   } else if(!tcstricmp(str, "STRINC") || !tcstricmp(str, "INC")){
1825     op = TDBQCSTRINC;
1826   } else if(!tcstricmp(str, "STRBW") || !tcstricmp(str, "BW")){
1827     op = TDBQCSTRBW;
1828   } else if(!tcstricmp(str, "STREW") || !tcstricmp(str, "EW")){
1829     op = TDBQCSTREW;
1830   } else if(!tcstricmp(str, "STRAND") || !tcstricmp(str, "AND")){
1831     op = TDBQCSTRAND;
1832   } else if(!tcstricmp(str, "STROR") || !tcstricmp(str, "OR")){
1833     op = TDBQCSTROR;
1834   } else if(!tcstricmp(str, "STROREQ") || !tcstricmp(str, "OREQ")){
1835     op = TDBQCSTROREQ;
1836   } else if(!tcstricmp(str, "STRRX") || !tcstricmp(str, "RX")){
1837     op = TDBQCSTRRX;
1838   } else if(!tcstricmp(str, "NUMEQ") || !tcstricmp(str, "NUM") ||
1839             !tcstricmp(str, "=") || !tcstricmp(str, "==")){
1840     op = TDBQCNUMEQ;
1841   } else if(!tcstricmp(str, "NUMGT") || !tcstricmp(str, ">")){
1842     op = TDBQCNUMGT;
1843   } else if(!tcstricmp(str, "NUMGE") || !tcstricmp(str, ">=")){
1844     op = TDBQCNUMGE;
1845   } else if(!tcstricmp(str, "NUMLT") || !tcstricmp(str, "<")){
1846     op = TDBQCNUMLT;
1847   } else if(!tcstricmp(str, "NUMLE") || !tcstricmp(str, "<=")){
1848     op = TDBQCNUMLE;
1849   } else if(!tcstricmp(str, "NUMBT")){
1850     op = TDBQCNUMBT;
1851   } else if(!tcstricmp(str, "NUMOREQ")){
1852     op = TDBQCNUMOREQ;
1853   } else if(!tcstricmp(str, "FTSPH") || !tcstricmp(str, "FTS")){
1854     op = TDBQCFTSPH;
1855   } else if(!tcstricmp(str, "FTSAND")){
1856     op = TDBQCFTSAND;
1857   } else if(!tcstricmp(str, "FTSOR")){
1858     op = TDBQCFTSOR;
1859   } else if(!tcstricmp(str, "FTSEX")){
1860     op = TDBQCFTSEX;
1861   } else if(tcstrisnum(str)){
1862     op = tcatoi(str);
1863   }
1864   return op | flags;
1865 }
1866 
1867 
1868 /* Convert a string into the query order type number. */
tctdbqrystrtoordertype(const char * str)1869 int tctdbqrystrtoordertype(const char *str){
1870   assert(str);
1871   int type = -1;
1872   if(!tcstricmp(str, "STRASC") || !tcstricmp(str, "STR") || !tcstricmp(str, "ASC")){
1873     type = TDBQOSTRASC;
1874   } else if(!tcstricmp(str, "STRDESC") || !tcstricmp(str, "DESC")){
1875     type = TDBQOSTRDESC;
1876   } else if(!tcstricmp(str, "NUMASC") || !tcstricmp(str, "NUM")){
1877     type = TDBQONUMASC;
1878   } else if(!tcstricmp(str, "NUMDESC")){
1879     type = TDBQONUMDESC;
1880   } else if(tcstrisnum(str)){
1881     type = tcatoi(str);
1882   }
1883   return type;
1884 }
1885 
1886 
1887 /* Convert a string into the set operation type number. */
tctdbmetastrtosettype(const char * str)1888 int tctdbmetastrtosettype(const char *str){
1889   assert(str);
1890   int type = -1;
1891   if(!tcstricmp(str, "UNION") || !tcstricmp(str, "CUP") || !tcstricmp(str, "+")){
1892     type = TDBMSUNION;
1893   } else if(!tcstricmp(str, "ISECT") || !tcstricmp(str, "INTERSECTION") ||
1894             !tcstricmp(str, "CAP") || !tcstricmp(str, "*")){
1895     type = TDBMSISECT;
1896   } else if(!tcstricmp(str, "DIFF") || !tcstricmp(str, "DIFFERENCE") ||
1897             !tcstricmp(str, "MINUS") || !tcstricmp(str, "-")){
1898     type = TDBMSDIFF;
1899   } else if(tcstrisnum(str)){
1900     type = tcatoi(str);
1901   }
1902   return type;
1903 }
1904 
1905 
1906 
1907 /*************************************************************************************************
1908  * private features
1909  *************************************************************************************************/
1910 
1911 
1912 /* Clear all members.
1913    `tdb' specifies the table database object. */
tctdbclear(TCTDB * tdb)1914 static void tctdbclear(TCTDB *tdb){
1915   assert(tdb);
1916   tdb->mmtx = NULL;
1917   tdb->hdb = NULL;
1918   tdb->open = false;
1919   tdb->wmode = false;
1920   tdb->opts = 0;
1921   tdb->lcnum = TDBDEFLCNUM;
1922   tdb->ncnum = TDBDEFNCNUM;
1923   tdb->iccmax = TDBIDXICCMAX;
1924   tdb->iccsync = TDBIDXICCSYNC;
1925   tdb->idxs = NULL;
1926   tdb->inum = 0;
1927   tdb->tran = false;
1928 }
1929 
1930 
1931 /* Open a database file and connect a table database object.
1932    `tdb' specifies the table database object.
1933    `path' specifies the path of the internal database file.
1934    `omode' specifies the connection mode.
1935    If successful, the return value is true, else, it is false. */
tctdbopenimpl(TCTDB * tdb,const char * path,int omode)1936 static bool tctdbopenimpl(TCTDB *tdb, const char *path, int omode){
1937   assert(tdb && path);
1938   int dbgfd = tchdbdbgfd(tdb->hdb);
1939   TCCODEC enc, dec;
1940   void *encop, *decop;
1941   tchdbcodecfunc(tdb->hdb, &enc, &encop, &dec, &decop);
1942   int homode = HDBOREADER;
1943   int bomode = BDBOREADER;
1944   if(omode & TDBOWRITER){
1945     homode = HDBOWRITER;
1946     bomode = BDBOWRITER;
1947     if(omode & TDBOCREAT){
1948       homode |= HDBOCREAT;
1949       bomode |= BDBOCREAT;
1950     }
1951     if(omode & TDBOTRUNC){
1952       homode |= HDBOTRUNC;
1953       bomode |= BDBOTRUNC;
1954     }
1955     tdb->wmode = true;
1956   } else {
1957     tdb->wmode = false;
1958   }
1959   if(omode & TDBONOLCK){
1960     homode |= HDBONOLCK;
1961     bomode |= BDBONOLCK;
1962   }
1963   if(omode & TDBOLCKNB){
1964     homode |= HDBOLCKNB;
1965     bomode |= BDBOLCKNB;
1966   }
1967   if(omode & TDBOTSYNC){
1968     homode |= HDBOTSYNC;
1969     bomode |= BDBOTSYNC;
1970   }
1971   tchdbsettype(tdb->hdb, TCDBTTABLE);
1972   if(!tchdbopen(tdb->hdb, path, homode)) return false;
1973   char *tpath = tcsprintf("%s%c%s%c*", path, MYEXTCHR, TDBIDXSUFFIX, MYEXTCHR);
1974   if((omode & TDBOWRITER) && (omode & TDBOTRUNC)){
1975     TCLIST *paths = tcglobpat(tpath);
1976     int pnum = TCLISTNUM(paths);
1977     for(int i = 0; i < pnum; i++){
1978       unlink(TCLISTVALPTR(paths, i));
1979     }
1980     tclistdel(paths);
1981   }
1982   TCLIST *paths = tcglobpat(tpath);
1983   int pnum = TCLISTNUM(paths);
1984   TCMALLOC(tdb->idxs, sizeof(tdb->idxs[0]) * pnum + 1);
1985   TDBIDX *idxs = tdb->idxs;
1986   int inum = 0;
1987   for(int i = 0; i < pnum; i++){
1988     const char *ipath = TCLISTVALPTR(paths, i);
1989     if(!tcstrfwm(ipath, path)) continue;
1990     const char *rp = ipath + strlen(path);
1991     if(*rp != MYEXTCHR) continue;
1992     rp++;
1993     if(!tcstrfwm(rp, TDBIDXSUFFIX)) continue;
1994     rp += strlen(TDBIDXSUFFIX);
1995     if(*rp != MYEXTCHR) continue;
1996     rp++;
1997     char *stem = tcstrdup(rp);
1998     char *ep = strrchr(stem, MYEXTCHR);
1999     if(!ep) continue;
2000     *(ep++) = '\0';
2001     int nsiz;
2002     char *name = tcurldecode(stem, &nsiz);
2003     if(!strcmp(ep, "lex") || !strcmp(ep, "dec") || !strcmp(ep, "tok") || !strcmp(ep, "qgr")){
2004       TCBDB *bdb = tcbdbnew();
2005       if(dbgfd >= 0) tcbdbsetdbgfd(bdb, dbgfd);
2006       if(tdb->mmtx) tcbdbsetmutex(bdb);
2007       if(enc && dec) tcbdbsetcodecfunc(bdb, enc, encop, dec, decop);
2008       tcbdbsetcache(bdb, tdb->lcnum, tdb->ncnum);
2009       tcbdbsetxmsiz(bdb, tchdbxmsiz(tdb->hdb));
2010       tcbdbsetdfunit(bdb, tchdbdfunit(tdb->hdb));
2011       tcbdbsetlsmax(bdb, TDBIDXLSMAX);
2012       if(tcbdbopen(bdb, ipath, bomode)){
2013         idxs[inum].name = tcstrdup(name);
2014         idxs[inum].type = TDBITLEXICAL;
2015         if(!strcmp(ep, "dec")){
2016           idxs[inum].type = TDBITDECIMAL;
2017         } else if(!strcmp(ep, "tok")){
2018           idxs[inum].type = TDBITTOKEN;
2019         } else if(!strcmp(ep, "qgr")){
2020           idxs[inum].type = TDBITQGRAM;
2021         }
2022         idxs[inum].db = bdb;
2023         idxs[inum].cc = NULL;
2024         if(idxs[inum].type == TDBITTOKEN){
2025           idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
2026         } else if(idxs[inum].type == TDBITQGRAM){
2027           idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
2028         }
2029         inum++;
2030       } else {
2031         tcbdbdel(bdb);
2032       }
2033     }
2034     TCFREE(name);
2035     TCFREE(stem);
2036   }
2037   tclistdel(paths);
2038   TCFREE(tpath);
2039   tdb->inum = inum;
2040   tdb->open = true;
2041   uint8_t hopts = tchdbopts(tdb->hdb);
2042   uint8_t opts = 0;
2043   if(hopts & HDBTLARGE) opts |= TDBTLARGE;
2044   if(hopts & HDBTDEFLATE) opts |= TDBTDEFLATE;
2045   if(hopts & HDBTBZIP) opts |= TDBTBZIP;
2046   if(hopts & HDBTTCBS) opts |= TDBTTCBS;
2047   if(hopts & HDBTEXCODEC) opts |= TDBTEXCODEC;
2048   tdb->opts = opts;
2049   tdb->tran = false;
2050   return true;
2051 }
2052 
2053 
2054 /* Close a table database object.
2055    `tdb' specifies the table database object.
2056    If successful, the return value is true, else, it is false. */
tctdbcloseimpl(TCTDB * tdb)2057 static bool tctdbcloseimpl(TCTDB *tdb){
2058   assert(tdb);
2059   bool err = false;
2060   if(tdb->tran && !tctdbtranabortimpl(tdb)) err = true;
2061   TDBIDX *idxs = tdb->idxs;
2062   int inum = tdb->inum;
2063   for(int i = 0; i < inum; i++){
2064     TDBIDX *idx = idxs + i;
2065     switch(idx->type){
2066       case TDBITTOKEN:
2067       case TDBITQGRAM:
2068         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2069         tcmapdel(idx->cc);
2070         break;
2071     }
2072   }
2073   for(int i = 0; i < inum; i++){
2074     TDBIDX *idx = idxs + i;
2075     switch(idx->type){
2076       case TDBITLEXICAL:
2077       case TDBITDECIMAL:
2078       case TDBITTOKEN:
2079       case TDBITQGRAM:
2080         if(!tcbdbclose(idx->db)){
2081           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2082           err = true;
2083         }
2084         tcbdbdel(idx->db);
2085         break;
2086     }
2087     TCFREE(idx->name);
2088   }
2089   TCFREE(idxs);
2090   if(!tchdbclose(tdb->hdb)) err = true;
2091   tdb->open = false;
2092   return !err;
2093 }
2094 
2095 
2096 /* Store a record into a table database object.
2097    `tdb' specifies the table database object.
2098    `pkbuf' specifies the pointer to the region of the primary key.
2099    `pksiz' specifies the size of the region of the primary key.
2100    `cols' specifies a map object containing columns.
2101    `dmode' specifies behavior when the key overlaps.
2102    If successful, the return value is true, else, it is false. */
tctdbputimpl(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols,int dmode)2103 static bool tctdbputimpl(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols, int dmode){
2104   assert(tdb && pkbuf && pksiz >= 0 && cols);
2105   bool err = false;
2106   int osiz;
2107   char *obuf = tchdbget(tdb->hdb, pkbuf, pksiz, &osiz);
2108   if(obuf){
2109     if(dmode == TDBPDKEEP){
2110       tctdbsetecode(tdb, TCEKEEP, __FILE__, __LINE__, __func__);
2111       TCFREE(obuf);
2112       return false;
2113     }
2114     TCMAP *ocols = tcmapload(obuf, osiz);
2115     if(dmode == TDBPDCAT){
2116       TCMAP *ncols = tcmapnew2(TCMAPRNUM(cols) + 1);
2117       tcmapiterinit(cols);
2118       const char *kbuf;
2119       int ksiz;
2120       while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
2121         int vsiz;
2122         const char *vbuf = tcmapiterval(kbuf, &vsiz);
2123         if(tcmapputkeep(ocols, kbuf, ksiz, vbuf, vsiz)) tcmapput(ncols, kbuf, ksiz, vbuf, vsiz);
2124       }
2125       if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true;
2126       tcmapdel(ncols);
2127       int csiz;
2128       char *cbuf = tcmapdump(ocols, &csiz);
2129       if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
2130       TCFREE(cbuf);
2131     } else {
2132       TCMAP *ncols = tcmapnew2(TCMAPRNUM(cols) + 1);
2133       tcmapiterinit(cols);
2134       const char *kbuf;
2135       int ksiz;
2136       while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
2137         int vsiz;
2138         const char *vbuf = tcmapiterval(kbuf, &vsiz);
2139         int osiz;
2140         const char *obuf = tcmapget(ocols, kbuf, ksiz, &osiz);
2141         if(obuf && osiz == vsiz && !memcmp(obuf, vbuf, osiz)){
2142           tcmapout(ocols, kbuf, ksiz);
2143         } else {
2144           tcmapput(ncols, kbuf, ksiz, vbuf, vsiz);
2145         }
2146       }
2147       if(!tctdbidxout(tdb, pkbuf, pksiz, ocols)) err = true;
2148       if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true;
2149       tcmapdel(ncols);
2150       int csiz;
2151       char *cbuf = tcmapdump(cols, &csiz);
2152       if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
2153       TCFREE(cbuf);
2154     }
2155     tcmapdel(ocols);
2156     TCFREE(obuf);
2157   } else {
2158     if(!tctdbidxput(tdb, pkbuf, pksiz, cols)) err = true;
2159     int csiz;
2160     char *cbuf = tcmapdump(cols, &csiz);
2161     if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
2162     TCFREE(cbuf);
2163   }
2164   return !err;
2165 }
2166 
2167 
2168 /* Remove a record of a table database object.
2169    `tdb' specifies the table database object.
2170    `pkbuf' specifies the pointer to the region of the primary key.
2171    `pksiz' specifies the size of the region of the primary key.
2172    If successful, the return value is true, else, it is false. */
tctdboutimpl(TCTDB * tdb,const char * pkbuf,int pksiz)2173 static bool tctdboutimpl(TCTDB *tdb, const char *pkbuf, int pksiz){
2174   assert(tdb && pkbuf && pksiz >= 0);
2175   int csiz;
2176   char *cbuf = tchdbget(tdb->hdb, pkbuf, pksiz, &csiz);
2177   if(!cbuf) return false;
2178   bool err = false;
2179   TCMAP *cols = tcmapload(cbuf, csiz);
2180   if(!tctdbidxout(tdb, pkbuf, pksiz, cols)) err = true;
2181   if(!tchdbout(tdb->hdb, pkbuf, pksiz)) err = true;
2182   tcmapdel(cols);
2183   TCFREE(cbuf);
2184   return !err;
2185 }
2186 
2187 
2188 /* Retrieve a record in a table database object.
2189    `tdb' specifies the table database object.
2190    `pkbuf' specifies the pointer to the region of the primary key.
2191    `pksiz' specifies the size of the region of the primary key.
2192    If successful, the return value is a map object of the columns of the corresponding record. */
tctdbgetimpl(TCTDB * tdb,const void * pkbuf,int pksiz)2193 static TCMAP *tctdbgetimpl(TCTDB *tdb, const void *pkbuf, int pksiz){
2194   assert(tdb && pkbuf && pksiz >= 0);
2195   int csiz;
2196   char *cbuf = tchdbget(tdb->hdb, pkbuf, pksiz, &csiz);
2197   if(!cbuf) return NULL;
2198   TCMAP *cols = tcmapload(cbuf, csiz);
2199   TCFREE(cbuf);
2200   return cols;
2201 }
2202 
2203 
2204 /* Retrieve the value of a column of a record in a table database object.
2205    `tdb' specifies the table database object.
2206    `pkbuf' specifies the pointer to the region of the primary key.
2207    `pksiz' specifies the size of the region of the primary key.
2208    `nbuf' specifies the pointer to the region of the column name.
2209    `nsiz' specifies the size of the region of the column name.
2210    `sp' specifies the pointer to the variable into which the size of the region of the return
2211    value is assigned.
2212    If successful, the return value is the pointer to the region of the value of the column of the
2213    corresponding record. */
tctdbgetonecol(TCTDB * tdb,const void * pkbuf,int pksiz,const void * nbuf,int nsiz,int * sp)2214 static char *tctdbgetonecol(TCTDB *tdb, const void *pkbuf, int pksiz,
2215                             const void *nbuf, int nsiz, int *sp){
2216   assert(tdb && pkbuf && pksiz >= 0 && nbuf && nsiz >= 0 && sp);
2217   int csiz;
2218   char *cbuf = tchdbget(tdb->hdb, pkbuf, pksiz, &csiz);
2219   if(!cbuf) return NULL;
2220   void *rv = tcmaploadone(cbuf, csiz, nbuf, nsiz, sp);
2221   TCFREE(cbuf);
2222   return rv;
2223 }
2224 
2225 
2226 /* Add a real number to a column of a record in a table database object.
2227    `tdb' specifies the table database object.
2228    `pkbuf' specifies the pointer to the region of the primary key.
2229    `pksiz' specifies the size of the region of the primary key.
2230    `num' specifies the additional value.
2231    If successful, the return value is the summation value, else, it is Not-a-Number. */
tctdbaddnumber(TCTDB * tdb,const void * pkbuf,int pksiz,double num)2232 static double tctdbaddnumber(TCTDB *tdb, const void *pkbuf, int pksiz, double num){
2233   assert(tdb && pkbuf && pksiz >= 0);
2234   int csiz;
2235   char *cbuf = tchdbget(tdb->hdb, pkbuf, pksiz, &csiz);
2236   TCMAP *cols = cbuf ? tcmapload(cbuf, csiz) : tcmapnew2(1);
2237   if(cbuf){
2238     const char *vbuf = tcmapget2(cols, TDBNUMCNTCOL);
2239     if(vbuf) num += tctdbatof(vbuf);
2240     TCFREE(cbuf);
2241   }
2242   char numbuf[TDBCOLBUFSIZ];
2243   int len = snprintf(numbuf, TDBCOLBUFSIZ - 1, "%f", num);
2244   if(len > TDBCOLBUFSIZ - 1){
2245     tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
2246     num = nan("");
2247   } else {
2248     while(--len > 0){
2249       if(numbuf[len] != '0') break;
2250       numbuf[len] = '\0';
2251     }
2252     if(numbuf[len] == '.') numbuf[len] = '\0';
2253     tcmapput2(cols, TDBNUMCNTCOL, numbuf);
2254     if(!tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER)) num = nan("");
2255   }
2256   tcmapdel(cols);
2257 
2258   return num;
2259 }
2260 
2261 
2262 /* Optimize the file of a table database object.
2263    `tdb' specifies the table database object.
2264    `bnum' specifies the number of elements of the bucket array.
2265    `apow' specifies the size of record alignment by power of 2.
2266    `fpow' specifies the maximum number of elements of the free block pool by power of 2.
2267    `opts' specifies options by bitwise-or.
2268    If successful, the return value is true, else, it is false. */
tctdboptimizeimpl(TCTDB * tdb,int64_t bnum,int8_t apow,int8_t fpow,uint8_t opts)2269 static bool tctdboptimizeimpl(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
2270   assert(tdb);
2271   bool err = false;
2272   TCHDB *hdb = tdb->hdb;
2273   TDBIDX *idxs = tdb->idxs;
2274   int inum = tdb->inum;
2275   for(int i = 0; i < inum; i++){
2276     TDBIDX *idx = idxs + i;
2277     switch(idx->type){
2278       case TDBITTOKEN:
2279       case TDBITQGRAM:
2280         tcmapclear(idx->cc);
2281         break;
2282     }
2283   }
2284   for(int i = 0; i < inum; i++){
2285     TDBIDX *idx = idxs + i;
2286     switch(idx->type){
2287       case TDBITLEXICAL:
2288       case TDBITDECIMAL:
2289       case TDBITTOKEN:
2290       case TDBITQGRAM:
2291         if(!tcbdbvanish(idx->db)){
2292           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2293           err = true;
2294         }
2295         break;
2296     }
2297   }
2298   const char *path = tchdbpath(tdb->hdb);
2299   char *tpath = tcsprintf("%s%ctmp%c%llu", path, MYEXTCHR, MYEXTCHR, tchdbinode(tdb->hdb));
2300   TCHDB *thdb = tchdbnew();
2301   tchdbsettype(thdb, TCDBTTABLE);
2302   int dbgfd = tchdbdbgfd(tdb->hdb);
2303   if(dbgfd >= 0) tchdbsetdbgfd(thdb, dbgfd);
2304   TCCODEC enc, dec;
2305   void *encop, *decop;
2306   tchdbcodecfunc(hdb, &enc, &encop, &dec, &decop);
2307   if(enc && dec) tchdbsetcodecfunc(thdb, enc, encop, dec, decop);
2308   if(bnum < 1) bnum = tchdbrnum(hdb) * 2 + 1;
2309   if(apow < 0) apow = tclog2l(tchdbalign(hdb));
2310   if(fpow < 0) fpow = tclog2l(tchdbfbpmax(hdb));
2311   if(opts == UINT8_MAX) opts = tdb->opts;
2312   uint8_t hopts = 0;
2313   if(opts & TDBTLARGE) hopts |= HDBTLARGE;
2314   if(opts & TDBTDEFLATE) hopts |= HDBTDEFLATE;
2315   if(opts & TDBTBZIP) hopts |= HDBTBZIP;
2316   if(opts & TDBTTCBS) hopts |= HDBTTCBS;
2317   if(opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
2318   tchdbtune(thdb, bnum, apow, fpow, hopts);
2319   if(tchdbopen(thdb, tpath, HDBOWRITER | HDBOCREAT | HDBOTRUNC)){
2320     memcpy(tchdbopaque(thdb), tchdbopaque(hdb), TDBOPAQUESIZ + TDBLEFTOPQSIZ);
2321     if(!tchdbiterinit(hdb)) err = true;
2322     TCXSTR *kxstr = tcxstrnew();
2323     TCXSTR *vxstr = tcxstrnew();
2324     while(tchdbiternext3(hdb, kxstr, vxstr)){
2325       TCMAP *cols = tcmapload(TCXSTRPTR(vxstr), TCXSTRSIZE(vxstr));
2326       if(!tctdbidxput(tdb, TCXSTRPTR(kxstr), TCXSTRSIZE(kxstr), cols)) err = true;
2327       tcmapdel(cols);
2328       if(!tchdbput(thdb, TCXSTRPTR(kxstr), TCXSTRSIZE(kxstr),
2329                    TCXSTRPTR(vxstr), TCXSTRSIZE(vxstr))){
2330         tctdbsetecode(tdb, tchdbecode(thdb), __FILE__, __LINE__, __func__);
2331         err = true;
2332       }
2333     }
2334     tcxstrdel(vxstr);
2335     tcxstrdel(kxstr);
2336     if(!tchdbclose(thdb)){
2337       tctdbsetecode(tdb, tchdbecode(thdb), __FILE__, __LINE__, __func__);
2338       err = true;
2339     }
2340     if(!err){
2341       if(unlink(path) == -1){
2342         tctdbsetecode(tdb, TCEUNLINK, __FILE__, __LINE__, __func__);
2343         err = true;
2344       }
2345       if(rename(tpath, path) == -1){
2346         tctdbsetecode(tdb, TCERENAME, __FILE__, __LINE__, __func__);
2347         err = true;
2348       }
2349       char *npath = tcstrdup(path);
2350       int omode = (tchdbomode(hdb) & ~HDBOCREAT) & ~HDBOTRUNC;
2351       if(!tchdbclose(hdb)) err = true;
2352       if(!tchdbopen(hdb, npath, omode)) err = true;
2353       TCFREE(npath);
2354     }
2355   } else {
2356     tctdbsetecode(tdb, tchdbecode(thdb), __FILE__, __LINE__, __func__);
2357     err = true;
2358   }
2359   tchdbdel(thdb);
2360   TCFREE(tpath);
2361   for(int i = 0; i < inum; i++){
2362     TDBIDX *idx = idxs + i;
2363     switch(idx->type){
2364       case TDBITTOKEN:
2365       case TDBITQGRAM:
2366         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2367         break;
2368     }
2369   }
2370   for(int i = 0; i < inum; i++){
2371     TDBIDX *idx = idxs + i;
2372     switch(idx->type){
2373       case TDBITLEXICAL:
2374       case TDBITDECIMAL:
2375       case TDBITTOKEN:
2376       case TDBITQGRAM:
2377         if(!tcbdboptimize(idx->db, -1, -1, -1, -1, -1, UINT8_MAX)){
2378           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2379           err = true;
2380         }
2381         break;
2382     }
2383   }
2384   return !err;
2385 }
2386 
2387 
2388 /* Remove all records of a table database object.
2389    `tdb' specifies the table database object.
2390    If successful, the return value is true, else, it is false. */
tctdbvanishimpl(TCTDB * tdb)2391 static bool tctdbvanishimpl(TCTDB *tdb){
2392   assert(tdb);
2393   bool err = false;
2394   if(!tchdbvanish(tdb->hdb)) err = true;
2395   TDBIDX *idxs = tdb->idxs;
2396   int inum = tdb->inum;
2397   for(int i = 0; i < inum; i++){
2398     TDBIDX *idx = idxs + i;
2399     switch(idx->type){
2400       case TDBITTOKEN:
2401       case TDBITQGRAM:
2402         tcmapclear(idx->cc);
2403         break;
2404     }
2405   }
2406   for(int i = 0; i < inum; i++){
2407     TDBIDX *idx = idxs + i;
2408     switch(idx->type){
2409       case TDBITLEXICAL:
2410       case TDBITDECIMAL:
2411       case TDBITTOKEN:
2412       case TDBITQGRAM:
2413         if(!tcbdbvanish(idx->db)){
2414           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2415           err = true;
2416         }
2417         break;
2418     }
2419   }
2420   return !err;
2421 }
2422 
2423 
2424 /* Copy the database file of a table database object.
2425    `tdb' specifies the table database object.
2426    `path' specifies the path of the destination file.
2427    If successful, the return value is true, else, it is false. */
tctdbcopyimpl(TCTDB * tdb,const char * path)2428 static bool tctdbcopyimpl(TCTDB *tdb, const char *path){
2429   assert(tdb);
2430   bool err = false;
2431   if(!tchdbcopy(tdb->hdb, path)) err = true;
2432   const char *opath = tchdbpath(tdb->hdb);
2433   TDBIDX *idxs = tdb->idxs;
2434   int inum = tdb->inum;
2435   for(int i = 0; i < inum; i++){
2436     TDBIDX *idx = idxs + i;
2437     switch(idx->type){
2438       case TDBITTOKEN:
2439       case TDBITQGRAM:
2440         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2441         break;
2442     }
2443   }
2444   for(int i = 0; i < inum; i++){
2445     TDBIDX *idx = idxs + i;
2446     const char *ipath;
2447     switch(idx->type){
2448       case TDBITLEXICAL:
2449       case TDBITDECIMAL:
2450       case TDBITTOKEN:
2451       case TDBITQGRAM:
2452         if(*path == '@'){
2453           if(!tcbdbcopy(idx->db, path)){
2454             tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2455             err = true;
2456           }
2457         } else {
2458           ipath = tcbdbpath(idx->db);
2459           if(tcstrfwm(ipath, opath)){
2460             char *tpath = tcsprintf("%s%s", path, ipath + strlen(opath));
2461             if(!tcbdbcopy(idx->db, tpath)){
2462               tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2463               err = true;
2464             }
2465             TCFREE(tpath);
2466           } else {
2467             tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
2468             err = true;
2469           }
2470         }
2471         break;
2472     }
2473   }
2474   return !err;
2475 }
2476 
2477 
2478 /* Begin the transaction of a table database object.
2479    `tdb' specifies the table database object.
2480    If successful, the return value is true, else, it is false. */
tctdbtranbeginimpl(TCTDB * tdb)2481 static bool tctdbtranbeginimpl(TCTDB *tdb){
2482   assert(tdb);
2483   if(!tctdbmemsync(tdb, false)) return false;
2484   if(!tchdbtranbegin(tdb->hdb)) return false;
2485   bool err = false;
2486   TDBIDX *idxs = tdb->idxs;
2487   int inum = tdb->inum;
2488   for(int i = 0; i < inum; i++){
2489     TDBIDX *idx = idxs + i;
2490     switch(idx->type){
2491       case TDBITTOKEN:
2492       case TDBITQGRAM:
2493         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2494         break;
2495     }
2496   }
2497   for(int i = 0; i < inum; i++){
2498     TDBIDX *idx = idxs + i;
2499     switch(idx->type){
2500       case TDBITLEXICAL:
2501       case TDBITDECIMAL:
2502       case TDBITTOKEN:
2503       case TDBITQGRAM:
2504         if(!tcbdbtranbegin(idx->db)){
2505           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2506           err = true;
2507         }
2508         break;
2509     }
2510   }
2511   return !err;
2512 }
2513 
2514 
2515 /* Commit the transaction of a table database object.
2516    `tdb' specifies the table database object.
2517    If successful, the return value is true, else, it is false. */
tctdbtrancommitimpl(TCTDB * tdb)2518 static bool tctdbtrancommitimpl(TCTDB *tdb){
2519   assert(tdb);
2520   bool err = false;
2521   if(!tctdbmemsync(tdb, false)) err = true;
2522   if(!tchdbtrancommit(tdb->hdb)) err = true;
2523   TDBIDX *idxs = tdb->idxs;
2524   int inum = tdb->inum;
2525   for(int i = 0; i < inum; i++){
2526     TDBIDX *idx = idxs + i;
2527     switch(idx->type){
2528       case TDBITTOKEN:
2529       case TDBITQGRAM:
2530         if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2531         break;
2532     }
2533   }
2534   for(int i = 0; i < inum; i++){
2535     TDBIDX *idx = idxs + i;
2536     switch(idx->type){
2537       case TDBITLEXICAL:
2538       case TDBITDECIMAL:
2539       case TDBITTOKEN:
2540       case TDBITQGRAM:
2541         if(!tcbdbtrancommit(idx->db)){
2542           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2543           err = true;
2544         }
2545         break;
2546     }
2547   }
2548   return !err;
2549 }
2550 
2551 
2552 /* Abort the transaction of a table database object.
2553    `tdb' specifies the table database object.
2554    If successful, the return value is true, else, it is false. */
tctdbtranabortimpl(TCTDB * tdb)2555 static bool tctdbtranabortimpl(TCTDB *tdb){
2556   assert(tdb);
2557   bool err = false;
2558   if(!tchdbtranabort(tdb->hdb)) err = true;
2559   TDBIDX *idxs = tdb->idxs;
2560   int inum = tdb->inum;
2561   for(int i = 0; i < inum; i++){
2562     TDBIDX *idx = idxs + i;
2563     switch(idx->type){
2564       case TDBITTOKEN:
2565       case TDBITQGRAM:
2566         tcmapclear(idx->cc);
2567         break;
2568     }
2569   }
2570   for(int i = 0; i < inum; i++){
2571     TDBIDX *idx = idxs + i;
2572     switch(idx->type){
2573       case TDBITLEXICAL:
2574       case TDBITDECIMAL:
2575       case TDBITTOKEN:
2576       case TDBITQGRAM:
2577         if(!tcbdbtranabort(idx->db)){
2578           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2579           err = true;
2580         }
2581         break;
2582     }
2583   }
2584   return !err;
2585 }
2586 
2587 
2588 /* Set a column index to a table database object.
2589    `tdb' specifies the table database object. connected as a writer.
2590    `name' specifies the name of a column.
2591    `type' specifies the index type.
2592    If successful, the return value is true, else, it is false. */
tctdbsetindeximpl(TCTDB * tdb,const char * name,int type)2593 static bool tctdbsetindeximpl(TCTDB *tdb, const char *name, int type){
2594   assert(tdb && name);
2595   bool err = false;
2596   bool keep = false;
2597   if(type & TDBITKEEP){
2598     type &= ~TDBITKEEP;
2599     keep = true;
2600   }
2601   bool done = false;
2602   TDBIDX *idxs = tdb->idxs;
2603   int inum = tdb->inum;
2604   for(int i = 0; i < inum; i++){
2605     TDBIDX *idx = idxs + i;
2606     const char *path;
2607     if(!strcmp(idx->name, name)){
2608       if(keep){
2609         tctdbsetecode(tdb, TCEKEEP, __FILE__, __LINE__, __func__);
2610         return false;
2611       }
2612       if(type == TDBITOPT){
2613         switch(idx->type){
2614           case TDBITTOKEN:
2615           case TDBITQGRAM:
2616             if(!tctdbidxsyncicc(tdb, idx, true)) err = true;
2617             break;
2618         }
2619         switch(idx->type){
2620           case TDBITLEXICAL:
2621           case TDBITDECIMAL:
2622           case TDBITTOKEN:
2623           case TDBITQGRAM:
2624             if(!tcbdboptimize(idx->db, -1, -1, -1, -1, -1, UINT8_MAX)){
2625               tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2626               err = true;
2627             }
2628             break;
2629         }
2630         done = true;
2631         break;
2632       }
2633       switch(idx->type){
2634         case TDBITTOKEN:
2635         case TDBITQGRAM:
2636           tcmapdel(idx->cc);
2637           break;
2638       }
2639       switch(idx->type){
2640         case TDBITLEXICAL:
2641         case TDBITDECIMAL:
2642         case TDBITTOKEN:
2643         case TDBITQGRAM:
2644           path = tcbdbpath(idx->db);
2645           if(path && unlink(path)){
2646             tctdbsetecode(tdb, TCEUNLINK, __FILE__, __LINE__, __func__);
2647             err = true;
2648           }
2649           tcbdbdel(idx->db);
2650           break;
2651       }
2652       TCFREE(idx->name);
2653       tdb->inum--;
2654       inum = tdb->inum;
2655       memmove(idxs + i, idxs + i + 1, sizeof(*idxs) * (inum - i));
2656       done = true;
2657       break;
2658     }
2659   }
2660   if(type == TDBITOPT || type == TDBITVOID){
2661     if(!done){
2662       tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
2663       err = true;
2664     }
2665     return !err;
2666   }
2667   TCXSTR *pbuf = tcxstrnew();
2668   tcxstrprintf(pbuf, "%s%c%s%c%?", tchdbpath(tdb->hdb), MYEXTCHR, TDBIDXSUFFIX, MYEXTCHR, name);
2669   TCREALLOC(tdb->idxs, tdb->idxs, sizeof(tdb->idxs[0]) * (inum + 1));
2670   TDBIDX *idx = tdb->idxs + inum;
2671   int homode = tchdbomode(tdb->hdb);
2672   int bomode = BDBOWRITER | BDBOCREAT | BDBOTRUNC;
2673   if(homode & HDBONOLCK) bomode |= BDBONOLCK;
2674   if(homode & HDBOLCKNB) bomode |= BDBOLCKNB;
2675   if(homode & HDBOTSYNC) bomode |= BDBOTSYNC;
2676   int dbgfd = tchdbdbgfd(tdb->hdb);
2677   TCCODEC enc, dec;
2678   void *encop, *decop;
2679   tchdbcodecfunc(tdb->hdb, &enc, &encop, &dec, &decop);
2680   int64_t bbnum = (tchdbbnum(tdb->hdb) / TDBIDXLMEMB) * 4 + TDBIDXLMEMB;
2681   int64_t bxmsiz = tchdbxmsiz(tdb->hdb);
2682   uint8_t opts = tdb->opts;
2683   uint8_t bopts = 0;
2684   if(opts & TDBTLARGE) bopts |= BDBTLARGE;
2685   if(opts & TDBTDEFLATE) bopts |= BDBTDEFLATE;
2686   if(opts & TDBTBZIP) bopts |= BDBTBZIP;
2687   if(opts & TDBTTCBS) bopts |= BDBTTCBS;
2688   if(opts & TDBTEXCODEC) bopts |= BDBTEXCODEC;
2689   switch(type){
2690     case TDBITLEXICAL:
2691       idx->db = tcbdbnew();
2692       idx->name = tcstrdup(name);
2693       tcxstrprintf(pbuf, "%clex", MYEXTCHR);
2694       if(dbgfd >= 0) tcbdbsetdbgfd(idx->db, dbgfd);
2695       if(tdb->mmtx) tcbdbsetmutex(idx->db);
2696       if(enc && dec) tcbdbsetcodecfunc(idx->db, enc, encop, dec, decop);
2697       tcbdbtune(idx->db, TDBIDXLMEMB, TDBIDXNMEMB, bbnum, -1, -1, bopts);
2698       tcbdbsetcache(idx->db, tdb->lcnum, tdb->ncnum);
2699       tcbdbsetxmsiz(idx->db, bxmsiz);
2700       tcbdbsetdfunit(idx->db, tchdbdfunit(tdb->hdb));
2701       tcbdbsetlsmax(idx->db, TDBIDXLSMAX);
2702       if(!tcbdbopen(idx->db, TCXSTRPTR(pbuf), bomode)){
2703         tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2704         err = true;
2705       }
2706       tdb->inum++;
2707       break;
2708     case TDBITDECIMAL:
2709       idx->db = tcbdbnew();
2710       idx->name = tcstrdup(name);
2711       tcxstrprintf(pbuf, "%cdec", MYEXTCHR);
2712       if(dbgfd >= 0) tcbdbsetdbgfd(idx->db, dbgfd);
2713       if(tdb->mmtx) tcbdbsetmutex(idx->db);
2714       tcbdbsetcmpfunc(idx->db, tccmpdecimal, NULL);
2715       if(enc && dec) tcbdbsetcodecfunc(idx->db, enc, encop, dec, decop);
2716       tcbdbtune(idx->db, TDBIDXLMEMB, TDBIDXNMEMB, bbnum, -1, -1, bopts);
2717       tcbdbsetcache(idx->db, tdb->lcnum, tdb->ncnum);
2718       tcbdbsetxmsiz(idx->db, bxmsiz);
2719       tcbdbsetdfunit(idx->db, tchdbdfunit(tdb->hdb));
2720       tcbdbsetlsmax(idx->db, TDBIDXLSMAX);
2721       if(!tcbdbopen(idx->db, TCXSTRPTR(pbuf), bomode)){
2722         tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2723         err = true;
2724       }
2725       tdb->inum++;
2726       break;
2727     case TDBITTOKEN:
2728       idx->db = tcbdbnew();
2729       idx->cc = tcmapnew2(TDBIDXICCBNUM);
2730       idx->name = tcstrdup(name);
2731       tcxstrprintf(pbuf, "%ctok", MYEXTCHR);
2732       if(dbgfd >= 0) tcbdbsetdbgfd(idx->db, dbgfd);
2733       if(tdb->mmtx) tcbdbsetmutex(idx->db);
2734       if(enc && dec) tcbdbsetcodecfunc(idx->db, enc, encop, dec, decop);
2735       tcbdbtune(idx->db, TDBIDXLMEMB, TDBIDXNMEMB, bbnum, -1, -1, bopts);
2736       tcbdbsetcache(idx->db, tdb->lcnum, tdb->ncnum);
2737       tcbdbsetxmsiz(idx->db, bxmsiz);
2738       tcbdbsetdfunit(idx->db, tchdbdfunit(tdb->hdb));
2739       tcbdbsetlsmax(idx->db, TDBIDXLSMAX);
2740       if(!tcbdbopen(idx->db, TCXSTRPTR(pbuf), bomode)){
2741         tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2742         err = true;
2743       }
2744       tdb->inum++;
2745       break;
2746     case TDBITQGRAM:
2747       idx->db = tcbdbnew();
2748       idx->cc = tcmapnew2(TDBIDXICCBNUM);
2749       idx->name = tcstrdup(name);
2750       tcxstrprintf(pbuf, "%cqgr", MYEXTCHR);
2751       if(dbgfd >= 0) tcbdbsetdbgfd(idx->db, dbgfd);
2752       if(tdb->mmtx) tcbdbsetmutex(idx->db);
2753       if(enc && dec) tcbdbsetcodecfunc(idx->db, enc, encop, dec, decop);
2754       tcbdbtune(idx->db, TDBIDXLMEMB, TDBIDXNMEMB, bbnum, -1, -1, bopts);
2755       tcbdbsetcache(idx->db, tdb->lcnum, tdb->ncnum);
2756       tcbdbsetxmsiz(idx->db, bxmsiz);
2757       tcbdbsetdfunit(idx->db, tchdbdfunit(tdb->hdb));
2758       tcbdbsetlsmax(idx->db, TDBIDXLSMAX);
2759       if(!tcbdbopen(idx->db, TCXSTRPTR(pbuf), bomode)){
2760         tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
2761         err = true;
2762       }
2763       tdb->inum++;
2764       break;
2765     default:
2766       tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
2767       err = true;
2768       break;
2769   }
2770   idx->type = type;
2771   if(!err){
2772     TCHDB *hdb = tdb->hdb;
2773     if(!tchdbiterinit(hdb)) err = true;
2774     void *db = idx->db;
2775     TCXSTR *kxstr = tcxstrnew();
2776     TCXSTR *vxstr = tcxstrnew();
2777     int nsiz = strlen(name);
2778     while(tchdbiternext3(hdb, kxstr, vxstr)){
2779       if(nsiz < 1){
2780         const char *pkbuf = TCXSTRPTR(kxstr);
2781         int pksiz = TCXSTRSIZE(kxstr);
2782         switch(type){
2783           case TDBITLEXICAL:
2784           case TDBITDECIMAL:
2785             if(!tcbdbput(db, pkbuf, pksiz, pkbuf, pksiz)){
2786               tctdbsetecode(tdb, tcbdbecode(db), __FILE__, __LINE__, __func__);
2787               err = true;
2788             }
2789             break;
2790           case TDBITTOKEN:
2791             if(!tctdbidxputtoken(tdb, idx, pkbuf, pksiz, pkbuf, pksiz)) err = true;
2792             break;
2793           case TDBITQGRAM:
2794             if(!tctdbidxputqgram(tdb, idx, pkbuf, pksiz, pkbuf, pksiz)) err = true;
2795             break;
2796         }
2797       } else {
2798         const char *pkbuf = TCXSTRPTR(kxstr);
2799         int pksiz = TCXSTRSIZE(kxstr);
2800         uint16_t hash = tctdbidxhash(pkbuf, pksiz);
2801         int vsiz;
2802         char *vbuf = tcmaploadone(TCXSTRPTR(vxstr), TCXSTRSIZE(vxstr), name, nsiz, &vsiz);
2803         if(vbuf){
2804           switch(type){
2805             case TDBITLEXICAL:
2806             case TDBITDECIMAL:
2807               if(!tctdbidxputone(tdb, idx, pkbuf, pksiz, hash, vbuf, vsiz)) err = true;
2808               break;
2809             case TDBITTOKEN:
2810               if(!tctdbidxputtoken(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
2811               break;
2812             case TDBITQGRAM:
2813               if(!tctdbidxputqgram(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
2814               break;
2815           }
2816           TCFREE(vbuf);
2817         }
2818       }
2819     }
2820     tcxstrdel(vxstr);
2821     tcxstrdel(kxstr);
2822   }
2823   tcxstrdel(pbuf);
2824   return !err;
2825 }
2826 
2827 
2828 /* Generate a unique ID number.
2829    `tdb' specifies the table database object.
2830    `inc' specifies the increment of the seed.
2831    The return value is the new unique ID number or -1 on failure. */
tctdbgenuidimpl(TCTDB * tdb,int64_t inc)2832 static int64_t tctdbgenuidimpl(TCTDB *tdb, int64_t inc){
2833   assert(tdb);
2834   void *opq = tchdbopaque(tdb->hdb);
2835   uint64_t llnum, uid;
2836   if(inc < 0){
2837     uid = -inc - 1;
2838   } else {
2839     memcpy(&llnum, opq, sizeof(llnum));
2840     if(inc == 0) return TCITOHLL(llnum);
2841     uid = TCITOHLL(llnum) + inc;
2842   }
2843   llnum = TCITOHLL(uid);
2844   memcpy(opq, &llnum, sizeof(llnum));
2845   return uid;
2846 }
2847 
2848 
2849 /* Execute the search of a query object.
2850    `qry' specifies the query object.
2851    The return value is a list object of the primary keys of the corresponding records. */
tctdbqrysearchimpl(TDBQRY * qry)2852 static TCLIST *tctdbqrysearchimpl(TDBQRY *qry){
2853   assert(qry);
2854   TCTDB *tdb = qry->tdb;
2855   TCHDB *hdb = tdb->hdb;
2856   TDBIDX *idxs = tdb->idxs;
2857   int inum = tdb->inum;
2858   TDBCOND *conds = qry->conds;
2859   int cnum = qry->cnum;
2860   int acnum = cnum;
2861   int max = qry->max;
2862   if(max < INT_MAX - qry->skip) max += qry->skip;
2863   const char *oname = qry->oname;
2864   int otype = qry->otype;
2865   TCXSTR *hint = qry->hint;
2866   TCLIST *res = NULL;
2867   for(int i = 0; i < cnum; i++){
2868     TDBCOND *cond = conds + i;
2869     cond->alive = true;
2870   }
2871   tcxstrclear(hint);
2872   bool isord = oname != NULL;
2873   TDBCOND *mcond = NULL;
2874   TDBIDX *midx = NULL;
2875   TDBCOND *ncond = NULL;
2876   TDBIDX *nidx = NULL;
2877   TDBCOND *scond = NULL;
2878   TDBIDX *sidx = NULL;
2879   for(int i = 0; i < cnum; i++){
2880     TDBCOND *cond = conds + i;
2881     if(!cond->sign || cond->noidx) continue;
2882     for(int j = 0; j < inum; j++){
2883       TDBIDX *idx = idxs + j;
2884       if(!strcmp(cond->name, idx->name)){
2885         switch(idx->type){
2886           case TDBITLEXICAL:
2887             switch(cond->op){
2888               case TDBQCSTREQ:
2889               case TDBQCSTRBW:
2890               case TDBQCSTROREQ:
2891                 if(!mcond){
2892                   mcond = cond;
2893                   midx = idx;
2894                 } else if(!ncond){
2895                   ncond = cond;
2896                   nidx = idx;
2897                 }
2898                 break;
2899               default:
2900                 if(!scond){
2901                   scond = cond;
2902                   sidx = idx;
2903                 }
2904                 break;
2905             }
2906             break;
2907           case TDBITDECIMAL:
2908             switch(cond->op){
2909               case TDBQCNUMEQ:
2910               case TDBQCNUMGT:
2911               case TDBQCNUMGE:
2912               case TDBQCNUMLT:
2913               case TDBQCNUMLE:
2914               case TDBQCNUMBT:
2915               case TDBQCNUMOREQ:
2916                 if(!mcond){
2917                   mcond = cond;
2918                   midx = idx;
2919                 } else if(!ncond){
2920                   ncond = cond;
2921                   nidx = idx;
2922                 }
2923                 break;
2924               default:
2925                 if(!scond){
2926                   scond = cond;
2927                   sidx = idx;
2928                 }
2929                 break;
2930             }
2931             break;
2932           case TDBITTOKEN:
2933             switch(cond->op){
2934               case TDBQCSTRAND:
2935               case TDBQCSTROR:
2936                 if(!mcond){
2937                   mcond = cond;
2938                   midx = idx;
2939                 } else if(!ncond){
2940                   ncond = cond;
2941                   nidx = idx;
2942                 }
2943                 break;
2944             }
2945             break;
2946           case TDBITQGRAM:
2947             switch(cond->op){
2948               case TDBQCFTSPH:
2949                 if(!mcond){
2950                   mcond = cond;
2951                   midx = idx;
2952                 } else if(!ncond){
2953                   ncond = cond;
2954                   nidx = idx;
2955                 }
2956                 break;
2957             }
2958             break;
2959         }
2960       }
2961     }
2962   }
2963   if(mcond){
2964     res = tclistnew();
2965     mcond->alive = false;
2966     acnum--;
2967     TCMAP *nmap = NULL;
2968     if(ncond){
2969       ncond->alive = false;
2970       acnum--;
2971       nmap = tctdbqryidxfetch(qry, ncond, nidx);
2972       max = tclmin(max, TCMAPRNUM(nmap));
2973     }
2974     const char *expr = mcond->expr;
2975     int esiz = mcond->esiz;
2976     TDBCOND *ucond = NULL;
2977     for(int i = 0; i < cnum; i++){
2978       TDBCOND *cond = conds + i;
2979       if(!cond->alive) continue;
2980       if(ucond){
2981         ucond = NULL;
2982         break;
2983       }
2984       ucond = cond;
2985     }
2986     bool trim = *midx->name != '\0';
2987     if(mcond->op == TDBQCSTREQ){
2988       tcxstrprintf(hint, "using an index: \"%s\" asc (STREQ)\n", mcond->name);
2989       BDBCUR *cur = tcbdbcurnew(midx->db);
2990       tcbdbcurjump(cur, expr, esiz + trim);
2991       if(oname && !strcmp(oname, mcond->name)) oname = NULL;
2992       bool all = oname != NULL;
2993       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
2994       trim = *midx->name != '\0';
2995       const char *kbuf;
2996       int ksiz;
2997       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
2998         if(trim) ksiz -= 3;
2999         if(ksiz == esiz && !memcmp(kbuf, expr, esiz)){
3000           int vsiz;
3001           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3002           int nsiz;
3003           if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3004             if(acnum < 1){
3005               TCLISTPUSH(res, vbuf, vsiz);
3006             } else if(ucond){
3007               if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3008             } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3009               TCLISTPUSH(res, vbuf, vsiz);
3010             }
3011           }
3012         } else {
3013           break;
3014         }
3015         tcbdbcurnext(cur);
3016       }
3017       tcbdbcurdel(cur);
3018     } else if(mcond->op == TDBQCSTRBW){
3019       tcxstrprintf(hint, "using an index: \"%s\" asc (STRBW)\n", mcond->name);
3020       BDBCUR *cur = tcbdbcurnew(midx->db);
3021       tcbdbcurjump(cur, expr, esiz + trim);
3022       bool all = oname && (strcmp(oname, mcond->name) || otype != TDBQOSTRASC);
3023       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3024       const char *kbuf;
3025       int ksiz;
3026       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3027         if(trim) ksiz -= 3;
3028         if(ksiz >= esiz && !memcmp(kbuf, expr, esiz)){
3029           int vsiz;
3030           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3031           int nsiz;
3032           if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3033             if(acnum < 1){
3034               TCLISTPUSH(res, vbuf, vsiz);
3035             } else if(ucond){
3036               if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3037             } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3038               TCLISTPUSH(res, vbuf, vsiz);
3039             }
3040           }
3041         } else {
3042           break;
3043         }
3044         tcbdbcurnext(cur);
3045       }
3046       tcbdbcurdel(cur);
3047       if(oname && !strcmp(oname, mcond->name)){
3048         if(otype == TDBQOSTRASC){
3049           oname = NULL;
3050         } else if(otype == TDBQOSTRDESC){
3051           tclistinvert(res);
3052           oname = NULL;
3053         }
3054       }
3055     } else if(mcond->op == TDBQCSTROREQ){
3056       tcxstrprintf(hint, "using an index: \"%s\" skip (STROREQ)\n", mcond->name);
3057       BDBCUR *cur = tcbdbcurnew(midx->db);
3058       TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
3059       tclistsort(tokens);
3060       for(int i = 1; i < TCLISTNUM(tokens); i++){
3061         if(!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))){
3062           TCFREE(tclistremove2(tokens, i));
3063           i--;
3064         }
3065       }
3066       if(oname && !strcmp(oname, mcond->name)){
3067         if(otype == TDBQOSTRASC){
3068           oname = NULL;
3069         } else if(otype == TDBQOSTRDESC){
3070           tclistinvert(tokens);
3071           oname = NULL;
3072         }
3073       }
3074       int tnum = TCLISTNUM(tokens);
3075       bool all = oname != NULL;
3076       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3077       for(int i = 0; (all || TCLISTNUM(res) < max) && i < tnum; i++){
3078         const char *token;
3079         int tsiz;
3080         TCLISTVAL(token, tokens, i, tsiz);
3081         if(tsiz < 1) continue;
3082         tcbdbcurjump(cur, token, tsiz + trim);
3083         const char *kbuf;
3084         int ksiz;
3085         while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3086           if(trim) ksiz -= 3;
3087           if(ksiz == tsiz && !memcmp(kbuf, token, tsiz)){
3088             int vsiz;
3089             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3090             int nsiz;
3091             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3092               if(acnum < 1){
3093                 TCLISTPUSH(res, vbuf, vsiz);
3094               } else if(ucond){
3095                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3096               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3097                 TCLISTPUSH(res, vbuf, vsiz);
3098               }
3099             }
3100           } else {
3101             break;
3102           }
3103           tcbdbcurnext(cur);
3104         }
3105       }
3106       tclistdel(tokens);
3107       tcbdbcurdel(cur);
3108     } else if(mcond->op == TDBQCNUMEQ){
3109       tcxstrprintf(hint, "using an index: \"%s\" asc (NUMEQ)\n", mcond->name);
3110       BDBCUR *cur = tcbdbcurnew(midx->db);
3111       if(oname && !strcmp(oname, mcond->name)) oname = NULL;
3112       long double xnum = tctdbatof(expr);
3113       tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3114       bool all = oname != NULL;
3115       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3116       const char *kbuf;
3117       int ksiz;
3118       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3119         if(tctdbatof(kbuf) == xnum){
3120           int vsiz;
3121           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3122           int nsiz;
3123           if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3124             if(acnum < 1){
3125               TCLISTPUSH(res, vbuf, vsiz);
3126             } else if(ucond){
3127               if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3128             } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3129               TCLISTPUSH(res, vbuf, vsiz);
3130             }
3131           }
3132         } else {
3133           break;
3134         }
3135         tcbdbcurnext(cur);
3136       }
3137       tcbdbcurdel(cur);
3138     } else if(mcond->op == TDBQCNUMGT || mcond->op == TDBQCNUMGE){
3139       if(oname && !strcmp(oname, mcond->name) && otype == TDBQONUMDESC){
3140         tcxstrprintf(hint, "using an index: \"%s\" desc (NUMGT/NUMGE)\n", mcond->name);
3141         long double xnum = tctdbatof(expr);
3142         BDBCUR *cur = tcbdbcurnew(midx->db);
3143         tcbdbcurlast(cur);
3144         if(max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3145         const char *kbuf;
3146         int ksiz;
3147         while(TCLISTNUM(res) < max && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3148           long double knum = tctdbatof(kbuf);
3149           if(knum < xnum) break;
3150           if(knum > xnum || (knum >= xnum && mcond->op == TDBQCNUMGE)){
3151             int vsiz;
3152             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3153             int nsiz;
3154             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3155               if(acnum < 1){
3156                 TCLISTPUSH(res, vbuf, vsiz);
3157               } else if(ucond){
3158                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3159               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3160                 TCLISTPUSH(res, vbuf, vsiz);
3161               }
3162             }
3163           }
3164           tcbdbcurprev(cur);
3165         }
3166         tcbdbcurdel(cur);
3167         oname = NULL;
3168       } else {
3169         tcxstrprintf(hint, "using an index: \"%s\" asc (NUMGT/NUMGE)\n", mcond->name);
3170         long double xnum = tctdbatof(expr);
3171         BDBCUR *cur = tcbdbcurnew(midx->db);
3172         tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3173         bool all = oname && (strcmp(oname, mcond->name) || otype != TDBQONUMASC);
3174         if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3175         const char *kbuf;
3176         int ksiz;
3177         while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3178           long double knum = tctdbatof(kbuf);
3179           if(knum > xnum || (knum >= xnum && mcond->op == TDBQCNUMGE)){
3180             int vsiz;
3181             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3182             int nsiz;
3183             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3184               if(acnum < 1){
3185                 TCLISTPUSH(res, vbuf, vsiz);
3186               } else if(ucond){
3187                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3188               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3189                 TCLISTPUSH(res, vbuf, vsiz);
3190               }
3191             }
3192           }
3193           tcbdbcurnext(cur);
3194         }
3195         tcbdbcurdel(cur);
3196         if(!all) oname = NULL;
3197       }
3198     } else if(mcond->op == TDBQCNUMLT || mcond->op == TDBQCNUMLE){
3199       if(oname && !strcmp(oname, mcond->name) && otype == TDBQONUMASC){
3200         tcxstrprintf(hint, "using an index: \"%s\" asc (NUMLT/NUMLE)\n", mcond->name);
3201         long double xnum = tctdbatof(expr);
3202         BDBCUR *cur = tcbdbcurnew(midx->db);
3203         tcbdbcurfirst(cur);
3204         if(max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3205         const char *kbuf;
3206         int ksiz;
3207         while(TCLISTNUM(res) < max && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3208           long double knum = tctdbatof(kbuf);
3209           if(knum > xnum) break;
3210           if(knum < xnum || (knum <= xnum && mcond->op == TDBQCNUMLE)){
3211             int vsiz;
3212             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3213             int nsiz;
3214             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3215               if(acnum < 1){
3216                 TCLISTPUSH(res, vbuf, vsiz);
3217               } else if(ucond){
3218                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3219               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3220                 TCLISTPUSH(res, vbuf, vsiz);
3221               }
3222             }
3223           }
3224           tcbdbcurnext(cur);
3225         }
3226         tcbdbcurdel(cur);
3227         oname = NULL;
3228       } else {
3229         tcxstrprintf(hint, "using an index: \"%s\" desc (NUMLT/NUMLE)\n", mcond->name);
3230         long double xnum = tctdbatof(expr);
3231         BDBCUR *cur = tcbdbcurnew(midx->db);
3232         tctdbqryidxcurjumpnum(cur, expr, esiz, false);
3233         bool all = oname && (strcmp(oname, mcond->name) || otype != TDBQONUMDESC);
3234         if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3235         const char *kbuf;
3236         int ksiz;
3237         while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3238           long double knum = tctdbatof(kbuf);
3239           if(knum < xnum || (knum <= xnum && mcond->op == TDBQCNUMLE)){
3240             int vsiz;
3241             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3242             int nsiz;
3243             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3244               if(acnum < 1){
3245                 TCLISTPUSH(res, vbuf, vsiz);
3246               } else if(ucond){
3247                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3248               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3249                 TCLISTPUSH(res, vbuf, vsiz);
3250               }
3251             }
3252           }
3253           tcbdbcurprev(cur);
3254         }
3255         tcbdbcurdel(cur);
3256         if(!all) oname = NULL;
3257       }
3258     } else if(mcond->op == TDBQCNUMBT){
3259       tcxstrprintf(hint, "using an index: \"%s\" asc (NUMBT)\n", mcond->name);
3260       while(*expr == ' ' || *expr == ','){
3261         expr++;
3262       }
3263       const char *pv = expr;
3264       while(*pv != '\0' && *pv != ' ' && *pv != ','){
3265         pv++;
3266       }
3267       esiz = pv - expr;
3268       if(*pv != ' ' && *pv != ',') pv = " ";
3269       pv++;
3270       while(*pv == ' ' || *pv == ','){
3271         pv++;
3272       }
3273       long double lower = tctdbatof(expr);
3274       long double upper = tctdbatof(pv);
3275       if(lower > upper){
3276         expr = pv;
3277         esiz = strlen(expr);
3278         long double swap = lower;
3279         lower = upper;
3280         upper = swap;
3281       }
3282       BDBCUR *cur = tcbdbcurnew(midx->db);
3283       tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3284       bool all = oname && (strcmp(oname, mcond->name) || otype != TDBQONUMASC);
3285       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3286       const char *kbuf;
3287       int ksiz;
3288       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3289         if(tctdbatof(kbuf) > upper) break;
3290         int vsiz;
3291         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3292         int nsiz;
3293         if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3294           if(acnum < 1){
3295             TCLISTPUSH(res, vbuf, vsiz);
3296           } else if(ucond){
3297             if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3298           } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3299             TCLISTPUSH(res, vbuf, vsiz);
3300           }
3301         }
3302         tcbdbcurnext(cur);
3303       }
3304       tcbdbcurdel(cur);
3305       if(oname && !strcmp(oname, mcond->name)){
3306         if(otype == TDBQONUMASC){
3307           oname = NULL;
3308         } else if(otype == TDBQONUMDESC){
3309           tclistinvert(res);
3310           oname = NULL;
3311         }
3312       }
3313     } else if(mcond->op == TDBQCNUMOREQ){
3314       tcxstrprintf(hint, "using an index: \"%s\" skip (NUMOREQ)\n", mcond->name);
3315       BDBCUR *cur = tcbdbcurnew(midx->db);
3316       TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
3317       tclistsortex(tokens, tdbcmppkeynumasc);
3318       for(int i = 1; i < TCLISTNUM(tokens); i++){
3319         if(tctdbatof(TCLISTVALPTR(tokens, i)) == tctdbatof(TCLISTVALPTR(tokens, i - 1))){
3320           TCFREE(tclistremove2(tokens, i));
3321           i--;
3322         }
3323       }
3324       if(oname && !strcmp(oname, mcond->name)){
3325         if(otype == TDBQONUMASC){
3326           oname = NULL;
3327         } else if(otype == TDBQONUMDESC){
3328           tclistinvert(tokens);
3329           oname = NULL;
3330         }
3331       }
3332       int tnum = TCLISTNUM(tokens);
3333       bool all = oname != NULL;
3334       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3335       for(int i = 0; (all || TCLISTNUM(res) < max) && i < tnum; i++){
3336         const char *token;
3337         int tsiz;
3338         TCLISTVAL(token, tokens, i, tsiz);
3339         if(tsiz < 1) continue;
3340         long double xnum = tctdbatof(token);
3341         tctdbqryidxcurjumpnum(cur, token, tsiz, true);
3342         const char *kbuf;
3343         int ksiz;
3344         while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3345           if(tctdbatof(kbuf) == xnum){
3346             int vsiz;
3347             const char *vbuf = tcbdbcurval3(cur, &vsiz);
3348             int nsiz;
3349             if(!nmap || tcmapget(nmap, vbuf, vsiz, &nsiz)){
3350               if(acnum < 1){
3351                 TCLISTPUSH(res, vbuf, vsiz);
3352               } else if(ucond){
3353                 if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3354               } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3355                 TCLISTPUSH(res, vbuf, vsiz);
3356               }
3357             }
3358           } else {
3359             break;
3360           }
3361           tcbdbcurnext(cur);
3362         }
3363       }
3364       tclistdel(tokens);
3365       tcbdbcurdel(cur);
3366     } else if(mcond->op == TDBQCSTRAND || mcond->op == TDBQCSTROR){
3367       tcxstrprintf(hint, "using an index: \"%s\" inverted (%s)\n",
3368                    mcond->name, mcond->op == TDBQCSTRAND ? "STRAND" : "STROR");
3369       bool all = oname != NULL;
3370       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3371       TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
3372       tclistsort(tokens);
3373       for(int i = 1; i < TCLISTNUM(tokens); i++){
3374         if(!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))){
3375           TCFREE(tclistremove2(tokens, i));
3376           i--;
3377         }
3378       }
3379       TCMAP *tres = tctdbidxgetbytokens(tdb, midx, tokens, mcond->op, hint);
3380       tcmapiterinit(tres);
3381       const char *kbuf;
3382       int ksiz;
3383       while((all || TCLISTNUM(res) < max) && (kbuf = tcmapiternext(tres, &ksiz)) != NULL){
3384         int nsiz;
3385         if(!nmap || tcmapget(nmap, kbuf, ksiz, &nsiz)){
3386           if(acnum < 1){
3387             TCLISTPUSH(res, kbuf, ksiz);
3388           } else if(ucond){
3389             if(tctdbqryonecondmatch(qry, ucond, kbuf, ksiz)) TCLISTPUSH(res, kbuf, ksiz);
3390           } else if(tctdbqryallcondmatch(qry, kbuf, ksiz)){
3391             TCLISTPUSH(res, kbuf, ksiz);
3392           }
3393         }
3394       }
3395       tcmapdel(tres);
3396       tclistdel(tokens);
3397     } else if(mcond->op == TDBQCFTSPH){
3398       tcxstrprintf(hint, "using an index: \"%s\" inverted (FTS)\n", mcond->name);
3399       TCMAP *tres = tctdbidxgetbyfts(tdb, midx, mcond, hint);
3400       bool all = oname != NULL;
3401       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3402       tcmapiterinit(tres);
3403       const char *kbuf;
3404       int ksiz;
3405       while((all || TCLISTNUM(res) < max) && (kbuf = tcmapiternext(tres, &ksiz)) != NULL){
3406         int nsiz;
3407         if(!nmap || tcmapget(nmap, kbuf, ksiz, &nsiz)){
3408           if(acnum < 1){
3409             TCLISTPUSH(res, kbuf, ksiz);
3410           } else if(ucond){
3411             if(tctdbqryonecondmatch(qry, ucond, kbuf, ksiz)) TCLISTPUSH(res, kbuf, ksiz);
3412           } else if(tctdbqryallcondmatch(qry, kbuf, ksiz)){
3413             TCLISTPUSH(res, kbuf, ksiz);
3414           }
3415         }
3416       }
3417       tcmapdel(tres);
3418     }
3419     if(nmap) tcmapdel(nmap);
3420   }
3421   if(!res && scond){
3422     res = tclistnew();
3423     scond->alive = false;
3424     acnum--;
3425     TDBCOND *ucond = NULL;
3426     for(int i = 0; i < cnum; i++){
3427       TDBCOND *cond = conds + i;
3428       if(!cond->alive) continue;
3429       if(ucond){
3430         ucond = NULL;
3431         break;
3432       }
3433       ucond = cond;
3434     }
3435     bool trim = *sidx->name != '\0';
3436     bool asc = true;
3437     bool all = true;
3438     if(!oname){
3439       all = false;
3440     } else if(!strcmp(oname, scond->name)){
3441       switch(sidx->type){
3442         case TDBITLEXICAL:
3443           switch(otype){
3444             case TDBQOSTRASC:
3445               asc = true;
3446               all = false;
3447               break;
3448             case TDBQOSTRDESC:
3449               asc = false;
3450               all = false;
3451               break;
3452           }
3453           break;
3454         case TDBITDECIMAL:
3455           switch(otype){
3456             case TDBQONUMASC:
3457               asc = true;
3458               all = false;
3459               break;
3460             case TDBQONUMDESC:
3461               asc = false;
3462               all = false;
3463               break;
3464           }
3465           break;
3466       }
3467     }
3468     if(asc){
3469       tcxstrprintf(hint, "using an index: \"%s\" asc (all)\n", scond->name);
3470       BDBCUR *cur = tcbdbcurnew(sidx->db);
3471       tcbdbcurfirst(cur);
3472       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3473       const char *kbuf;
3474       int ksiz;
3475       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3476         if(trim) ksiz -= 3;
3477         if(ksiz < 0) break;
3478         if(tctdbqrycondmatch(scond, kbuf, ksiz)){
3479           int vsiz;
3480           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3481           if(acnum < 1){
3482             TCLISTPUSH(res, vbuf, vsiz);
3483           } else if(ucond){
3484             if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3485           } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3486             TCLISTPUSH(res, vbuf, vsiz);
3487           }
3488         }
3489         tcbdbcurnext(cur);
3490       }
3491       tcbdbcurdel(cur);
3492     } else {
3493       tcxstrprintf(hint, "using an index: \"%s\" desc (all)\n", scond->name);
3494       BDBCUR *cur = tcbdbcurnew(sidx->db);
3495       tcbdbcurlast(cur);
3496       if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3497       const char *kbuf;
3498       int ksiz;
3499       while((all || TCLISTNUM(res) < max) && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3500         if(trim) ksiz -= 3;
3501         if(ksiz < 0) break;
3502         if(tctdbqrycondmatch(scond, kbuf, ksiz)){
3503           int vsiz;
3504           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3505           if(acnum < 1){
3506             TCLISTPUSH(res, vbuf, vsiz);
3507           } else if(ucond){
3508             if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3509           } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3510             TCLISTPUSH(res, vbuf, vsiz);
3511           }
3512         }
3513         tcbdbcurprev(cur);
3514       }
3515       tcbdbcurdel(cur);
3516     }
3517     if(!all) oname = NULL;
3518   }
3519   if(!res && oname && max < tchdbrnum(hdb) * TDBORDRATIO){
3520     TDBIDX *oidx = NULL;
3521     bool asc = true;
3522     for(int i = 0; !oidx && i < inum; i++){
3523       TDBIDX *idx = idxs + i;
3524       if(strcmp(idx->name, oname)) continue;
3525       switch(idx->type){
3526         case TDBITLEXICAL:
3527           switch(otype){
3528             case TDBQOSTRASC:
3529               oidx = idx;
3530               asc = true;
3531               break;
3532             case TDBQOSTRDESC:
3533               oidx = idx;
3534               asc = false;
3535               break;
3536           }
3537           break;
3538         case TDBITDECIMAL:
3539           switch(otype){
3540             case TDBQONUMASC:
3541               oidx = idx;
3542               asc = true;
3543               break;
3544             case TDBQONUMDESC:
3545               oidx = idx;
3546               asc = false;
3547               break;
3548           }
3549           break;
3550       }
3551     }
3552     if(oidx){
3553       res = tclistnew();
3554       TDBCOND *ucond = NULL;
3555       for(int i = 0; i < cnum; i++){
3556         TDBCOND *cond = conds + i;
3557         if(!cond->alive) continue;
3558         if(ucond){
3559           ucond = NULL;
3560           break;
3561         }
3562         ucond = cond;
3563       }
3564       bool trim = *oidx->name != '\0';
3565       if(asc){
3566         tcxstrprintf(hint, "using an index: \"%s\" asc (order)\n", oname);
3567         BDBCUR *cur = tcbdbcurnew(oidx->db);
3568         tcbdbcurfirst(cur);
3569         tcxstrprintf(hint, "limited matching: %d\n", max);
3570         const char *kbuf;
3571         int ksiz;
3572         while(TCLISTNUM(res) < max && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3573           if(trim) ksiz -= 3;
3574           if(ksiz < 0) break;
3575           int vsiz;
3576           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3577           if(acnum < 1){
3578             TCLISTPUSH(res, vbuf, vsiz);
3579           } else if(ucond){
3580             if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3581           } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3582             TCLISTPUSH(res, vbuf, vsiz);
3583           }
3584           tcbdbcurnext(cur);
3585         }
3586         tcbdbcurdel(cur);
3587       } else {
3588         tcxstrprintf(hint, "using an index: \"%s\" desc (order)\n", oname);
3589         BDBCUR *cur = tcbdbcurnew(oidx->db);
3590         tcbdbcurlast(cur);
3591         tcxstrprintf(hint, "limited matching: %d\n", max);
3592         const char *kbuf;
3593         int ksiz;
3594         while(TCLISTNUM(res) < max && (kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3595           if(trim) ksiz -= 3;
3596           if(ksiz < 0) break;
3597           int vsiz;
3598           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3599           if(acnum < 1){
3600             TCLISTPUSH(res, vbuf, vsiz);
3601           } else if(ucond){
3602             if(tctdbqryonecondmatch(qry, ucond, vbuf, vsiz)) TCLISTPUSH(res, vbuf, vsiz);
3603           } else if(tctdbqryallcondmatch(qry, vbuf, vsiz)){
3604             TCLISTPUSH(res, vbuf, vsiz);
3605           }
3606           tcbdbcurprev(cur);
3607         }
3608         tcbdbcurdel(cur);
3609       }
3610       int rnum = TCLISTNUM(res);
3611       if(rnum >= max || tcbdbrnum(oidx->db) >= tchdbrnum(hdb)){
3612         oname = NULL;
3613       } else {
3614         tcxstrprintf(hint, "abort the result: %d\n", rnum);
3615         tclistdel(res);
3616         res = NULL;
3617       }
3618     }
3619   }
3620   if(!res){
3621     tcxstrprintf(hint, "scanning the whole table\n");
3622     res = tclistnew();
3623     TDBCOND *ucond = NULL;
3624     for(int i = 0; i < cnum; i++){
3625       TDBCOND *cond = conds + i;
3626       if(!cond->alive) continue;
3627       if(ucond){
3628         ucond = NULL;
3629         break;
3630       }
3631       ucond = cond;
3632     }
3633     char *lkbuf = NULL;
3634     int lksiz = 0;
3635     char *pkbuf;
3636     int pksiz;
3637     const char *cbuf;
3638     int csiz;
3639     bool all = oname != NULL;
3640     if(!all && max < INT_MAX) tcxstrprintf(hint, "limited matching: %d\n", max);
3641     while((all || TCLISTNUM(res) < max) &&
3642           (pkbuf = tchdbgetnext3(hdb, lkbuf, lksiz, &pksiz, &cbuf, &csiz)) != NULL){
3643       if(ucond){
3644         if(ucond->nsiz < 1){
3645           char *tkbuf;
3646           TCMEMDUP(tkbuf, pkbuf, pksiz);
3647           if(tctdbqrycondmatch(ucond, tkbuf, pksiz) == ucond->sign)
3648             TCLISTPUSH(res, pkbuf, pksiz);
3649           TCFREE(tkbuf);
3650         } else {
3651           int vsiz;
3652           char *vbuf = tcmaploadone(cbuf, csiz, ucond->name, ucond->nsiz, &vsiz);
3653           if(vbuf){
3654             if(tctdbqrycondmatch(ucond, vbuf, vsiz) == ucond->sign)
3655               TCLISTPUSH(res, pkbuf, pksiz);
3656             TCFREE(vbuf);
3657           } else {
3658             if(!ucond->sign) TCLISTPUSH(res, pkbuf, pksiz);
3659           }
3660         }
3661       } else {
3662         TCMAP *cols = tcmapload(cbuf, csiz);
3663         bool ok = true;
3664         for(int i = 0; i < cnum; i++){
3665           TDBCOND *cond = conds + i;
3666           if(cond->nsiz < 1){
3667             char *tkbuf;
3668             TCMEMDUP(tkbuf, pkbuf, pksiz);
3669             if(tctdbqrycondmatch(cond, tkbuf, pksiz) != cond->sign){
3670               TCFREE(tkbuf);
3671               ok = false;
3672               break;
3673             }
3674             TCFREE(tkbuf);
3675           } else {
3676             int vsiz;
3677             const char *vbuf = tcmapget(cols, cond->name, cond->nsiz, &vsiz);
3678             if(vbuf){
3679               if(tctdbqrycondmatch(cond, vbuf, vsiz) != cond->sign){
3680                 ok = false;
3681                 break;
3682               }
3683             } else {
3684               if(cond->sign){
3685                 ok = false;
3686                 break;
3687               }
3688             }
3689           }
3690         }
3691         if(ok) TCLISTPUSH(res, pkbuf, pksiz);
3692         tcmapdel(cols);
3693       }
3694       TCFREE(lkbuf);
3695       lkbuf = pkbuf;
3696       lksiz = pksiz;
3697     }
3698     TCFREE(lkbuf);
3699   }
3700   int rnum = TCLISTNUM(res);
3701   tcxstrprintf(hint, "result set size: %d\n", rnum);
3702   if(oname){
3703     if(*oname == '\0'){
3704       tcxstrprintf(hint, "sorting the result set: \"%s\"\n", oname);
3705       switch(otype){
3706         case TDBQOSTRASC:
3707           tclistsort(res);
3708           break;
3709         case TDBQOSTRDESC:
3710           tclistsort(res);
3711           tclistinvert(res);
3712           break;
3713         case TDBQONUMASC:
3714           tclistsortex(res, tdbcmppkeynumasc);
3715           break;
3716         case TDBQONUMDESC:
3717           tclistsortex(res, tdbcmppkeynumdesc);
3718           break;
3719       }
3720     } else {
3721       tcxstrprintf(hint, "sorting the result set: \"%s\"\n", oname);
3722       TDBSORTREC *keys;
3723       TCMALLOC(keys, sizeof(*keys) * rnum + 1);
3724       int onsiz = strlen(oname);
3725       for(int i = 0; i < rnum; i++){
3726         TDBSORTREC *key = keys + i;
3727         const char *kbuf;
3728         int ksiz;
3729         TCLISTVAL(kbuf, res, i, ksiz);
3730         char *vbuf = NULL;
3731         int vsiz = 0;
3732         int csiz;
3733         char *cbuf = tchdbget(hdb, kbuf, ksiz, &csiz);
3734         if(cbuf){
3735           vbuf = tcmaploadone(cbuf, csiz, oname, onsiz, &vsiz);
3736           TCFREE(cbuf);
3737         }
3738         key->kbuf = kbuf;
3739         key->ksiz = ksiz;
3740         key->vbuf = vbuf;
3741         key->vsiz = vsiz;
3742       }
3743       int (*compar)(const TDBSORTREC *a, const TDBSORTREC *b) = NULL;
3744       switch(otype){
3745         case TDBQOSTRASC:
3746           compar = tdbcmpsortrecstrasc;
3747           break;
3748         case TDBQOSTRDESC:
3749           compar = tdbcmpsortrecstrdesc;
3750           break;
3751         case TDBQONUMASC:
3752           compar = tdbcmpsortrecnumasc;
3753           break;
3754         case TDBQONUMDESC:
3755           compar = tdbcmpsortrecnumdesc;
3756           break;
3757       }
3758       if(compar){
3759         if(max <= rnum / 16){
3760           tctopsort(keys, rnum, sizeof(*keys), max, (int (*)(const void *, const void *))compar);
3761         } else {
3762           qsort(keys, rnum, sizeof(*keys), (int (*)(const void *, const void *))compar);
3763         }
3764       }
3765       TCLIST *nres = tclistnew2(rnum);
3766       for(int i = 0; i < rnum; i++){
3767         TDBSORTREC *key = keys + i;
3768         TCLISTPUSH(nres, key->kbuf, key->ksiz);
3769         TCFREE(key->vbuf);
3770       }
3771       tclistdel(res);
3772       res = nres;
3773       TCFREE(keys);
3774     }
3775   } else if(isord){
3776     tcxstrprintf(hint, "leaving the index order\n");
3777   } else {
3778     tcxstrprintf(hint, "leaving the natural order\n");
3779   }
3780   if(qry->skip > 0){
3781     int left = tclmin(TCLISTNUM(res), qry->skip);
3782     while(left-- > 0){
3783       int rsiz;
3784       TCFREE(tclistshift(res, &rsiz));
3785     }
3786     max -= qry->skip;
3787   }
3788   if(TCLISTNUM(res) > max){
3789     int left = TCLISTNUM(res) - max;
3790     while(left-- > 0){
3791       int rsiz;
3792       TCFREE(tclistpop(res, &rsiz));
3793     }
3794   }
3795   qry->count = TCLISTNUM(res);
3796   return res;
3797 }
3798 
3799 
3800 /* Fetch record keys from an index matching to a condition.
3801    `qry' specifies the query object.
3802    `cond' specifies the condition object.
3803    `idx' specifies an index object.
3804    The return value is a map object containing primary keys of the corresponding records. */
tctdbqryidxfetch(TDBQRY * qry,TDBCOND * cond,TDBIDX * idx)3805 static TCMAP *tctdbqryidxfetch(TDBQRY *qry, TDBCOND *cond, TDBIDX *idx){
3806   assert(qry && cond && idx);
3807   TCTDB *tdb = qry->tdb;
3808   TCHDB *hdb = tdb->hdb;
3809   TCXSTR *hint = qry->hint;
3810   const char *expr = cond->expr;
3811   int esiz = cond->esiz;
3812   bool trim = *idx->name != '\0';
3813   TCMAP *nmap = tcmapnew2(tclmin(TDBDEFBNUM, tchdbrnum(hdb)) / 4 + 1);
3814   if(cond->op == TDBQCSTREQ){
3815     tcxstrprintf(hint, "using an auxiliary index: \"%s\" one (STREQ)\n", cond->name);
3816     BDBCUR *cur = tcbdbcurnew(idx->db);
3817     tcbdbcurjump(cur, expr, esiz + trim);
3818     const char *kbuf;
3819     int ksiz;
3820     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3821       if(trim) ksiz -= 3;
3822       if(ksiz == esiz && !memcmp(kbuf, expr, esiz)){
3823         int vsiz;
3824         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3825         tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3826       } else {
3827         break;
3828       }
3829       tcbdbcurnext(cur);
3830     }
3831     tcbdbcurdel(cur);
3832   } else if(cond->op == TDBQCSTRBW){
3833     tcxstrprintf(hint, "using an auxiliary index: \"%s\" asc (STRBW)\n", cond->name);
3834     BDBCUR *cur = tcbdbcurnew(idx->db);
3835     tcbdbcurjump(cur, expr, esiz + trim);
3836     const char *kbuf;
3837     int ksiz;
3838     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3839       if(trim) ksiz -= 3;
3840       if(ksiz >= esiz && !memcmp(kbuf, expr, esiz)){
3841         int vsiz;
3842         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3843         tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3844       } else {
3845         break;
3846       }
3847       tcbdbcurnext(cur);
3848     }
3849     tcbdbcurdel(cur);
3850   } else if(cond->op == TDBQCSTROREQ){
3851     tcxstrprintf(hint, "using an auxiliary index: \"%s\" skip (STROREQ)\n", cond->name);
3852     TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
3853     tclistsort(tokens);
3854     for(int i = 1; i < TCLISTNUM(tokens); i++){
3855       if(!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))){
3856         TCFREE(tclistremove2(tokens, i));
3857         i--;
3858       }
3859     }
3860     int tnum = TCLISTNUM(tokens);
3861     for(int i = 0; i < tnum; i++){
3862       const char *token;
3863       int tsiz;
3864       TCLISTVAL(token, tokens, i, tsiz);
3865       if(tsiz < 1) continue;
3866       BDBCUR *cur = tcbdbcurnew(idx->db);
3867       tcbdbcurjump(cur, token, tsiz + trim);
3868       const char *kbuf;
3869       int ksiz;
3870       while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3871         if(trim) ksiz -= 3;
3872         if(ksiz == tsiz && !memcmp(kbuf, token, tsiz)){
3873           int vsiz;
3874           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3875           tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3876         } else {
3877           break;
3878         }
3879         tcbdbcurnext(cur);
3880       }
3881       tcbdbcurdel(cur);
3882     }
3883     tclistdel(tokens);
3884   } else if(cond->op == TDBQCNUMEQ){
3885     tcxstrprintf(hint, "using an auxiliary index: \"%s\" asc (NUMEQ)\n", cond->name);
3886     long double xnum = tctdbatof(expr);
3887     BDBCUR *cur = tcbdbcurnew(idx->db);
3888     tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3889     const char *kbuf;
3890     int ksiz;
3891     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3892       if(tctdbatof(kbuf) == xnum){
3893         int vsiz;
3894         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3895         tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3896       } else {
3897         break;
3898       }
3899       tcbdbcurnext(cur);
3900     }
3901     tcbdbcurdel(cur);
3902   } else if(cond->op == TDBQCNUMGT || cond->op == TDBQCNUMGE){
3903     tcxstrprintf(hint, "using an auxiliary index: \"%s\" asc (NUMGT/NUMGE)\n", cond->name);
3904     long double xnum = tctdbatof(expr);
3905     BDBCUR *cur = tcbdbcurnew(idx->db);
3906     tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3907     const char *kbuf;
3908     int ksiz;
3909     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3910       long double knum = tctdbatof(kbuf);
3911       if(knum > xnum || (knum >= xnum && cond->op == TDBQCNUMGE)){
3912         int vsiz;
3913         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3914         tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3915       }
3916       tcbdbcurnext(cur);
3917     }
3918     tcbdbcurdel(cur);
3919   } else if(cond->op == TDBQCNUMLT || cond->op == TDBQCNUMLE){
3920     tcxstrprintf(hint, "using an auxiliary index: \"%s\" desc (NUMLT/NUMLE)\n", cond->name);
3921     long double xnum = tctdbatof(expr);
3922     BDBCUR *cur = tcbdbcurnew(idx->db);
3923     tctdbqryidxcurjumpnum(cur, expr, esiz, false);
3924     const char *kbuf;
3925     int ksiz;
3926     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3927       long double knum = tctdbatof(kbuf);
3928       if(knum < xnum || (knum <= xnum && cond->op == TDBQCNUMLE)){
3929         int vsiz;
3930         const char *vbuf = tcbdbcurval3(cur, &vsiz);
3931         tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3932       }
3933       tcbdbcurprev(cur);
3934     }
3935     tcbdbcurdel(cur);
3936   } else if(cond->op == TDBQCNUMBT){
3937     tcxstrprintf(hint, "using an auxiliary index: \"%s\" asc (NUMBT)\n", cond->name);
3938     while(*expr == ' ' || *expr == ','){
3939       expr++;
3940     }
3941     const char *pv = expr;
3942     while(*pv != '\0' && *pv != ' ' && *pv != ','){
3943       pv++;
3944     }
3945     esiz = pv - expr;
3946     if(*pv != ' ' && *pv != ',') pv = " ";
3947     pv++;
3948     while(*pv == ' ' || *pv == ','){
3949       pv++;
3950     }
3951     long double lower = tctdbatof(expr);
3952     long double upper = tctdbatof(pv);
3953     if(lower > upper){
3954       expr = pv;
3955       esiz = strlen(expr);
3956       long double swap = lower;
3957       lower = upper;
3958       upper = swap;
3959     }
3960     BDBCUR *cur = tcbdbcurnew(idx->db);
3961     tctdbqryidxcurjumpnum(cur, expr, esiz, true);
3962     const char *kbuf;
3963     int ksiz;
3964     while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3965       if(tctdbatof(kbuf) > upper) break;
3966       int vsiz;
3967       const char *vbuf = tcbdbcurval3(cur, &vsiz);
3968       tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3969       tcbdbcurnext(cur);
3970     }
3971     tcbdbcurdel(cur);
3972   } else if(cond->op == TDBQCNUMOREQ){
3973     tcxstrprintf(hint, "using an auxiliary index: \"%s\" skip (NUMOREQ)\n", cond->name);
3974     BDBCUR *cur = tcbdbcurnew(idx->db);
3975     TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
3976     tclistsortex(tokens, tdbcmppkeynumasc);
3977     for(int i = 1; i < TCLISTNUM(tokens); i++){
3978       if(tctdbatof(TCLISTVALPTR(tokens, i)) == tctdbatof(TCLISTVALPTR(tokens, i - 1))){
3979         TCFREE(tclistremove2(tokens, i));
3980         i--;
3981       }
3982     }
3983     int tnum = TCLISTNUM(tokens);
3984     for(int i = 0; i < tnum; i++){
3985       const char *token;
3986       int tsiz;
3987       TCLISTVAL(token, tokens, i, tsiz);
3988       if(tsiz < 1) continue;
3989       long double xnum = tctdbatof(token);
3990       tctdbqryidxcurjumpnum(cur, token, tsiz, true);
3991       const char *kbuf;
3992       int ksiz;
3993       while((kbuf = tcbdbcurkey3(cur, &ksiz)) != NULL){
3994         if(tctdbatof(kbuf) == xnum){
3995           int vsiz;
3996           const char *vbuf = tcbdbcurval3(cur, &vsiz);
3997           tcmapputkeep(nmap, vbuf, vsiz, "", 0);
3998         } else {
3999           break;
4000         }
4001         tcbdbcurnext(cur);
4002       }
4003     }
4004     tclistdel(tokens);
4005     tcbdbcurdel(cur);
4006   } else if(cond->op == TDBQCSTRAND || cond->op == TDBQCSTROR){
4007     tcxstrprintf(hint, "using an auxiliary index: \"%s\" inverted (%s)\n",
4008                  cond->name, cond->op == TDBQCSTRAND ? "STRAND" : "STROR");
4009     TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
4010     tclistsort(tokens);
4011     for(int i = 1; i < TCLISTNUM(tokens); i++){
4012       if(!strcmp(TCLISTVALPTR(tokens, i), TCLISTVALPTR(tokens, i - 1))){
4013         TCFREE(tclistremove2(tokens, i));
4014         i--;
4015       }
4016     }
4017     tcmapdel(nmap);
4018     nmap = tctdbidxgetbytokens(tdb, idx, tokens, cond->op, hint);
4019     tclistdel(tokens);
4020   } else if(cond->op == TDBQCFTSPH){
4021     tcxstrprintf(hint, "using an auxiliary index: \"%s\" inverted (FTS)\n", cond->name);
4022     tcmapdel(nmap);
4023     nmap = tctdbidxgetbyfts(tdb, idx, cond, hint);
4024   }
4025   tcxstrprintf(hint, "auxiliary result set size: %lld\n", (long long)TCMAPRNUM(nmap));
4026   return nmap;
4027 }
4028 
4029 
4030 /* Convert a string to a real number.
4031    `str' specifies the string.
4032    The return value is the real number. */
tctdbatof(const char * str)4033 static long double tctdbatof(const char *str){
4034   assert(str);
4035   while(*str > '\0' && *str <= ' '){
4036     str++;
4037   }
4038   int sign = 1;
4039   if(*str == '-'){
4040     str++;
4041     sign = -1;
4042   } else if(*str == '+'){
4043     str++;
4044   }
4045   if(tcstrifwm(str, "inf")) return HUGE_VALL * sign;
4046   if(tcstrifwm(str, "nan")) return nanl("");
4047   long double num = 0;
4048   int col = 0;
4049   while(*str != '\0'){
4050     if(*str < '0' || *str > '9') break;
4051     num = num * 10 + *str - '0';
4052     str++;
4053     if(num > 0) col++;
4054   }
4055   if(*str == '.'){
4056     str++;
4057     long double fract = 0.0;
4058     long double base = 10;
4059     while(col < TDBNUMCOLMAX && *str != '\0'){
4060       if(*str < '0' || *str > '9') break;
4061       fract += (*str - '0') / base;
4062       str++;
4063       col++;
4064       base *= 10;
4065     }
4066     num += fract;
4067   }
4068   return num * sign;
4069 }
4070 
4071 
4072 /* Jump a cursor to the record of a key.
4073    `cur' specifies the cursor object.
4074    `expr' specifies the expression.
4075    `esiz' specifies the size of the expression.
4076    `first' specifies whether to jump the first candidate.
4077    If successful, the return value is true, else, it is false. */
tctdbqryidxcurjumpnum(BDBCUR * cur,const char * expr,int esiz,bool first)4078 static bool tctdbqryidxcurjumpnum(BDBCUR *cur, const char *expr, int esiz, bool first){
4079   assert(cur && expr && esiz >= 0);
4080   char stack[TCNUMBUFSIZ], *rbuf;
4081   if(esiz < sizeof(stack)){
4082     rbuf = stack;
4083   } else {
4084     TCMALLOC(rbuf, esiz + 1);
4085   }
4086   rbuf[0] = first ? 0x00 : 0x7f;
4087   memcpy(rbuf + 1, expr, esiz);
4088   bool err = false;
4089   if(first){
4090     if(!tcbdbcurjump(cur, rbuf, esiz + 1)) err = true;
4091   } else {
4092     if(!tcbdbcurjumpback(cur, rbuf, esiz + 1)) err = true;
4093   }
4094   if(rbuf != stack) TCFREE(rbuf);
4095   return !err;
4096 }
4097 
4098 
4099 /* Check matching of one condition and a record.
4100    `qry' specifies the query object.
4101    `cond' specifies the condition object.
4102    `pkbuf' specifies the pointer to the region of the primary key.
4103    `pksiz' specifies the size of the region of the primary key.
4104    If they matches, the return value is true, else it is false. */
tctdbqryonecondmatch(TDBQRY * qry,TDBCOND * cond,const char * pkbuf,int pksiz)4105 static bool tctdbqryonecondmatch(TDBQRY *qry, TDBCOND *cond, const char *pkbuf, int pksiz){
4106   assert(qry && cond && pkbuf && pksiz >= 0);
4107   if(cond->nsiz < 1) return tctdbqrycondmatch(cond, pkbuf, pksiz) == cond->sign;
4108   int csiz;
4109   char *cbuf = tchdbget(qry->tdb->hdb, pkbuf, pksiz, &csiz);
4110   if(!cbuf) return false;
4111   bool rv;
4112   int vsiz;
4113   char *vbuf = tcmaploadone(cbuf, csiz, cond->name, cond->nsiz, &vsiz);
4114   if(vbuf){
4115     rv = tctdbqrycondmatch(cond, vbuf, vsiz) == cond->sign;
4116     TCFREE(vbuf);
4117   } else {
4118     rv = !cond->sign;
4119   }
4120   TCFREE(cbuf);
4121   return rv;
4122 }
4123 
4124 
4125 /* Check matching of all conditions and a record.
4126    `qry' specifies the query object.
4127    `pkbuf' specifies the pointer to the region of the primary key.
4128    `pksiz' specifies the size of the region of the primary key.
4129    If they matches, the return value is true, else it is false. */
tctdbqryallcondmatch(TDBQRY * qry,const char * pkbuf,int pksiz)4130 static bool tctdbqryallcondmatch(TDBQRY *qry, const char *pkbuf, int pksiz){
4131   assert(qry && pkbuf && pksiz >= 0);
4132   TCTDB *tdb = qry->tdb;
4133   TDBCOND *conds = qry->conds;
4134   int cnum = qry->cnum;
4135   int csiz;
4136   char *cbuf = tchdbget(tdb->hdb, pkbuf, pksiz, &csiz);
4137   if(!cbuf) return false;
4138   TCMAP *cols = tcmapload(cbuf, csiz);
4139   bool ok = true;
4140   for(int i = 0; i < cnum; i++){
4141     TDBCOND *cond = conds + i;
4142     if(!cond->alive) continue;
4143     if(cond->nsiz < 1){
4144       if(tctdbqrycondmatch(cond, pkbuf, pksiz) != cond->sign){
4145         ok = false;
4146         break;
4147       }
4148     } else {
4149       int vsiz;
4150       const char *vbuf = tcmapget(cols, cond->name, cond->nsiz, &vsiz);
4151       if(vbuf){
4152         if(tctdbqrycondmatch(cond, vbuf, vsiz) != cond->sign){
4153           ok = false;
4154           break;
4155         }
4156       } else {
4157         if(cond->sign){
4158           ok = false;
4159           break;
4160         }
4161       }
4162     }
4163   }
4164   tcmapdel(cols);
4165   TCFREE(cbuf);
4166   return ok;
4167 }
4168 
4169 
4170 /* Check matching of a operand expression and a column value.
4171    `cond' specifies the condition object.
4172    `vbuf' specifies the column value.
4173    `vsiz' specifies the size of the column value.
4174    If they matches, the return value is true, else it is false. */
tctdbqrycondmatch(TDBCOND * cond,const char * vbuf,int vsiz)4175 static bool tctdbqrycondmatch(TDBCOND *cond, const char *vbuf, int vsiz){
4176   assert(cond && vbuf && vsiz >= 0);
4177   bool hit = false;
4178   switch(cond->op){
4179     case TDBQCSTREQ:
4180       hit = vsiz == cond->esiz && !memcmp(vbuf, cond->expr, cond->esiz);
4181       break;
4182     case TDBQCSTRINC:
4183       hit = strstr(vbuf, cond->expr) != NULL;
4184       break;
4185     case TDBQCSTRBW:
4186       hit = tcstrfwm(vbuf, cond->expr);
4187       break;
4188     case TDBQCSTREW:
4189       hit = tcstrbwm(vbuf, cond->expr);
4190       break;
4191     case TDBQCSTRAND:
4192       hit = tctdbqrycondcheckstrand(vbuf, cond->expr);
4193       break;
4194     case TDBQCSTROR:
4195       hit = tctdbqrycondcheckstror(vbuf, cond->expr);
4196       break;
4197     case TDBQCSTROREQ:
4198       hit = tctdbqrycondcheckstroreq(vbuf, cond->expr);
4199       break;
4200     case TDBQCSTRRX:
4201       hit = cond->regex && regexec(cond->regex, vbuf, 0, NULL, 0) == 0;
4202       break;
4203     case TDBQCNUMEQ:
4204       hit = tctdbatof(vbuf) == tctdbatof(cond->expr);
4205       break;
4206     case TDBQCNUMGT:
4207       hit = tctdbatof(vbuf) > tctdbatof(cond->expr);
4208       break;
4209     case TDBQCNUMGE:
4210       hit = tctdbatof(vbuf) >= tctdbatof(cond->expr);
4211       break;
4212     case TDBQCNUMLT:
4213       hit = tctdbatof(vbuf) < tctdbatof(cond->expr);
4214       break;
4215     case TDBQCNUMLE:
4216       hit = tctdbatof(vbuf) <= tctdbatof(cond->expr);
4217       break;
4218     case TDBQCNUMBT:
4219       hit = tctdbqrycondchecknumbt(vbuf, cond->expr);
4220       break;
4221     case TDBQCNUMOREQ:
4222       hit = tctdbqrycondchecknumoreq(vbuf, cond->expr);
4223       break;
4224     case TDBQCFTSPH:
4225       hit = tctdbqrycondcheckfts(vbuf, vsiz, cond);
4226       break;
4227   }
4228   return hit;
4229 }
4230 
4231 
4232 /* Check whether a string includes all tokens in another string.
4233    `vbuf' specifies the column value.
4234    `expr' specifies the operand expression.
4235    If they matches, the return value is true, else it is false. */
tctdbqrycondcheckstrand(const char * vbuf,const char * expr)4236 static bool tctdbqrycondcheckstrand(const char *vbuf, const char *expr){
4237   assert(vbuf && expr);
4238   const unsigned char *sp = (unsigned char *)expr;
4239   while(*sp != '\0'){
4240     while((*sp != '\0' && *sp <= ' ') || *sp == ','){
4241       sp++;
4242     }
4243     const unsigned char *ep = sp;
4244     while(*ep > ' ' && *ep != ','){
4245       ep++;
4246     }
4247     if(ep > sp){
4248       bool hit = false;
4249       const unsigned char *rp = (unsigned char *)vbuf;
4250       while(*rp != '\0'){
4251         const unsigned char *pp;
4252         for(pp = sp; pp < ep; pp++, rp++){
4253           if(*pp != *rp) break;
4254         }
4255         if(pp == ep && (*rp <= ' ' || *rp == ',')){
4256           hit = true;
4257           break;
4258         }
4259         while(*rp > ' ' && *rp != ','){
4260           rp++;
4261         }
4262         while((*rp != '\0' && *rp <= ' ') || *rp == ','){
4263           rp++;
4264         }
4265       }
4266       if(!hit) return false;
4267     }
4268     sp = ep;
4269   }
4270   return true;
4271 }
4272 
4273 
4274 /* Check whether a string includes at least one token in another string.
4275    `vbuf' specifies the target value.
4276    `expr' specifies the operation value.
4277    If they matches, the return value is true, else it is false. */
tctdbqrycondcheckstror(const char * vbuf,const char * expr)4278 static bool tctdbqrycondcheckstror(const char *vbuf, const char *expr){
4279   assert(vbuf && expr);
4280   const unsigned char *sp = (unsigned char *)expr;
4281   while(*sp != '\0'){
4282     while((*sp != '\0' && *sp <= ' ') || *sp == ','){
4283       sp++;
4284     }
4285     const unsigned char *ep = sp;
4286     while(*ep > ' ' && *ep != ','){
4287       ep++;
4288     }
4289     if(ep > sp){
4290       bool hit = false;
4291       const unsigned char *rp = (unsigned char *)vbuf;
4292       while(*rp != '\0'){
4293         const unsigned char *pp;
4294         for(pp = sp; pp < ep; pp++, rp++){
4295           if(*pp != *rp) break;
4296         }
4297         if(pp == ep && (*rp <= ' ' || *rp == ',')){
4298           hit = true;
4299           break;
4300         }
4301         while(*rp > ' ' && *rp != ','){
4302           rp++;
4303         }
4304         while((*rp != '\0' && *rp <= ' ') || *rp == ','){
4305           rp++;
4306         }
4307       }
4308       if(hit) return true;
4309     }
4310     sp = ep;
4311   }
4312   return false;
4313 }
4314 
4315 
4316 /* Check whether a string is equal to at least one token in another string.
4317    `vbuf' specifies the target value.
4318    `expr' specifies the operation value.
4319    If they matches, the return value is true, else it is false. */
tctdbqrycondcheckstroreq(const char * vbuf,const char * expr)4320 static bool tctdbqrycondcheckstroreq(const char *vbuf, const char *expr){
4321   assert(vbuf && expr);
4322   const unsigned char *sp = (unsigned char *)expr;
4323   while(*sp != '\0'){
4324     while((*sp != '\0' && *sp <= ' ') || *sp == ','){
4325       sp++;
4326     }
4327     const unsigned char *ep = sp;
4328     while(*ep > ' ' && *ep != ','){
4329       ep++;
4330     }
4331     if(ep > sp){
4332       const unsigned char *rp;
4333       for(rp = (unsigned char *)vbuf; *rp != '\0'; rp++){
4334         if(*sp != *rp || sp >= ep) break;
4335         sp++;
4336       }
4337       if(*rp == '\0' && sp == ep) return true;
4338     }
4339     sp = ep;
4340   }
4341   return false;
4342 }
4343 
4344 
4345 /* Check whether a decimal string is between two tokens in another string.
4346    `vbuf' specifies the target value.
4347    `expr' specifies the operation value.
4348    If they matches, the return value is true, else it is false. */
tctdbqrycondchecknumbt(const char * vbuf,const char * expr)4349 static bool tctdbqrycondchecknumbt(const char *vbuf, const char *expr){
4350   assert(vbuf && expr);
4351   while(*expr == ' ' || *expr == ','){
4352     expr++;
4353   }
4354   const char *pv = expr;
4355   while(*pv != '\0' && *pv != ' ' && *pv != ','){
4356     pv++;
4357   }
4358   if(*pv != ' ' && *pv != ',') pv = " ";
4359   pv++;
4360   while(*pv == ' ' || *pv == ','){
4361     pv++;
4362   }
4363   long double val = tctdbatof(vbuf);
4364   long double lower = tctdbatof(expr);
4365   long double upper = tctdbatof(pv);
4366   if(lower > upper){
4367     long double swap = lower;
4368     lower = upper;
4369     upper = swap;
4370   }
4371   return val >= lower && val <= upper;
4372 }
4373 
4374 
4375 /* Check whether a number is equal to at least one token in another string.
4376    `vbuf' specifies the target value.
4377    `expr' specifies the operation value.
4378    If they matches, the return value is true, else it is false. */
tctdbqrycondchecknumoreq(const char * vbuf,const char * expr)4379 static bool tctdbqrycondchecknumoreq(const char *vbuf, const char *expr){
4380   assert(vbuf && expr);
4381   long double vnum = tctdbatof(vbuf);
4382   const char *sp = expr;
4383   while(*sp != '\0'){
4384     while(*sp == ' ' || *sp == ','){
4385       sp++;
4386     }
4387     const char *ep = sp;
4388     while(*ep != '\0' && *ep != ' ' && *ep != ','){
4389       ep++;
4390     }
4391     if(ep > sp && vnum == tctdbatof(sp)) return true;
4392     sp = ep;
4393   }
4394   return false;
4395 }
4396 
4397 
4398 /* Check whether a text matches a condition.
4399    `vbuf' specifies the target value.
4400    `vsiz' specifies the size of the target value.
4401    `cond' specifies the condition object.
4402    If they matches, the return value is true, else it is false. */
tctdbqrycondcheckfts(const char * vbuf,int vsiz,TDBCOND * cond)4403 static bool tctdbqrycondcheckfts(const char *vbuf, int vsiz, TDBCOND *cond){
4404   assert(vbuf && cond);
4405   TDBFTSUNIT *ftsunits = cond->ftsunits;
4406   int ftsnum = cond->ftsnum;
4407   if(ftsnum < 1) return false;
4408   if(!ftsunits[0].sign) return false;
4409   char astack[TDBCOLBUFSIZ];
4410   uint16_t *ary;
4411   int asiz = sizeof(*ary) * (vsiz + 1);
4412   if(asiz < sizeof(astack)){
4413     ary = (uint16_t *)astack;
4414   } else {
4415     TCMALLOC(ary, asiz + 1);
4416   }
4417   int anum;
4418   tcstrutftoucs(vbuf, ary, &anum);
4419   anum = tcstrucsnorm(ary, anum, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
4420   char sstack[TDBCOLBUFSIZ], *str;
4421   int ssiz = anum * 3 + 1;
4422   if(ssiz < sizeof(sstack)){
4423     str = sstack;
4424   } else {
4425     TCMALLOC(str, ssiz + 1);
4426   }
4427   tcstrucstoutf(ary, anum, str);
4428   bool ok = true;
4429   for(int i = 0; i < ftsnum; i++){
4430     TDBFTSUNIT *ftsunit = ftsunits + i;
4431     TCLIST *tokens = ftsunit->tokens;
4432     int tnum = TCLISTNUM(tokens);
4433     bool hit = false;
4434     for(int j = 0; j < tnum; j++){
4435       if(strstr(str, TCLISTVALPTR(tokens, j))){
4436         hit = true;
4437         break;
4438       }
4439     }
4440     if(hit != ftsunit->sign) ok = false;
4441   }
4442   if(str != sstack) TCFREE(str);
4443   if(ary != (uint16_t *)astack) TCFREE(ary);
4444   return ok;
4445 }
4446 
4447 
4448 /* Compare two primary keys by number ascending.
4449    `a' specifies a key.
4450    `b' specifies of the other key.
4451    The return value is positive if the former is big, negative if the latter is big, 0 if both
4452    are equivalent. */
tdbcmppkeynumasc(const TCLISTDATUM * a,const TCLISTDATUM * b)4453 static int tdbcmppkeynumasc(const TCLISTDATUM *a, const TCLISTDATUM *b){
4454   assert(a && b);
4455   return tccmpdecimal(a->ptr, a->size, b->ptr, b->size, NULL);
4456 }
4457 
4458 
4459 /* Compare two primary keys by number descending.
4460    `a' specifies a key.
4461    `b' specifies of the other key.
4462    The return value is positive if the former is big, negative if the latter is big, 0 if both
4463    are equivalent. */
tdbcmppkeynumdesc(const TCLISTDATUM * a,const TCLISTDATUM * b)4464 static int tdbcmppkeynumdesc(const TCLISTDATUM *a, const TCLISTDATUM *b){
4465   assert(a && b);
4466   return tccmpdecimal(b->ptr, b->size, a->ptr, a->size, NULL);
4467 }
4468 
4469 
4470 /* Compare two sort records by string ascending.
4471    `a' specifies a key.
4472    `b' specifies of the other key.
4473    The return value is positive if the former is big, negative if the latter is big, 0 if both
4474    are equivalent. */
tdbcmpsortrecstrasc(const TDBSORTREC * a,const TDBSORTREC * b)4475 static int tdbcmpsortrecstrasc(const TDBSORTREC *a, const TDBSORTREC *b){
4476   assert(a && b);
4477   if(!a->vbuf){
4478     if(!b->vbuf) return 0;
4479     return 1;
4480   }
4481   if(!b->vbuf){
4482     if(!a->vbuf) return 0;
4483     return -1;
4484   }
4485   int rv;
4486   TCCMPLEXICAL(rv, a->vbuf, a->vsiz, b->vbuf, b->vsiz);
4487   return rv;
4488 }
4489 
4490 
4491 /* Compare two sort records by string descending.
4492    `a' specifies a key.
4493    `b' specifies of the other key.
4494    The return value is positive if the former is big, negative if the latter is big, 0 if both
4495    are equivalent. */
tdbcmpsortrecstrdesc(const TDBSORTREC * a,const TDBSORTREC * b)4496 static int tdbcmpsortrecstrdesc(const TDBSORTREC *a, const TDBSORTREC *b){
4497   assert(a && b);
4498   if(!a->vbuf){
4499     if(!b->vbuf) return 0;
4500     return 1;
4501   }
4502   if(!b->vbuf){
4503     if(!a->vbuf) return 0;
4504     return -1;
4505   }
4506   int rv;
4507   TCCMPLEXICAL(rv, a->vbuf, a->vsiz, b->vbuf, b->vsiz);
4508   return -rv;
4509 }
4510 
4511 
4512 /* Compare two sort records by number ascending.
4513    `a' specifies a key.
4514    `b' specifies of the other key.
4515    The return value is positive if the former is big, negative if the latter is big, 0 if both
4516    are equivalent. */
tdbcmpsortrecnumasc(const TDBSORTREC * a,const TDBSORTREC * b)4517 static int tdbcmpsortrecnumasc(const TDBSORTREC *a, const TDBSORTREC *b){
4518   assert(a && b);
4519   if(!a->vbuf){
4520     if(!b->vbuf) return 0;
4521     return 1;
4522   }
4523   if(!b->vbuf){
4524     if(!a->vbuf) return 0;
4525     return -1;
4526   }
4527   long double anum = tctdbatof(a->vbuf);
4528   long double bnum = tctdbatof(b->vbuf);
4529   if(anum < bnum) return -1;
4530   if(anum > bnum) return 1;
4531   return 0;
4532 }
4533 
4534 
4535 /* Compare two sort records by number descending.
4536    `a' specifies a key.
4537    `b' specifies of the other key.
4538    The return value is positive if the former is big, negative if the latter is big, 0 if both
4539    are equivalent. */
tdbcmpsortrecnumdesc(const TDBSORTREC * a,const TDBSORTREC * b)4540 static int tdbcmpsortrecnumdesc(const TDBSORTREC *a, const TDBSORTREC *b){
4541   assert(a && b);
4542   if(!a->vbuf){
4543     if(!b->vbuf) return 0;
4544     return 1;
4545   }
4546   if(!b->vbuf){
4547     if(!a->vbuf) return 0;
4548     return -1;
4549   }
4550   long double anum = tctdbatof(a->vbuf);
4551   long double bnum = tctdbatof(b->vbuf);
4552   if(anum < bnum) return 1;
4553   if(anum > bnum) return -1;
4554   return 0;
4555 }
4556 
4557 
4558 /* Get the hash value of a record.
4559    `pkbuf' specifies the pointer to the region of the primary key.
4560    `pksiz' specifies the size of the region of the primary key.
4561    The return value is the hash value. */
tctdbidxhash(const char * pkbuf,int pksiz)4562 static uint16_t tctdbidxhash(const char *pkbuf, int pksiz){
4563   assert(pkbuf && pksiz && pksiz >= 0);
4564   uint32_t hash = 19780211;
4565   while(pksiz--){
4566     hash = hash * 37 + *(uint8_t *)pkbuf++;
4567   }
4568   return hash;
4569 }
4570 
4571 
4572 /* Add a record into indices of a table database object.
4573    `tdb' specifies the table database object.
4574    `pkbuf' specifies the pointer to the region of the primary key.
4575    `pksiz' specifies the size of the region of the primary key.
4576    `cols' specifies a map object containing columns.
4577    If successful, the return value is true, else, it is false. */
tctdbidxput(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols)4578 static bool tctdbidxput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
4579   assert(tdb && pkbuf && pksiz >= 0 && cols);
4580   bool err = false;
4581   uint16_t hash = tctdbidxhash(pkbuf, pksiz);
4582   TDBIDX *idxs = tdb->idxs;
4583   int inum = tdb->inum;
4584   for(int i = 0; i < inum; i++){
4585     TDBIDX *idx = idxs + i;
4586     if(*(idx->name) != '\0') continue;
4587     char stack[TDBCOLBUFSIZ], *rbuf;
4588     if(pksiz < sizeof(stack)){
4589       rbuf = stack;
4590     } else {
4591       TCMALLOC(rbuf, pksiz + 1);
4592     }
4593     memcpy(rbuf, pkbuf, pksiz);
4594     rbuf[pksiz] = '\0';
4595     switch(idx->type){
4596       case TDBITLEXICAL:
4597       case TDBITDECIMAL:
4598         if(!tcbdbput(idx->db, pkbuf, pksiz, rbuf, pksiz)){
4599           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4600           err = true;
4601         }
4602         break;
4603       case TDBITTOKEN:
4604         if(!tctdbidxputtoken(tdb, idx, pkbuf, pksiz, pkbuf, pksiz)) err = true;
4605         break;
4606       case TDBITQGRAM:
4607         if(!tctdbidxputqgram(tdb, idx, pkbuf, pksiz, pkbuf, pksiz)) err = true;
4608         break;
4609     }
4610     if(rbuf != stack) TCFREE(rbuf);
4611   }
4612   tcmapiterinit(cols);
4613   const char *kbuf;
4614   int ksiz;
4615   while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
4616     int vsiz;
4617     const char *vbuf = tcmapiterval(kbuf, &vsiz);
4618     for(int i = 0; i < inum; i++){
4619       TDBIDX *idx = idxs + i;
4620       if(strcmp(idx->name, kbuf)) continue;
4621       switch(idx->type){
4622         case TDBITLEXICAL:
4623         case TDBITDECIMAL:
4624           if(!tctdbidxputone(tdb, idx, pkbuf, pksiz, hash, vbuf, vsiz)) err = true;
4625           break;
4626         case TDBITTOKEN:
4627           if(!tctdbidxputtoken(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
4628           break;
4629         case TDBITQGRAM:
4630           if(!tctdbidxputqgram(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
4631           break;
4632       }
4633     }
4634   }
4635   return !err;
4636 }
4637 
4638 
4639 /* Add a column of a record into an index of a table database object.
4640    `tdb' specifies the table database object.
4641    `idx' specifies the index object.
4642    `pkbuf' specifies the pointer to the region of the primary key.
4643    `pksiz' specifies the size of the region of the primary key.
4644    `hash' specifies the hash value of the primary key.
4645    `vbuf' specifies the pointer to the region of the column value.
4646    `vsiz' specifies the size of the region of the column value.
4647    If successful, the return value is true, else, it is false. */
tctdbidxputone(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,uint16_t hash,const char * vbuf,int vsiz)4648 static bool tctdbidxputone(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz, uint16_t hash,
4649                            const char *vbuf, int vsiz){
4650   assert(tdb && pkbuf && pksiz >= 0 && vbuf && vsiz);
4651   bool err = false;
4652   char stack[TDBCOLBUFSIZ], *rbuf;
4653   int rsiz = vsiz + 3;
4654   if(rsiz <= sizeof(stack)){
4655     rbuf = stack;
4656   } else {
4657     TCMALLOC(rbuf, rsiz);
4658   }
4659   memcpy(rbuf, vbuf, vsiz);
4660   rbuf[vsiz] = '\0';
4661   rbuf[vsiz+1] = hash >> 8;
4662   rbuf[vsiz+2] = hash & 0xff;
4663   if(!tcbdbputdup(idx->db, rbuf, rsiz, pkbuf, pksiz)){
4664     tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4665     err = true;
4666   }
4667   if(rbuf != stack) TCFREE(rbuf);
4668   return !err;
4669 }
4670 
4671 
4672 /* Add a column of a record into an token inverted index of a table database object.
4673    `tdb' specifies the table database object.
4674    `idx' specifies the index object.
4675    `pkbuf' specifies the pointer to the region of the primary key.
4676    `pksiz' specifies the size of the region of the primary key.
4677    `vbuf' specifies the pointer to the region of the column value.
4678    `vsiz' specifies the size of the region of the column value.
4679    If successful, the return value is true, else, it is false. */
tctdbidxputtoken(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,const char * vbuf,int vsiz)4680 static bool tctdbidxputtoken(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
4681                              const char *vbuf, int vsiz){
4682   assert(tdb && idx && pkbuf && pksiz >= 0 && vbuf && vsiz >= 0);
4683   bool err = false;
4684   TCMAP *cc = idx->cc;
4685   char stack[TDBCOLBUFSIZ], *rbuf;
4686   int rsiz = pksiz + TCNUMBUFSIZ;
4687   if(rsiz < sizeof(stack)){
4688     rbuf = stack;
4689   } else {
4690     TCMALLOC(rbuf, rsiz);
4691   }
4692   uint64_t pkid = 0;
4693   for(int i = 0; i < pksiz; i++){
4694     int c = pkbuf[i];
4695     if(c >= '0' && c <= '9'){
4696       pkid = pkid * 10 + c - '0';
4697       if(pkid > INT64_MAX){
4698         pkid = 0;
4699         break;
4700       }
4701     } else {
4702       pkid = 0;
4703       break;
4704     }
4705   }
4706   if(pksiz > 0 && *pkbuf == '0') pkid = 0;
4707   if(pkid > 0){
4708     TCSETVNUMBUF64(rsiz, rbuf, pkid);
4709   } else {
4710     char *wp = rbuf;
4711     *(wp++) = '\0';
4712     TCSETVNUMBUF(rsiz, wp, pksiz);
4713     wp += rsiz;
4714     memcpy(wp, pkbuf, pksiz);
4715     wp += pksiz;
4716     rsiz = wp - rbuf;
4717   }
4718   const unsigned char *sp = (unsigned char *)vbuf;
4719   while(*sp != '\0'){
4720     while((*sp != '\0' && *sp <= ' ') || *sp == ','){
4721       sp++;
4722     }
4723     const unsigned char *ep = sp;
4724     while(*ep > ' ' && *ep != ','){
4725       ep++;
4726     }
4727     if(ep > sp) tcmapputcat3(cc, sp, ep - sp, rbuf, rsiz);
4728     sp = ep;
4729   }
4730   if(rbuf != stack) TCFREE(rbuf);
4731   if(tcmapmsiz(cc) > tdb->iccmax && !tctdbidxsyncicc(tdb, idx, false)) err = true;
4732   return !err;
4733 }
4734 
4735 
4736 /* Add a column of a record into an q-gram inverted index of a table database object.
4737    `tdb' specifies the table database object.
4738    `idx' specifies the index object.
4739    `pkbuf' specifies the pointer to the region of the primary key.
4740    `pksiz' specifies the size of the region of the primary key.
4741    `vbuf' specifies the pointer to the region of the column value.
4742    `vsiz' specifies the size of the region of the column value.
4743    If successful, the return value is true, else, it is false. */
tctdbidxputqgram(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,const char * vbuf,int vsiz)4744 static bool tctdbidxputqgram(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
4745                              const char *vbuf, int vsiz){
4746   assert(tdb && idx && pkbuf && pksiz >= 0 && vbuf && vsiz >= 0);
4747   bool err = false;
4748   TCMAP *cc = idx->cc;
4749   char stack[TDBCOLBUFSIZ], *rbuf;
4750   int rsiz = pksiz + TCNUMBUFSIZ * 2;
4751   if(rsiz < sizeof(stack)){
4752     rbuf = stack;
4753   } else {
4754     TCMALLOC(rbuf, rsiz);
4755   }
4756   uint64_t pkid = 0;
4757   for(int i = 0; i < pksiz; i++){
4758     int c = pkbuf[i];
4759     if(c >= '0' && c <= '9'){
4760       pkid = pkid * 10 + c - '0';
4761       if(pkid > INT64_MAX){
4762         pkid = 0;
4763         break;
4764       }
4765     } else {
4766       pkid = 0;
4767       break;
4768     }
4769   }
4770   if(pksiz > 0 && *pkbuf == '0') pkid = 0;
4771   if(pkid > 0){
4772     TCSETVNUMBUF64(rsiz, rbuf, pkid);
4773   } else {
4774     char *wp = rbuf;
4775     *(wp++) = '\0';
4776     TCSETVNUMBUF(rsiz, wp, pksiz);
4777     wp += rsiz;
4778     memcpy(wp, pkbuf, pksiz);
4779     wp += pksiz;
4780     rsiz = wp - rbuf;
4781   }
4782   uint16_t *ary;
4783   TCMALLOC(ary, sizeof(*ary) * (vsiz + TDBIDXQGUNIT));
4784   int anum;
4785   tcstrutftoucs(vbuf, ary, &anum);
4786   anum = tcstrucsnorm(ary, anum, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
4787   for(int i = 0; i < TDBIDXQGUNIT; i++){
4788     ary[anum+i] = 0;
4789   }
4790   char *wp = rbuf + rsiz;
4791   char token[TDBIDXQGUNIT*3+1];
4792   for(int i = 0; i < anum; i++){
4793     tcstrucstoutf(ary + i, TDBIDXQGUNIT, token);
4794     int step;
4795     TCSETVNUMBUF(step, wp, i);
4796     tcmapputcat3(cc, token, strlen(token), rbuf, rsiz + step);
4797   }
4798   TCFREE(ary);
4799   if(rbuf != stack) TCFREE(rbuf);
4800   if(tcmapmsiz(cc) > tdb->iccmax && !tctdbidxsyncicc(tdb, idx, false)) err = true;
4801   return !err;
4802 }
4803 
4804 
4805 /* Remove a record from indices of a table database object.
4806    `tdb' specifies the table database object.
4807    `pkbuf' specifies the pointer to the region of the primary key.
4808    `pksiz' specifies the size of the region of the primary key.
4809    `cols' specifies a map object containing columns.
4810    If successful, the return value is true, else, it is false. */
tctdbidxout(TCTDB * tdb,const void * pkbuf,int pksiz,TCMAP * cols)4811 static bool tctdbidxout(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
4812   assert(tdb && pkbuf && pksiz >= 0 && cols);
4813   bool err = false;
4814   uint16_t hash = tctdbidxhash(pkbuf, pksiz);
4815   TDBIDX *idxs = tdb->idxs;
4816   int inum = tdb->inum;
4817   for(int i = 0; i < inum; i++){
4818     TDBIDX *idx = idxs + i;
4819     if(*(idx->name) != '\0') continue;
4820     char stack[TDBCOLBUFSIZ], *rbuf;
4821     if(pksiz < sizeof(stack)){
4822       rbuf = stack;
4823     } else {
4824       TCMALLOC(rbuf, pksiz + 1);
4825     }
4826     memcpy(rbuf, pkbuf, pksiz);
4827     rbuf[pksiz] = '\0';
4828     switch(idx->type){
4829       case TDBITLEXICAL:
4830       case TDBITDECIMAL:
4831         if(!tcbdbout(idx->db, pkbuf, pksiz)){
4832           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4833           err = true;
4834         }
4835         break;
4836       case TDBITTOKEN:
4837         if(!tctdbidxouttoken(tdb, idx, pkbuf, pksiz, rbuf, pksiz)) err = true;
4838         break;
4839       case TDBITQGRAM:
4840         if(!tctdbidxoutqgram(tdb, idx, pkbuf, pksiz, rbuf, pksiz)) err = true;
4841         break;
4842     }
4843     if(rbuf != stack) TCFREE(rbuf);
4844   }
4845   tcmapiterinit(cols);
4846   const char *kbuf;
4847   int ksiz;
4848   while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
4849     int vsiz;
4850     const char *vbuf = tcmapiterval(kbuf, &vsiz);
4851     for(int i = 0; i < inum; i++){
4852       TDBIDX *idx = idxs + i;
4853       if(strcmp(idx->name, kbuf)) continue;
4854       switch(idx->type){
4855         case TDBITLEXICAL:
4856         case TDBITDECIMAL:
4857           if(!tctdbidxoutone(tdb, idx, pkbuf, pksiz, hash, vbuf, vsiz)) err = true;
4858           break;
4859         case TDBITTOKEN:
4860           if(!tctdbidxouttoken(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
4861           break;
4862         case TDBITQGRAM:
4863           if(!tctdbidxoutqgram(tdb, idx, pkbuf, pksiz, vbuf, vsiz)) err = true;
4864           break;
4865       }
4866     }
4867   }
4868   return !err;
4869 }
4870 
4871 
4872 /* Remove a column of a record from an index of a table database object.
4873    `tdb' specifies the table database object.
4874    `idx' specifies the index object.
4875    `pkbuf' specifies the pointer to the region of the primary key.
4876    `pksiz' specifies the size of the region of the primary key.
4877    `hash' specifies the hash value of the primary key.
4878    `vbuf' specifies the pointer to the region of the column value.
4879    `vsiz' specifies the size of the region of the column value.
4880    If successful, the return value is true, else, it is false. */
tctdbidxoutone(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,uint16_t hash,const char * vbuf,int vsiz)4881 static bool tctdbidxoutone(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz, uint16_t hash,
4882                            const char *vbuf, int vsiz){
4883   assert(tdb && idx && pkbuf && pksiz >= 0 && vbuf && vsiz >= 0);
4884   bool err = false;
4885   char stack[TDBCOLBUFSIZ], *rbuf;
4886   int rsiz = vsiz + 3;
4887   if(rsiz <= sizeof(stack)){
4888     rbuf = stack;
4889   } else {
4890     TCMALLOC(rbuf, rsiz);
4891   }
4892   memcpy(rbuf, vbuf, vsiz);
4893   rbuf[vsiz] = '\0';
4894   rbuf[vsiz+1] = hash >> 8;
4895   rbuf[vsiz+2] = hash & 0xff;
4896   int ovsiz;
4897   const char *ovbuf = tcbdbget3(idx->db, rbuf, rsiz, &ovsiz);
4898   if(ovbuf && ovsiz == pksiz && !memcmp(ovbuf, pkbuf, ovsiz)){
4899     if(!tcbdbout(idx->db, rbuf, rsiz)){
4900       tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4901       err = true;
4902     }
4903   } else {
4904     BDBCUR *cur = tcbdbcurnew(idx->db);
4905     if(tcbdbcurjump(cur, rbuf, rsiz)){
4906       int oksiz;
4907       const char *okbuf;
4908       while((okbuf = tcbdbcurkey3(cur, &oksiz)) != NULL){
4909         if(oksiz != rsiz || memcmp(okbuf, rbuf, oksiz)) break;
4910         ovbuf = tcbdbcurval3(cur, &ovsiz);
4911         if(ovsiz == pksiz && !memcmp(ovbuf, pkbuf, ovsiz)){
4912           if(!tcbdbcurout(cur)){
4913             tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4914             err = true;
4915           }
4916           break;
4917         }
4918         tcbdbcurnext(cur);
4919       }
4920     } else {
4921       tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
4922       err = true;
4923     }
4924     tcbdbcurdel(cur);
4925   }
4926   if(rbuf != stack) TCFREE(rbuf);
4927   return !err;
4928 }
4929 
4930 
4931 /* Remove a column of a record from a token inverted index of a table database object.
4932    `tdb' specifies the table database object.
4933    `idx' specifies the index object.
4934    `pkbuf' specifies the pointer to the region of the primary key.
4935    `pksiz' specifies the size of the region of the primary key.
4936    `vbuf' specifies the pointer to the region of the column value.
4937    `vsiz' specifies the size of the region of the column value.
4938    If successful, the return value is true, else, it is false. */
tctdbidxouttoken(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,const char * vbuf,int vsiz)4939 static bool tctdbidxouttoken(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
4940                              const char *vbuf, int vsiz){
4941   assert(tdb && idx && pkbuf && pksiz >= 0 && vbuf && vsiz >= 0);
4942   bool err = false;
4943   TCBDB *db = idx->db;
4944   TCMAP *cc = idx->cc;
4945   uint64_t pkid = 0;
4946   for(int i = 0; i < pksiz; i++){
4947     int c = pkbuf[i];
4948     if(c >= '0' && c <= '9'){
4949       pkid = pkid * 10 + c - '0';
4950       if(pkid > INT64_MAX){
4951         pkid = 0;
4952         break;
4953       }
4954     } else {
4955       pkid = 0;
4956       break;
4957     }
4958   }
4959   TCXSTR *xstr = tcxstrnew();
4960   const unsigned char *sp = (unsigned char *)vbuf;
4961   while(*sp != '\0'){
4962     while((*sp != '\0' && *sp <= ' ') || *sp == ','){
4963       sp++;
4964     }
4965     const unsigned char *ep = sp;
4966     while(*ep > ' ' && *ep != ','){
4967       ep++;
4968     }
4969     if(ep > sp){
4970       tcxstrclear(xstr);
4971       int len = ep - sp;
4972       int csiz;
4973       const char *cbuf = tcmapget(cc, sp, len, &csiz);
4974       if(cbuf){
4975         while(csiz > 0){
4976           const char *pv = cbuf;
4977           bool ok = true;
4978           if(*cbuf == '\0'){
4979             cbuf++;
4980             csiz--;
4981             int tsiz, step;
4982             TCREADVNUMBUF(cbuf, tsiz, step);
4983             cbuf += step;
4984             csiz -= step;
4985             if(tsiz == pksiz && !memcmp(cbuf, pkbuf, tsiz)) ok = false;
4986             cbuf += tsiz;
4987             csiz -= tsiz;
4988           } else {
4989             int64_t tid;
4990             int step;
4991             TCREADVNUMBUF64(cbuf, tid, step);
4992             if(tid == pkid) ok = false;
4993             cbuf += step;
4994             csiz -= step;
4995           }
4996           if(ok) TCXSTRCAT(xstr, pv, cbuf - pv);
4997         }
4998         if(csiz != 0){
4999           tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
5000           err = true;
5001         }
5002       }
5003       cbuf = tcbdbget3(db, sp, len, &csiz);
5004       if(cbuf){
5005         while(csiz > 0){
5006           const char *pv = cbuf;
5007           bool ok = true;
5008           if(*cbuf == '\0'){
5009             cbuf++;
5010             csiz--;
5011             int tsiz, step;
5012             TCREADVNUMBUF(cbuf, tsiz, step);
5013             cbuf += step;
5014             csiz -= step;
5015             if(tsiz == pksiz && !memcmp(cbuf, pkbuf, tsiz)) ok = false;
5016             cbuf += tsiz;
5017             csiz -= tsiz;
5018           } else {
5019             int64_t tid;
5020             int step;
5021             TCREADVNUMBUF64(cbuf, tid, step);
5022             if(tid == pkid) ok = false;
5023             cbuf += step;
5024             csiz -= step;
5025           }
5026           if(ok) TCXSTRCAT(xstr, pv, cbuf - pv);
5027         }
5028         if(csiz != 0){
5029           tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
5030           err = true;
5031         }
5032         if(!tcbdbout(db, sp, len)){
5033           tctdbsetecode(tdb, tcbdbecode(db), __FILE__, __LINE__, __func__);
5034           err = true;
5035         }
5036       }
5037       tcmapput(cc, sp, len, TCXSTRPTR(xstr), TCXSTRSIZE(xstr));
5038     }
5039     sp = ep;
5040   }
5041   tcxstrdel(xstr);
5042   if(tcmapmsiz(cc) > tdb->iccmax && !tctdbidxsyncicc(tdb, idx, false)) err = true;
5043   return !err;
5044 }
5045 
5046 
5047 /* Remove a column of a record from a q-gram inverted index of a table database object.
5048    `tdb' specifies the table database object.
5049    `idx' specifies the index object.
5050    `pkbuf' specifies the pointer to the region of the primary key.
5051    `pksiz' specifies the size of the region of the primary key.
5052    `vbuf' specifies the pointer to the region of the column value.
5053    `vsiz' specifies the size of the region of the column value.
5054    If successful, the return value is true, else, it is false. */
tctdbidxoutqgram(TCTDB * tdb,TDBIDX * idx,const char * pkbuf,int pksiz,const char * vbuf,int vsiz)5055 static bool tctdbidxoutqgram(TCTDB *tdb, TDBIDX *idx, const char *pkbuf, int pksiz,
5056                              const char *vbuf, int vsiz){
5057   assert(tdb && idx && pkbuf && pksiz >= 0 && vbuf && vsiz >= 0);
5058   bool err = false;
5059   TCBDB *db = idx->db;
5060   TCMAP *cc = idx->cc;
5061   uint64_t pkid = 0;
5062   for(int i = 0; i < pksiz; i++){
5063     int c = pkbuf[i];
5064     if(c >= '0' && c <= '9'){
5065       pkid = pkid * 10 + c - '0';
5066       if(pkid > INT64_MAX){
5067         pkid = 0;
5068         break;
5069       }
5070     } else {
5071       pkid = 0;
5072       break;
5073     }
5074   }
5075   TCXSTR *xstr = tcxstrnew();
5076   uint16_t *ary;
5077   TCMALLOC(ary, sizeof(*ary) * (vsiz + TDBIDXQGUNIT));
5078   int anum;
5079   tcstrutftoucs(vbuf, ary, &anum);
5080   anum = tcstrucsnorm(ary, anum, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
5081   for(int i = 0; i < TDBIDXQGUNIT; i++){
5082     ary[anum+i] = 0;
5083   }
5084   char token[TDBIDXQGUNIT*3+1];
5085   for(int i = 0; i < anum; i++){
5086     tcstrucstoutf(ary + i, TDBIDXQGUNIT, token);
5087     int tsiz = strlen(token);
5088     tcxstrclear(xstr);
5089     int csiz;
5090     const char *cbuf = tcmapget(cc, token, tsiz, &csiz);
5091     if(cbuf){
5092       while(csiz > 0){
5093         const char *pv = cbuf;
5094         bool ok = true;
5095         if(*cbuf == '\0'){
5096           cbuf++;
5097           csiz--;
5098           int tsiz, step;
5099           TCREADVNUMBUF(cbuf, tsiz, step);
5100           cbuf += step;
5101           csiz -= step;
5102           if(tsiz == pksiz && !memcmp(cbuf, pkbuf, tsiz)) ok = false;
5103           cbuf += tsiz;
5104           csiz -= tsiz;
5105         } else {
5106           int64_t tid;
5107           int step;
5108           TCREADVNUMBUF64(cbuf, tid, step);
5109           if(tid == pkid) ok = false;
5110           cbuf += step;
5111           csiz -= step;
5112         }
5113         if(csiz > 0){
5114           int off, step;
5115           TCREADVNUMBUF(cbuf, off, step);
5116           cbuf += step;
5117           csiz -= step;
5118           if(ok) TCXSTRCAT(xstr, pv, cbuf - pv);
5119         }
5120       }
5121       if(csiz != 0){
5122         tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
5123         err = true;
5124       }
5125     }
5126     cbuf = tcbdbget3(db, token, tsiz, &csiz);
5127     if(cbuf){
5128       while(csiz > 0){
5129         const char *pv = cbuf;
5130         bool ok = true;
5131         if(*cbuf == '\0'){
5132           cbuf++;
5133           csiz--;
5134           int tsiz, step;
5135           TCREADVNUMBUF(cbuf, tsiz, step);
5136           cbuf += step;
5137           csiz -= step;
5138           if(tsiz == pksiz && !memcmp(cbuf, pkbuf, tsiz)) ok = false;
5139           cbuf += tsiz;
5140           csiz -= tsiz;
5141         } else {
5142           int64_t tid;
5143           int step;
5144           TCREADVNUMBUF64(cbuf, tid, step);
5145           if(tid == pkid) ok = false;
5146           cbuf += step;
5147           csiz -= step;
5148         }
5149         if(csiz > 0){
5150           int off, step;
5151           TCREADVNUMBUF(cbuf, off, step);
5152           cbuf += step;
5153           csiz -= step;
5154           if(ok) TCXSTRCAT(xstr, pv, cbuf - pv);
5155         }
5156       }
5157       if(csiz != 0){
5158         tctdbsetecode(tdb, TCEMISC, __FILE__, __LINE__, __func__);
5159         err = true;
5160       }
5161       if(!tcbdbout(db, token, tsiz)){
5162         tctdbsetecode(tdb, tcbdbecode(db), __FILE__, __LINE__, __func__);
5163         err = true;
5164       }
5165     }
5166     tcmapput(cc, token, tsiz, TCXSTRPTR(xstr), TCXSTRSIZE(xstr));
5167   }
5168   TCFREE(ary);
5169   tcxstrdel(xstr);
5170   if(tcmapmsiz(cc) > tdb->iccmax && !tctdbidxsyncicc(tdb, idx, false)) err = true;
5171   return !err;
5172 }
5173 
5174 
5175 /* Synchronize updated contents of an inverted cache of a table database object.
5176    `tdb' specifies the table database object.
5177    `idx' specifies the index object.
5178    `all' specifies whether to sync all tokens.
5179    If successful, the return value is true, else, it is false. */
tctdbidxsyncicc(TCTDB * tdb,TDBIDX * idx,bool all)5180 static bool tctdbidxsyncicc(TCTDB *tdb, TDBIDX *idx, bool all){
5181   assert(tdb && idx);
5182   TCBDB *db = idx->db;
5183   TCMAP *cc = idx->cc;
5184   int rnum = TCMAPRNUM(cc);
5185   if(rnum < 1) return true;
5186   bool err = false;
5187   const char **keys;
5188   TCMALLOC(keys, sizeof(*keys) * rnum);
5189   int knum = 0;
5190   int64_t usiz = tcmapmsiz(cc) - sizeof(void *) * TDBIDXICCBNUM;
5191   int64_t max = all ? INT64_MAX : usiz * tdb->iccsync;
5192   int64_t sum = 0;
5193   const char *kbuf;
5194   int ksiz;
5195   tcmapiterinit(cc);
5196   while(sum < max && (kbuf = tcmapiternext(cc, &ksiz)) != NULL){
5197     int vsiz;
5198     tcmapiterval(kbuf, &vsiz);
5199     keys[knum++] = kbuf;
5200     sum += sizeof(TCMAPREC) + sizeof(void *) + ksiz + vsiz;
5201   }
5202   qsort(keys, knum, sizeof(*keys), (int(*)(const void *, const void *))tctdbidxcmpkey);
5203   for(int i = 0; i < knum; i++){
5204     const char *kbuf = keys[i];
5205     int ksiz = strlen(kbuf);
5206     int vsiz;
5207     const char *vbuf = tcmapget(cc, kbuf, ksiz, &vsiz);
5208     if(vsiz > 0 && !tcbdbputcat(db, kbuf, ksiz, vbuf, vsiz)){
5209       tctdbsetecode(tdb, tcbdbecode(db), __FILE__, __LINE__, __func__);
5210       err = true;
5211     }
5212     tcmapout(cc, kbuf, ksiz);
5213   }
5214   TCFREE(keys);
5215   return !err;
5216 }
5217 
5218 
5219 /* Compare two index search keys in lexical order.
5220    `a' specifies the pointer to one element.
5221    `b' specifies the pointer to the other element.
5222    The return value is positive if the former is big, negative if the latter is big, 0 if both
5223    are equivalent. */
tctdbidxcmpkey(const char ** a,const char ** b)5224 static int tctdbidxcmpkey(const char **a, const char **b){
5225   assert(a && b);
5226   const unsigned char *ap = (unsigned char *)*a;
5227   const unsigned char *bp = (unsigned char *)*b;
5228   while(true){
5229     if(*ap == '\0') return *bp == '\0' ? 0 : -1;
5230     if(*bp == '\0') return *ap == '\0' ? 0 : 1;
5231     if(*ap != *bp) return *ap - *bp;
5232     ap++;
5233     bp++;
5234   }
5235   return 0;
5236 }
5237 
5238 
5239 /* Retrieve records by a token inverted index of a table database object.
5240    `tdb' specifies the table database object.
5241    `idx' specifies the index object.
5242    `token' specifies the list object of tokens.
5243    `op' specifies the operation type.
5244    `hint' specifies the hint object.
5245    The return value is a map object of the primary keys of the corresponding records. */
tctdbidxgetbytokens(TCTDB * tdb,TDBIDX * idx,const TCLIST * tokens,int op,TCXSTR * hint)5246 static TCMAP *tctdbidxgetbytokens(TCTDB *tdb, TDBIDX *idx, const TCLIST *tokens, int op,
5247                                   TCXSTR *hint){
5248   assert(tdb && idx && tokens && hint);
5249   TCBDB *db = idx->db;
5250   TCMAP *cc = idx->cc;
5251   int tnum = TCLISTNUM(tokens);
5252   TCMAP *res = tcmapnew();
5253   int cnt = 0;
5254   for(int i = 0; i < tnum; i++){
5255     const char *token;
5256     int tsiz;
5257     TCLISTVAL(token, tokens, i, tsiz);
5258     if(tsiz < 1) continue;
5259     int onum = 0;
5260     TCMAP *wring = (cnt > 0 && op == TDBQCSTRAND) ? tcmapnew() : NULL;
5261     int csiz;
5262     const char *cbuf = tcmapget(cc, token, tsiz, &csiz);
5263     if(cbuf){
5264       while(csiz > 0){
5265         if(*cbuf == '\0'){
5266           cbuf++;
5267           csiz--;
5268           int tsiz, step;
5269           TCREADVNUMBUF(cbuf, tsiz, step);
5270           cbuf += step;
5271           csiz -= step;
5272           if(cnt < 1){
5273             tcmapput(res, cbuf, tsiz, "", 0);
5274           } else if(wring){
5275             int rsiz;
5276             if(tcmapget(res, cbuf, tsiz, &rsiz)) tcmapput(wring, cbuf, tsiz, "", 0);
5277           } else {
5278             tcmapput(res, cbuf, tsiz, "", 0);
5279           }
5280           cbuf += tsiz;
5281           csiz -= tsiz;
5282         } else {
5283           int64_t tid;
5284           int step;
5285           TCREADVNUMBUF64(cbuf, tid, step);
5286           char pkbuf[TCNUMBUFSIZ];
5287           int pksiz = sprintf(pkbuf, "%lld", (long long)tid);
5288           if(cnt < 1){
5289             tcmapput(res, pkbuf, pksiz, "", 0);
5290           } else if(wring){
5291             int rsiz;
5292             if(tcmapget(res, pkbuf, pksiz, &rsiz)) tcmapput(wring, pkbuf, pksiz, "", 0);
5293           } else {
5294             tcmapput(res, pkbuf, pksiz, "", 0);
5295           }
5296           cbuf += step;
5297           csiz -= step;
5298         }
5299         onum++;
5300       }
5301     }
5302     cbuf = tcbdbget3(db, token, tsiz, &csiz);
5303     if(cbuf){
5304       while(csiz > 0){
5305         if(*cbuf == '\0'){
5306           cbuf++;
5307           csiz--;
5308           int tsiz, step;
5309           TCREADVNUMBUF(cbuf, tsiz, step);
5310           cbuf += step;
5311           csiz -= step;
5312           if(cnt < 1){
5313             tcmapput(res, cbuf, tsiz, "", 0);
5314           } else if(wring){
5315             int rsiz;
5316             if(tcmapget(res, cbuf, tsiz, &rsiz)) tcmapput(wring, cbuf, tsiz, "", 0);
5317           } else {
5318             tcmapput(res, cbuf, tsiz, "", 0);
5319           }
5320           cbuf += tsiz;
5321           csiz -= tsiz;
5322         } else {
5323           int64_t tid;
5324           int step;
5325           TCREADVNUMBUF64(cbuf, tid, step);
5326           char pkbuf[TCNUMBUFSIZ];
5327           int pksiz = sprintf(pkbuf, "%lld", (long long)tid);
5328           if(cnt < 1){
5329             tcmapput(res, pkbuf, pksiz, "", 0);
5330           } else if(wring){
5331             int rsiz;
5332             if(tcmapget(res, pkbuf, pksiz, &rsiz)) tcmapput(wring, pkbuf, pksiz, "", 0);
5333           } else {
5334             tcmapput(res, pkbuf, pksiz, "", 0);
5335           }
5336           cbuf += step;
5337           csiz -= step;
5338         }
5339         onum++;
5340       }
5341     }
5342     if(wring){
5343       tcmapdel(res);
5344       res = wring;
5345     }
5346     tcxstrprintf(hint, "token occurrence: \"%s\" %d\n", token, onum);
5347     cnt++;
5348   }
5349   return res;
5350 }
5351 
5352 
5353 /* Retrieve records by a token inverted index of a table database object.
5354    `tdb' specifies the table database object.
5355    `idx' specifies the index object.
5356    `cond' specifies the condition object.
5357    `hint' specifies the hint object.
5358    The return value is a map object of the primary keys of the corresponding records. */
tctdbidxgetbyfts(TCTDB * tdb,TDBIDX * idx,TDBCOND * cond,TCXSTR * hint)5359 static TCMAP *tctdbidxgetbyfts(TCTDB *tdb, TDBIDX *idx, TDBCOND *cond, TCXSTR *hint){
5360   assert(tdb && idx && cond && hint);
5361   TDBFTSUNIT *ftsunits = cond->ftsunits;
5362   int ftsnum = cond->ftsnum;
5363   if(ftsnum < 1) return tcmapnew2(1);
5364   if(!ftsunits[0].sign) return tcmapnew2(1);
5365   TCMAP *res = tcmapnew();
5366   tctdbidxgetbyftsunion(idx, ftsunits->tokens, true, NULL, res, hint);
5367   for(int i = 1; i < ftsnum; i++){
5368     TDBFTSUNIT *ftsunit = ftsunits + i;
5369     if(ftsunit->sign){
5370       TCMAP *nres = tcmapnew2(TCMAPRNUM(res) + 1);
5371       tctdbidxgetbyftsunion(idx, ftsunit->tokens, true, res, nres, hint);
5372       tcmapdel(res);
5373       res = nres;
5374     } else {
5375       tctdbidxgetbyftsunion(idx, ftsunit->tokens, false, res, NULL, hint);
5376     }
5377   }
5378   return res;
5379 }
5380 
5381 
5382 /* Retrieve union records by a token inverted index of a table database object.
5383    `idx' specifies the index object.
5384    `tokens' specifies a list object of the union tokens.
5385    `sign' specifies the logical sign.
5386    `ores' specifies a map object of old primary keys.
5387    `nres' specifies a map object of new primary keys.
5388    `hint' specifies the hint object. */
tctdbidxgetbyftsunion(TDBIDX * idx,const TCLIST * tokens,bool sign,TCMAP * ores,TCMAP * nres,TCXSTR * hint)5389 static void tctdbidxgetbyftsunion(TDBIDX *idx, const TCLIST *tokens, bool sign,
5390                                   TCMAP *ores, TCMAP *nres, TCXSTR *hint){
5391   assert(idx && tokens && hint);
5392   TCBDB *db = idx->db;
5393   TCMAP *cc = idx->cc;
5394   int tnum = TCLISTNUM(tokens);
5395   for(int i = 0; i < tnum; i++){
5396     const char *word;
5397     int wsiz;
5398     TCLISTVAL(word, tokens, i, wsiz);
5399     uint16_t *ary;
5400     TCMALLOC(ary, sizeof(*ary) * (wsiz + TDBIDXQGUNIT));
5401     int anum;
5402     tcstrutftoucs(word, ary, &anum);
5403     for(int j = 0; j < TDBIDXQGUNIT; j++){
5404       ary[anum+j] = 0;
5405     }
5406     if(anum >= TDBIDXQGUNIT){
5407       TDBFTSSTROCR *socrs;
5408       TCMALLOC(socrs, TDBFTSOCRUNIT * sizeof(*socrs));
5409       int sonum = 0;
5410       int soanum = TDBFTSOCRUNIT;
5411       int sobase = 0;
5412       TDBFTSNUMOCR *nocrs;
5413       TCMALLOC(nocrs, TDBFTSOCRUNIT * sizeof(*nocrs));
5414       int nonum = 0;
5415       int noanum = TDBFTSOCRUNIT;
5416       int nobase = 0;
5417       TCBITMAP *pkmap = TCBITMAPNEW(TDBFTSBMNUM);
5418       char token[TDBIDXQGUNIT*3+1];
5419       uint16_t seq = 0;
5420       for(int j = 0; j < anum; j += TDBIDXQGUNIT){
5421         sobase = sonum;
5422         nobase = nonum;
5423         int diff = anum - j - TDBIDXQGUNIT;
5424         if(diff < 0){
5425           j += diff;
5426           diff = -diff;
5427         } else {
5428           diff = 0;
5429         }
5430         tcstrucstoutf(ary + j, TDBIDXQGUNIT, token);
5431         int tsiz = strlen(token);
5432         int csiz;
5433         const char *cbuf = tcmapget(cc, token, tsiz, &csiz);
5434         if(cbuf){
5435           while(csiz > 0){
5436             const char *pkbuf = NULL;
5437             int32_t pksiz = 0;
5438             int64_t pkid = 0;
5439             if(*cbuf == '\0'){
5440               cbuf++;
5441               csiz--;
5442               int step;
5443               TCREADVNUMBUF(cbuf, pksiz, step);
5444               cbuf += step;
5445               csiz -= step;
5446               pkbuf = cbuf;
5447               cbuf += pksiz;
5448               csiz -= pksiz;
5449             } else {
5450               int step;
5451               TCREADVNUMBUF64(cbuf, pkid, step);
5452               cbuf += step;
5453               csiz -= step;
5454             }
5455             if(csiz > 0){
5456               int off, step;
5457               TCREADVNUMBUF(cbuf, off, step);
5458               cbuf += step;
5459               csiz -= step;
5460               off += diff;
5461               if(pkbuf){
5462                 unsigned int hash = 19780211;
5463                 for(int k = 0; k < pksiz; k++){
5464                   hash = hash * 37 + ((unsigned char *)pkbuf)[k];
5465                 }
5466                 hash = hash % TDBFTSBMNUM;
5467                 if(j == 0 || TCBITMAPCHECK(pkmap, hash)){
5468                   if(sonum >= soanum){
5469                     soanum *= 2;
5470                     TCREALLOC(socrs, socrs, soanum * sizeof(*socrs));
5471                   }
5472                   TDBFTSSTROCR *ocr = socrs + sonum;
5473                   ocr->pkbuf = pkbuf;
5474                   ocr->pksiz = pksiz;
5475                   ocr->off = off;
5476                   ocr->seq = seq;
5477                   ocr->hash = hash;
5478                   sonum++;
5479                   if(j == 0) TCBITMAPON(pkmap, hash);
5480                 }
5481               } else {
5482                 unsigned int hash = pkid % TDBFTSBMNUM;
5483                 if(j == 0 || TCBITMAPCHECK(pkmap, hash)){
5484                   if(nonum >= noanum){
5485                     noanum *= 2;
5486                     TCREALLOC(nocrs, nocrs, noanum * sizeof(*nocrs));
5487                   }
5488                   TDBFTSNUMOCR *ocr = nocrs + nonum;
5489                   ocr->pkid = pkid;
5490                   ocr->off = off;
5491                   ocr->seq = seq;
5492                   ocr->hash = hash;
5493                   nonum++;
5494                   if(j == 0) TCBITMAPON(pkmap, hash);
5495                 }
5496               }
5497             }
5498           }
5499         }
5500         cbuf = tcbdbget3(db, token, tsiz, &csiz);
5501         if(cbuf){
5502           while(csiz > 0){
5503             const char *pkbuf = NULL;
5504             int32_t pksiz = 0;
5505             int64_t pkid = 0;
5506             if(*cbuf == '\0'){
5507               cbuf++;
5508               csiz--;
5509               int step;
5510               TCREADVNUMBUF(cbuf, pksiz, step);
5511               cbuf += step;
5512               csiz -= step;
5513               pkbuf = cbuf;
5514               cbuf += pksiz;
5515               csiz -= pksiz;
5516             } else {
5517               int step;
5518               TCREADVNUMBUF64(cbuf, pkid, step);
5519               cbuf += step;
5520               csiz -= step;
5521             }
5522             if(csiz > 0){
5523               int off, step;
5524               TCREADVNUMBUF(cbuf, off, step);
5525               cbuf += step;
5526               csiz -= step;
5527               off += diff;
5528               if(pkbuf){
5529                 unsigned int hash = 19780211;
5530                 for(int k = 0; k < pksiz; k++){
5531                   hash = hash * 37 + ((unsigned char *)pkbuf)[k];
5532                 }
5533                 hash = hash % TDBFTSBMNUM;
5534                 if(j == 0 || TCBITMAPCHECK(pkmap, hash)){
5535                   if(sonum >= soanum){
5536                     soanum *= 2;
5537                     TCREALLOC(socrs, socrs, soanum * sizeof(*socrs));
5538                   }
5539                   TDBFTSSTROCR *ocr = socrs + sonum;
5540                   ocr->pkbuf = pkbuf;
5541                   ocr->pksiz = pksiz;
5542                   ocr->off = off;
5543                   ocr->seq = seq;
5544                   ocr->hash = hash;
5545                   sonum++;
5546                   if(j == 0) TCBITMAPON(pkmap, hash);
5547                 }
5548               } else {
5549                 unsigned int hash = pkid % TDBFTSBMNUM;
5550                 if(j == 0 || TCBITMAPCHECK(pkmap, hash)){
5551                   if(nonum >= noanum){
5552                     noanum *= 2;
5553                     TCREALLOC(nocrs, nocrs, noanum * sizeof(*nocrs));
5554                   }
5555                   TDBFTSNUMOCR *ocr = nocrs + nonum;
5556                   ocr->pkid = pkid;
5557                   ocr->off = off;
5558                   ocr->seq = seq;
5559                   ocr->hash = hash;
5560                   nonum++;
5561                   if(j == 0) TCBITMAPON(pkmap, hash);
5562                 }
5563               }
5564             }
5565           }
5566         }
5567         seq++;
5568         if(sonum <= sobase && nonum <= nobase){
5569           sonum = 0;
5570           nonum = 0;
5571           break;
5572         }
5573       }
5574       TCBITMAPDEL(pkmap);
5575       if(seq > 1){
5576         if(sonum > UINT16_MAX){
5577           int flnum = sonum * 16 + 1;
5578           TCBITMAP *flmap = TCBITMAPNEW(flnum);
5579           for(int j = sobase; j < sonum; j++){
5580             TDBFTSSTROCR *ocr = socrs + j;
5581             uint32_t hash = (((uint32_t)ocr->off << 16) | ocr->hash) % flnum;
5582             TCBITMAPON(flmap, hash);
5583           }
5584           int wi = 0;
5585           for(int j = 0; j < sobase; j++){
5586             TDBFTSSTROCR *ocr = socrs + j;
5587             int rem = (seq - ocr->seq - 1) * TDBIDXQGUNIT;
5588             uint32_t hash = (((uint32_t)(ocr->off + rem) << 16) | ocr->hash) % flnum;
5589             if(TCBITMAPCHECK(flmap, hash)) socrs[wi++] = *ocr;
5590           }
5591           for(int j = sobase; j < sonum; j++){
5592             socrs[wi++] = socrs[j];
5593           }
5594           sonum = wi;
5595           TCBITMAPDEL(flmap);
5596         }
5597         if(sonum > UINT16_MAX * 2){
5598           TDBFTSSTROCR *rocrs;
5599           TCMALLOC(rocrs, sizeof(*rocrs) * sonum);
5600           uint32_t *counts;
5601           TCCALLOC(counts, sizeof(*counts), (UINT16_MAX + 1));
5602           for(int j = 0; j < sonum; j++){
5603             counts[socrs[j].hash]++;
5604           }
5605           for(int j = 0; j < UINT16_MAX; j++){
5606             counts[j+1] += counts[j];
5607           }
5608           for(int j = sonum - 1; j >= 0; j--){
5609             rocrs[--counts[socrs[j].hash]] = socrs[j];
5610           }
5611           for(int j = 0; j < UINT16_MAX; j++){
5612             int num = counts[j+1] - counts[j];
5613             if(num > 1) qsort(rocrs + counts[j], num, sizeof(*rocrs),
5614                               (int (*)(const void *, const void *))tctdbidxftscmpstrocr);
5615           }
5616           int num = sonum - counts[UINT16_MAX];
5617           if(num > 1) qsort(rocrs + counts[UINT16_MAX], num, sizeof(*rocrs),
5618                             (int (*)(const void *, const void *))tctdbidxftscmpstrocr);
5619           TCFREE(counts);
5620           TCFREE(socrs);
5621           socrs = rocrs;
5622         } else if(sonum > 1){
5623           qsort(socrs, sonum, sizeof(*socrs),
5624                 (int (*)(const void *, const void *))tctdbidxftscmpstrocr);
5625         }
5626         if(nonum > UINT16_MAX){
5627           int flnum = nonum * 16 + 1;
5628           TCBITMAP *flmap = TCBITMAPNEW(flnum);
5629           for(int j = nobase; j < nonum; j++){
5630             TDBFTSNUMOCR *ocr = nocrs + j;
5631             uint32_t hash = (((uint32_t)ocr->off << 16) | ocr->hash) % flnum;
5632             TCBITMAPON(flmap, hash);
5633           }
5634           int wi = 0;
5635           for(int j = 0; j < nobase; j++){
5636             TDBFTSNUMOCR *ocr = nocrs + j;
5637             int rem = (seq - ocr->seq - 1) * TDBIDXQGUNIT;
5638             uint32_t hash = (((uint32_t)(ocr->off + rem) << 16) | ocr->hash) % flnum;
5639             if(TCBITMAPCHECK(flmap, hash)) nocrs[wi++] = *ocr;
5640           }
5641           for(int j = nobase; j < nonum; j++){
5642             nocrs[wi++] = nocrs[j];
5643           }
5644           nonum = wi;
5645           TCBITMAPDEL(flmap);
5646         }
5647         if(nonum > UINT16_MAX * 2){
5648           TDBFTSNUMOCR *rocrs;
5649           TCMALLOC(rocrs, sizeof(*rocrs) * nonum);
5650           uint32_t *counts;
5651           TCCALLOC(counts, sizeof(*counts), (UINT16_MAX + 1));
5652           for(int j = 0; j < nonum; j++){
5653             counts[nocrs[j].hash]++;
5654           }
5655           for(int j = 0; j < UINT16_MAX; j++){
5656             counts[j+1] += counts[j];
5657           }
5658           for(int j = nonum - 1; j >= 0; j--){
5659             rocrs[--counts[nocrs[j].hash]] = nocrs[j];
5660           }
5661           for(int j = 0; j < UINT16_MAX; j++){
5662             int num = counts[j+1] - counts[j];
5663             if(num > 1) qsort(rocrs + counts[j], num, sizeof(*rocrs),
5664                               (int (*)(const void *, const void *))tctdbidxftscmpnumocr);
5665           }
5666           int num = nonum - counts[UINT16_MAX];
5667           if(num > 1) qsort(rocrs + counts[UINT16_MAX], num, sizeof(*rocrs),
5668                             (int (*)(const void *, const void *))tctdbidxftscmpnumocr);
5669           TCFREE(counts);
5670           TCFREE(nocrs);
5671           nocrs = rocrs;
5672         } else if(nonum > 1){
5673           qsort(nocrs, nonum, sizeof(*nocrs),
5674                 (int (*)(const void *, const void *))tctdbidxftscmpnumocr);
5675         }
5676       }
5677       int rem = (seq - 1) * TDBIDXQGUNIT;
5678       int onum = 0;
5679       int ri = 0;
5680       while(ri < sonum){
5681         TDBFTSSTROCR *ocr = socrs + ri;
5682         ri++;
5683         if(ocr->seq > 0) continue;
5684         const char *pkbuf = ocr->pkbuf;
5685         int32_t pksiz = ocr->pksiz;
5686         int32_t off = ocr->off;
5687         uint16_t seq = 1;
5688         for(int j = ri; j < sonum; j++){
5689           TDBFTSSTROCR *tocr = socrs + j;
5690           if(!tocr->pkbuf || tocr->pksiz != pksiz || memcmp(tocr->pkbuf, pkbuf, pksiz) ||
5691              tocr->off > off + TDBIDXQGUNIT) break;
5692           if(tocr->seq == seq && tocr->off == off + TDBIDXQGUNIT){
5693             off = tocr->off;
5694             seq++;
5695           }
5696         }
5697         if(off == ocr->off + rem){
5698           onum++;
5699           if(ores){
5700             int rsiz;
5701             if(tcmapget(ores, pkbuf, pksiz, &rsiz)){
5702               if(sign){
5703                 tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5704               } else {
5705                 tcmapout(ores, pkbuf, pksiz);
5706               }
5707             }
5708           } else {
5709             tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5710           }
5711           while(ri < sonum){
5712             ocr = socrs + ri;
5713             if(!ocr->pkbuf || ocr->pksiz != pksiz || memcmp(ocr->pkbuf, pkbuf, pksiz)) break;
5714             ri++;
5715           }
5716         }
5717       }
5718       ri = 0;
5719       while(ri < nonum){
5720         TDBFTSNUMOCR *ocr = nocrs + ri;
5721         ri++;
5722         if(ocr->seq > 0) continue;
5723         int64_t pkid = ocr->pkid;
5724         int32_t off = ocr->off;
5725         uint16_t seq = 1;
5726         for(int j = ri; j < nonum; j++){
5727           TDBFTSNUMOCR *tocr = nocrs + j;
5728           if(tocr->pkid != pkid || tocr->off > off + TDBIDXQGUNIT) break;
5729           if(tocr->seq == seq && tocr->off == off + TDBIDXQGUNIT){
5730             off = tocr->off;
5731             seq++;
5732           }
5733         }
5734         if(off == ocr->off + rem){
5735           onum++;
5736           char pkbuf[TCNUMBUFSIZ];
5737           int pksiz = sprintf(pkbuf, "%lld", (long long)pkid);
5738           if(ores){
5739             int rsiz;
5740             if(tcmapget(ores, pkbuf, pksiz, &rsiz)){
5741               if(sign){
5742                 tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5743               } else {
5744                 tcmapout(ores, pkbuf, pksiz);
5745               }
5746             }
5747           } else {
5748             tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5749           }
5750           while(ri < nonum && nocrs[ri].pkid == pkid){
5751             ri++;
5752           }
5753         }
5754       }
5755       tcxstrprintf(hint, "token occurrence: \"%s\" %d\n", word, onum);
5756       TCFREE(nocrs);
5757       TCFREE(socrs);
5758     } else {
5759       int onum = 0;
5760       TCMAP *uniq = (i > 0 || ores) ? tcmapnew2(UINT16_MAX) : NULL;
5761       tcmapiterinit(cc);
5762       const char *kbuf;
5763       int ksiz;
5764       while((kbuf = tcmapiternext(cc, &ksiz)) != NULL){
5765         if(ksiz < wsiz || memcmp(kbuf, word, wsiz)) continue;
5766         int csiz;
5767         const char *cbuf = tcmapiterval(kbuf, &csiz);
5768         while(csiz > 0){
5769           const char *pkbuf = NULL;
5770           int32_t pksiz = 0;
5771           int64_t pkid = 0;
5772           if(*cbuf == '\0'){
5773             cbuf++;
5774             csiz--;
5775             int step;
5776             TCREADVNUMBUF(cbuf, pksiz, step);
5777             cbuf += step;
5778             csiz -= step;
5779             pkbuf = cbuf;
5780             cbuf += pksiz;
5781             csiz -= pksiz;
5782           } else {
5783             int step;
5784             TCREADVNUMBUF64(cbuf, pkid, step);
5785             cbuf += step;
5786             csiz -= step;
5787           }
5788           if(csiz > 0){
5789             int off, step;
5790             TCREADVNUMBUF(cbuf, off, step);
5791             cbuf += step;
5792             csiz -= step;
5793             if(pkbuf){
5794               if(ores){
5795                 int rsiz;
5796                 if(tcmapget(ores, pkbuf, pksiz, &rsiz)){
5797                   if(sign){
5798                     tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5799                   } else {
5800                     tcmapout(ores, pkbuf, pksiz);
5801                   }
5802                 }
5803               } else {
5804                 if(tcmapputkeep(nres, pkbuf, pksiz, "", 0)) onum++;
5805               }
5806               if(uniq) tcmapputkeep(uniq, pkbuf, pksiz, "", 0);
5807             } else {
5808               char numbuf[TCNUMBUFSIZ];
5809               int pksiz = sprintf(numbuf, "%lld", (long long)pkid);
5810               if(ores){
5811                 int rsiz;
5812                 if(tcmapget(ores, numbuf, pksiz, &rsiz)){
5813                   if(sign){
5814                     tcmapputkeep(nres, numbuf, pksiz, "", 0);
5815                   } else {
5816                     tcmapout(ores, numbuf, pksiz);
5817                   }
5818                 }
5819               } else {
5820                 if(tcmapputkeep(nres, numbuf, pksiz, "", 0)) onum++;
5821               }
5822               if(uniq) tcmapputkeep(uniq, numbuf, pksiz, "", 0);
5823             }
5824           }
5825         }
5826       }
5827       BDBCUR *cur = tcbdbcurnew(db);
5828       tcbdbcurjump(cur, word, wsiz);
5829       TCXSTR *key = tcxstrnew();
5830       TCXSTR *val = tcxstrnew();
5831       while(tcbdbcurrec(cur, key, val)){
5832         const char *kbuf = TCXSTRPTR(key);
5833         int ksiz = TCXSTRSIZE(key);
5834         if(ksiz < wsiz || memcmp(kbuf, word, wsiz)) break;
5835         const char *cbuf = TCXSTRPTR(val);
5836         int csiz = TCXSTRSIZE(val);
5837         while(csiz > 0){
5838           const char *pkbuf = NULL;
5839           int32_t pksiz = 0;
5840           int64_t pkid = 0;
5841           if(*cbuf == '\0'){
5842             cbuf++;
5843             csiz--;
5844             int step;
5845             TCREADVNUMBUF(cbuf, pksiz, step);
5846             cbuf += step;
5847             csiz -= step;
5848             pkbuf = cbuf;
5849             cbuf += pksiz;
5850             csiz -= pksiz;
5851           } else {
5852             int step;
5853             TCREADVNUMBUF64(cbuf, pkid, step);
5854             cbuf += step;
5855             csiz -= step;
5856           }
5857           if(csiz > 0){
5858             int off, step;
5859             TCREADVNUMBUF(cbuf, off, step);
5860             cbuf += step;
5861             csiz -= step;
5862             if(pkbuf){
5863               if(ores){
5864                 int rsiz;
5865                 if(tcmapget(ores, pkbuf, pksiz, &rsiz)){
5866                   if(sign){
5867                     tcmapputkeep(nres, pkbuf, pksiz, "", 0);
5868                   } else {
5869                     tcmapout(ores, pkbuf, pksiz);
5870                   }
5871                 }
5872               } else {
5873                 if(tcmapputkeep(nres, pkbuf, pksiz, "", 0)) onum++;
5874               }
5875               if(uniq) tcmapputkeep(uniq, pkbuf, pksiz, "", 0);
5876             } else {
5877               char numbuf[TCNUMBUFSIZ];
5878               int pksiz = sprintf(numbuf, "%lld", (long long)pkid);
5879               if(ores){
5880                 int rsiz;
5881                 if(tcmapget(ores, numbuf, pksiz, &rsiz)){
5882                   if(sign){
5883                     tcmapputkeep(nres, numbuf, pksiz, "", 0);
5884                   } else {
5885                     tcmapout(ores, numbuf, pksiz);
5886                   }
5887                 }
5888               } else {
5889                 if(tcmapputkeep(nres, numbuf, pksiz, "", 0)) onum++;
5890               }
5891               if(uniq) tcmapputkeep(uniq, numbuf, pksiz, "", 0);
5892             }
5893           }
5894         }
5895         tcbdbcurnext(cur);
5896       }
5897       tcxstrdel(val);
5898       tcxstrdel(key);
5899       tcbdbcurdel(cur);
5900       tcxstrprintf(hint, "token occurrence: \"%s\" %d\n",
5901                    word, uniq ? (int)tcmaprnum(uniq) : onum);
5902       if(uniq) tcmapdel(uniq);
5903     }
5904     TCFREE(ary);
5905   }
5906 }
5907 
5908 
5909 /* Compare two string occurrences of full-text search in identical order.
5910    `a' specifies the pointer to one occurrence.
5911    `b' specifies the pointer to the other occurrence.
5912    The return value is positive if the former is big, negative if the latter is big, 0 if both
5913    are equivalent. */
tctdbidxftscmpstrocr(TDBFTSSTROCR * a,TDBFTSSTROCR * b)5914 static int tctdbidxftscmpstrocr(TDBFTSSTROCR *a, TDBFTSSTROCR *b){
5915   assert(a && b);
5916   if(a->pksiz > b->pksiz) return 1;
5917   if(a->pksiz < b->pksiz) return -1;
5918   int diff = memcmp(a->pkbuf, b->pkbuf, a->pksiz);
5919   if(diff != 0) return diff;
5920   return a->off - b->off;
5921 }
5922 
5923 
5924 /* Compare two number occurrences of full-text search in identical order.
5925    `a' specifies the pointer to one occurrence.
5926    `b' specifies the pointer to the other occurrence.
5927    The return value is positive if the former is big, negative if the latter is big, 0 if both
5928    are equivalent. */
tctdbidxftscmpnumocr(TDBFTSNUMOCR * a,TDBFTSNUMOCR * b)5929 static int tctdbidxftscmpnumocr(TDBFTSNUMOCR *a, TDBFTSNUMOCR *b){
5930   assert(a && b);
5931   if(a->pkid > b->pkid) return 1;
5932   if(a->pkid < b->pkid) return -1;
5933   return a->off - b->off;
5934 }
5935 
5936 
5937 /* Parse an expression of full-text search.
5938    `expr' specifies the expression.
5939    `esiz' specifies the size of the expression.
5940    `np' specifies the pointer to a variable into which the number of elements of the return value
5941    is assigned.
5942    `op' specifies the operation type.
5943    The return value is the pointer to the array of full-text search units. */
tctdbftsparseexpr(const char * expr,int esiz,int op,int * np)5944 static TDBFTSUNIT *tctdbftsparseexpr(const char *expr, int esiz, int op, int *np){
5945   assert(expr && esiz >= 0 && np);
5946   TDBFTSUNIT *ftsunits;
5947   TCMALLOC(ftsunits, TDBFTSUNITMAX * sizeof(*ftsunits));
5948   int ftsnum = 0;
5949   uint16_t *ary;
5950   TCMALLOC(ary, sizeof(*ary) * esiz + 1);
5951   int anum;
5952   tcstrutftoucs(expr, ary, &anum);
5953   anum = tcstrucsnorm(ary, anum, TCUNSPACE | TCUNLOWER | TCUNNOACC | TCUNWIDTH);
5954   char *str;
5955   TCMALLOC(str, esiz + 1);
5956   tcstrucstoutf(ary, anum, str);
5957   if(op == TDBQCFTSPH){
5958     TCLIST *tokens = tclistnew2(1);
5959     tclistpush2(tokens, str);
5960     ftsunits[ftsnum].tokens = tokens;
5961     ftsunits[ftsnum].sign = true;
5962     ftsnum++;
5963   } else if(op == TDBQCFTSAND){
5964     TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
5965     int tnum = TCLISTNUM(tokens);
5966     for(int i = 0; i < tnum && ftsnum < TDBFTSUNITMAX; i++){
5967       const char *token = TCLISTVALPTR(tokens, i);
5968       if(*token == '\0') continue;
5969       TCLIST *ttokens = tclistnew2(1);
5970       tclistpush2(ttokens, token);
5971       ftsunits[ftsnum].tokens = ttokens;
5972       ftsunits[ftsnum].sign = true;
5973       ftsnum++;
5974     }
5975     tclistdel(tokens);
5976   } else if(op == TDBQCFTSOR){
5977     TCLIST *tokens = tcstrsplit(expr, "\t\n\r ,");
5978     int tnum = TCLISTNUM(tokens);
5979     TCLIST *ttokens = tclistnew2(tnum);
5980     for(int i = 0; i < tnum; i++){
5981       const char *token = TCLISTVALPTR(tokens, i);
5982       if(*token == '\0') continue;
5983       tclistpush2(ttokens, token);
5984     }
5985     ftsunits[ftsnum].tokens = ttokens;
5986     ftsunits[ftsnum].sign = true;
5987     ftsnum++;
5988     tclistdel(tokens);
5989   } else if(op == TDBQCFTSEX){
5990     TCLIST *tokens = tcstrtokenize(str);
5991     int op = 0;
5992     for(int i = 0; i < tclistnum(tokens); i++){
5993       const char *token = TCLISTVALPTR(tokens, i);
5994       if(!strcmp(token, "&&")){
5995         op = 0;
5996       } else if(!strcmp(token, "||")){
5997         op = 1;
5998       } else if(!strcmp(token, "!!")){
5999         op = 2;
6000       } else {
6001         if(op == 0 || op == 2){
6002           if(ftsnum >= TDBFTSUNITMAX) break;
6003           TCLIST *ttokens = tclistnew2(2);
6004           tclistpush2(ttokens, token);
6005           ftsunits[ftsnum].tokens = ttokens;
6006           ftsunits[ftsnum].sign = op == 0;
6007           ftsnum++;
6008         } else if(op == 1){
6009           if(ftsnum < 1){
6010             ftsunits[ftsnum].tokens = tclistnew2(2);
6011             ftsunits[ftsnum].sign = op == 0;
6012             ftsnum++;
6013           }
6014           TCLIST *ttokens = ftsunits[ftsnum-1].tokens;
6015           tclistpush2(ttokens, token);
6016         }
6017         op = 0;
6018       }
6019     }
6020     tclistdel(tokens);
6021   }
6022   TCFREE(str);
6023   TCFREE(ary);
6024   *np = ftsnum;
6025   return ftsunits;
6026 }
6027 
6028 
6029 /* Perform dynamic defragmentation of a table database object.
6030    `tdb' specifies the table database object.
6031    `step' specifie the number of steps.
6032    If successful, the return value is true, else, it is false. */
tctdbdefragimpl(TCTDB * tdb,int64_t step)6033 static bool tctdbdefragimpl(TCTDB *tdb, int64_t step){
6034   assert(tdb);
6035   bool err = false;
6036   TCHDB *hdb = tdb->hdb;
6037   TDBIDX *idxs = tdb->idxs;
6038   int inum = tdb->inum;
6039   if(!tchdbdefrag(hdb, step)) err = true;
6040   for(int i = 0; i < inum; i++){
6041     TDBIDX *idx = idxs + i;
6042     switch(idx->type){
6043       case TDBITLEXICAL:
6044       case TDBITDECIMAL:
6045       case TDBITTOKEN:
6046       case TDBITQGRAM:
6047         if(!tcbdbdefrag(idx->db, step)){
6048           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
6049           err = true;
6050         }
6051         break;
6052     }
6053   }
6054   return !err;
6055 }
6056 
6057 
6058 /* Clear the cache of a table tree database object.
6059    `tdb' specifies the table tree database object.
6060    If successful, the return value is true, else, it is false. */
tctdbcacheclearimpl(TCTDB * tdb)6061 static bool tctdbcacheclearimpl(TCTDB *tdb){
6062   assert(tdb);
6063   bool err = false;
6064   TCHDB *hdb = tdb->hdb;
6065   TDBIDX *idxs = tdb->idxs;
6066   int inum = tdb->inum;
6067   if(!tchdbcacheclear(hdb)) err = true;
6068   for(int i = 0; i < inum; i++){
6069     TDBIDX *idx = idxs + i;
6070     switch(idx->type){
6071       case TDBITLEXICAL:
6072       case TDBITDECIMAL:
6073       case TDBITTOKEN:
6074       case TDBITQGRAM:
6075         if(!tcbdbcacheclear(idx->db)){
6076           tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
6077           err = true;
6078         }
6079         break;
6080     }
6081   }
6082   return !err;
6083 }
6084 
6085 
6086 /* Process each record atomically of a table database object.
6087    `tdb' specifies the table database object.
6088    `func' specifies the pointer to the iterator function called for each record.
6089    `op' specifies an arbitrary pointer to be given as a parameter of the iterator function.
6090    If successful, the return value is true, else, it is false. */
tctdbforeachimpl(TCTDB * tdb,TCITER iter,void * op)6091 static bool tctdbforeachimpl(TCTDB *tdb, TCITER iter, void *op){
6092   assert(tdb && iter);
6093   TCHDB *hdb = tdb->hdb;
6094   char *lkbuf = NULL;
6095   int lksiz = 0;
6096   char *pkbuf, stack[TDBPAGEBUFSIZ], *rbuf;
6097   int pksiz;
6098   const char *cbuf;
6099   int csiz;
6100   while((pkbuf = tchdbgetnext3(hdb, lkbuf, lksiz, &pksiz, &cbuf, &csiz)) != NULL){
6101     if(pksiz < TDBPAGEBUFSIZ){
6102       rbuf = stack;
6103     } else {
6104       TCMALLOC(rbuf, pksiz + 1);
6105     }
6106     memcpy(rbuf, pkbuf, pksiz);
6107     stack[pksiz] = '\0';
6108     TCMAP *cols = tcmapload(cbuf, csiz);
6109     int zsiz;
6110     char *zbuf = tcstrjoin4(cols, &zsiz);
6111     bool rv = iter(rbuf, pksiz, zbuf, zsiz, op);
6112     TCFREE(zbuf);
6113     if(rbuf != stack) TCFREE(rbuf);
6114     tcmapdel(cols);
6115     TCFREE(lkbuf);
6116     lkbuf = pkbuf;
6117     lksiz = pksiz;
6118     if(!rv) break;
6119   }
6120   TCFREE(lkbuf);
6121   return true;
6122 }
6123 
6124 
6125 /* Answer to remove for each record of a query.
6126    `pkbuf' is ignored.
6127    `pksiz' is ignored.
6128    `op' is ignored.
6129    The return value is always `TDBQPOUT'. */
tctdbqryprocoutcb(const void * pkbuf,int pksiz,TCMAP * cols,void * op)6130 static int tctdbqryprocoutcb(const void *pkbuf, int pksiz, TCMAP *cols, void *op){
6131   assert(pkbuf && pksiz >= 0 && cols);
6132   return TDBQPOUT;
6133 }
6134 
6135 
6136 /* Lock a method of the table database object.
6137    `tdb' specifies the table database object.
6138    `wr' specifies whether the lock is writer or not.
6139    If successful, the return value is true, else, it is false. */
tctdblockmethod(TCTDB * tdb,bool wr)6140 static bool tctdblockmethod(TCTDB *tdb, bool wr){
6141   assert(tdb);
6142   if(wr ? pthread_rwlock_wrlock(tdb->mmtx) != 0 : pthread_rwlock_rdlock(tdb->mmtx) != 0){
6143     tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
6144     return false;
6145   }
6146   TCTESTYIELD();
6147   return true;
6148 }
6149 
6150 
6151 /* Unlock a method of the table database object.
6152    `tdb' specifies the table database object.
6153    If successful, the return value is true, else, it is false. */
tctdbunlockmethod(TCTDB * tdb)6154 static bool tctdbunlockmethod(TCTDB *tdb){
6155   assert(tdb);
6156   if(pthread_rwlock_unlock(tdb->mmtx) != 0){
6157     tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
6158     return false;
6159   }
6160   TCTESTYIELD();
6161   return true;
6162 }
6163 
6164 
6165 
6166 /*************************************************************************************************
6167  * debugging functions
6168  *************************************************************************************************/
6169 
6170 
6171 /* Print meta data of the header into the debugging output.
6172    `tdb' specifies the table database object. */
tctdbprintmeta(TCTDB * tdb)6173 void tctdbprintmeta(TCTDB *tdb){
6174   assert(tdb);
6175   int dbgfd = tchdbdbgfd(tdb->hdb);
6176   if(dbgfd < 0) return;
6177   if(dbgfd == UINT16_MAX) dbgfd = 1;
6178   char buf[TDBPAGEBUFSIZ];
6179   char *wp = buf;
6180   wp += sprintf(wp, "META:");
6181   wp += sprintf(wp, " mmtx=%p", (void *)tdb->mmtx);
6182   wp += sprintf(wp, " hdb=%p", (void *)tdb->hdb);
6183   wp += sprintf(wp, " open=%d", tdb->open);
6184   wp += sprintf(wp, " wmode=%d", tdb->wmode);
6185   wp += sprintf(wp, " opts=%u", tdb->opts);
6186   wp += sprintf(wp, " lcnum=%d", tdb->lcnum);
6187   wp += sprintf(wp, " ncnum=%d", tdb->ncnum);
6188   wp += sprintf(wp, " iccmax=%lld", (long long)tdb->iccmax);
6189   wp += sprintf(wp, " iccsync=%f", tdb->iccsync);
6190   wp += sprintf(wp, " idxs=%p", (void *)tdb->idxs);
6191   wp += sprintf(wp, " inum=%d", tdb->inum);
6192   wp += sprintf(wp, " tran=%d", tdb->tran);
6193   *(wp++) = '\n';
6194   tcwrite(dbgfd, buf, wp - buf);
6195 }
6196 
6197 
6198 
6199 // END OF FILE
6200