1 /* 2 ** 2014 May 31 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ****************************************************************************** 12 ** 13 ** Interfaces to extend FTS5. Using the interfaces defined in this file, 14 ** FTS5 may be extended with: 15 ** 16 ** * custom tokenizers, and 17 ** * custom auxiliary functions. 18 */ 19 20 21 #ifndef _FTS5_H 22 #define _FTS5_H 23 24 #include "sqlite3.h" 25 26 #ifdef __cplusplus 27 extern "C" { 28 #endif 29 30 /************************************************************************* 31 ** CUSTOM AUXILIARY FUNCTIONS 32 ** 33 ** Virtual table implementations may overload SQL functions by implementing 34 ** the sqlite3_module.xFindFunction() method. 35 */ 36 37 typedef struct Fts5ExtensionApi Fts5ExtensionApi; 38 typedef struct Fts5Context Fts5Context; 39 typedef struct Fts5PhraseIter Fts5PhraseIter; 40 41 typedef void (*fts5_extension_function)( 42 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ 43 Fts5Context *pFts, /* First arg to pass to pApi functions */ 44 sqlite3_context *pCtx, /* Context for returning result/error */ 45 int nVal, /* Number of values in apVal[] array */ 46 sqlite3_value **apVal /* Array of trailing arguments */ 47 ); 48 49 struct Fts5PhraseIter { 50 const unsigned char *a; 51 const unsigned char *b; 52 }; 53 54 /* 55 ** EXTENSION API FUNCTIONS 56 ** 57 ** xUserData(pFts): 58 ** Return a copy of the context pointer the extension function was 59 ** registered with. 60 ** 61 ** xColumnTotalSize(pFts, iCol, pnToken): 62 ** If parameter iCol is less than zero, set output variable *pnToken 63 ** to the total number of tokens in the FTS5 table. Or, if iCol is 64 ** non-negative but less than the number of columns in the table, return 65 ** the total number of tokens in column iCol, considering all rows in 66 ** the FTS5 table. 67 ** 68 ** If parameter iCol is greater than or equal to the number of columns 69 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. 70 ** an OOM condition or IO error), an appropriate SQLite error code is 71 ** returned. 72 ** 73 ** xColumnCount(pFts): 74 ** Return the number of columns in the table. 75 ** 76 ** xColumnSize(pFts, iCol, pnToken): 77 ** If parameter iCol is less than zero, set output variable *pnToken 78 ** to the total number of tokens in the current row. Or, if iCol is 79 ** non-negative but less than the number of columns in the table, set 80 ** *pnToken to the number of tokens in column iCol of the current row. 81 ** 82 ** If parameter iCol is greater than or equal to the number of columns 83 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. 84 ** an OOM condition or IO error), an appropriate SQLite error code is 85 ** returned. 86 ** 87 ** This function may be quite inefficient if used with an FTS5 table 88 ** created with the "columnsize=0" option. 89 ** 90 ** xColumnText: 91 ** This function attempts to retrieve the text of column iCol of the 92 ** current document. If successful, (*pz) is set to point to a buffer 93 ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes 94 ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, 95 ** if an error occurs, an SQLite error code is returned and the final values 96 ** of (*pz) and (*pn) are undefined. 97 ** 98 ** xPhraseCount: 99 ** Returns the number of phrases in the current query expression. 100 ** 101 ** xPhraseSize: 102 ** Returns the number of tokens in phrase iPhrase of the query. Phrases 103 ** are numbered starting from zero. 104 ** 105 ** xInstCount: 106 ** Set *pnInst to the total number of occurrences of all phrases within 107 ** the query within the current row. Return SQLITE_OK if successful, or 108 ** an error code (i.e. SQLITE_NOMEM) if an error occurs. 109 ** 110 ** This API can be quite slow if used with an FTS5 table created with the 111 ** "detail=none" or "detail=column" option. If the FTS5 table is created 112 ** with either "detail=none" or "detail=column" and "content=" option 113 ** (i.e. if it is a contentless table), then this API always returns 0. 114 ** 115 ** xInst: 116 ** Query for the details of phrase match iIdx within the current row. 117 ** Phrase matches are numbered starting from zero, so the iIdx argument 118 ** should be greater than or equal to zero and smaller than the value 119 ** output by xInstCount(). 120 ** 121 ** Usually, output parameter *piPhrase is set to the phrase number, *piCol 122 ** to the column in which it occurs and *piOff the token offset of the 123 ** first token of the phrase. The exception is if the table was created 124 ** with the offsets=0 option specified. In this case *piOff is always 125 ** set to -1. 126 ** 127 ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 128 ** if an error occurs. 129 ** 130 ** This API can be quite slow if used with an FTS5 table created with the 131 ** "detail=none" or "detail=column" option. 132 ** 133 ** xRowid: 134 ** Returns the rowid of the current row. 135 ** 136 ** xTokenize: 137 ** Tokenize text using the tokenizer belonging to the FTS5 table. 138 ** 139 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): 140 ** This API function is used to query the FTS table for phrase iPhrase 141 ** of the current query. Specifically, a query equivalent to: 142 ** 143 ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid 144 ** 145 ** with $p set to a phrase equivalent to the phrase iPhrase of the 146 ** current query is executed. For each row visited, the callback function 147 ** passed as the fourth argument is invoked. The context and API objects 148 ** passed to the callback function may be used to access the properties of 149 ** each matched row. Invoking Api.xUserData() returns a copy of the pointer 150 ** passed as the third argument to pUserData. 151 ** 152 ** If the callback function returns any value other than SQLITE_OK, the 153 ** query is abandoned and the xQueryPhrase function returns immediately. 154 ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. 155 ** Otherwise, the error code is propagated upwards. 156 ** 157 ** If the query runs to completion without incident, SQLITE_OK is returned. 158 ** Or, if some error occurs before the query completes or is aborted by 159 ** the callback, an SQLite error code is returned. 160 ** 161 ** 162 ** xSetAuxdata(pFts5, pAux, xDelete) 163 ** 164 ** Save the pointer passed as the second argument as the extension functions 165 ** "auxiliary data". The pointer may then be retrieved by the current or any 166 ** future invocation of the same fts5 extension function made as part of 167 ** of the same MATCH query using the xGetAuxdata() API. 168 ** 169 ** Each extension function is allocated a single auxiliary data slot for 170 ** each FTS query (MATCH expression). If the extension function is invoked 171 ** more than once for a single FTS query, then all invocations share a 172 ** single auxiliary data context. 173 ** 174 ** If there is already an auxiliary data pointer when this function is 175 ** invoked, then it is replaced by the new pointer. If an xDelete callback 176 ** was specified along with the original pointer, it is invoked at this 177 ** point. 178 ** 179 ** The xDelete callback, if one is specified, is also invoked on the 180 ** auxiliary data pointer after the FTS5 query has finished. 181 ** 182 ** If an error (e.g. an OOM condition) occurs within this function, an 183 ** the auxiliary data is set to NULL and an error code returned. If the 184 ** xDelete parameter was not NULL, it is invoked on the auxiliary data 185 ** pointer before returning. 186 ** 187 ** 188 ** xGetAuxdata(pFts5, bClear) 189 ** 190 ** Returns the current auxiliary data pointer for the fts5 extension 191 ** function. See the xSetAuxdata() method for details. 192 ** 193 ** If the bClear argument is non-zero, then the auxiliary data is cleared 194 ** (set to NULL) before this function returns. In this case the xDelete, 195 ** if any, is not invoked. 196 ** 197 ** 198 ** xRowCount(pFts5, pnRow) 199 ** 200 ** This function is used to retrieve the total number of rows in the table. 201 ** In other words, the same value that would be returned by: 202 ** 203 ** SELECT count(*) FROM ftstable; 204 ** 205 ** xPhraseFirst() 206 ** This function is used, along with type Fts5PhraseIter and the xPhraseNext 207 ** method, to iterate through all instances of a single query phrase within 208 ** the current row. This is the same information as is accessible via the 209 ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient 210 ** to use, this API may be faster under some circumstances. To iterate 211 ** through instances of phrase iPhrase, use the following code: 212 ** 213 ** Fts5PhraseIter iter; 214 ** int iCol, iOff; 215 ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); 216 ** iCol>=0; 217 ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) 218 ** ){ 219 ** // An instance of phrase iPhrase at offset iOff of column iCol 220 ** } 221 ** 222 ** The Fts5PhraseIter structure is defined above. Applications should not 223 ** modify this structure directly - it should only be used as shown above 224 ** with the xPhraseFirst() and xPhraseNext() API methods (and by 225 ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). 226 ** 227 ** This API can be quite slow if used with an FTS5 table created with the 228 ** "detail=none" or "detail=column" option. If the FTS5 table is created 229 ** with either "detail=none" or "detail=column" and "content=" option 230 ** (i.e. if it is a contentless table), then this API always iterates 231 ** through an empty set (all calls to xPhraseFirst() set iCol to -1). 232 ** 233 ** xPhraseNext() 234 ** See xPhraseFirst above. 235 ** 236 ** xPhraseFirstColumn() 237 ** This function and xPhraseNextColumn() are similar to the xPhraseFirst() 238 ** and xPhraseNext() APIs described above. The difference is that instead 239 ** of iterating through all instances of a phrase in the current row, these 240 ** APIs are used to iterate through the set of columns in the current row 241 ** that contain one or more instances of a specified phrase. For example: 242 ** 243 ** Fts5PhraseIter iter; 244 ** int iCol; 245 ** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); 246 ** iCol>=0; 247 ** pApi->xPhraseNextColumn(pFts, &iter, &iCol) 248 ** ){ 249 ** // Column iCol contains at least one instance of phrase iPhrase 250 ** } 251 ** 252 ** This API can be quite slow if used with an FTS5 table created with the 253 ** "detail=none" option. If the FTS5 table is created with either 254 ** "detail=none" "content=" option (i.e. if it is a contentless table), 255 ** then this API always iterates through an empty set (all calls to 256 ** xPhraseFirstColumn() set iCol to -1). 257 ** 258 ** The information accessed using this API and its companion 259 ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext 260 ** (or xInst/xInstCount). The chief advantage of this API is that it is 261 ** significantly more efficient than those alternatives when used with 262 ** "detail=column" tables. 263 ** 264 ** xPhraseNextColumn() 265 ** See xPhraseFirstColumn above. 266 */ 267 struct Fts5ExtensionApi { 268 int iVersion; /* Currently always set to 3 */ 269 270 void *(*xUserData)(Fts5Context*); 271 272 int (*xColumnCount)(Fts5Context*); 273 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); 274 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); 275 276 int (*xTokenize)(Fts5Context*, 277 const char *pText, int nText, /* Text to tokenize */ 278 void *pCtx, /* Context passed to xToken() */ 279 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ 280 ); 281 282 int (*xPhraseCount)(Fts5Context*); 283 int (*xPhraseSize)(Fts5Context*, int iPhrase); 284 285 int (*xInstCount)(Fts5Context*, int *pnInst); 286 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); 287 288 sqlite3_int64 (*xRowid)(Fts5Context*); 289 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); 290 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); 291 292 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, 293 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) 294 ); 295 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); 296 void *(*xGetAuxdata)(Fts5Context*, int bClear); 297 298 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); 299 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); 300 301 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); 302 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); 303 }; 304 305 /* 306 ** CUSTOM AUXILIARY FUNCTIONS 307 *************************************************************************/ 308 309 /************************************************************************* 310 ** CUSTOM TOKENIZERS 311 ** 312 ** Applications may also register custom tokenizer types. A tokenizer 313 ** is registered by providing fts5 with a populated instance of the 314 ** following structure. All structure methods must be defined, setting 315 ** any member of the fts5_tokenizer struct to NULL leads to undefined 316 ** behaviour. The structure methods are expected to function as follows: 317 ** 318 ** xCreate: 319 ** This function is used to allocate and inititalize a tokenizer instance. 320 ** A tokenizer instance is required to actually tokenize text. 321 ** 322 ** The first argument passed to this function is a copy of the (void*) 323 ** pointer provided by the application when the fts5_tokenizer object 324 ** was registered with FTS5 (the third argument to xCreateTokenizer()). 325 ** The second and third arguments are an array of nul-terminated strings 326 ** containing the tokenizer arguments, if any, specified following the 327 ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used 328 ** to create the FTS5 table. 329 ** 330 ** The final argument is an output variable. If successful, (*ppOut) 331 ** should be set to point to the new tokenizer handle and SQLITE_OK 332 ** returned. If an error occurs, some value other than SQLITE_OK should 333 ** be returned. In this case, fts5 assumes that the final value of *ppOut 334 ** is undefined. 335 ** 336 ** xDelete: 337 ** This function is invoked to delete a tokenizer handle previously 338 ** allocated using xCreate(). Fts5 guarantees that this function will 339 ** be invoked exactly once for each successful call to xCreate(). 340 ** 341 ** xTokenize: 342 ** This function is expected to tokenize the nText byte string indicated 343 ** by argument pText. pText may or may not be nul-terminated. The first 344 ** argument passed to this function is a pointer to an Fts5Tokenizer object 345 ** returned by an earlier call to xCreate(). 346 ** 347 ** The second argument indicates the reason that FTS5 is requesting 348 ** tokenization of the supplied text. This is always one of the following 349 ** four values: 350 ** 351 ** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into 352 ** or removed from the FTS table. The tokenizer is being invoked to 353 ** determine the set of tokens to add to (or delete from) the 354 ** FTS index. 355 ** 356 ** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed 357 ** against the FTS index. The tokenizer is being called to tokenize 358 ** a bareword or quoted string specified as part of the query. 359 ** 360 ** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as 361 ** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is 362 ** followed by a "*" character, indicating that the last token 363 ** returned by the tokenizer will be treated as a token prefix. 364 ** 365 ** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to 366 ** satisfy an fts5_api.xTokenize() request made by an auxiliary 367 ** function. Or an fts5_api.xColumnSize() request made by the same 368 ** on a columnsize=0 database. 369 ** </ul> 370 ** 371 ** For each token in the input string, the supplied callback xToken() must 372 ** be invoked. The first argument to it should be a copy of the pointer 373 ** passed as the second argument to xTokenize(). The third and fourth 374 ** arguments are a pointer to a buffer containing the token text, and the 375 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets 376 ** of the first byte of and first byte immediately following the text from 377 ** which the token is derived within the input. 378 ** 379 ** The second argument passed to the xToken() callback ("tflags") should 380 ** normally be set to 0. The exception is if the tokenizer supports 381 ** synonyms. In this case see the discussion below for details. 382 ** 383 ** FTS5 assumes the xToken() callback is invoked for each token in the 384 ** order that they occur within the input text. 385 ** 386 ** If an xToken() callback returns any value other than SQLITE_OK, then 387 ** the tokenization should be abandoned and the xTokenize() method should 388 ** immediately return a copy of the xToken() return value. Or, if the 389 ** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, 390 ** if an error occurs with the xTokenize() implementation itself, it 391 ** may abandon the tokenization and return any error code other than 392 ** SQLITE_OK or SQLITE_DONE. 393 ** 394 ** SYNONYM SUPPORT 395 ** 396 ** Custom tokenizers may also support synonyms. Consider a case in which a 397 ** user wishes to query for a phrase such as "first place". Using the 398 ** built-in tokenizers, the FTS5 query 'first + place' will match instances 399 ** of "first place" within the document set, but not alternative forms 400 ** such as "1st place". In some applications, it would be better to match 401 ** all instances of "first place" or "1st place" regardless of which form 402 ** the user specified in the MATCH query text. 403 ** 404 ** There are several ways to approach this in FTS5: 405 ** 406 ** <ol><li> By mapping all synonyms to a single token. In this case, the 407 ** In the above example, this means that the tokenizer returns the 408 ** same token for inputs "first" and "1st". Say that token is in 409 ** fact "first", so that when the user inserts the document "I won 410 ** 1st place" entries are added to the index for tokens "i", "won", 411 ** "first" and "place". If the user then queries for '1st + place', 412 ** the tokenizer substitutes "first" for "1st" and the query works 413 ** as expected. 414 ** 415 ** <li> By adding multiple synonyms for a single term to the FTS index. 416 ** In this case, when tokenizing query text, the tokenizer may 417 ** provide multiple synonyms for a single term within the document. 418 ** FTS5 then queries the index for each synonym individually. For 419 ** example, faced with the query: 420 ** 421 ** <codeblock> 422 ** ... MATCH 'first place'</codeblock> 423 ** 424 ** the tokenizer offers both "1st" and "first" as synonyms for the 425 ** first token in the MATCH query and FTS5 effectively runs a query 426 ** similar to: 427 ** 428 ** <codeblock> 429 ** ... MATCH '(first OR 1st) place'</codeblock> 430 ** 431 ** except that, for the purposes of auxiliary functions, the query 432 ** still appears to contain just two phrases - "(first OR 1st)" 433 ** being treated as a single phrase. 434 ** 435 ** <li> By adding multiple synonyms for a single term to the FTS index. 436 ** Using this method, when tokenizing document text, the tokenizer 437 ** provides multiple synonyms for each token. So that when a 438 ** document such as "I won first place" is tokenized, entries are 439 ** added to the FTS index for "i", "won", "first", "1st" and 440 ** "place". 441 ** 442 ** This way, even if the tokenizer does not provide synonyms 443 ** when tokenizing query text (it should not - to do would be 444 ** inefficient), it doesn't matter if the user queries for 445 ** 'first + place' or '1st + place', as there are entires in the 446 ** FTS index corresponding to both forms of the first token. 447 ** </ol> 448 ** 449 ** Whether it is parsing document or query text, any call to xToken that 450 ** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit 451 ** is considered to supply a synonym for the previous token. For example, 452 ** when parsing the document "I won first place", a tokenizer that supports 453 ** synonyms would call xToken() 5 times, as follows: 454 ** 455 ** <codeblock> 456 ** xToken(pCtx, 0, "i", 1, 0, 1); 457 ** xToken(pCtx, 0, "won", 3, 2, 5); 458 ** xToken(pCtx, 0, "first", 5, 6, 11); 459 ** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); 460 ** xToken(pCtx, 0, "place", 5, 12, 17); 461 **</codeblock> 462 ** 463 ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time 464 ** xToken() is called. Multiple synonyms may be specified for a single token 465 ** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. 466 ** There is no limit to the number of synonyms that may be provided for a 467 ** single token. 468 ** 469 ** In many cases, method (1) above is the best approach. It does not add 470 ** extra data to the FTS index or require FTS5 to query for multiple terms, 471 ** so it is efficient in terms of disk space and query speed. However, it 472 ** does not support prefix queries very well. If, as suggested above, the 473 ** token "first" is subsituted for "1st" by the tokenizer, then the query: 474 ** 475 ** <codeblock> 476 ** ... MATCH '1s*'</codeblock> 477 ** 478 ** will not match documents that contain the token "1st" (as the tokenizer 479 ** will probably not map "1s" to any prefix of "first"). 480 ** 481 ** For full prefix support, method (3) may be preferred. In this case, 482 ** because the index contains entries for both "first" and "1st", prefix 483 ** queries such as 'fi*' or '1s*' will match correctly. However, because 484 ** extra entries are added to the FTS index, this method uses more space 485 ** within the database. 486 ** 487 ** Method (2) offers a midpoint between (1) and (3). Using this method, 488 ** a query such as '1s*' will match documents that contain the literal 489 ** token "1st", but not "first" (assuming the tokenizer is not able to 490 ** provide synonyms for prefixes). However, a non-prefix query like '1st' 491 ** will match against "1st" and "first". This method does not require 492 ** extra disk space, as no extra entries are added to the FTS index. 493 ** On the other hand, it may require more CPU cycles to run MATCH queries, 494 ** as separate queries of the FTS index are required for each synonym. 495 ** 496 ** When using methods (2) or (3), it is important that the tokenizer only 497 ** provide synonyms when tokenizing document text (method (2)) or query 498 ** text (method (3)), not both. Doing so will not cause any errors, but is 499 ** inefficient. 500 */ 501 typedef struct Fts5Tokenizer Fts5Tokenizer; 502 typedef struct fts5_tokenizer fts5_tokenizer; 503 struct fts5_tokenizer { 504 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); 505 void (*xDelete)(Fts5Tokenizer*); 506 int (*xTokenize)(Fts5Tokenizer*, 507 void *pCtx, 508 int flags, /* Mask of FTS5_TOKENIZE_* flags */ 509 const char *pText, int nText, 510 int (*xToken)( 511 void *pCtx, /* Copy of 2nd argument to xTokenize() */ 512 int tflags, /* Mask of FTS5_TOKEN_* flags */ 513 const char *pToken, /* Pointer to buffer containing token */ 514 int nToken, /* Size of token in bytes */ 515 int iStart, /* Byte offset of token within input text */ 516 int iEnd /* Byte offset of end of token within input text */ 517 ) 518 ); 519 }; 520 521 /* Flags that may be passed as the third argument to xTokenize() */ 522 #define FTS5_TOKENIZE_QUERY 0x0001 523 #define FTS5_TOKENIZE_PREFIX 0x0002 524 #define FTS5_TOKENIZE_DOCUMENT 0x0004 525 #define FTS5_TOKENIZE_AUX 0x0008 526 527 /* Flags that may be passed by the tokenizer implementation back to FTS5 528 ** as the third argument to the supplied xToken callback. */ 529 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ 530 531 /* 532 ** END OF CUSTOM TOKENIZERS 533 *************************************************************************/ 534 535 /************************************************************************* 536 ** FTS5 EXTENSION REGISTRATION API 537 */ 538 typedef struct fts5_api fts5_api; 539 struct fts5_api { 540 int iVersion; /* Currently always set to 2 */ 541 542 /* Create a new tokenizer */ 543 int (*xCreateTokenizer)( 544 fts5_api *pApi, 545 const char *zName, 546 void *pContext, 547 fts5_tokenizer *pTokenizer, 548 void (*xDestroy)(void*) 549 ); 550 551 /* Find an existing tokenizer */ 552 int (*xFindTokenizer)( 553 fts5_api *pApi, 554 const char *zName, 555 void **ppContext, 556 fts5_tokenizer *pTokenizer 557 ); 558 559 /* Create a new auxiliary function */ 560 int (*xCreateFunction)( 561 fts5_api *pApi, 562 const char *zName, 563 void *pContext, 564 fts5_extension_function xFunction, 565 void (*xDestroy)(void*) 566 ); 567 }; 568 569 /* 570 ** END OF REGISTRATION API 571 *************************************************************************/ 572 573 #ifdef __cplusplus 574 } /* end of the 'extern "C"' block */ 575 #endif 576 577 #endif /* _FTS5_H */ 578 579