1 /*
2 ** 2006 July 10
3 **
4 ** The author disclaims copyright to this source code.
5 **
6 *************************************************************************
7 ** Defines the interface to tokenizers used by fulltext-search.  There
8 ** are three basic components:
9 **
10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer
11 ** interface functions.  This is essentially the class structure for
12 ** tokenizers.
13 **
14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
15 ** including customization information defined at creation time.
16 **
17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
18 ** tokens from a particular input.
19 */
20 #ifndef _TOKENIZER_H_
21 #define _TOKENIZER_H_
22 
23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then
25 ** we will need a way to register the API consistently.
26 */
27 #include "sqlite3.h"
28 
29 /*
30 ** Structures used by the tokenizer interface.
31 */
32 typedef struct sqlite3_tokenizer sqlite3_tokenizer;
33 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
34 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
35 
36 struct sqlite3_tokenizer_module {
37   int iVersion;                  /* currently 0 */
38 
39   /*
40   ** Create and destroy a tokenizer.  argc/argv are passed down from
41   ** the fulltext virtual table creation to allow customization.
42   */
43   int (*xCreate)(int argc, const char **argv,
44                  sqlite3_tokenizer **ppTokenizer);
45   int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
46 
47   /*
48   ** Tokenize a particular input.  Call xOpen() to prepare to
49   ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
50   ** xClose() to free any internal state.  The pInput passed to
51   ** xOpen() must exist until the cursor is closed.  The ppToken
52   ** result from xNext() is only valid until the next call to xNext()
53   ** or until xClose() is called.
54   */
55   /* TODO(shess) current implementation requires pInput to be
56   ** nul-terminated.  This should either be fixed, or pInput/nBytes
57   ** should be converted to zInput.
58   */
59   int (*xOpen)(sqlite3_tokenizer *pTokenizer,
60                const char *pInput, int nBytes,
61                sqlite3_tokenizer_cursor **ppCursor);
62   int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
63   int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
64                const char **ppToken, int *pnBytes,
65                int *piStartOffset, int *piEndOffset, int *piPosition);
66 };
67 
68 struct sqlite3_tokenizer {
69   sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
70   /* Tokenizer implementations will typically add additional fields */
71 };
72 
73 struct sqlite3_tokenizer_cursor {
74   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
75   /* Tokenizer implementations will typically add additional fields */
76 };
77 
78 /*
79 ** Get the module for a tokenizer which generates tokens based on a
80 ** set of non-token characters.  The default is to break tokens at any
81 ** non-alnum character, though the set of delimiters can also be
82 ** specified by the first argv argument to xCreate().
83 */
84 /* TODO(shess) This doesn't belong here.  Need some sort of
85 ** registration process.
86 */
87 void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
88 
89 #endif /* _TOKENIZER_H_ */
90