1 #ifndef __FORWARD_INDEX_H__
2 #define __FORWARD_INDEX_H__
3 #include "redisearch.h"
4 #include "util/block_alloc.h"
5 #include "util/khtable.h"
6 #include "util/mempool.h"
7 #include "dep/triemap/triemap.h"
8 #include "varint.h"
9 #include "tokenize.h"
10 #include "document.h"
11
12 typedef struct ForwardIndexEntry {
13 struct ForwardIndexEntry *next;
14 t_docId docId;
15
16 uint32_t freq;
17 t_fieldMask fieldMask;
18
19 const char *term;
20 uint32_t len;
21 uint32_t hash;
22 VarintVectorWriter *vw;
23 } ForwardIndexEntry;
24
25 // the quantizationn factor used to encode normalized (0..1) frquencies in the index
26 #define FREQ_QUANTIZE_FACTOR 0xFFFF
27
28 typedef struct ForwardIndex {
29 KHTable *hits;
30 uint32_t maxFreq;
31 uint32_t totalFreq;
32 uint32_t idxFlags;
33 Stemmer *stemmer;
34 SynonymMap *smap;
35 BlkAlloc terms;
36 BlkAlloc entries;
37 mempool_t *vvwPool;
38
39 } ForwardIndex;
40
41 typedef struct {
42 const char *doc;
43 VarintVectorWriter *allOffsets;
44 ForwardIndex *idx;
45 t_fieldId fieldId;
46 float fieldScore;
47 } ForwardIndexTokenizerCtx;
48
ForwardIndexTokenizerCtx_Init(ForwardIndexTokenizerCtx * ctx,ForwardIndex * idx,const char * doc,VarintVectorWriter * vvw,t_fieldId fieldId,float score)49 static inline void ForwardIndexTokenizerCtx_Init(ForwardIndexTokenizerCtx *ctx, ForwardIndex *idx,
50 const char *doc, VarintVectorWriter *vvw,
51 t_fieldId fieldId, float score) {
52 ctx->idx = idx;
53 ctx->fieldId = fieldId;
54 ctx->fieldScore = score;
55 ctx->doc = doc;
56 ctx->allOffsets = vvw;
57 }
58
59 typedef struct {
60 KHTable *hits;
61 KHTableEntry *curEnt;
62 uint32_t curBucketIdx;
63 } ForwardIndexIterator;
64
65 int forwardIndexTokenFunc(void *ctx, const Token *tokInfo);
66 void ForwardIndexFree(ForwardIndex *idx);
67
68 void ForwardIndex_Reset(ForwardIndex *idx, Document *doc, uint32_t idxFlags);
69
70 ForwardIndex *NewForwardIndex(Document *doc, uint32_t idxFlags);
71 ForwardIndexIterator ForwardIndex_Iterate(ForwardIndex *i);
72 ForwardIndexEntry *ForwardIndexIterator_Next(ForwardIndexIterator *iter);
73
74 // Find an existing entry within the index
75 ForwardIndexEntry *ForwardIndex_Find(ForwardIndex *i, const char *s, size_t n, uint32_t hash);
76
77 void ForwardIndex_NormalizeFreq(ForwardIndex *, ForwardIndexEntry *);
78
79 #endif
80