1 #ifndef __DOC_TABLE_H__
2 #define __DOC_TABLE_H__
3 #include <stdlib.h>
4 #include <string.h>
5 #include "redismodule.h"
6 #include "dep/triemap/triemap.h"
7 #include "redisearch.h"
8 #include "sortable.h"
9 #include "byte_offsets.h"
10 #include "rmutil/sds.h"
11 #include "util/dict.h"
12
13 #ifdef __cplusplus
14 extern "C" {
15 #endif
16 // Retrieves the pointer and length for the document's key.
DMD_KeyPtrLen(const RSDocumentMetadata * dmd,size_t * len)17 static inline const char *DMD_KeyPtrLen(const RSDocumentMetadata *dmd, size_t *len) {
18 if (len) {
19 *len = sdslen(dmd->keyPtr);
20 }
21 return dmd->keyPtr;
22 }
23
24 // Convenience function to create a RedisModuleString from the document's key
DMD_CreateKeyString(const RSDocumentMetadata * dmd,RedisModuleCtx * ctx)25 static inline RedisModuleString *DMD_CreateKeyString(const RSDocumentMetadata *dmd,
26 RedisModuleCtx *ctx) {
27 return RedisModule_CreateString(ctx, dmd->keyPtr, sdslen(dmd->keyPtr));
28 }
29
30 /* Map between external id an incremental id */
31 typedef struct {
32 TrieMap *tm;
33 } DocIdMap;
34
35 DocIdMap NewDocIdMap();
36 /* Get docId from a did-map. Returns 0 if the key is not in the map */
37 t_docId DocIdMap_Get(const DocIdMap *m, const char *s, size_t n);
38
39 /* Put a new doc id in the map if it does not already exist */
40 void DocIdMap_Put(DocIdMap *m, const char *s, size_t n, t_docId docId);
41
42 int DocIdMap_Delete(DocIdMap *m, const char *s, size_t n);
43 /* Free the doc id map */
44 void DocIdMap_Free(DocIdMap *m);
45
46 /* The DocTable is a simple mapping between incremental ids and the original document key and
47 * metadata. It is also responsible for storing the id incrementor for the index and assigning
48 * new
49 * incremental ids to inserted keys.
50 *
51 * NOTE: Currently there is no deduplication on the table so we do not prevent dual insertion of
52 * the
53 * same key. This may result in document duplication in results */
54
55 typedef struct {
56 DLLIST2 lroot;
57 } DMDChain;
58
59 typedef struct {
60 size_t size;
61 // the maximum size this table is allowed to grow to
62 t_docId maxSize;
63 t_docId maxDocId;
64 size_t cap;
65 size_t memsize;
66 size_t sortablesSize;
67
68 DMDChain *buckets;
69 DocIdMap dim;
70 } DocTable;
71
72 /* increasing the ref count of the given dmd */
73 #define DMD_Incref(md) \
74 if (md) ++md->ref_count;
75
76 #define DOCTABLE_FOREACH(dt, code) \
77 for (size_t i = 0; i < dt->cap; ++i) { \
78 DMDChain *chain = &dt->buckets[i]; \
79 if (DLLIST2_IS_EMPTY(&chain->lroot)) { \
80 continue; \
81 } \
82 DLLIST2_FOREACH(it, &chain->lroot) { \
83 RSDocumentMetadata *dmd = DLLIST_ITEM(it, RSDocumentMetadata, llnode); \
84 code; \
85 } \
86 }
87
88 /* Creates a new DocTable with a given capacity */
89 DocTable NewDocTable(size_t cap, size_t max_size);
90
91 #define DocTable_New(cap) NewDocTable(cap, RSGlobalConfig.maxDocTableSize)
92
93 /* Get the metadata for a doc Id from the DocTable.
94 * If docId is not inside the table, we return NULL */
95 RSDocumentMetadata *DocTable_Get(const DocTable *t, t_docId docId);
96
97 RSDocumentMetadata *DocTable_GetByKeyR(const DocTable *r, RedisModuleString *s);
98
99 /* Put a new document into the table, assign it an incremental id and store the metadata in the
100 * table.
101 *
102 * NOTE: Currently there is no deduplication on the table so we do not prevent dual insertion of the
103 * same key. This may result in document duplication in results */
104 t_docId DocTable_Put(DocTable *t, const char *s, size_t n, double score, u_char flags,
105 const char *payload, size_t payloadSize);
106
107 /* Get the "real" external key for an incremental i
108 * If the document ID is not in the table, the returned key's `str` member will
109 * be NULL
110 */
111 const char *DocTable_GetKey(DocTable *t, t_docId docId, size_t *n);
112
113 /* Get the score for a document from the table. Returns 0 if docId is not in the table. */
114 float DocTable_GetScore(DocTable *t, t_docId docId);
115
116 /* Set the payload for a document. Returns 1 if we set the payload, 0 if we couldn't find the
117 * document */
118 int DocTable_SetPayload(DocTable *t, t_docId docId, const char *data, size_t len);
119
120 int DocTable_Exists(const DocTable *t, t_docId docId);
121
122 /* Set the sorting vector for a document. If the vector is NULL we mark the doc as not having a
123 * vector. Returns 1 on success, 0 if the document does not exist. No further validation is done */
124 int DocTable_SetSortingVector(DocTable *t, t_docId docId, RSSortingVector *v);
125
126 /* Set the offset vector for a document. This contains the byte offsets of each token found in
127 * the document. This is used for highlighting
128 */
129 int DocTable_SetByteOffsets(DocTable *t, t_docId docId, RSByteOffsets *offsets);
130
131 /* Get the payload for a document, if any was set. If no payload has been set or the document id is
132 * not found, we return NULL */
133 RSPayload *DocTable_GetPayload(DocTable *t, t_docId dodcId);
134
135 /** Get the docId of a key if it exists in the table, or 0 if it doesnt */
136 t_docId DocTable_GetId(const DocTable *dt, const char *s, size_t n);
137
138 #define STRVARS_FROM_RSTRING(r) \
139 size_t n; \
140 const char *s = RedisModule_StringPtrLen(r, &n);
141
DocTable_GetIdR(const DocTable * dt,RedisModuleString * r)142 static inline t_docId DocTable_GetIdR(const DocTable *dt, RedisModuleString *r) {
143 STRVARS_FROM_RSTRING(r);
144 return DocTable_GetId(dt, s, n);
145 }
146
147 /* Free the table and all the keys of documents */
148 void DocTable_Free(DocTable *t);
149
150 int DocTable_Delete(DocTable *t, const char *key, size_t n);
DocTable_DeleteR(DocTable * t,RedisModuleString * r)151 static inline int DocTable_DeleteR(DocTable *t, RedisModuleString *r) {
152 STRVARS_FROM_RSTRING(r);
153 return DocTable_Delete(t, s, n);
154 }
155
156 RSDocumentMetadata *DocTable_Pop(DocTable *t, const char *s, size_t n);
DocTable_PopR(DocTable * t,RedisModuleString * r)157 static inline RSDocumentMetadata *DocTable_PopR(DocTable *t, RedisModuleString *r) {
158 STRVARS_FROM_RSTRING(r);
159 return DocTable_Pop(t, s, n);
160 }
161
DocTable_GetByKey(DocTable * dt,const char * key)162 static inline RSDocumentMetadata *DocTable_GetByKey(DocTable *dt, const char *key) {
163 t_docId id = DocTable_GetId(dt, key, strlen(key));
164 if (id == 0) {
165 return NULL;
166 }
167 return DocTable_Get(dt, id);
168 }
169
170 /* Change name of document hash in the same spec without reindexing */
171 int DocTable_Replace(DocTable *t, const char *from_str, size_t from_len, const char *to_str,
172 size_t to_len);
173
174 /* don't use this function directly. Use DMD_Decref */
175 void DMD_Free(RSDocumentMetadata *);
176
177 /* Decrement the refcount of the DMD object, freeing it if we're the last reference */
DMD_Decref(RSDocumentMetadata * dmd)178 static inline void DMD_Decref(RSDocumentMetadata *dmd) {
179 if (dmd && !--dmd->ref_count) {
180 DMD_Free(dmd);
181 }
182 }
183
184 /* Save the table to RDB. Called from the owning index */
185 void DocTable_RdbSave(DocTable *t, RedisModuleIO *rdb);
186
187 void DocTable_LegacyRdbLoad(DocTable *t, RedisModuleIO *rdb, int encver);
188
189 /* Load the table from RDB */
190 void DocTable_RdbLoad(DocTable *t, RedisModuleIO *rdb, int encver);
191
192 #ifdef __cplusplus
193 }
194 #endif
195 #endif
196