1 #include "../../src/query.h"
2 #include "../../src/query_parser/tokenizer.h"
3 #include "../../src/stopwords.h"
4 #include "../../src/extension.h"
5 #include "../../src/ext/default.h"
6 #include <stdio.h>
7 #include <gtest/gtest.h>
8
9 #define QUERY_PARSE_CTX(ctx, qt, opts) NewQueryParseCtx(&ctx, qt, strlen(qt), &opts);
10
11 struct SearchOptionsCXX : RSSearchOptions {
SearchOptionsCXXSearchOptionsCXX12 SearchOptionsCXX() {
13 memset(this, 0, sizeof(*this));
14 flags = RS_DEFAULT_QUERY_FLAGS;
15 fieldmask = RS_FIELDMASK_ALL;
16 language = DEFAULT_LANGUAGE;
17 stopwords = DefaultStopWordList();
18 }
19 };
20
21 class QASTCXX : public QueryAST {
22 SearchOptionsCXX m_opts;
23 QueryError m_status = {QueryErrorCode(0)};
24 RedisSearchCtx *sctx = NULL;
25
26 public:
QASTCXX()27 QASTCXX() {
28 memset(static_cast<QueryAST *>(this), 0, sizeof(QueryAST));
29 }
QASTCXX(RedisSearchCtx & sctx)30 QASTCXX(RedisSearchCtx &sctx) : QASTCXX() {
31 setContext(&sctx);
32 }
setContext(RedisSearchCtx * sctx)33 void setContext(RedisSearchCtx *sctx) {
34 this->sctx = sctx;
35 }
36
parse(const char * s)37 bool parse(const char *s) {
38 QueryError_ClearError(&m_status);
39 QAST_Destroy(this);
40
41 int rc = QAST_Parse(this, sctx, &m_opts, s, strlen(s), &m_status);
42 return rc == REDISMODULE_OK && !QueryError_HasError(&m_status) && root != NULL;
43 }
44
print() const45 void print() const {
46 QAST_Print(this, sctx->spec);
47 }
48
getError() const49 const char *getError() const {
50 return QueryError_GetError(&m_status);
51 }
52
~QASTCXX()53 ~QASTCXX() {
54 QueryError_ClearError(&m_status);
55 QAST_Destroy(this);
56 }
57 };
58
isValidQuery(const char * qt,RedisSearchCtx & ctx)59 bool isValidQuery(const char *qt, RedisSearchCtx &ctx) {
60 QASTCXX ast;
61 ast.setContext(&ctx);
62 return ast.parse(qt);
63
64 // if (err) {
65 // Query_Free(q);
66 // fprintf(stderr, "Error parsing query '%s': %s\n", qt, err);
67 // free(err);
68 // return 1;
69 // }
70 // Query_Free(q);
71
72 // return 0;
73 }
74
75 #define assertValidQuery(qt, ctx) ASSERT_TRUE(isValidQuery(qt, ctx))
76 #define assertInvalidQuery(qt, ctx) ASSERT_FALSE(isValidQuery(qt, ctx))
77
78 class QueryTest : public ::testing::Test {};
79
TEST_F(QueryTest,testParser)80 TEST_F(QueryTest, testParser) {
81 RedisSearchCtx ctx;
82 static const char *args[] = {"SCHEMA", "title", "text", "weight", "0.1",
83 "body", "text", "weight", "2.0", "bar",
84 "numeric", "loc", "geo", "tags", "tag"};
85 QueryError err = {QueryErrorCode(0)};
86 ctx.spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
87 ASSERT_FALSE(QueryError_HasError(&err)) << QueryError_GetError(&err);
88
89 // test some valid queries
90 assertValidQuery("hello", ctx);
91
92 assertValidQuery("hello wor*", ctx);
93 assertValidQuery("hello world", ctx);
94 assertValidQuery("hello (world)", ctx);
95
96 assertValidQuery("\"hello world\"", ctx);
97 assertValidQuery("\"hello\"", ctx);
98
99 assertValidQuery("\"hello world\" \"foo bar\"", ctx);
100 assertValidQuery("\"hello world\"|\"foo bar\"", ctx);
101 assertValidQuery("\"hello world\" (\"foo bar\")", ctx);
102 assertValidQuery("hello \"foo bar\" world", ctx);
103 assertValidQuery("hello|hallo|yellow world", ctx);
104 assertValidQuery("(hello|world|foo) bar baz 123", ctx);
105 assertValidQuery("(hello|world|foo) (bar baz)", ctx);
106 // assertValidQuery("(hello world|foo \"bar baz\") \"bar baz\" bbbb");
107 assertValidQuery("@title:(barack obama) @body:us|president", ctx);
108 assertValidQuery("@ti_tle:barack obama @body:us", ctx);
109 assertValidQuery("@title:barack @body:obama", ctx);
110 assertValidQuery("@tit_le|bo_dy:barack @body|title|url|something_else:obama", ctx);
111 assertValidQuery("hello world&good+bye foo.bar", ctx);
112 assertValidQuery("@BusinessName:\"Wells Fargo Bank, National Association\"", ctx);
113 // escaping and unicode in field names
114 assertValidQuery("@Business\\:\\-\\ Name:Wells Fargo", ctx);
115 assertValidQuery("@שלום:Wells Fargo", ctx);
116
117 assertValidQuery("foo -bar -(bar baz)", ctx);
118 assertValidQuery("(hello world)|(goodbye moon)", ctx);
119 assertInvalidQuery("@title:", ctx);
120 assertInvalidQuery("@body:@title:", ctx);
121 assertInvalidQuery("@body|title:@title:", ctx);
122 assertInvalidQuery("@body|title", ctx);
123 assertValidQuery("hello ~world ~war", ctx);
124 assertValidQuery("hello ~(world war)", ctx);
125 assertValidQuery("-foo", ctx);
126 assertValidQuery("@title:-foo", ctx);
127 assertValidQuery("-@title:foo", ctx);
128
129 // some geo queries
130 assertValidQuery("@loc:[15.1 -15 30 km]", ctx);
131 assertValidQuery("@loc:[15 -15.1 30 m]", ctx);
132 assertValidQuery("@loc:[15.03 -15.45 30 mi]", ctx);
133 assertValidQuery("@loc:[15.65 -15.65 30 ft]", ctx);
134 assertValidQuery("hello world @loc:[15.65 -15.65 30 ft]", ctx);
135 assertValidQuery("hello world -@loc:[15.65 -15.65 30 ft]", ctx);
136 assertValidQuery("hello world ~@loc:[15.65 -15.65 30 ft]", ctx);
137 assertValidQuery("@title:hello world ~@loc:[15.65 -15.65 30 ft]", ctx);
138 assertValidQuery("@loc:[15.65 -15.65 30 ft] @loc:[15.65 -15.65 30 ft]", ctx);
139 assertValidQuery("@loc:[15.65 -15.65 30 ft]|@loc:[15.65 -15.65 30 ft]", ctx);
140 assertValidQuery("hello (world @loc:[15.65 -15.65 30 ft])", ctx);
141
142 assertInvalidQuery("@loc:[190.65 -100.65 30 ft])", ctx);
143 assertInvalidQuery("@loc:[50 50 -1 ft])", ctx);
144 assertInvalidQuery("@loc:[50 50 1 quoops])", ctx);
145 assertInvalidQuery("@loc:[50 50 1 ftps])", ctx);
146 assertInvalidQuery("@loc:[50 50 1 1])", ctx);
147 assertInvalidQuery("@loc:[50 50 1])", ctx);
148 // numeric
149 assertValidQuery("@number:[100 200]", ctx);
150 assertValidQuery("@number:[100 -200]", ctx);
151 assertValidQuery("@number:[(100 (200]", ctx);
152 assertValidQuery("@number:[100 inf]", ctx);
153 assertValidQuery("@number:[100 -inf]", ctx);
154 assertValidQuery("@number:[-inf +inf]", ctx);
155 assertValidQuery("@number:[-inf +inf]|@number:[100 200]", ctx);
156
157 assertInvalidQuery("@number:[100 foo]", ctx);
158
159 // Tag queries
160 assertValidQuery("@tags:{foo}", ctx);
161 assertValidQuery("@tags:{foo|bar baz|boo}", ctx);
162 assertValidQuery("@tags:{foo|bar\\ baz|boo}", ctx);
163 assertValidQuery("@tags:{foo*}", ctx);
164 assertValidQuery("@tags:{foo\\-*}", ctx);
165 assertValidQuery("@tags:{bar | foo*}", ctx);
166 assertValidQuery("@tags:{bar* | foo}", ctx);
167 assertValidQuery("@tags:{bar* | foo*}", ctx);
168
169 assertInvalidQuery("@tags:{foo|bar\\ baz|}", ctx);
170 assertInvalidQuery("@tags:{foo|bar\\ baz|", ctx);
171 assertInvalidQuery("{foo|bar\\ baz}", ctx);
172
173 assertInvalidQuery("(foo", ctx);
174 assertInvalidQuery("\"foo", ctx);
175 assertValidQuery("", ctx);
176 assertInvalidQuery("()", ctx);
177
178 // test stopwords
179 assertValidQuery("a for is", ctx);
180 assertValidQuery("a|for|is", ctx);
181 assertValidQuery("a little bit of party", ctx);
182 assertValidQuery("no-as", ctx);
183 assertValidQuery("~no~as", ctx);
184 assertValidQuery("(no -as) =>{$weight: 0.5}", ctx);
185 assertValidQuery("@foo:-as", ctx);
186
187 // test utf-8 query
188 assertValidQuery("שלום עולם", ctx);
189
190 // Test attribute
191 assertValidQuery("(foo bar) => {$weight: 0.5; $slop: 2}", ctx);
192 assertValidQuery("foo => {$weight: 0.5} bar => {$weight: 0.1}", ctx);
193
194 assertValidQuery("@title:(foo bar) => {$weight: 0.5; $slop: 2}", ctx);
195 assertValidQuery(
196 "@title:(foo bar) => {$weight: 0.5; $slop: 2} @body:(foo bar) => {$weight: 0.5; $slop: 2}",
197 ctx);
198 assertValidQuery("(foo => {$weight: 0.5;}) | ((bar) => {$weight: 0.5})", ctx);
199 assertValidQuery("(foo => {$weight: 0.5;}) ((bar) => {}) => {}", ctx);
200 assertValidQuery("@tag:{foo | bar} => {$weight: 0.5;} ", ctx);
201 assertValidQuery("@num:[0 100] => {$weight: 0.5;} ", ctx);
202 assertInvalidQuery("@tag:{foo | bar} => {$weight: -0.5;} ", ctx);
203 assertInvalidQuery("@tag:{foo | bar} => {$great: 0.5;} ", ctx);
204 assertInvalidQuery("@tag:{foo | bar} => {$great:;} ", ctx);
205 assertInvalidQuery("@tag:{foo | bar} => {$:1;} ", ctx);
206
207 assertInvalidQuery(" => {$weight: 0.5;} ", ctx);
208
209 const char *qt = "(hello|world) and \"another world\" (foo is bar) -(baz boo*)";
210 QASTCXX ast;
211 ast.setContext(&ctx);
212 ASSERT_TRUE(ast.parse(qt));
213 QueryNode *n = ast.root;
214 QAST_Print(&ast, ctx.spec);
215 ASSERT_TRUE(n != NULL);
216 ASSERT_EQ(n->type, QN_PHRASE);
217 ASSERT_EQ(n->pn.exact, 0);
218 ASSERT_EQ(QueryNode_NumChildren(n), 4);
219 ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
220
221 ASSERT_TRUE(n->children[0]->type == QN_UNION);
222 ASSERT_STREQ("hello", n->children[0]->children[0]->tn.str);
223 ASSERT_STREQ("world", n->children[0]->children[1]->tn.str);
224
225 QueryNode *_n = n->children[1];
226
227 ASSERT_TRUE(_n->type == QN_PHRASE);
228 ASSERT_TRUE(_n->pn.exact == 1);
229 ASSERT_EQ(QueryNode_NumChildren(_n), 2);
230 ASSERT_STREQ("another", _n->children[0]->tn.str);
231 ASSERT_STREQ("world", _n->children[1]->tn.str);
232
233 _n = n->children[2];
234 ASSERT_TRUE(_n->type == QN_PHRASE);
235
236 ASSERT_TRUE(_n->pn.exact == 0);
237 ASSERT_EQ(QueryNode_NumChildren(_n), 2);
238 ASSERT_STREQ("foo", _n->children[0]->tn.str);
239 ASSERT_STREQ("bar", _n->children[1]->tn.str);
240
241 _n = n->children[3];
242 ASSERT_TRUE(_n->type == QN_NOT);
243 _n = QueryNode_GetChild(_n, 0);
244 ASSERT_TRUE(_n->pn.exact == 0);
245 ASSERT_EQ(2, QueryNode_NumChildren(_n));
246 ASSERT_STREQ("baz", _n->children[0]->tn.str);
247
248 ASSERT_EQ(_n->children[1]->type, QN_PREFX);
249 ASSERT_STREQ("boo", _n->children[1]->pfx.str);
250 QAST_Destroy(&ast);
251 IndexSpec_Free(ctx.spec);
252 }
253
TEST_F(QueryTest,testPureNegative)254 TEST_F(QueryTest, testPureNegative) {
255 const char *qs[] = {"-@title:hello", "-hello", "@title:-hello", "-(foo)", "-foo", "(-foo)", NULL};
256 static const char *args[] = {"SCHEMA", "title", "text", "weight", "0.1", "body",
257 "text", "weight", "2.0", "bar", "numeric"};
258 QueryError err = {QueryErrorCode(0)};
259 IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
260 RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
261 for (size_t i = 0; qs[i] != NULL; i++) {
262 QASTCXX ast;
263 ast.setContext(&ctx);
264 ASSERT_TRUE(ast.parse(qs[i])) << ast.getError();
265 QueryNode *n = ast.root;
266 ASSERT_TRUE(n != NULL);
267 ASSERT_EQ(n->type, QN_NOT);
268 ASSERT_TRUE(QueryNode_GetChild(n, 0) != NULL);
269 }
270 IndexSpec_Free(ctx.spec);
271 }
272
TEST_F(QueryTest,testGeoQuery)273 TEST_F(QueryTest, testGeoQuery) {
274 static const char *args[] = {"SCHEMA", "title", "text", "loc", "geo"};
275 QueryError err = {QueryErrorCode(0)};
276 IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
277 RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
278 const char *qt = "@title:hello world @loc:[31.52 32.1342 10.01 km]";
279 QASTCXX ast;
280 ast.setContext(&ctx);
281 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
282 QueryNode *n = ast.root;
283 ASSERT_EQ(n->type, QN_PHRASE);
284 ASSERT_TRUE((n->opts.fieldMask == RS_FIELDMASK_ALL));
285 ASSERT_EQ(QueryNode_NumChildren(n), 2);
286
287 QueryNode *gn = n->children[1];
288 ASSERT_EQ(gn->type, QN_GEO);
289 ASSERT_STREQ(gn->gn.gf->property, "loc");
290 ASSERT_EQ(gn->gn.gf->unitType, GEO_DISTANCE_KM);
291 ASSERT_EQ(gn->gn.gf->lon, 31.52);
292 ASSERT_EQ(gn->gn.gf->lat, 32.1342);
293 ASSERT_EQ(gn->gn.gf->radius, 10.01);
294 IndexSpec_Free(ctx.spec);
295 }
296
TEST_F(QueryTest,testFieldSpec)297 TEST_F(QueryTest, testFieldSpec) {
298 static const char *args[] = {"SCHEMA", "title", "text", "weight", "0.1", "body",
299 "text", "weight", "2.0", "bar", "numeric"};
300 QueryError err = {QUERY_OK};
301 IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
302 RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
303 const char *qt = "@title:hello world";
304 QASTCXX ast(ctx);
305 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
306 ast.print();
307 QueryNode *n = ast.root;
308 ASSERT_EQ(n->type, QN_PHRASE);
309 ASSERT_EQ(n->opts.fieldMask, 0x01);
310
311 qt = "(@title:hello) (@body:world)";
312 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
313 n = ast.root;
314
315 ASSERT_TRUE(n != NULL);
316 printf("%s ====> ", qt);
317 ast.print();
318 ASSERT_EQ(n->type, QN_PHRASE);
319 ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
320 ASSERT_EQ(n->children[0]->opts.fieldMask, 0x01);
321 ASSERT_EQ(n->children[1]->opts.fieldMask, 0x02);
322
323 // test field modifiers
324 qt = "@title:(hello world) @body:(world apart) @adas_dfsd:fofofof";
325 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
326 n = ast.root;
327 printf("%s ====> ", qt);
328 ast.print();
329 ASSERT_EQ(n->type, QN_PHRASE);
330 ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
331 ASSERT_EQ(QueryNode_NumChildren(n), 3);
332 ASSERT_EQ(n->children[0]->opts.fieldMask, 0x01);
333 ASSERT_EQ(n->children[1]->opts.fieldMask, 0x02);
334 ASSERT_EQ(n->children[2]->opts.fieldMask, 0x00);
335 // ASSERT_EQ(n->children[2]->fieldMask, 0x00)
336
337 // test numeric ranges
338 qt = "@num:[0.4 (500]";
339 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
340 n = ast.root;
341 ASSERT_EQ(n->type, QN_NUMERIC);
342 ASSERT_EQ(n->nn.nf->min, 0.4);
343 ASSERT_EQ(n->nn.nf->max, 500.0);
344 ASSERT_EQ(n->nn.nf->inclusiveMin, 1);
345 ASSERT_EQ(n->nn.nf->inclusiveMax, 0);
346 IndexSpec_Free(ctx.spec);
347 }
348
TEST_F(QueryTest,testAttributes)349 TEST_F(QueryTest, testAttributes) {
350 static const char *args[] = {"SCHEMA", "title", "text", "body", "text"};
351 QueryError err = {QUERY_OK};
352 IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
353 RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
354
355 const char *qt =
356 "(@title:(foo bar) => {$weight: 0.5} @body:lol => {$weight: 0.2}) => "
357 "{$weight:0.3; $slop:2; $inorder:true}";
358 QASTCXX ast(ctx);
359 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
360 QueryNode *n = ast.root;
361 ASSERT_EQ(0.3, n->opts.weight);
362 ASSERT_EQ(2, n->opts.maxSlop);
363 ASSERT_EQ(1, n->opts.inOrder);
364
365 ASSERT_EQ(n->type, QN_PHRASE);
366 ASSERT_EQ(QueryNode_NumChildren(n), 2);
367 ASSERT_EQ(0.5, n->children[0]->opts.weight);
368 ASSERT_EQ(0.2, n->children[1]->opts.weight);
369 IndexSpec_Free(ctx.spec);
370 }
371
TEST_F(QueryTest,testTags)372 TEST_F(QueryTest, testTags) {
373 static const char *args[] = {"SCHEMA", "title", "text", "tags", "tag", "separator", ";"};
374 QueryError err = {QUERY_OK};
375 IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
376 RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
377
378 const char *qt = "@tags:{hello world |foo| שלום| lorem\\ ipsum }";
379 QASTCXX ast(ctx);
380 ASSERT_TRUE(ast.parse(qt)) << ast.getError();
381 ast.print();
382 QueryNode *n = ast.root;
383 ASSERT_EQ(n->type, QN_TAG);
384 ASSERT_EQ(4, QueryNode_NumChildren(n));
385 ASSERT_EQ(QN_PHRASE, n->children[0]->type);
386 ASSERT_STREQ("hello", n->children[0]->children[0]->tn.str);
387 ASSERT_STREQ("world", n->children[0]->children[1]->tn.str);
388
389 ASSERT_EQ(QN_TOKEN, n->children[1]->type);
390 ASSERT_STREQ("foo", n->children[1]->tn.str);
391
392 ASSERT_EQ(QN_TOKEN, n->children[2]->type);
393 ASSERT_STREQ("שלום", n->children[2]->tn.str);
394
395 ASSERT_EQ(QN_TOKEN, n->children[3]->type);
396 ASSERT_STREQ("lorem\\ ipsum", n->children[3]->tn.str);
397 IndexSpec_Free(ctx.spec);
398 }
399