1 #include "../../src/query.h"
2 #include "../../src/query_parser/tokenizer.h"
3 #include "../../src/stopwords.h"
4 #include "../../src/extension.h"
5 #include "../../src/ext/default.h"
6 #include <stdio.h>
7 #include <gtest/gtest.h>
8 
9 #define QUERY_PARSE_CTX(ctx, qt, opts) NewQueryParseCtx(&ctx, qt, strlen(qt), &opts);
10 
11 struct SearchOptionsCXX : RSSearchOptions {
SearchOptionsCXXSearchOptionsCXX12   SearchOptionsCXX() {
13     memset(this, 0, sizeof(*this));
14     flags = RS_DEFAULT_QUERY_FLAGS;
15     fieldmask = RS_FIELDMASK_ALL;
16     language = DEFAULT_LANGUAGE;
17     stopwords = DefaultStopWordList();
18   }
19 };
20 
21 class QASTCXX : public QueryAST {
22   SearchOptionsCXX m_opts;
23   QueryError m_status = {QueryErrorCode(0)};
24   RedisSearchCtx *sctx = NULL;
25 
26  public:
QASTCXX()27   QASTCXX() {
28     memset(static_cast<QueryAST *>(this), 0, sizeof(QueryAST));
29   }
QASTCXX(RedisSearchCtx & sctx)30   QASTCXX(RedisSearchCtx &sctx) : QASTCXX() {
31     setContext(&sctx);
32   }
setContext(RedisSearchCtx * sctx)33   void setContext(RedisSearchCtx *sctx) {
34     this->sctx = sctx;
35   }
36 
parse(const char * s)37   bool parse(const char *s) {
38     QueryError_ClearError(&m_status);
39     QAST_Destroy(this);
40 
41     int rc = QAST_Parse(this, sctx, &m_opts, s, strlen(s), &m_status);
42     return rc == REDISMODULE_OK && !QueryError_HasError(&m_status) && root != NULL;
43   }
44 
print() const45   void print() const {
46     QAST_Print(this, sctx->spec);
47   }
48 
getError() const49   const char *getError() const {
50     return QueryError_GetError(&m_status);
51   }
52 
~QASTCXX()53   ~QASTCXX() {
54     QueryError_ClearError(&m_status);
55     QAST_Destroy(this);
56   }
57 };
58 
isValidQuery(const char * qt,RedisSearchCtx & ctx)59 bool isValidQuery(const char *qt, RedisSearchCtx &ctx) {
60   QASTCXX ast;
61   ast.setContext(&ctx);
62   return ast.parse(qt);
63 
64   // if (err) {
65   //   Query_Free(q);
66   //   fprintf(stderr, "Error parsing query '%s': %s\n", qt, err);
67   //   free(err);
68   //   return 1;
69   // }
70   // Query_Free(q);
71 
72   // return 0;
73 }
74 
75 #define assertValidQuery(qt, ctx) ASSERT_TRUE(isValidQuery(qt, ctx))
76 #define assertInvalidQuery(qt, ctx) ASSERT_FALSE(isValidQuery(qt, ctx))
77 
78 class QueryTest : public ::testing::Test {};
79 
TEST_F(QueryTest,testParser)80 TEST_F(QueryTest, testParser) {
81   RedisSearchCtx ctx;
82   static const char *args[] = {"SCHEMA",  "title", "text",   "weight", "0.1",
83                                "body",    "text",  "weight", "2.0",    "bar",
84                                "numeric", "loc",   "geo",    "tags",   "tag"};
85   QueryError err = {QueryErrorCode(0)};
86   ctx.spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
87   ASSERT_FALSE(QueryError_HasError(&err)) << QueryError_GetError(&err);
88 
89   // test some valid queries
90   assertValidQuery("hello", ctx);
91 
92   assertValidQuery("hello wor*", ctx);
93   assertValidQuery("hello world", ctx);
94   assertValidQuery("hello (world)", ctx);
95 
96   assertValidQuery("\"hello world\"", ctx);
97   assertValidQuery("\"hello\"", ctx);
98 
99   assertValidQuery("\"hello world\" \"foo bar\"", ctx);
100   assertValidQuery("\"hello world\"|\"foo bar\"", ctx);
101   assertValidQuery("\"hello world\" (\"foo bar\")", ctx);
102   assertValidQuery("hello \"foo bar\" world", ctx);
103   assertValidQuery("hello|hallo|yellow world", ctx);
104   assertValidQuery("(hello|world|foo) bar baz 123", ctx);
105   assertValidQuery("(hello|world|foo) (bar baz)", ctx);
106   // assertValidQuery("(hello world|foo \"bar baz\") \"bar baz\" bbbb");
107   assertValidQuery("@title:(barack obama)  @body:us|president", ctx);
108   assertValidQuery("@ti_tle:barack obama  @body:us", ctx);
109   assertValidQuery("@title:barack @body:obama", ctx);
110   assertValidQuery("@tit_le|bo_dy:barack @body|title|url|something_else:obama", ctx);
111   assertValidQuery("hello world&good+bye foo.bar", ctx);
112   assertValidQuery("@BusinessName:\"Wells Fargo Bank, National Association\"", ctx);
113   // escaping and unicode in field names
114   assertValidQuery("@Business\\:\\-\\ Name:Wells Fargo", ctx);
115   assertValidQuery("@שלום:Wells Fargo", ctx);
116 
117   assertValidQuery("foo -bar -(bar baz)", ctx);
118   assertValidQuery("(hello world)|(goodbye moon)", ctx);
119   assertInvalidQuery("@title:", ctx);
120   assertInvalidQuery("@body:@title:", ctx);
121   assertInvalidQuery("@body|title:@title:", ctx);
122   assertInvalidQuery("@body|title", ctx);
123   assertValidQuery("hello ~world ~war", ctx);
124   assertValidQuery("hello ~(world war)", ctx);
125   assertValidQuery("-foo", ctx);
126   assertValidQuery("@title:-foo", ctx);
127   assertValidQuery("-@title:foo", ctx);
128 
129   // some geo queries
130   assertValidQuery("@loc:[15.1 -15 30 km]", ctx);
131   assertValidQuery("@loc:[15 -15.1 30 m]", ctx);
132   assertValidQuery("@loc:[15.03 -15.45 30 mi]", ctx);
133   assertValidQuery("@loc:[15.65 -15.65 30 ft]", ctx);
134   assertValidQuery("hello world @loc:[15.65 -15.65 30 ft]", ctx);
135   assertValidQuery("hello world -@loc:[15.65 -15.65 30 ft]", ctx);
136   assertValidQuery("hello world ~@loc:[15.65 -15.65 30 ft]", ctx);
137   assertValidQuery("@title:hello world ~@loc:[15.65 -15.65 30 ft]", ctx);
138   assertValidQuery("@loc:[15.65 -15.65 30 ft] @loc:[15.65 -15.65 30 ft]", ctx);
139   assertValidQuery("@loc:[15.65 -15.65 30 ft]|@loc:[15.65 -15.65 30 ft]", ctx);
140   assertValidQuery("hello (world @loc:[15.65 -15.65 30 ft])", ctx);
141 
142   assertInvalidQuery("@loc:[190.65 -100.65 30 ft])", ctx);
143   assertInvalidQuery("@loc:[50 50 -1 ft])", ctx);
144   assertInvalidQuery("@loc:[50 50 1 quoops])", ctx);
145   assertInvalidQuery("@loc:[50 50 1 ftps])", ctx);
146   assertInvalidQuery("@loc:[50 50 1 1])", ctx);
147   assertInvalidQuery("@loc:[50 50 1])", ctx);
148   // numeric
149   assertValidQuery("@number:[100 200]", ctx);
150   assertValidQuery("@number:[100 -200]", ctx);
151   assertValidQuery("@number:[(100 (200]", ctx);
152   assertValidQuery("@number:[100 inf]", ctx);
153   assertValidQuery("@number:[100 -inf]", ctx);
154   assertValidQuery("@number:[-inf +inf]", ctx);
155   assertValidQuery("@number:[-inf +inf]|@number:[100 200]", ctx);
156 
157   assertInvalidQuery("@number:[100 foo]", ctx);
158 
159   // Tag queries
160   assertValidQuery("@tags:{foo}", ctx);
161   assertValidQuery("@tags:{foo|bar baz|boo}", ctx);
162   assertValidQuery("@tags:{foo|bar\\ baz|boo}", ctx);
163   assertValidQuery("@tags:{foo*}", ctx);
164   assertValidQuery("@tags:{foo\\-*}", ctx);
165   assertValidQuery("@tags:{bar | foo*}", ctx);
166   assertValidQuery("@tags:{bar* | foo}", ctx);
167   assertValidQuery("@tags:{bar* | foo*}", ctx);
168 
169   assertInvalidQuery("@tags:{foo|bar\\ baz|}", ctx);
170   assertInvalidQuery("@tags:{foo|bar\\ baz|", ctx);
171   assertInvalidQuery("{foo|bar\\ baz}", ctx);
172 
173   assertInvalidQuery("(foo", ctx);
174   assertInvalidQuery("\"foo", ctx);
175   assertValidQuery("", ctx);
176   assertInvalidQuery("()", ctx);
177 
178   // test stopwords
179   assertValidQuery("a for is", ctx);
180   assertValidQuery("a|for|is", ctx);
181   assertValidQuery("a little bit of party", ctx);
182   assertValidQuery("no-as", ctx);
183   assertValidQuery("~no~as", ctx);
184   assertValidQuery("(no -as) =>{$weight: 0.5}", ctx);
185   assertValidQuery("@foo:-as", ctx);
186 
187   // test utf-8 query
188   assertValidQuery("שלום עולם", ctx);
189 
190   // Test attribute
191   assertValidQuery("(foo bar) => {$weight: 0.5; $slop: 2}", ctx);
192   assertValidQuery("foo => {$weight: 0.5} bar => {$weight: 0.1}", ctx);
193 
194   assertValidQuery("@title:(foo bar) => {$weight: 0.5; $slop: 2}", ctx);
195   assertValidQuery(
196       "@title:(foo bar) => {$weight: 0.5; $slop: 2} @body:(foo bar) => {$weight: 0.5; $slop: 2}",
197       ctx);
198   assertValidQuery("(foo => {$weight: 0.5;}) | ((bar) => {$weight: 0.5})", ctx);
199   assertValidQuery("(foo => {$weight: 0.5;})  ((bar) => {}) => {}", ctx);
200   assertValidQuery("@tag:{foo | bar} => {$weight: 0.5;} ", ctx);
201   assertValidQuery("@num:[0 100] => {$weight: 0.5;} ", ctx);
202   assertInvalidQuery("@tag:{foo | bar} => {$weight: -0.5;} ", ctx);
203   assertInvalidQuery("@tag:{foo | bar} => {$great: 0.5;} ", ctx);
204   assertInvalidQuery("@tag:{foo | bar} => {$great:;} ", ctx);
205   assertInvalidQuery("@tag:{foo | bar} => {$:1;} ", ctx);
206 
207   assertInvalidQuery(" => {$weight: 0.5;} ", ctx);
208 
209   const char *qt = "(hello|world) and \"another world\" (foo is bar) -(baz boo*)";
210   QASTCXX ast;
211   ast.setContext(&ctx);
212   ASSERT_TRUE(ast.parse(qt));
213   QueryNode *n = ast.root;
214   QAST_Print(&ast, ctx.spec);
215   ASSERT_TRUE(n != NULL);
216   ASSERT_EQ(n->type, QN_PHRASE);
217   ASSERT_EQ(n->pn.exact, 0);
218   ASSERT_EQ(QueryNode_NumChildren(n), 4);
219   ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
220 
221   ASSERT_TRUE(n->children[0]->type == QN_UNION);
222   ASSERT_STREQ("hello", n->children[0]->children[0]->tn.str);
223   ASSERT_STREQ("world", n->children[0]->children[1]->tn.str);
224 
225   QueryNode *_n = n->children[1];
226 
227   ASSERT_TRUE(_n->type == QN_PHRASE);
228   ASSERT_TRUE(_n->pn.exact == 1);
229   ASSERT_EQ(QueryNode_NumChildren(_n), 2);
230   ASSERT_STREQ("another", _n->children[0]->tn.str);
231   ASSERT_STREQ("world", _n->children[1]->tn.str);
232 
233   _n = n->children[2];
234   ASSERT_TRUE(_n->type == QN_PHRASE);
235 
236   ASSERT_TRUE(_n->pn.exact == 0);
237   ASSERT_EQ(QueryNode_NumChildren(_n), 2);
238   ASSERT_STREQ("foo", _n->children[0]->tn.str);
239   ASSERT_STREQ("bar", _n->children[1]->tn.str);
240 
241   _n = n->children[3];
242   ASSERT_TRUE(_n->type == QN_NOT);
243   _n = QueryNode_GetChild(_n, 0);
244   ASSERT_TRUE(_n->pn.exact == 0);
245   ASSERT_EQ(2, QueryNode_NumChildren(_n));
246   ASSERT_STREQ("baz", _n->children[0]->tn.str);
247 
248   ASSERT_EQ(_n->children[1]->type, QN_PREFX);
249   ASSERT_STREQ("boo", _n->children[1]->pfx.str);
250   QAST_Destroy(&ast);
251   IndexSpec_Free(ctx.spec);
252 }
253 
TEST_F(QueryTest,testPureNegative)254 TEST_F(QueryTest, testPureNegative) {
255   const char *qs[] = {"-@title:hello", "-hello", "@title:-hello", "-(foo)", "-foo", "(-foo)", NULL};
256   static const char *args[] = {"SCHEMA", "title",  "text", "weight", "0.1",    "body",
257                                "text",   "weight", "2.0",  "bar",    "numeric"};
258   QueryError err = {QueryErrorCode(0)};
259   IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
260   RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
261   for (size_t i = 0; qs[i] != NULL; i++) {
262     QASTCXX ast;
263     ast.setContext(&ctx);
264     ASSERT_TRUE(ast.parse(qs[i])) << ast.getError();
265     QueryNode *n = ast.root;
266     ASSERT_TRUE(n != NULL);
267     ASSERT_EQ(n->type, QN_NOT);
268     ASSERT_TRUE(QueryNode_GetChild(n, 0) != NULL);
269   }
270   IndexSpec_Free(ctx.spec);
271 }
272 
TEST_F(QueryTest,testGeoQuery)273 TEST_F(QueryTest, testGeoQuery) {
274   static const char *args[] = {"SCHEMA", "title", "text", "loc", "geo"};
275   QueryError err = {QueryErrorCode(0)};
276   IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
277   RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
278   const char *qt = "@title:hello world @loc:[31.52 32.1342 10.01 km]";
279   QASTCXX ast;
280   ast.setContext(&ctx);
281   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
282   QueryNode *n = ast.root;
283   ASSERT_EQ(n->type, QN_PHRASE);
284   ASSERT_TRUE((n->opts.fieldMask == RS_FIELDMASK_ALL));
285   ASSERT_EQ(QueryNode_NumChildren(n), 2);
286 
287   QueryNode *gn = n->children[1];
288   ASSERT_EQ(gn->type, QN_GEO);
289   ASSERT_STREQ(gn->gn.gf->property, "loc");
290   ASSERT_EQ(gn->gn.gf->unitType, GEO_DISTANCE_KM);
291   ASSERT_EQ(gn->gn.gf->lon, 31.52);
292   ASSERT_EQ(gn->gn.gf->lat, 32.1342);
293   ASSERT_EQ(gn->gn.gf->radius, 10.01);
294   IndexSpec_Free(ctx.spec);
295 }
296 
TEST_F(QueryTest,testFieldSpec)297 TEST_F(QueryTest, testFieldSpec) {
298   static const char *args[] = {"SCHEMA", "title",  "text", "weight", "0.1",    "body",
299                                "text",   "weight", "2.0",  "bar",    "numeric"};
300   QueryError err = {QUERY_OK};
301   IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
302   RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
303   const char *qt = "@title:hello world";
304   QASTCXX ast(ctx);
305   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
306   ast.print();
307   QueryNode *n = ast.root;
308   ASSERT_EQ(n->type, QN_PHRASE);
309   ASSERT_EQ(n->opts.fieldMask, 0x01);
310 
311   qt = "(@title:hello) (@body:world)";
312   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
313   n = ast.root;
314 
315   ASSERT_TRUE(n != NULL);
316   printf("%s ====> ", qt);
317   ast.print();
318   ASSERT_EQ(n->type, QN_PHRASE);
319   ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
320   ASSERT_EQ(n->children[0]->opts.fieldMask, 0x01);
321   ASSERT_EQ(n->children[1]->opts.fieldMask, 0x02);
322 
323   // test field modifiers
324   qt = "@title:(hello world) @body:(world apart) @adas_dfsd:fofofof";
325   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
326   n = ast.root;
327   printf("%s ====> ", qt);
328   ast.print();
329   ASSERT_EQ(n->type, QN_PHRASE);
330   ASSERT_EQ(n->opts.fieldMask, RS_FIELDMASK_ALL);
331   ASSERT_EQ(QueryNode_NumChildren(n), 3);
332   ASSERT_EQ(n->children[0]->opts.fieldMask, 0x01);
333   ASSERT_EQ(n->children[1]->opts.fieldMask, 0x02);
334   ASSERT_EQ(n->children[2]->opts.fieldMask, 0x00);
335   // ASSERT_EQ(n->children[2]->fieldMask, 0x00)
336 
337   // test numeric ranges
338   qt = "@num:[0.4 (500]";
339   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
340   n = ast.root;
341   ASSERT_EQ(n->type, QN_NUMERIC);
342   ASSERT_EQ(n->nn.nf->min, 0.4);
343   ASSERT_EQ(n->nn.nf->max, 500.0);
344   ASSERT_EQ(n->nn.nf->inclusiveMin, 1);
345   ASSERT_EQ(n->nn.nf->inclusiveMax, 0);
346   IndexSpec_Free(ctx.spec);
347 }
348 
TEST_F(QueryTest,testAttributes)349 TEST_F(QueryTest, testAttributes) {
350   static const char *args[] = {"SCHEMA", "title", "text", "body", "text"};
351   QueryError err = {QUERY_OK};
352   IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
353   RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
354 
355   const char *qt =
356       "(@title:(foo bar) => {$weight: 0.5} @body:lol => {$weight: 0.2}) => "
357       "{$weight:0.3; $slop:2; $inorder:true}";
358   QASTCXX ast(ctx);
359   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
360   QueryNode *n = ast.root;
361   ASSERT_EQ(0.3, n->opts.weight);
362   ASSERT_EQ(2, n->opts.maxSlop);
363   ASSERT_EQ(1, n->opts.inOrder);
364 
365   ASSERT_EQ(n->type, QN_PHRASE);
366   ASSERT_EQ(QueryNode_NumChildren(n), 2);
367   ASSERT_EQ(0.5, n->children[0]->opts.weight);
368   ASSERT_EQ(0.2, n->children[1]->opts.weight);
369   IndexSpec_Free(ctx.spec);
370 }
371 
TEST_F(QueryTest,testTags)372 TEST_F(QueryTest, testTags) {
373   static const char *args[] = {"SCHEMA", "title", "text", "tags", "tag", "separator", ";"};
374   QueryError err = {QUERY_OK};
375   IndexSpec *spec = IndexSpec_Parse("idx", args, sizeof(args) / sizeof(const char *), &err);
376   RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, spec);
377 
378   const char *qt = "@tags:{hello world  |foo| שלום|  lorem\\ ipsum    }";
379   QASTCXX ast(ctx);
380   ASSERT_TRUE(ast.parse(qt)) << ast.getError();
381   ast.print();
382   QueryNode *n = ast.root;
383   ASSERT_EQ(n->type, QN_TAG);
384   ASSERT_EQ(4, QueryNode_NumChildren(n));
385   ASSERT_EQ(QN_PHRASE, n->children[0]->type);
386   ASSERT_STREQ("hello", n->children[0]->children[0]->tn.str);
387   ASSERT_STREQ("world", n->children[0]->children[1]->tn.str);
388 
389   ASSERT_EQ(QN_TOKEN, n->children[1]->type);
390   ASSERT_STREQ("foo", n->children[1]->tn.str);
391 
392   ASSERT_EQ(QN_TOKEN, n->children[2]->type);
393   ASSERT_STREQ("שלום", n->children[2]->tn.str);
394 
395   ASSERT_EQ(QN_TOKEN, n->children[3]->type);
396   ASSERT_STREQ("lorem\\ ipsum", n->children[3]->tn.str);
397   IndexSpec_Free(ctx.spec);
398 }
399