1# -*- coding: utf-8 -*-
2
3import redis
4import unittest
5from hotels import hotels
6import random
7import time
8from RLTest import Env
9from includes import *
10from common import *
11
12# this tests is not longer relevant
13# def testAdd(env):
14#     if env.is_cluster():
15#         raise unittest.SkipTest()
16
17#     r = env
18#     env.assertOk(r.execute_command(
19#         'ft.create', 'idx', 'schema', 'title', 'text', 'body', 'text'))
20#     env.assertTrue(r.exists('idx:idx'))
21#     env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1.0, 'fields',
22#                                     'title', 'hello world',
23#                                     'body', 'lorem ist ipsum'))
24
25#     for _ in r.retry_with_rdb_reload():
26#         prefix = 'ft'
27#         env.assertExists(prefix + ':idx/hello')
28#         env.assertExists(prefix + ':idx/world')
29#         env.assertExists(prefix + ':idx/lorem')
30
31def testAddErrors(env):
32    env.expect('ft.create idx ON HASH schema foo text bar numeric sortable').equal('OK')
33    env.expect('ft.add idx doc1 1 redis 4').error().contains('Unknown keyword')
34    env.expect('ft.add idx doc1').error().contains("wrong number of arguments")
35    env.expect('ft.add idx doc1 42').error().contains("Score must be between 0 and 1")
36    env.expect('ft.add idx doc1 1.0').error().contains("No field list found")
37    env.expect('ft.add fake_idx doc1 1.0 fields foo bar').error().contains("Unknown index name")
38
39def assertEqualIgnoreCluster(env, val1, val2):
40    # todo: each test that uses this function should be switch back to env.assertEqual once fix
41    # issues on coordinator
42    if env.isCluster():
43        return
44    env.assertEqual(val1, val2)
45
46def testConditionalUpdate(env):
47    env.assertOk(env.cmd(
48        'ft.create', 'idx','ON', 'HASH',
49        'schema', 'foo', 'text', 'bar', 'numeric', 'sortable'))
50    env.assertOk(env.cmd('ft.add', 'idx', '1', '1',
51                           'fields', 'foo', 'hello', 'bar', '123'))
52    env.assertOk(env.cmd('ft.add', 'idx', '1', '1', 'replace', 'if',
53                           '@foo == "hello"', 'fields', 'foo', 'world', 'bar', '123'))
54    env.assertEqual('NOADD', env.cmd('ft.add', 'idx', '1', '1', 'replace',
55                                       'if', '@foo == "hello"', 'fields', 'foo', 'world', 'bar', '123'))
56    env.assertEqual('NOADD', env.cmd('ft.add', 'idx', '1', '1', 'replace',
57                                       'if', '1 == 2', 'fields', 'foo', 'world', 'bar', '123'))
58    env.assertOk(env.cmd('ft.add', 'idx', '1', '1', 'replace', 'partial', 'if',
59                           '@foo == "world"', 'fields', 'bar', '234'))
60    env.assertOk(env.cmd('ft.add', 'idx', '1', '1', 'replace', 'if',
61                           '@bar == 234', 'fields', 'foo', 'hello', 'bar', '123'))
62
63    # Ensure that conditionals are ignored if the document doesn't exist
64    env.assertOk(env.cmd('FT.ADD', 'idx', '666', '1',
65                           'IF', '@bar > 42', 'FIELDS', 'bar', '15'))
66    # Ensure that it fails if we try again, because it already exists
67    env.assertEqual('NOADD', env.cmd('FT.ADD', 'idx', '666', '1',
68                                       'REPLACE', 'IF', '@bar > 42', 'FIELDS', 'bar', '15'))
69    # Ensure that it fails because we're not using 'REPLACE'
70    with env.assertResponseError():
71        env.assertOk(env.cmd('FT.ADD', 'idx', '666', '1',
72                               'IF', '@bar > 42', 'FIELDS', 'bar', '15'))
73
74def testUnionIdList(env):
75    # Regression test for https://github.com/RediSearch/RediSearch/issues/306
76    r = env
77    N = 100
78    env.assertOk(r.execute_command(
79        "ft.create", "test", 'ON', 'HASH',
80        "SCHEMA",  "tags", "TAG", "waypoint", "GEO"))
81    env.assertOk(r.execute_command(
82        "ft.add", "test", "1", "1", "FIELDS", "tags", "alberta", "waypoint", "-113.524,53.5244"))
83    env.assertOk(r.execute_command(
84        "ft.add", "test", "2", "1", "FIELDS", "tags", "ontario", "waypoint", "-79.395,43.661667"))
85
86    r.cmd('ft.search', 'test', '@tags:{ontario}')
87
88    res = r.execute_command(
89        'ft.search', 'test', "@waypoint:[-113.52 53.52 20 mi]|@tags:{ontario}", 'nocontent')
90    env.assertEqual(res, [2, '2', '1'])
91
92def testAttributes(env):
93    env.assertOk(env.cmd('ft.create', 'idx','ON', 'HASH',
94                         'schema', 'title', 'text', 'body', 'text'))
95    env.assertOk(env.cmd('ft.add', 'idx', 'doc1', 1.0, 'fields',
96                                            'title', 't1 t2', 'body', 't3 t4 t5'))
97    env.assertOk(env.cmd('ft.add', 'idx', 'doc2', 1.0, 'fields',
98                           'body', 't1 t2', 'title', 't3 t5'))
99
100    res = env.cmd(
101        'ft.search', 'idx', '(@title:(t1 t2) => {$weight: 0.2}) |(@body:(t1 t2) => {$weight: 0.5})', 'nocontent')
102    env.assertListEqual([2L, 'doc2', 'doc1'], res)
103    res = env.cmd(
104        'ft.search', 'idx', '(@title:(t1 t2) => {$weight: 2.5}) |(@body:(t1 t2) => {$weight: 0.5})', 'nocontent')
105    env.assertListEqual([2L, 'doc1', 'doc2'], res)
106
107    res = env.cmd(
108        'ft.search', 'idx', '(t3 t5) => {$slop: 4}', 'nocontent')
109    env.assertListEqual([2L, 'doc2', 'doc1'], res)
110    res = env.cmd(
111        'ft.search', 'idx', '(t5 t3) => {$slop: 0}', 'nocontent')
112    env.assertListEqual([1L, 'doc2'], res)
113    res = env.cmd(
114        'ft.search', 'idx', '(t5 t3) => {$slop: 0; $inorder:true}', 'nocontent')
115    env.assertListEqual([0], res)
116
117def testUnion(env):
118    N = 100
119    r = env
120    env.assertOk(r.execute_command(
121        'ft.create', 'idx','ON', 'HASH', 'schema', 'f', 'text'))
122    for i in range(N):
123
124        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
125                                        'f', 'hello world' if i % 2 == 0 else 'hallo werld'))
126
127    for _ in r.retry_with_rdb_reload():
128        waitForIndex(r, 'idx')
129        res = r.execute_command(
130            'ft.search', 'idx', 'hello|hallo', 'nocontent', 'limit', '0', '100')
131        env.assertEqual(N + 1, len(res))
132        env.assertEqual(N, res[0])
133
134        res = r.execute_command(
135            'ft.search', 'idx', 'hello|world', 'nocontent', 'limit', '0', '100')
136        env.assertEqual(51, len(res))
137        env.assertEqual(50, res[0])
138
139        res = r.execute_command('ft.search', 'idx', '(hello|hello)(world|world)',
140                                'nocontent', 'verbatim', 'limit', '0', '100')
141        env.assertEqual(51, len(res))
142        env.assertEqual(50, res[0])
143
144        res = r.execute_command(
145            'ft.search', 'idx', '(hello|hallo)(werld|world)', 'nocontent', 'verbatim', 'limit', '0', '100')
146        env.assertEqual(101, len(res))
147        env.assertEqual(100, res[0])
148
149        res = r.execute_command(
150            'ft.search', 'idx', '(hallo|hello)(world|werld)', 'nocontent', 'verbatim', 'limit', '0', '100')
151        env.assertEqual(101, len(res))
152        env.assertEqual(100, res[0])
153
154        res = r.execute_command(
155            'ft.search', 'idx', '(hello|werld)(hallo|world)', 'nocontent', 'verbatim', 'limit', '0', '100')
156        env.assertEqual(101, len(res))
157        env.assertEqual(100, res[0])
158
159        res = r.execute_command(
160            'ft.search', 'idx', '(hello|hallo) world', 'nocontent', 'verbatim', 'limit', '0', '100')
161        env.assertEqual(51, len(res))
162        env.assertEqual(50, res[0])
163
164        res = r.execute_command(
165            'ft.search', 'idx', '(hello world)|((hello world)|(hallo world|werld) | hello world werld)',
166            'nocontent', 'verbatim', 'limit', '0', '100')
167        env.assertEqual(101, len(res))
168        env.assertEqual(100, res[0])
169
170def testSearch(env):
171    r = env
172    r.expect('ft.create', 'idx', 'ON', 'HASH',
173             'schema', 'title', 'text', 'weight', 10.0, 'body', 'text').ok()
174    r.expect('ft.add', 'idx', 'doc1', 0.5,
175             'fields','title', 'hello world', 'body', 'lorem ist ipsum').ok()
176    r.expect('ft.add', 'idx', 'doc2', 1.0,
177             'fields', 'title', 'hello another world', 'body', 'lorem ist ipsum lorem lorem').ok()
178    # order of documents might change after reload
179    for _ in r.retry_with_rdb_reload():
180        waitForIndex(env, 'idx')
181        res = r.execute_command('ft.search', 'idx', 'hello')
182        expected = [2L, 'doc2', ['title', 'hello another world', 'body', 'lorem ist ipsum lorem lorem'],
183                    'doc1', ['title', 'hello world', 'body', 'lorem ist ipsum']]
184        env.assertEqual(toSortedFlatList(res), toSortedFlatList(expected))
185
186        # Test empty query
187        res = r.execute_command('ft.search', 'idx', '')
188        env.assertListEqual([0], res)
189
190        # Test searching with no content
191        res = r.execute_command(
192            'ft.search', 'idx', 'hello', 'nocontent')
193        env.assertTrue(len(res) == 3)
194        expected = ['doc2', 'doc1']
195        env.assertEqual(res[0], 2L)
196        for item in expected:
197            env.assertIn(item, res)
198
199        # Test searching WITHSCORES
200        res = r.execute_command('ft.search', 'idx', 'hello', 'WITHSCORES')
201        env.assertEqual(len(res), 7)
202        env.assertEqual(res[0], 2L)
203        for item in expected:
204            env.assertIn(item, res)
205        env.assertTrue(float(res[2]) > 0)
206        env.assertTrue(float(res[5]) > 0)
207
208        # Test searching WITHSCORES NOCONTENT
209        res = r.execute_command('ft.search', 'idx', 'hello', 'WITHSCORES', 'NOCONTENT')
210        env.assertEqual(len(res), 5)
211        env.assertEqual(res[0], 2L)
212        for item in expected:
213            env.assertIn(item, res)
214        env.assertTrue(float(res[2]) > 0)
215        env.assertTrue(float(res[4]) > 0)
216
217def testGet(env):
218    r = env
219    env.assertOk(r.execute_command(
220        'ft.create', 'idx', 'ON', 'HASH',
221        'schema', 'foo', 'text', 'bar', 'text'))
222
223    env.expect('ft.get').error().contains("wrong number of arguments")
224    env.expect('ft.get', 'idx').error().contains("wrong number of arguments")
225    env.expect('ft.get', 'idx', 'foo', 'bar').error().contains("wrong number of arguments")
226    env.expect('ft.mget').error().contains("wrong number of arguments")
227    env.expect('ft.mget', 'idx').error().contains("wrong number of arguments")
228    env.expect('ft.mget', 'fake_idx').error().contains("wrong number of arguments")
229
230    env.expect('ft.get fake_idx foo').error().contains("Unknown Index name")
231    env.expect('ft.mget fake_idx foo').error().contains("Unknown Index name")
232
233    for i in range(100):
234        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
235                                        'foo', 'hello world', 'bar', 'wat wat'))
236
237    for i in range(100):
238        res = r.execute_command('ft.get', 'idx', 'doc%d' % i)
239        env.assertIsNotNone(res)
240        env.assertEqual(set(['foo', 'hello world', 'bar', 'wat wat']), set(res))
241        env.assertIsNone(r.execute_command(
242            'ft.get', 'idx', 'doc%dsdfsd' % i))
243    env.expect('ft.get', 'no_idx', 'doc0').error().contains("Unknown Index name")
244
245    rr = r.execute_command(
246        'ft.mget', 'idx', *('doc%d' % i for i in range(100)))
247    env.assertEqual(len(rr), 100)
248    for res in rr:
249        env.assertIsNotNone(res)
250        env.assertEqual(set(['foo', 'hello world', 'bar', 'wat wat']), set(res))
251    rr = r.execute_command(
252        'ft.mget', 'idx', *('doc-%d' % i for i in range(100)))
253    env.assertEqual(len(rr), 100)
254    for res in rr:
255        env.assertIsNone(res)
256
257    # Verify that when a document is deleted, GET returns NULL
258    r.cmd('ft.del', 'idx', 'doc10') # But we still keep the document
259    r.cmd('ft.del', 'idx', 'doc11')
260    assert r.cmd('ft.del', 'idx', 'coverage') == 0
261    res = r.cmd('ft.get', 'idx', 'doc10')
262    r.assertEqual(None, res)
263    res = r.cmd('ft.mget', 'idx', 'doc10')
264    r.assertEqual([None], res)
265    res = r.cmd('ft.mget', 'idx', 'doc10', 'doc11', 'doc12')
266    r.assertIsNone(res[0])
267    r.assertIsNone(res[1])
268    r.assertTrue(not not res[2])
269
270    env.expect('ft.add idx doc 0.1 language arabic payload redislabs fields foo foo').ok()
271    env.expect('ft.get idx doc').equal(['foo', 'foo'])
272    res = env.cmd('hgetall doc')
273    env.assertEqual(set(res), set(['foo', 'foo', '__score', '0.1', '__language', 'arabic', '__payload', 'redislabs']))
274
275
276def testDelete(env):
277    r = env
278    env.assertOk(r.execute_command(
279        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'f', 'text'))
280
281    for i in range(100):
282        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
283                                        'f', 'hello world'))
284
285    env.expect('ft.del', 'fake_idx', 'doc1').error()
286
287    for i in range(100):
288        # the doc hash should exist now
289        r.expect('ft.get', 'idx', 'doc%d' % i).notRaiseError()
290        # Delete the actual docs only half of the time
291        env.assertEqual(1, r.execute_command(
292           'ft.del', 'idx', 'doc%d' % i, 'DD' if i % 2 == 0 else ''))
293        # second delete should return 0
294        env.assertEqual(0, r.execute_command(
295            'ft.del', 'idx', 'doc%d' % i))
296        # second delete should return 0
297
298        # TODO: return 0 if doc wasn't found
299        #env.assertEqual(0, r.execute_command(
300        #    'ft.del', 'idx', 'doc%d' % i))
301
302        # After del with DD the doc hash should not exist
303        if i % 2 == 0:
304            env.assertFalse(r.exists('doc%d' % i))
305        else:
306            r.expect('ft.get', 'idx', 'doc%d' % i).notRaiseError()
307        res = r.execute_command(
308            'ft.search', 'idx', 'hello', 'nocontent', 'limit', 0, 100)
309        env.assertNotIn('doc%d' % i, res)
310        env.assertEqual(res[0], 100 - i - 1)
311        env.assertEqual(len(res), 100 - i)
312
313        # test reinsertion
314        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
315                                        'f', 'hello world'))
316        res = r.execute_command(
317            'ft.search', 'idx', 'hello', 'nocontent', 'limit', 0, 100)
318        env.assertIn('doc%d' % i, res)
319        env.assertEqual(1, r.execute_command(
320            'ft.del', 'idx', 'doc%d' % i))
321    for _ in r.retry_with_rdb_reload():
322        waitForIndex(env, 'idx')
323        did = 'rrrr'
324        env.assertOk(r.execute_command('ft.add', 'idx', did, 1, 'fields',
325                                        'f', 'hello world'))
326        env.assertEqual(1, r.execute_command('ft.del', 'idx', did))
327        env.assertEqual(0, r.execute_command('ft.del', 'idx', did))
328        env.assertOk(r.execute_command('ft.add', 'idx', did, 1, 'fields',
329                                        'f', 'hello world'))
330        env.assertEqual(1, r.execute_command('ft.del', 'idx', did))
331        env.assertEqual(0, r.execute_command('ft.del', 'idx', did))
332
333def testReplace(env):
334    r = env
335
336    env.assertOk(r.execute_command(
337        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'f', 'text'))
338
339    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1.0, 'fields',
340                                    'f', 'hello world'))
341    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1.0, 'fields',
342                                    'f', 'hello world'))
343    res = r.execute_command(
344        'ft.search', 'idx', 'hello world')
345    env.assertEqual(2, res[0])
346
347    with env.assertResponseError():
348        # make sure we can't insert a doc twice
349        res = r.execute_command('ft.add', 'idx', 'doc1', 1.0, 'fields',
350                                'f', 'hello world')
351
352    # now replace doc1 with a different content
353    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1.0, 'replace', 'fields',
354                                    'f', 'goodbye universe'))
355
356    for _ in r.retry_with_rdb_reload():
357        waitForIndex(env, 'idx')
358        # make sure the query for hello world does not return the replaced
359        # document
360        res = r.execute_command(
361            'ft.search', 'idx', 'hello world', 'nocontent')
362        env.assertEqual(1, res[0])
363        env.assertEqual('doc2', res[1])
364
365        # search for the doc's new content
366        res = r.execute_command(
367            'ft.search', 'idx', 'goodbye universe', 'nocontent')
368        env.assertEqual(1, res[0])
369        env.assertEqual('doc1', res[1])
370
371def testDrop(env):
372    r = env
373    env.assertOk(r.execute_command(
374        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'f', 'text', 'n', 'numeric', 't', 'tag', 'g', 'geo'))
375
376    for i in range(100):
377        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
378                                        'f', 'hello world', 'n', 666, 't', 'foo bar',
379                                        'g', '19.04,47.497'))
380    keys = r.keys('*')
381    env.assertGreaterEqual(len(keys), 100)
382
383    env.assertOk(r.execute_command('ft.drop', 'idx'))
384    keys = r.keys('*')
385
386    env.assertEqual(0, len(keys))
387    env.flush()
388
389    # Now do the same with KEEPDOCS
390    env.assertOk(r.execute_command(
391        'ft.create', 'idx', 'ON', 'HASH',
392        'schema', 'f', 'text', 'n', 'numeric', 't', 'tag', 'g', 'geo'))
393
394    for i in range(100):
395        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
396                                        'f', 'hello world', 'n', 666, 't', 'foo bar',
397                                        'g', '19.04,47.497'))
398    keys = r.keys('*')
399    env.assertGreaterEqual(len(keys), 100)
400
401    if not env.is_cluster():
402        env.assertOk(r.execute_command('ft.drop', 'idx', 'KEEPDOCS'))
403        keys = r.keys('*')
404        env.assertListEqual(['doc0', 'doc1', 'doc10', 'doc11', 'doc12', 'doc13', 'doc14', 'doc15', 'doc16', 'doc17', 'doc18', 'doc19', 'doc2', 'doc20', 'doc21', 'doc22', 'doc23', 'doc24', 'doc25', 'doc26', 'doc27', 'doc28', 'doc29', 'doc3', 'doc30', 'doc31', 'doc32', 'doc33', 'doc34', 'doc35', 'doc36', 'doc37', 'doc38', 'doc39', 'doc4', 'doc40', 'doc41', 'doc42', 'doc43', 'doc44', 'doc45', 'doc46', 'doc47', 'doc48', 'doc49', 'doc5', 'doc50', 'doc51', 'doc52', 'doc53',
405                              'doc54', 'doc55', 'doc56', 'doc57', 'doc58', 'doc59', 'doc6', 'doc60', 'doc61', 'doc62', 'doc63', 'doc64', 'doc65', 'doc66', 'doc67', 'doc68', 'doc69', 'doc7', 'doc70', 'doc71', 'doc72', 'doc73', 'doc74', 'doc75', 'doc76', 'doc77', 'doc78', 'doc79', 'doc8', 'doc80', 'doc81', 'doc82', 'doc83', 'doc84', 'doc85', 'doc86', 'doc87', 'doc88', 'doc89', 'doc9', 'doc90', 'doc91', 'doc92', 'doc93', 'doc94', 'doc95', 'doc96', 'doc97', 'doc98', 'doc99'], sorted(keys))
406
407    env.expect('FT.DROP', 'idx', 'KEEPDOCS', '666').error().contains("wrong number of arguments")
408
409def testDelete(env):
410    r = env
411    r.expect('ft.create', 'idx', 'ON', 'HASH', 'schema', 'f', 'text', 'n', 'numeric', 't', 'tag', 'g', 'geo').ok()
412
413    for i in range(100):
414        r.expect('ft.add', 'idx', 'doc%d' % i, 1.0,
415                 'fields', 'f', 'hello world', 'n', 666, 't', 'foo bar',
416                 'g', '19.04,47.497').ok()
417    keys = r.keys('*')
418    env.assertGreaterEqual(len(keys), 100)
419
420    r.expect('FT.DROPINDEX', 'idx', 'dd').ok()
421    keys = r.keys('*')
422
423    env.assertEqual(0, len(keys))
424    env.flush()
425
426    # Now do the same with KEEPDOCS
427    env.expect('ft.create', 'idx', 'ON', 'HASH',
428               'schema', 'f', 'text', 'n', 'numeric', 't', 'tag', 'g', 'geo').ok()
429
430    for i in range(100):
431        r.expect('ft.add', 'idx', 'doc%d' % i, 1.0,
432                 'fields', 'f', 'hello world', 'n', 666, 't', 'foo bar',
433                 'g', '19.04,47.497').ok()
434    keys = r.keys('*')
435    env.assertGreaterEqual(len(keys), 100)
436
437    if not env.is_cluster():
438        r.expect('FT.DROPINDEX', 'idx').ok()
439        keys = r.keys('*')
440        env.assertListEqual(sorted("doc%d" %k for k in range(100)), sorted(keys))
441
442    env.expect('FT.DROPINDEX', 'idx', 'dd', '666').error().contains("wrong number of arguments")
443
444def testCustomStopwords(env):
445    r = env
446    # Index with default stopwords
447    env.assertOk(r.execute_command(
448        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text'))
449
450    # Index with custom stopwords
451    env.assertOk(r.execute_command('ft.create', 'idx2', 'ON', 'HASH', 'stopwords', 2, 'hello', 'world',
452                                    'schema', 'foo', 'text'))
453    assertInfoField(env, 'idx2', 'stopwords_list', ['hello', 'world'])
454
455    # Index with NO stopwords
456    env.assertOk(r.execute_command('ft.create', 'idx3', 'ON', 'HASH', 'stopwords', 0,
457                                    'schema', 'foo', 'text'))
458    assertInfoField(env, 'idx3', 'stopwords_list', [])
459
460    # 2nd Index with NO stopwords - check global is used and freed
461    env.assertOk(r.execute_command('ft.create', 'idx4', 'ON', 'HASH', 'stopwords', 0,
462                                    'schema', 'foo', 'text'))
463
464    #for idx in ('idx', 'idx2', 'idx3'):
465    env.assertOk(r.execute_command(
466        'ft.add', 'idx', 'doc1', 1.0, 'fields', 'foo', 'hello world'))
467    env.assertOk(r.execute_command(
468        'ft.add', 'idx', 'doc2', 1.0, 'fields', 'foo', 'to be or not to be'))
469
470    for _ in r.retry_with_rdb_reload():
471        waitForIndex(r, 'idx')
472        # Normal index should return results just for 'hello world'
473        env.assertEqual([1, 'doc1'],  r.execute_command(
474            'ft.search', 'idx', 'hello world', 'nocontent'))
475        env.assertEqual([0],  r.execute_command(
476            'ft.search', 'idx', 'to be or not', 'nocontent'))
477
478        # Custom SW index should return results just for 'to be or not'
479        env.assertEqual([0],  r.execute_command(
480            'ft.search', 'idx2', 'hello world', 'nocontent'))
481        env.assertEqual([1, 'doc2'],  r.execute_command(
482            'ft.search', 'idx2', 'to be or not', 'nocontent'))
483
484        # No SW index should return results for both
485        env.assertEqual([1, 'doc1'],  r.execute_command(
486            'ft.search', 'idx3', 'hello world', 'nocontent'))
487        env.assertEqual([1, 'doc2'],  r.execute_command(
488            'ft.search', 'idx3', 'to be or not', 'nocontent'))
489
490def testStopwords(env):
491    # This test was taken from Python's tests, and failed due to some changes
492    # made earlier
493    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'stopwords', 3, 'foo',
494             'bar', 'baz', 'schema', 'txt', 'text')
495    env.cmd('ft.add', 'idx', 'doc1', 1.0, 'fields', 'txt', 'foo bar')
496    env.cmd('ft.add', 'idx', 'doc2', 1.0, 'fields', 'txt', 'hello world')
497
498    r1 = env.cmd('ft.search', 'idx', 'foo bar', 'nocontent')
499    r2 = env.cmd('ft.search', 'idx', 'foo bar hello world', 'nocontent')
500    env.assertEqual(0, r1[0])
501    env.assertEqual(1, r2[0])
502
503def testNoStopwords(env):
504    # This test taken from Java's test suite
505    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text')
506    for i in range(100):
507        env.cmd('ft.add', 'idx', 'doc{}'.format(i), 1.0, 'fields',
508                 'title', 'hello world' if i % 2 == 0 else 'hello worlds')
509
510    res = env.cmd('ft.search', 'idx', 'hello a world', 'NOCONTENT')
511    env.assertEqual(100, res[0])
512
513    res = env.cmd('ft.search', 'idx', 'hello a world',
514                   'VERBATIM', 'NOCONTENT')
515    env.assertEqual(50, res[0])
516
517    res = env.cmd('ft.search', 'idx', 'hello a world', 'NOSTOPWORDS')
518    env.assertEqual(0, res[0])
519
520def testOptional(env):
521    r = env
522    env.assertOk(r.execute_command(
523        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text'))
524    env.assertOk(r.execute_command('ft.add', 'idx',
525                                    'doc1', 1.0, 'fields', 'foo', 'hello wat woot'))
526    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2',
527                                    1.0, 'fields', 'foo', 'hello world woot'))
528    env.assertOk(r.execute_command('ft.add', 'idx', 'doc3',
529                                    1.0, 'fields', 'foo', 'hello world werld'))
530
531    res = r.execute_command('ft.search', 'idx', 'hello', 'nocontent')
532    env.assertEqual([3L, 'doc3', 'doc2', 'doc1'], res)
533    res = r.execute_command(
534        'ft.search', 'idx', 'hello world', 'nocontent', 'scorer', 'DISMAX')
535    env.assertEqual([2L, 'doc3', 'doc2'], res)
536    res = r.execute_command(
537        'ft.search', 'idx', 'hello ~world', 'nocontent', 'scorer', 'DISMAX')
538    env.assertEqual([3L, 'doc3', 'doc2', 'doc1'], res)
539    res = r.execute_command(
540        'ft.search', 'idx', 'hello ~world ~werld', 'nocontent', 'scorer', 'DISMAX')
541    env.assertEqual([3L, 'doc3', 'doc2', 'doc1'], res)
542    res = r.execute_command(
543        'ft.search', 'idx', '~world ~werld hello', 'nocontent', 'scorer', 'DISMAX')
544    env.assertEqual([3L, 'doc3', 'doc2', 'doc1'], res)
545
546def testExplain(env):
547
548    r = env
549    env.assertOk(r.execute_command(
550        'ft.create', 'idx', 'ON', 'HASH',
551        'schema', 'foo', 'text', 'bar', 'numeric', 'sortable'))
552    q = '(hello world) "what what" hello|world @bar:[10 100]|@bar:[200 300]'
553    res = r.execute_command('ft.explain', 'idx', q)
554    # print res.replace('\n', '\\n')
555    # expected = """INTERSECT {\n  UNION {\n    hello\n    +hello(expanded)\n  }\n  UNION {\n    world\n    +world(expanded)\n  }\n  EXACT {\n    what\n    what\n  }\n  UNION {\n    UNION {\n      hello\n      +hello(expanded)\n    }\n    UNION {\n      world\n      +world(expanded)\n    }\n  }\n  UNION {\n    NUMERIC {10.000000 <= @bar <= 100.000000}\n    NUMERIC {200.000000 <= @bar <= 300.000000}\n  }\n}\n"""
556    # expected = """INTERSECT {\n  UNION {\n    hello\n    <HL(expanded)\n    +hello(expanded)\n  }\n  UNION {\n    world\n    <ARLT(expanded)\n    +world(expanded)\n  }\n  EXACT {\n    what\n    what\n  }\n  UNION {\n    UNION {\n      hello\n      <HL(expanded)\n      +hello(expanded)\n    }\n    UNION {\n      world\n      <ARLT(expanded)\n      +world(expanded)\n    }\n  }\n  UNION {\n    NUMERIC {10.000000 <= @bar <= 100.000000}\n    NUMERIC {200.000000 <= @bar <= 300.000000}\n  }\n}\n"""
557    expected = """INTERSECT {\n  UNION {\n    hello\n    +hello(expanded)\n  }\n  UNION {\n    world\n    +world(expanded)\n  }\n  EXACT {\n    what\n    what\n  }\n  UNION {\n    UNION {\n      hello\n      +hello(expanded)\n    }\n    UNION {\n      world\n      +world(expanded)\n    }\n  }\n  UNION {\n    NUMERIC {10.000000 <= @bar <= 100.000000}\n    NUMERIC {200.000000 <= @bar <= 300.000000}\n  }\n}\n"""
558    env.assertEqual(res, expected)
559
560
561    # expected = ['INTERSECT {', '  UNION {', '    hello', '    <HL(expanded)', '    +hello(expanded)', '  }', '  UNION {', '    world', '    <ARLT(expanded)', '    +world(expanded)', '  }', '  EXACT {', '    what', '    what', '  }', '  UNION {', '    UNION {', '      hello', '      <HL(expanded)', '      +hello(expanded)', '    }', '    UNION {', '      world', '      <ARLT(expanded)', '      +world(expanded)', '    }', '  }', '  UNION {', '    NUMERIC {10.000000 <= @bar <= 100.000000}', '    NUMERIC {200.000000 <= @bar <= 300.000000}', '  }', '}', '']
562    if env.is_cluster():
563        raise unittest.SkipTest()
564    res = env.cmd('ft.explainCli', 'idx', q)
565    expected = ['INTERSECT {', '  UNION {', '    hello', '    +hello(expanded)', '  }', '  UNION {', '    world', '    +world(expanded)', '  }', '  EXACT {', '    what', '    what', '  }', '  UNION {', '    UNION {', '      hello', '      +hello(expanded)', '    }', '    UNION {', '      world', '      +world(expanded)', '    }', '  }', '  UNION {', '    NUMERIC {10.000000 <= @bar <= 100.000000}', '    NUMERIC {200.000000 <= @bar <= 300.000000}', '  }', '}', '']
566    env.assertEqual(expected, res)
567
568def testNoIndex(env):
569    r = env
570    env.assertOk(r.execute_command(
571        'ft.create', 'idx', 'ON', 'HASH', 'schema',
572        'foo', 'text',
573        'num', 'numeric', 'sortable', 'noindex',
574        'extra', 'text', 'noindex', 'sortable'))
575
576    if not env.isCluster():
577        # to specific check on cluster, todo : change it to be generic enough
578        res = env.cmd('ft.info', 'idx')
579        env.assertEqual(res[7][1][4], 'NOINDEX')
580        env.assertEqual(res[7][2][6], 'NOINDEX')
581
582    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', '0.1', 'fields',
583                                    'foo', 'hello world', 'num', 1, 'extra', 'hello lorem ipsum'))
584    res = r.execute_command(
585        'ft.search', 'idx', 'hello world', 'nocontent')
586    env.assertListEqual([1, 'doc1'], res)
587    res = r.execute_command(
588        'ft.search', 'idx', 'lorem ipsum', 'nocontent')
589    env.assertListEqual([0], res)
590    res = r.execute_command(
591        'ft.search', 'idx', '@extra:hello', 'nocontent')
592    env.assertListEqual([0], res)
593    res = r.execute_command(
594        'ft.search', 'idx', '@num:[1 1]', 'nocontent')
595    env.assertListEqual([0], res)
596
597def testPartial(env):
598    r = env
599    env.assertOk(r.execute_command(
600        'ft.create', 'idx', 'ON', 'HASH',  'SCORE_FIELD', '__score',
601        'schema',
602        'foo', 'text',
603        'num', 'numeric', 'sortable', 'noindex',
604        'extra', 'text', 'noindex'))
605    # print r.execute_command('ft.info', 'idx')
606
607    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', '0.1', 'fields',
608                                    'foo', 'hello world', 'num', 1, 'extra', 'lorem ipsum'))
609    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', '0.1', 'fields',
610                                    'foo', 'hello world', 'num', 2, 'extra', 'abba'))
611    res = r.execute_command('ft.search', 'idx', 'hello world',
612                            'sortby', 'num', 'asc', 'nocontent', 'withsortkeys')
613    env.assertListEqual([2L, 'doc1', '#1', 'doc2', '#2'], res)
614    res = r.execute_command('ft.search', 'idx', 'hello world',
615                            'sortby', 'num', 'desc', 'nocontent', 'withsortkeys')
616    env.assertListEqual([2L, 'doc2', '#2', 'doc1', '#1'], res)
617
618    # Updating non indexed fields doesn't affect search results
619    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', '0.1', 'replace', 'partial',
620                                    'fields', 'num', 3, 'extra', 'jorem gipsum'))
621    env.expect('ft.add', 'idx', 'doc12', '0.1', 'replace', 'partial',
622                                    'fields', 'num1', 'redis').equal('OK')
623
624    res = r.execute_command(
625        'ft.search', 'idx', 'hello world', 'sortby', 'num', 'desc',)
626    assertResultsEqual(env, [2L, 'doc1', ['foo', 'hello world', 'num', '3','extra', 'jorem gipsum'],
627        'doc2', ['foo', 'hello world', 'num', '2', 'extra', 'abba']], res)
628    res = r.execute_command(
629        'ft.search', 'idx', 'hello', 'nocontent', 'withscores')
630    # Updating only indexed field affects search results
631    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', '0.1', 'replace', 'partial',
632                                    'fields', 'foo', 'wat wet'))
633    res = r.execute_command(
634        'ft.search', 'idx', 'hello world', 'nocontent')
635    env.assertListEqual([1L, 'doc2'], res)
636    res = r.execute_command('ft.search', 'idx', 'wat', 'nocontent')
637    env.assertListEqual([1L, 'doc1'], res)
638
639    # Test updating of score and no fields
640    res = r.execute_command(
641        'ft.search', 'idx', 'wat', 'nocontent', 'withscores')
642    env.assertLess(float(res[2]), 1)
643    # env.assertListEqual([1L, 'doc1'], res)
644    env.assertOk(r.execute_command('ft.add', 'idx',
645                                    'doc1', '1.0', 'replace', 'partial', 'fields'))
646    res = r.execute_command(
647        'ft.search', 'idx', 'wat', 'nocontent', 'withscores')
648    # We reindex though no new fields, just score is updated. this effects score
649    env.assertEqual(float(res[2]), 1)
650
651    # Test updating payloads
652    res = r.execute_command(
653        'ft.search', 'idx', 'wat', 'nocontent', 'withpayloads')
654    env.assertIsNone(res[2])
655    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', '1.0',
656                                    'replace', 'partial', 'payload', 'foobar', 'fields'))
657    res = r.execute_command(
658        'ft.search', 'idx', 'wat', 'nocontent', 'withpayloads')
659    env.assertEqual('foobar', res[2])
660
661def testPaging(env):
662    r = env
663    env.assertOk(r.execute_command(
664        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text', 'bar', 'numeric', 'sortable'))
665    N = 100
666    for i in range(N):
667        env.assertOk(r.execute_command('ft.add', 'idx', '%d' % i, 1, 'fields',
668                                        'foo', 'hello', 'bar', i))
669
670    chunk = 7
671    offset = 0
672    while True:
673
674        res = r.execute_command(
675            'ft.search', 'idx', 'hello', 'nocontent', 'sortby', 'bar', 'desc', 'limit', offset, chunk)
676        env.assertEqual(res[0], N)
677
678        if offset + chunk > N:
679            env.assertTrue(len(res) - 1 <= chunk)
680            break
681        env.assertEqual(len(res), chunk + 1)
682        for n, id in enumerate(res[1:]):
683            env.assertEqual(int(id), N - 1 - (offset + n))
684        offset += chunk
685        chunk = random.randrange(1, 10)
686    res = r.execute_command(
687        'ft.search', 'idx', 'hello', 'nocontent', 'sortby', 'bar', 'asc', 'limit', N, 10)
688    env.assertEqual(res[0], N)
689    env.assertEqual(len(res), 1)
690
691    with env.assertResponseError():
692        r.execute_command(
693            'ft.search', 'idx', 'hello', 'nocontent', 'limit', 0, -1)
694    with env.assertResponseError():
695        r.execute_command(
696            'ft.search', 'idx', 'hello', 'nocontent', 'limit', -1, 10)
697    with env.assertResponseError():
698        r.execute_command(
699            'ft.search', 'idx', 'hello', 'nocontent', 'limit', 0, 2000000)
700
701def testPrefix(env):
702    r = env
703    env.assertOk(r.execute_command(
704        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text'))
705    N = 100
706    for i in range(N):
707        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
708                                        'foo', 'constant term%d' % (random.randrange(0, 5))))
709    for _ in r.retry_with_rdb_reload():
710        waitForIndex(r, 'idx')
711        res = r.execute_command(
712            'ft.search', 'idx', 'constant term', 'nocontent')
713        env.assertEqual([0], res)
714        res = r.execute_command(
715            'ft.search', 'idx', 'constant term*', 'nocontent')
716        env.assertEqual(N, res[0])
717        res = r.execute_command(
718            'ft.search', 'idx', 'const* term*', 'nocontent')
719        env.assertEqual(N, res[0])
720        res = r.execute_command(
721            'ft.search', 'idx', 'constant term1*', 'nocontent')
722        env.assertGreater(res[0], 2)
723        res = r.execute_command(
724            'ft.search', 'idx', 'const* -term*', 'nocontent')
725        env.assertEqual([0], res)
726        res = r.execute_command(
727            'ft.search', 'idx', 'constant term9*', 'nocontent')
728        env.assertEqual([0], res)
729
730def testSortBy(env):
731    r = env
732    env.assertOk(r.execute_command(
733        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text', 'sortable', 'bar', 'numeric', 'sortable'))
734    N = 100
735    for i in range(N):
736        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
737                                        'foo', 'hello%03d world' % i, 'bar', 100 - i))
738    for _ in r.retry_with_rdb_reload():
739        waitForIndex(r, 'idx')
740        res = r.execute_command(
741            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'foo')
742        env.assertEqual([100L, 'doc0', 'doc1', 'doc2', 'doc3',
743                          'doc4', 'doc5', 'doc6', 'doc7', 'doc8', 'doc9'], res)
744        res = r.execute_command(
745            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'foo', 'desc')
746        env.assertEqual([100L, 'doc99', 'doc98', 'doc97', 'doc96',
747                          'doc95', 'doc94', 'doc93', 'doc92', 'doc91', 'doc90'], res)
748        res = r.execute_command(
749            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'bar', 'desc')
750        env.assertEqual([100L, 'doc0', 'doc1', 'doc2', 'doc3',
751                          'doc4', 'doc5', 'doc6', 'doc7', 'doc8', 'doc9'], res)
752        res = r.execute_command(
753            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'bar', 'asc')
754        env.assertEqual([100L, 'doc99', 'doc98', 'doc97', 'doc96',
755                          'doc95', 'doc94', 'doc93', 'doc92', 'doc91', 'doc90'], res)
756
757        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
758                                'sortby', 'bar', 'desc', 'withscores', 'limit', '2', '5')
759        env.assertEqual(
760            [100L, 'doc2', '1', 'doc3', '1', 'doc4', '1', 'doc5', '1', 'doc6', '1'], res)
761
762        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
763                                'sortby', 'bar', 'desc', 'withsortkeys', 'limit', 0, 5)
764        env.assertListEqual(
765            [100L, 'doc0', '#100', 'doc1', '#99', 'doc2', '#98', 'doc3', '#97', 'doc4', '#96'], res)
766        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
767                                'sortby', 'foo', 'desc', 'withsortkeys', 'limit', 0, 5)
768        env.assertListEqual([100L, 'doc99', '$hello099 world', 'doc98', '$hello098 world', 'doc97', '$hello097 world', 'doc96',
769                              '$hello096 world', 'doc95', '$hello095 world'], res)
770
771def testSortByWithoutSortable(env):
772    r = env
773    env.assertOk(r.execute_command(
774        'ft.create', 'idx', 'schema', 'foo', 'text', 'bar', 'numeric', 'baz', 'text', 'sortable'))
775    N = 100
776    for i in range(N):
777        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
778                                        'foo', 'hello%03d world' % i, 'bar', 100 - i))
779    for _ in r.retry_with_rdb_reload():
780        waitForIndex(r, 'idx')
781
782        # test text
783        res = r.execute_command(
784            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'foo')
785        env.assertEqual([100L, 'doc0', 'doc1', 'doc2', 'doc3',
786                          'doc4', 'doc5', 'doc6', 'doc7', 'doc8', 'doc9'], res)
787        res = r.execute_command(
788            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'foo', 'desc')
789        env.assertEqual([100L, 'doc99', 'doc98', 'doc97', 'doc96',
790                          'doc95', 'doc94', 'doc93', 'doc92', 'doc91', 'doc90'], res)
791        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
792                                'sortby', 'foo', 'desc', 'withsortkeys', 'limit', 0, 5)
793        env.assertListEqual([100L, 'doc99', '$hello099 world', 'doc98', '$hello098 world', 'doc97', '$hello097 world', 'doc96',
794                              '$hello096 world', 'doc95', '$hello095 world'], res)
795
796        # test numeric
797        res = r.execute_command(
798            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'bar', 'desc')
799        env.assertEqual([100L, 'doc0', 'doc1', 'doc2', 'doc3',
800                          'doc4', 'doc5', 'doc6', 'doc7', 'doc8', 'doc9'], res)
801        res = r.execute_command(
802            'ft.search', 'idx', 'world', 'nocontent', 'sortby', 'bar', 'asc')
803        env.assertEqual([100L, 'doc99', 'doc98', 'doc97', 'doc96',
804                          'doc95', 'doc94', 'doc93', 'doc92', 'doc91', 'doc90'], res)
805
806        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
807                                'sortby', 'bar', 'desc', 'withscores', 'limit', '2', '5')
808        env.assertEqual(
809            [100L, 'doc2', '1', 'doc3', '1', 'doc4', '1', 'doc5', '1', 'doc6', '1'], res)
810
811        res = r.execute_command('ft.search', 'idx', 'world', 'nocontent',
812                                'sortby', 'bar', 'desc', 'withsortkeys', 'limit', 0, 5)
813        env.assertListEqual(
814            [100L, 'doc0', '#100', 'doc1', '#99', 'doc2', '#98', 'doc3', '#97', 'doc4', '#96'], res)
815
816def testNot(env):
817    r = env
818    env.assertOk(r.execute_command(
819        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text'))
820    N = 10
821    for i in range(N):
822        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
823                                        'foo', 'constant term%d' % (random.randrange(0, 5))))
824
825    for i in range(5):
826        inclusive = r.execute_command(
827            'ft.search', 'idx', 'constant term%d' % i, 'nocontent', 'limit', 0, N)
828
829        exclusive = r.execute_command(
830            'ft.search', 'idx', 'constant -term%d' % i, 'nocontent', 'limit', 0, N)
831        exclusive2 = r.execute_command(
832            'ft.search', 'idx', '-(term%d)' % i, 'nocontent', 'limit', 0, N)
833        exclusive3 = r.execute_command(
834            'ft.search', 'idx', '(-term%d) (constant)' % i, 'nocontent', 'limit', 0, N)
835
836        env.assertNotEqual(inclusive[0], N)
837        env.assertEqual(inclusive[0] + exclusive[0], N)
838        env.assertEqual(exclusive3[0], exclusive2[0])
839        env.assertEqual(exclusive3[0], exclusive[0])
840
841        s1, s2, s3, s4 = set(inclusive[1:]), set(
842            exclusive[1:]), set(exclusive2[1:]), set(exclusive3[1:])
843        env.assertTrue(s1.difference(s2) == s1)
844        env.assertTrue(s1.difference(s3) == s1)
845        env.assertTrue(s1.difference(s4) == s1)
846        env.assertTrue(s2 == s3)
847        env.assertTrue(s2 == s4)
848        env.assertTrue(s2.intersection(s1) == set())
849        env.assertTrue(s3.intersection(s1) == set())
850        env.assertTrue(s4.intersection(s1) == set())
851
852    # NOT on a non existing term
853    env.assertEqual(r.execute_command(
854        'ft.search', 'idx', 'constant -dasdfasdf', 'nocontent')[0], N)
855    # not on env term
856    env.assertEqual(r.execute_command(
857        'ft.search', 'idx', 'constant -constant', 'nocontent'), [0])
858
859    env.assertEqual(r.execute_command(
860        'ft.search', 'idx', 'constant -(term0|term1|term2|term3|term4|nothing)', 'nocontent'), [0])
861    # env.assertEqual(r.execute_command('ft.search', 'idx', 'constant -(term1 term2)', 'nocontent')[0], N)
862
863def testNestedIntersection(env):
864    r = env
865    env.assertOk(r.execute_command(
866        'ft.create', 'idx', 'ON', 'HASH',
867        'schema', 'a', 'text', 'b', 'text', 'c', 'text', 'd', 'text'))
868    for i in range(20):
869        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
870                                        'a', 'foo', 'b', 'bar', 'c', 'baz', 'd', 'gaz'))
871    res = [
872        r.execute_command('ft.search', 'idx',
873                          'foo bar baz gaz', 'nocontent'),
874        r.execute_command('ft.search', 'idx',
875                          '@a:foo @b:bar @c:baz @d:gaz', 'nocontent'),
876        r.execute_command('ft.search', 'idx',
877                          '@b:bar @a:foo @c:baz @d:gaz', 'nocontent'),
878        r.execute_command('ft.search', 'idx',
879                          '@c:baz @b:bar @a:foo @d:gaz', 'nocontent'),
880        r.execute_command('ft.search', 'idx',
881                          '@d:gaz @c:baz @b:bar @a:foo', 'nocontent'),
882        r.execute_command(
883            'ft.search', 'idx', '@a:foo (@b:bar (@c:baz @d:gaz))', 'nocontent'),
884        r.execute_command(
885            'ft.search', 'idx', '@c:baz (@a:foo (@b:bar (@c:baz @d:gaz)))', 'nocontent'),
886        r.execute_command(
887            'ft.search', 'idx', '@b:bar (@a:foo (@c:baz @d:gaz))', 'nocontent'),
888        r.execute_command(
889            'ft.search', 'idx', '@d:gaz (@a:foo (@c:baz @b:bar))', 'nocontent'),
890        r.execute_command('ft.search', 'idx',
891                          'foo (bar baz gaz)', 'nocontent'),
892        r.execute_command('ft.search', 'idx',
893                          'foo (bar (baz gaz))', 'nocontent'),
894        r.execute_command('ft.search', 'idx',
895                          'foo (bar (foo bar) (foo bar))', 'nocontent'),
896        r.execute_command('ft.search', 'idx',
897                          'foo (foo (bar baz (gaz)))', 'nocontent'),
898        r.execute_command('ft.search', 'idx', 'foo (foo (bar (baz (gaz (foo bar (gaz))))))', 'nocontent')]
899
900    for i, r in enumerate(res):
901        # print i, res[0], r
902        env.assertListEqual(res[0], r)
903
904def testInKeys(env):
905    r = env
906    env.assertOk(r.execute_command(
907        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text'))
908
909    for i in range(200):
910        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1.0, 'fields',
911                                        'foo', 'hello world'))
912
913    for _ in r.retry_with_rdb_reload():
914        waitForIndex(env, 'idx')
915        for keys in (
916            ['doc%d' % i for i in range(10)], ['doc%d' % i for i in range(0, 30, 2)], [
917                'doc%d' % i for i in range(99, 0, -5)]
918        ):
919            res = r.execute_command(
920                'ft.search', 'idx', 'hello world', 'NOCONTENT', 'LIMIT', 0, 100, 'INKEYS', len(keys), *keys)
921            env.assertEqual(len(keys), res[0])
922            env.assertTrue(all((k in res for k in keys)))
923
924        env.assertEqual(0, r.execute_command(
925            'ft.search', 'idx', 'hello world', 'NOCONTENT', 'LIMIT', 0, 100, 'INKEYS', 3, 'foo', 'bar', 'baz')[0])
926
927    with env.assertResponseError():
928        env.cmd('ft.search', 'idx', 'hello', 'INKEYS', 99)
929    with env.assertResponseError():
930        env.cmd('ft.search', 'idx', 'hello', 'INKEYS', -1)
931    with env.assertResponseError():
932        env.cmd('ft.search', 'idx', 'hello', 'inkeys', 4, 'foo')
933
934def testSlopInOrder(env):
935    r = env
936    env.assertOk(r.execute_command(
937        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text'))
938    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1, 'fields',
939                                    'title', 't1 t2'))
940    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1, 'fields',
941                                    'title', 't1 t3 t2'))
942    env.assertOk(r.execute_command('ft.add', 'idx', 'doc3', 1, 'fields',
943                                    'title', 't1 t3 t4 t2'))
944    env.assertOk(r.execute_command('ft.add', 'idx', 'doc4', 1, 'fields',
945                                    'title', 't1 t3 t4 t5 t2'))
946
947    res = r.execute_command(
948        'ft.search', 'idx', 't1|t4 t3|t2', 'slop', '0', 'inorder', 'nocontent')
949    env.assertEqual({'doc3', 'doc4', 'doc2', 'doc1'}, set(res[1:]))
950    res = r.execute_command(
951        'ft.search', 'idx', 't2 t1', 'slop', '0', 'nocontent')
952    env.assertEqual(1, res[0])
953    env.assertEqual('doc1', res[1])
954    env.assertEqual(0, r.execute_command(
955        'ft.search', 'idx', 't2 t1', 'slop', '0', 'inorder')[0])
956    env.assertEqual(1, r.execute_command(
957        'ft.search', 'idx', 't1 t2', 'slop', '0', 'inorder')[0])
958
959    env.assertEqual(2, r.execute_command(
960        'ft.search', 'idx', 't1 t2', 'slop', '1', 'inorder')[0])
961    env.assertEqual(3, r.execute_command(
962        'ft.search', 'idx', 't1 t2', 'slop', '2', 'inorder')[0])
963    env.assertEqual(4, r.execute_command(
964        'ft.search', 'idx', 't1 t2', 'slop', '3', 'inorder')[0])
965    env.assertEqual(4, r.execute_command(
966        'ft.search', 'idx', 't1 t2', 'inorder')[0])
967    env.assertEqual(0, r.execute_command(
968        'ft.search', 'idx', 't t1', 'inorder')[0])
969    env.assertEqual(2, r.execute_command(
970        'ft.search', 'idx', 't1 t2 t3 t4')[0])
971    env.assertEqual(0, r.execute_command(
972        'ft.search', 'idx', 't1 t2 t3 t4', 'inorder')[0])
973
974
975def testSlopInOrderIssue1986(env):
976    r = env
977    # test with qsort optimization on intersect iterator
978    env.assertOk(r.execute_command(
979        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text'))
980
981    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1, 'fields',
982                                    'title', 't1 t2'))
983    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1, 'fields',
984                                    'title', 't2 t1'))
985    env.assertOk(r.execute_command('ft.add', 'idx', 'doc3', 1, 'fields',
986                                    'title', 't1'))
987
988    # before fix, both queries returned `doc2`
989    env.assertEqual([1L, 'doc2', ['title', 't2 t1']], r.execute_command(
990        'ft.search', 'idx', 't2 t1', 'slop', '0', 'inorder'))
991    env.assertEqual([1L, 'doc1', ['title', 't1 t2']], r.execute_command(
992        'ft.search', 'idx', 't1 t2', 'slop', '0', 'inorder'))
993
994def testExact(env):
995    r = env
996    env.assertOk(r.execute_command(
997        'ft.create', 'idx', 'ON', 'HASH',
998        'schema', 'title', 'text', 'weight', 10.0, 'body', 'text'))
999    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 0.5, 'fields',
1000                                    'title', 'hello world',
1001                                    'body', 'lorem ist ipsum'))
1002    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1.0, 'fields',
1003                                    'title', 'hello another world',
1004                                    'body', 'lorem ist ipsum lorem lorem'))
1005
1006    res = r.execute_command(
1007        'ft.search', 'idx', '"hello world"', 'verbatim')
1008    env.assertEqual(3, len(res))
1009    env.assertEqual(1, res[0])
1010    env.assertEqual("doc1", res[1])
1011
1012    res = r.execute_command(
1013        'ft.search', 'idx', "hello \"another world\"", 'verbatim')
1014    env.assertEqual(3, len(res))
1015    env.assertEqual(1, res[0])
1016    env.assertEqual("doc2", res[1])
1017
1018
1019def testGeoErrors(env):
1020    env.expect('flushall')
1021    env.expect('ft.create idx ON HASH schema name text location geo').equal('OK')
1022    env.expect('ft.add idx hotel 1.0 fields name hill location -0.1757,51.5156').equal('OK')
1023    env.expect('ft.search idx hilton geofilter location -0.1757 51.5156 1 km').equal([0L])
1024
1025    # Insert error - works fine with out of keyspace implementation
1026    # env.expect('ft.add', 'idx', 'hotel1', 1, 'fields', 'name', '_hotel1', 'location', '1, 1').error()   \
1027    #        .contains('Could not index geo value')
1028
1029    # Query errors
1030    env.expect('ft.search idx hilton geofilter location lon 51.5156 1 km').error()   \
1031            .contains('Bad arguments for <lon>: Could not convert argument to expected type')
1032    env.expect('ft.search idx hilton geofilter location 51.5156 lat 1 km').error()   \
1033            .contains('Bad arguments for <lat>: Could not convert argument to expected type')
1034    env.expect('ft.search idx hilton geofilter location -0.1757 51.5156 radius km').error()   \
1035            .contains('Bad arguments for <radius>: Could not convert argument to expected type')
1036    env.expect('ft.search idx hilton geofilter location -0.1757 51.5156 1 fake').error()   \
1037            .contains('Unknown distance unit fake')
1038    env.expect('ft.search idx hilton geofilter location -0.1757 51.5156 1').error()   \
1039            .contains('GEOFILTER requires 5 arguments')
1040
1041def testGeo(env):
1042    r = env
1043    gsearch = lambda query, lon, lat, dist, unit='km': r.execute_command(
1044        'ft.search', 'idx', query, 'geofilter', 'location', lon, lat, dist, unit, 'LIMIT', 0, 20)
1045
1046    gsearch_inline = lambda query, lon, lat, dist, unit='km': r.execute_command(
1047        'ft.search', 'idx', '{} @location:[{} {} {} {}]'.format(query,  lon, lat, dist, unit), 'LIMIT', 0, 20)
1048
1049    env.assertOk(r.execute_command('ft.create', 'idx', 'ON', 'HASH',
1050                                    'schema', 'name', 'text', 'location', 'geo'))
1051
1052    for i, hotel in enumerate(hotels):
1053        env.assertOk(r.execute_command('ft.add', 'idx', 'hotel{}'.format(i), 1.0, 'fields', 'name',
1054                                        hotel[0], 'location', '{},{}'.format(hotel[2], hotel[1])))
1055
1056    for _ in r.retry_with_rdb_reload():
1057        waitForIndex(env, 'idx')
1058        res = r.execute_command('ft.search', 'idx', 'hilton')
1059        env.assertEqual(len(hotels), res[0])
1060
1061        res = gsearch('hilton', "-0.1757", "51.5156", '1')
1062        env.assertEqual(3, res[0])
1063        env.assertIn('hotel2', res)
1064        env.assertIn('hotel21', res)
1065        env.assertIn('hotel79', res)
1066        res2 = gsearch_inline('hilton', "-0.1757", "51.5156", '1')
1067        env.assertListEqual(sorted(res), sorted(res2))
1068
1069        res = gsearch('hilton', "-0.1757", "51.5156", '10')
1070        env.assertEqual(14, res[0])
1071
1072        res2 = gsearch('hilton', "-0.1757", "51.5156", '10000', 'm')
1073        env.assertListEqual(sorted(res), sorted(res2))
1074        res2 = gsearch_inline('hilton', "-0.1757", "51.5156", '10')
1075        env.assertListEqual(sorted(res), sorted(res2))
1076
1077        res = gsearch('heathrow', -0.44155, 51.45865, '10', 'm')
1078        env.assertEqual(1, res[0])
1079        env.assertEqual('hotel94', res[1])
1080        res2 = gsearch_inline(
1081            'heathrow', -0.44155, 51.45865, '10', 'm')
1082        env.assertListEqual(sorted(res), sorted(res2))
1083
1084        res = gsearch('heathrow', -0.44155, 51.45865, '10', 'km')
1085        env.assertEqual(5, res[0])
1086        env.assertIn('hotel94', res)
1087        res2 = gsearch_inline(
1088            'heathrow', -0.44155, 51.45865, '10', 'km')
1089        env.assertEqual(5, res2[0])
1090        env.assertListEqual(sorted(res), sorted(res2))
1091
1092        res = gsearch('heathrow', -0.44155, 51.45865, '5', 'km')
1093        env.assertEqual(3, res[0])
1094        env.assertIn('hotel94', res)
1095        res2 = gsearch_inline(
1096            'heathrow', -0.44155, 51.45865, '5', 'km')
1097        env.assertListEqual(sorted(res), sorted(res2))
1098
1099def testTagErrors(env):
1100    env.expect("ft.create", "test", 'ON', 'HASH',
1101                "SCHEMA",  "tags", "TAG").equal('OK')
1102    env.expect("ft.add", "test", "1", "1", "FIELDS", "tags", "alberta").equal('OK')
1103    env.expect("ft.add", "test", "2", "1", "FIELDS", "tags", "ontario. alberta").equal('OK')
1104
1105def testGeoDeletion(env):
1106    if env.is_cluster():
1107        raise unittest.SkipTest()
1108        # Can't properly test if deleted on cluster
1109
1110    env.expect('ft.config', 'set', 'FORK_GC_CLEAN_THRESHOLD', 0).ok()
1111    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema',
1112            'g1', 'geo', 'g2', 'geo', 't1', 'text')
1113    env.cmd('ft.add', 'idx', 'doc1', 1.0, 'fields',
1114            'g1', "-0.1757,51.5156",
1115            'g2', "-0.1757,51.5156",
1116            't1', "hello")
1117    env.cmd('ft.add', 'idx', 'doc2', 1.0, 'fields',
1118            'g1', "-0.1757,51.5156",
1119            'g2', "-0.1757,51.5156",
1120            't1', "hello")
1121    env.cmd('ft.add', 'idx', 'doc3', 1.0, 'fields',
1122            'g1', "-0.1757,51.5156",
1123            't1', "hello")
1124
1125    # keys are: "geo:idx/g1" and "geo:idx/g2"
1126    env.assertEqual(3, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g1')[0]))
1127    env.assertEqual(2, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g2')[0]))
1128
1129    # Remove the first doc
1130    env.cmd('ft.del', 'idx', 'doc1')
1131    for _ in range(100):
1132        forceInvokeGC(env, 'idx')
1133    env.assertEqual(2, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g1')[0]))
1134    env.assertEqual(1, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g2')[0]))
1135
1136    # Replace the other one:
1137    env.cmd('ft.add', 'idx', 'doc2', 1.0,
1138            'replace', 'fields',
1139            't1', 'just text here')
1140    for _ in range(100):
1141        forceInvokeGC(env, 'idx')
1142    env.assertEqual(1, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g1')[0]))
1143    env.assertEqual(0, len(env.cmd('FT.DEBUG DUMP_NUMIDX idx g2')[0]))
1144
1145def testInfields(env):
1146    r = env
1147    env.assertOk(r.execute_command(
1148        'ft.create', 'idx', 'ON', 'HASH',
1149        'schema', 'title', 'text', 'weight', 10.0, 'body', 'text', 'weight', 1.0))
1150    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 0.5, 'fields',
1151                                    'title', 'hello world',
1152                                    'body', 'lorem ipsum'))
1153
1154    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1.0, 'fields',
1155                                    'title', 'hello world lorem ipsum',
1156                                    'body', 'hello world'))
1157
1158    res = r.execute_command(
1159        'ft.search', 'idx', 'hello world', 'verbatim', "infields", 1, "title", "nocontent")
1160    env.assertEqual(3, len(res))
1161    env.assertEqual(2, res[0])
1162    env.assertEqual("doc2", res[1])
1163    env.assertEqual("doc1", res[2])
1164
1165    res = r.execute_command(
1166        'ft.search', 'idx', 'hello world', 'verbatim', "infields", 1, "body", "nocontent")
1167    env.assertEqual(2, len(res))
1168    env.assertEqual(1, res[0])
1169    env.assertEqual("doc2", res[1])
1170
1171    res = r.execute_command(
1172        'ft.search', 'idx', 'hello', 'verbatim', "infields", 1, "body", "nocontent")
1173    env.assertEqual(2, len(res))
1174    env.assertEqual(1, res[0])
1175    env.assertEqual("doc2", res[1])
1176
1177    res = r.execute_command(
1178        'ft.search', 'idx',  '\"hello world\"', 'verbatim', "infields", 1, "body", "nocontent")
1179
1180    env.assertEqual(2, len(res))
1181    env.assertEqual(1, res[0])
1182    env.assertEqual("doc2", res[1])
1183
1184    res = r.execute_command(
1185        'ft.search', 'idx', '\"lorem ipsum\"', 'verbatim', "infields", 1, "body", "nocontent")
1186    env.assertEqual(2, len(res))
1187    env.assertEqual(1, res[0])
1188    env.assertEqual("doc1", res[1])
1189
1190    res = r.execute_command(
1191        'ft.search', 'idx', 'lorem ipsum', "infields", 2, "body", "title", "nocontent")
1192    env.assertEqual(3, len(res))
1193    env.assertEqual(2, res[0])
1194    env.assertEqual("doc2", res[1])
1195    env.assertEqual("doc1", res[2])
1196
1197def testScorerSelection(env):
1198    r = env
1199    env.assertOk(r.execute_command(
1200        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text', 'body', 'text'))
1201
1202    # this is the default scorer
1203    res = r.execute_command(
1204        'ft.search', 'idx', 'foo', 'scorer', 'TFIDF')
1205    env.assertEqual(res, [0])
1206    with env.assertResponseError():
1207        res = r.execute_command(
1208            'ft.search', 'idx', 'foo', 'scorer', 'NOSUCHSCORER')
1209
1210def testFieldSelectors(env):
1211    r = env
1212    env.assertOk(r.execute_command(
1213        'ft.create', 'idx', 'ON', 'HASH', 'PREFIX', 1, 'doc',
1214        'schema', 'TiTle', 'text', 'BoDy', 'text', "יוניקוד", 'text', 'field.with,punct', 'text'))
1215    #todo: document as breaking change, ft.add fields name are not case insentive
1216    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 1, 'fields',
1217                                    'TiTle', 'hello world', 'BoDy', 'foo bar', 'יוניקוד', 'unicode', 'field.with,punct', 'punt'))
1218    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 0.5, 'fields',
1219                                    'BoDy', 'hello world', 'TiTle', 'foo bar', 'יוניקוד', 'unicode', 'field.with,punct', 'punt'))
1220
1221    res = r.execute_command(
1222        'ft.search', 'idx', '@TiTle:hello world', 'nocontent')
1223    env.assertEqual(res, [1, 'doc1'])
1224    res = r.execute_command(
1225        'ft.search', 'idx', '@BoDy:hello world', 'nocontent')
1226    env.assertEqual(res, [1, 'doc2'])
1227
1228    res = r.execute_command(
1229        'ft.search', 'idx', '@BoDy:hello @TiTle:world', 'nocontent')
1230    env.assertEqual(res, [0])
1231
1232    res = r.execute_command(
1233        'ft.search', 'idx', '@BoDy:hello world @TiTle:world', 'nocontent')
1234    env.assertEqual(res, [0])
1235    res = r.execute_command(
1236        'ft.search', 'idx', '@BoDy:(hello|foo) @TiTle:(world|bar)', 'nocontent')
1237    env.assertEqual(sorted(res), sorted([2, 'doc1', 'doc2']))
1238
1239    res = r.execute_command(
1240        'ft.search', 'idx', '@BoDy:(hello|foo world|bar)', 'nocontent')
1241    env.assertEqual(sorted(res), sorted([2, 'doc1', 'doc2']))
1242
1243    res = r.execute_command(
1244        'ft.search', 'idx', '@BoDy|TiTle:(hello world)', 'nocontent')
1245    env.assertEqual(sorted(res), sorted([2, 'doc1', 'doc2']))
1246
1247    res = r.execute_command(
1248        'ft.search', 'idx', '@יוניקוד:(unicode)', 'nocontent')
1249    env.assertEqual(sorted(res), sorted([2, 'doc1', 'doc2']))
1250
1251    res = r.execute_command(
1252        'ft.search', 'idx', '@field\\.with\\,punct:(punt)', 'nocontent')
1253    env.assertEqual(sorted(res), sorted([2, 'doc1', 'doc2']))
1254
1255def testStemming(env):
1256    r = env
1257    env.assertOk(r.execute_command(
1258        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text'))
1259    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 0.5, 'fields',
1260                                    'title', 'hello kitty'))
1261    env.assertOk(r.execute_command('ft.add', 'idx', 'doc2', 1.0, 'fields',
1262                                    'title', 'hello kitties'))
1263
1264    res = r.execute_command(
1265        'ft.search', 'idx', 'hello kitty', "nocontent")
1266    env.assertEqual(3, len(res))
1267    env.assertEqual(2, res[0])
1268
1269    res = r.execute_command(
1270        'ft.search', 'idx', 'hello kitty', "nocontent", "verbatim")
1271    env.assertEqual(2, len(res))
1272    env.assertEqual(1, res[0])
1273
1274    # test for unknown language
1275    with env.assertResponseError():
1276        res = r.execute_command(
1277            'ft.search', 'idx', 'hello kitty', "nocontent", "language", "foofoofian")
1278
1279def testExpander(env):
1280    r = env
1281    env.assertOk(r.execute_command(
1282        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text'))
1283    env.assertOk(r.execute_command('ft.add', 'idx', 'doc1', 0.5, 'fields',
1284                                    'title', 'hello kitty'))
1285    res = r.execute_command(
1286        'ft.search', 'idx', 'kitties',
1287        "nocontent",
1288        "expander", "SBSTEM"
1289        )
1290    env.assertEqual(2, len(res))
1291    env.assertEqual(1, res[0])
1292
1293    res = r.execute_command(
1294        'ft.search', 'idx', 'kitties', "nocontent", "expander", "noexpander")
1295    env.assertEqual(1, len(res))
1296    env.assertEqual(0, res[0])
1297
1298    res = r.execute_command(
1299        'ft.search', 'idx', 'kitti', "nocontent")
1300    env.assertEqual(2, len(res))
1301    env.assertEqual(1, res[0])
1302
1303    res = r.execute_command(
1304        'ft.search', 'idx', 'kitti', "nocontent", 'verbatim')
1305    env.assertEqual(1, len(res))
1306    env.assertEqual(0, res[0])
1307
1308    # Calling a stem directly works even with VERBATIM.
1309    # You need to use the + prefix escaped
1310    res = r.execute_command(
1311        'ft.search', 'idx', '\\+kitti', "nocontent", 'verbatim')
1312    env.assertEqual(2, len(res))
1313    env.assertEqual(1, res[0])
1314
1315def testNumericRange(env):
1316    r = env
1317    env.assertOk(r.execute_command(
1318        'ft.create', 'idx', 'ON', 'HASH', 'schema', 'title', 'text', 'score', 'numeric', 'price', 'numeric'))
1319
1320    env.expect('ft.search', 'idx', 'hello kitty', 'filter', 'score', 5).error().contains("FILTER requires 3 arguments")
1321    env.expect('ft.search', 'idx', 'hello kitty', 'filter', 'score', 5, 'inf').error().contains("Bad upper range: inf")
1322    env.expect('ft.search', 'idx', 'hello kitty', 'filter', 'score', 'inf', 5).error().contains("Bad lower range: inf")
1323
1324    for i in xrange(100):
1325        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1, 'fields',
1326                                        'title', 'hello kitty', 'score', i, 'price', 100 + 10 * i))
1327
1328    for _ in r.retry_with_rdb_reload():
1329        waitForIndex(env, 'idx')
1330        res = r.execute_command('ft.search', 'idx', 'hello kitty', "nocontent",
1331                                "filter", "score", 0, 100)
1332
1333        env.assertEqual(11, len(res))
1334        env.assertEqual(100, res[0])
1335
1336        res = r.execute_command('ft.search', 'idx', 'hello kitty', "nocontent",
1337                                "filter", "score", 0, 50)
1338        env.assertEqual(51, res[0])
1339
1340        res = r.execute_command('ft.search', 'idx', 'hello kitty', 'verbatim', "nocontent", "limit", 0, 100,
1341                                "filter", "score", "(0", "(50")
1342
1343        env.assertEqual(49, res[0])
1344        res = r.execute_command('ft.search', 'idx', 'hello kitty', "nocontent",
1345                                "filter", "score", "-inf", "+inf")
1346        env.assertEqual(100, res[0])
1347
1348        # test multi filters
1349        scrange = (19, 90)
1350        prrange = (290, 385)
1351        res = r.execute_command('ft.search', 'idx', 'hello kitty',
1352                                "filter", "score", scrange[
1353                                    0], scrange[1],
1354                                "filter", "price", prrange[0], prrange[1])
1355
1356        # print res
1357        for doc in res[2::2]:
1358
1359            sc = int(doc[doc.index('score') + 1])
1360            pr = int(doc[doc.index('price') + 1])
1361
1362            env.assertTrue(sc >= scrange[0] and sc <= scrange[1])
1363            env.assertGreaterEqual(pr, prrange[0])
1364            env.assertLessEqual(pr, prrange[1])
1365
1366        env.assertEqual(10, res[0])
1367
1368        res = r.execute_command('ft.search', 'idx', 'hello kitty',
1369                                "filter", "score", "19", "90",
1370                                "filter", "price", "90", "185")
1371
1372        env.assertEqual(0, res[0])
1373
1374        # Test numeric ranges as part of query syntax
1375        res = r.execute_command(
1376            'ft.search', 'idx', 'hello kitty @score:[0 100]', "nocontent")
1377
1378        env.assertEqual(11, len(res))
1379        env.assertEqual(100, res[0])
1380
1381        res = r.execute_command(
1382            'ft.search', 'idx', 'hello kitty  @score:[0 50]', "nocontent")
1383        env.assertEqual(51, res[0])
1384        res = r.execute_command(
1385            'ft.search', 'idx', 'hello kitty @score:[(0 (50]', 'verbatim', "nocontent")
1386        env.assertEqual(49, res[0])
1387        res = r.execute_command(
1388            'ft.search', 'idx', '@score:[(0 (50]', 'verbatim', "nocontent")
1389        env.assertEqual(49, res[0])
1390        res = r.execute_command(
1391            'ft.search', 'idx', 'hello kitty -@score:[(0 (50]', 'verbatim', "nocontent")
1392        env.assertEqual(51, res[0])
1393        res = r.execute_command(
1394            'ft.search', 'idx', 'hello kitty @score:[-inf +inf]', "nocontent")
1395        env.assertEqual(100, res[0])
1396
1397def testPayload(env):
1398    r = env
1399    env.expect('ft.create', 'idx', 'ON', 'HASH', 'PAYLOAD_FIELD', '__payload', 'schema', 'f', 'text').ok()
1400    for i in range(10):
1401        r.expect('ft.add', 'idx', '%d' % i, 1.0,
1402                 'payload', 'payload %d' % i,
1403                 'fields', 'f', 'hello world').ok()
1404
1405    for x in r.retry_with_rdb_reload():
1406        waitForIndex(env, 'idx')
1407        res = r.execute_command('ft.search', 'idx', 'hello world')
1408        r.assertEqual(21, len(res))
1409
1410        res = r.execute_command('ft.search', 'idx', 'hello world', 'withpayloads')
1411        r.assertEqual(31, len(res))
1412        r.assertEqual(10, res[0])
1413        for i in range(1, 30, 3):
1414            r.assertEqual(res[i + 1], 'payload %s' % res[i])
1415
1416def testGarbageCollector(env):
1417    env.skipOnCluster()
1418    if env.moduleArgs is not None and 'GC_POLICY FORK' in env.moduleArgs:
1419        # this test is not relevent for fork gc cause its not cleaning the last block
1420        raise unittest.SkipTest()
1421    N = 100
1422    r = env
1423    r.expect('ft.create', 'idx', 'ON', 'HASH', 'schema', 'foo', 'text').ok()
1424    waitForIndex(r, 'idx')
1425    for i in range(N):
1426        r.expect('ft.add', 'idx', 'doc%d' % i, 1.0,
1427                 'fields', 'foo', ' '.join(('term%d' % random.randrange(0, 10) for i in range(10)))).ok()
1428
1429    def get_stats(r):
1430        res = r.execute_command('ft.info', 'idx')
1431        d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
1432        gc_stats = {d['gc_stats'][x]: float(
1433            d['gc_stats'][x + 1]) for x in range(0, len(d['gc_stats']), 2)}
1434        d['gc_stats'] = gc_stats
1435        return d
1436
1437    stats = get_stats(r)
1438    if 'current_hz' in stats['gc_stats']:
1439        env.assertGreater(stats['gc_stats']['current_hz'], 8)
1440    env.assertEqual(0, stats['gc_stats']['bytes_collected'])
1441    env.assertGreater(int(stats['num_records']), 0)
1442
1443    initialIndexSize = float(stats['inverted_sz_mb']) * 1024 * 1024
1444    for i in range(N):
1445        r.expect('ft.del', 'idx', 'doc%d' % i).equal(1)
1446
1447    for _ in range(100):
1448        # gc is random so we need to do it long enough times for it to work
1449        forceInvokeGC(env, 'idx')
1450
1451    stats = get_stats(r)
1452
1453    env.assertEqual(0, int(stats['num_docs']))
1454    env.assertEqual(0, int(stats['num_records']))
1455    if not env.is_cluster():
1456        env.assertEqual(100, int(stats['max_doc_id']))
1457        if 'current_hz' in stats['gc_stats']:
1458            env.assertGreater(stats['gc_stats']['current_hz'], 30)
1459        currentIndexSize = float(stats['inverted_sz_mb']) * 1024 * 1024
1460        # print initialIndexSize, currentIndexSize,
1461        # stats['gc_stats']['bytes_collected']
1462        env.assertGreater(initialIndexSize, currentIndexSize)
1463        env.assertGreater(stats['gc_stats'][
1464            'bytes_collected'], currentIndexSize)
1465
1466    for i in range(10):
1467
1468        res = r.execute_command('ft.search', 'idx', 'term%d' % i)
1469        env.assertEqual([0], res)
1470
1471def testReturning(env):
1472    env.assertCmdOk('ft.create', 'idx', 'ON', 'HASH', 'schema',
1473                     'f1', 'text',
1474                     'f2', 'text',
1475                     'n1', 'numeric', 'sortable',
1476                     'f3', 'text')
1477    for i in range(10):
1478        env.assertCmdOk('ft.add', 'idx', 'DOC_{0}'.format(i), 1.0, 'fields',
1479                         'f2', 'val2', 'f1', 'val1', 'f3', 'val3',
1480                         'n1', i)
1481
1482    # RETURN 0. Simplest case
1483    for x in env.retry_with_reload():
1484        waitForIndex(env, 'idx')
1485        res = env.cmd('ft.search', 'idx', 'val*', 'return', '0')
1486        env.assertEqual(11, len(res))
1487        env.assertEqual(10, res[0])
1488        for r in res[1:]:
1489            env.assertTrue(r.startswith('DOC_'))
1490
1491    for field in ('f1', 'f2', 'f3', 'n1'):
1492        res = env.cmd('ft.search', 'idx', 'val*', 'return', 1, field)
1493        env.assertEqual(21, len(res))
1494        env.assertEqual(10, res[0])
1495        for pair in grouper(res[1:], 2):
1496            docname, fields = pair
1497            env.assertEqual(2, len(fields))
1498            env.assertEqual(field, fields[0])
1499            env.assertTrue(docname.startswith('DOC_'))
1500
1501    # Test that we don't return SORTBY fields if they weren't specified
1502    # also in RETURN
1503    res = env.cmd('ft.search', 'idx', 'val*', 'return', 1, 'f1',
1504        'sortby', 'n1', 'ASC')
1505    row = res[2]
1506    # get the first result
1507    env.assertEqual(['f1', 'val1'], row)
1508
1509    # Test when field is not found
1510    res = env.cmd('ft.search', 'idx', 'val*', 'return', 1, 'nonexist')
1511    env.assertEqual(21, len(res))
1512    env.assertEqual(10, res[0])
1513
1514    # # Test that we don't crash if we're given the wrong number of fields
1515    with env.assertResponseError():
1516        res = env.cmd('ft.search', 'idx', 'val*', 'return', 700, 'nonexist')
1517
1518def _test_create_options_real(env, *options):
1519    options = [x for x in options if x]
1520    has_offsets = 'NOOFFSETS' not in options
1521    has_fields = 'NOFIELDS' not in options
1522    has_freqs = 'NOFREQS' not in options
1523
1524    try:
1525        env.cmd('ft.drop', 'idx')
1526        # RS 2.0 ft.drop does not remove documents
1527        env.flush()
1528    except Exception as e:
1529        pass
1530
1531    options = ['idx'] + options + ['ON', 'HASH', 'schema', 'f1', 'text', 'f2', 'text']
1532    env.assertCmdOk('ft.create', *options)
1533    for i in range(10):
1534        env.assertCmdOk('ft.add', 'idx', 'doc{}'.format(
1535            i), 0.5, 'fields', 'f1', 'value for {}'.format(i))
1536
1537    # Query
1538#     res = env.cmd('ft.search', 'idx', "value for 3")
1539#     if not has_offsets:
1540#         env.assertIsNone(res)
1541#     else:
1542#         env.assertIsNotNone(res)
1543
1544    # Frequencies:
1545    env.assertCmdOk('ft.add', 'idx', 'doc100',
1546                     1.0, 'fields', 'f1', 'foo bar')
1547    env.assertCmdOk('ft.add', 'idx', 'doc200', 1.0,
1548                     'fields', 'f1', ('foo ' * 10) + ' bar')
1549    res = env.cmd('ft.search', 'idx', 'foo')
1550    env.assertEqual(2, res[0])
1551    if has_offsets:
1552        docname = res[1]
1553        if has_freqs:
1554            env.assertEqual('doc200', docname)
1555        else:
1556            env.assertEqual('doc100', docname)
1557
1558    env.assertCmdOk('ft.add', 'idx', 'doc300',
1559                     1.0, 'fields', 'f1', 'Hello')
1560    res = env.cmd('ft.search', 'idx', '@f2:Hello')
1561    if has_fields:
1562        env.assertEqual(1, len(res))
1563    else:
1564        env.assertEqual(3, len(res))
1565
1566def testCreationOptions(env):
1567    from itertools import combinations
1568    for x in range(1, 5):
1569        for combo in combinations(('NOOFFSETS', 'NOFREQS', 'NOFIELDS', ''), x):
1570            _test_create_options_real(env, *combo)
1571
1572    env.expect('ft.create', 'idx').error()
1573
1574def testInfoCommand(env):
1575    from itertools import combinations
1576    r = env
1577    env.assertOk(r.execute_command(
1578        'ft.create', 'idx', 'ON', 'HASH', 'NOFIELDS', 'schema', 'title', 'text'))
1579    N = 50
1580    for i in xrange(N):
1581        env.assertOk(r.execute_command('ft.add', 'idx', 'doc%d' % i, 1, 'replace', 'fields',
1582                                        'title', 'hello term%d' % i))
1583    for _ in r.retry_with_rdb_reload():
1584        waitForIndex(env, 'idx')
1585
1586        res = r.execute_command('ft.info', 'idx')
1587        d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
1588
1589        env.assertEqual(d['index_name'], 'idx')
1590        env.assertEqual(d['index_options'], ['NOFIELDS'])
1591        env.assertListEqual(
1592            d['fields'], [['title', 'type', 'TEXT', 'WEIGHT', '1']])
1593
1594        if not env.is_cluster():
1595            env.assertEquals(int(d['num_docs']), N)
1596            env.assertEquals(int(d['num_terms']), N + 1)
1597            env.assertEquals(int(d['max_doc_id']), N)
1598            env.assertEquals(int(d['records_per_doc_avg']), 2)
1599            env.assertEquals(int(d['num_records']), N * 2)
1600
1601            env.assertGreater(float(d['offset_vectors_sz_mb']), 0)
1602            env.assertGreater(float(d['key_table_size_mb']), 0)
1603            env.assertGreater(float(d['inverted_sz_mb']), 0)
1604            env.assertGreater(float(d['bytes_per_record_avg']), 0)
1605            env.assertGreater(float(d['doc_table_size_mb']), 0)
1606
1607    for x in range(1, 5):
1608        for combo in combinations(('NOOFFSETS', 'NOFREQS', 'NOFIELDS', ''), x):
1609            combo = list(filter(None, combo))
1610            options = combo + ['schema', 'f1', 'text']
1611            try:
1612                env.cmd('ft.drop', 'idx')
1613            except:
1614                pass
1615            env.assertCmdOk('ft.create', 'idx', 'ON', 'HASH', *options)
1616            info = env.cmd('ft.info', 'idx')
1617            ix = info.index('index_options')
1618            env.assertFalse(ix == -1)
1619
1620            opts = info[ix + 1]
1621            # make sure that an empty opts string returns no options in
1622            # info
1623            if not combo:
1624                env.assertListEqual([], opts)
1625
1626            for option in filter(None, combo):
1627                env.assertTrue(option in opts)
1628
1629def testNoStem(env):
1630    env.cmd('ft.create', 'idx', 'ON', 'HASH',
1631            'schema', 'body', 'text', 'name', 'text', 'nostem')
1632    if not env.isCluster():
1633        # todo: change it to be more generic to pass on is_cluster
1634        res = env.cmd('ft.info', 'idx')
1635        env.assertEqual(res[7][1][5], 'NOSTEM')
1636    for _ in env.retry_with_reload():
1637        waitForIndex(env, 'idx')
1638        try:
1639            env.cmd('ft.del', 'idx', 'doc')
1640        except redis.ResponseError:
1641            pass
1642
1643        # Insert a document
1644        env.assertCmdOk('ft.add', 'idx', 'doc', 1.0, 'fields',
1645                         'body', "located",
1646                         'name', "located")
1647
1648        # Now search for the fields
1649        res_body = env.cmd('ft.search', 'idx', '@body:location')
1650        res_name = env.cmd('ft.search', 'idx', '@name:location')
1651        env.assertEqual(0, res_name[0])
1652        env.assertEqual(1, res_body[0])
1653
1654def testSortbyMissingField(env):
1655    # GH Issue 131
1656    env.cmd('ft.create', 'ix', 'ON', 'HASH', 'schema', 'txt',
1657             'text', 'num', 'numeric', 'sortable')
1658    env.cmd('ft.add', 'ix', 'doc1', 1.0, 'fields', 'txt', 'foo')
1659    env.cmd('ft.search', 'ix', 'foo', 'sortby', 'num')
1660
1661def testParallelIndexing(env):
1662    # GH Issue 207
1663    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'txt', 'text')
1664    from threading import Thread
1665    env.getConnection()
1666    ndocs = 100
1667
1668    def runner(tid):
1669        cli = env.getConnection()
1670        for num in range(ndocs):
1671            cli.execute_command('ft.add', 'idx', 'doc{}_{}'.format(tid, num), 1.0,
1672                                'fields', 'txt', 'hello world' * 20)
1673    ths = []
1674    for tid in range(10):
1675        ths.append(Thread(target=runner, args=(tid,)))
1676
1677    [th.start() for th in ths]
1678    [th.join() for th in ths]
1679    res = env.cmd('ft.info', 'idx')
1680    d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
1681    env.assertEqual(1000, int(d['num_docs']))
1682
1683def testDoubleAdd(env):
1684    # Tests issue #210
1685    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'txt', 'text')
1686    env.cmd('ft.add', 'idx', 'doc1', 1.0, 'fields', 'txt', 'hello world')
1687    with env.assertResponseError():
1688        env.cmd('ft.add', 'idx', 'doc1', 1.0,
1689                 'fields', 'txt', 'goodbye world')
1690
1691    env.assertEqual('hello world', env.cmd('ft.get', 'idx', 'doc1')[1])
1692    env.assertEqual(0, env.cmd('ft.search', 'idx', 'goodbye')[0])
1693    env.assertEqual(1, env.cmd('ft.search', 'idx', 'hello')[0])
1694
1695    # Now with replace
1696    env.cmd('ft.add', 'idx', 'doc1', 1.0, 'replace',
1697             'fields', 'txt', 'goodbye world')
1698    env.assertEqual(1, env.cmd('ft.search', 'idx', 'goodbye')[0])
1699    env.assertEqual(0, env.cmd('ft.search', 'idx', 'hello')[0])
1700    env.assertEqual('goodbye world', env.cmd('ft.get', 'idx', 'doc1')[1])
1701
1702def testConcurrentErrors(env):
1703    from multiprocessing import Process
1704    import random
1705
1706    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'txt', 'text')
1707    docs_per_thread = 100
1708    num_threads = 50
1709
1710    docIds = ['doc{}'.format(x) for x in range(docs_per_thread)]
1711
1712    def thrfn():
1713        myIds = docIds[::]
1714        random.shuffle(myIds)
1715        cli = env.getConnection()
1716        with cli.pipeline(transaction=False) as pl:
1717            for x in myIds:
1718                pl.execute_command('ft.add', 'idx', x, 1.0,
1719                                   'fields', 'txt', ' hello world ' * 50)
1720            try:
1721                pl.execute()
1722            except Exception as e:
1723                pass
1724                # print e
1725
1726    thrs = [Process(target=thrfn) for x in range(num_threads)]
1727    [th.start() for th in thrs]
1728    [th.join() for th in thrs]
1729    res = env.cmd('ft.info', 'idx')
1730    d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
1731    env.assertEqual(100, int(d['num_docs']))
1732
1733def testBinaryKeys(env):
1734    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'txt', 'text')
1735    # Insert a document
1736    env.cmd('ft.add', 'idx', 'Hello', 1.0, 'fields', 'txt', 'NoBin match')
1737    env.cmd('ft.add', 'idx', 'Hello\x00World', 1.0, 'fields', 'txt', 'Bin match')
1738    for _ in env.reloading_iterator():
1739        waitForIndex(env, 'idx')
1740        exp = [2L, 'Hello\x00World', ['txt', 'Bin match'], 'Hello', ['txt', 'NoBin match']]
1741        res = env.cmd('ft.search', 'idx', 'match')
1742        for r in res:
1743            env.assertIn(r, exp)
1744
1745def testNonDefaultDb(env):
1746    if env.is_cluster():
1747        raise unittest.SkipTest()
1748
1749    # Should be ok
1750    env.cmd('FT.CREATE', 'idx1', 'ON', 'HASH', 'schema', 'txt', 'text')
1751    try:
1752        env.cmd('SELECT 1')
1753    except redis.ResponseError:
1754        return
1755
1756    # Should fail
1757    with env.assertResponseError():
1758        env.cmd('FT.CREATE', 'idx2', 'ON', 'HASH', 'schema', 'txt', 'text')
1759
1760def testDuplicateNonspecFields(env):
1761    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'schema', 'txt', 'text').ok()
1762    env.expect('FT.ADD', 'idx', 'doc', 1.0, 'fields',
1763                'txt', 'foo', 'f1', 'f1val', 'f1', 'f1val2', 'F1', 'f1Val3').ok()
1764    res = env.cmd('ft.get', 'idx', 'doc')
1765    res = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
1766    env.assertTrue(res['f1'] in ('f1val', 'f1val2'))
1767    env.assertEqual('f1Val3', res['F1'])
1768
1769def testDuplicateFields(env):
1770    # As of RS 2.0 it is allowed. only latest field will be saved and indexed
1771    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH',
1772            'SCHEMA', 'txt', 'TEXT', 'num', 'NUMERIC', 'SORTABLE')
1773
1774    env.expect('FT.ADD', 'idx', 'doc', 1.0, 'FIELDS',
1775        'txt', 'foo', 'txt', 'bar', 'txt', 'baz').ok()
1776    env.expect('FT.SEARCH idx *').equal([1L, 'doc', ['txt', 'baz']])
1777
1778def testDuplicateSpec(env):
1779    with env.assertResponseError():
1780        env.cmd('FT.CREATE', 'idx', 'ON', 'HASH',
1781                'SCHEMA', 'f1', 'text', 'n1', 'numeric', 'f1', 'text')
1782
1783def testSortbyMissingFieldSparse(env):
1784    # Note, the document needs to have one present sortable field in
1785    # order for the indexer to give it a sort vector
1786    env.cmd('ft.create', 'idx', 'ON', 'HASH',
1787            'SCHEMA', 'lastName', 'text', 'SORTABLE', 'firstName', 'text', 'SORTABLE')
1788    env.cmd('ft.add', 'idx', 'doc1', 1.0, 'fields', 'lastName', 'mark')
1789    res = env.cmd('ft.search', 'idx', 'mark', 'WITHSORTKEYS', "SORTBY",
1790                   "firstName", "ASC", "limit", 0, 100)
1791    # commented because we don't filter out exclusive sortby fields
1792    # env.assertEqual([1L, 'doc1', None, ['lastName', 'mark']], res)
1793
1794def testLuaAndMulti(env):
1795    env.skip() # addhash isn't supported
1796    if env.is_cluster():
1797        raise unittest.SkipTest()
1798    # Ensure we can work in Lua and Multi environments without crashing
1799    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'f1', 'text', 'n1', 'numeric')
1800    env.cmd('HMSET', 'hashDoc', 'f1', 'v1', 'n1', 4)
1801    env.cmd('HMSET', 'hashDoc2', 'f1', 'v1', 'n1', 5)
1802
1803    r = env.getConnection()
1804
1805    r.eval("return redis.call('ft.add', 'idx', 'doc1', 1.0, 'fields', 'f1', 'bar')", "0")
1806    r.eval("return redis.call('ft.addhash', 'idx', 'hashDoc', 1.0)", 0)
1807
1808    # Try in a pipeline:
1809    with r.pipeline(transaction=True) as pl:
1810        pl.execute_command('ft.add', 'idx', 'doc2',
1811                           1.0, 'fields', 'f1', 'v3')
1812        pl.execute_command('ft.add', 'idx', 'doc3',
1813                           1.0, 'fields', 'f1', 'v4')
1814        pl.execute_command('ft.addhash', 'idx', 'hashdoc2', 1.0)
1815    pl.execute()
1816
1817def testLanguageField(env):
1818    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'language', 'TEXT')
1819    env.cmd('FT.ADD', 'idx', 'doc1', 1.0,
1820             'FIELDS', 'language', 'gibberish')
1821    res = env.cmd('FT.SEARCH', 'idx', 'gibberish')
1822    env.assertEqual([1L, 'doc1', ['language', 'gibberish']], res)
1823    # The only way I can verify that LANGUAGE is parsed twice is ensuring we
1824    # provide a wrong language. This is much easier to test than trying to
1825    # figure out how a given word is stemmed
1826    with env.assertResponseError():
1827        env.cmd('FT.ADD', 'idx', 'doc1', 1.0, 'LANGUAGE',
1828                 'blah', 'FIELDS', 'language', 'gibber')
1829
1830def testUninitSortvector(env):
1831    # This would previously crash
1832    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'f1', 'TEXT')
1833    for x in range(2000):
1834        env.cmd('FT.ADD', 'idx', 'doc{}'.format(
1835            x), 1.0, 'FIELDS', 'f1', 'HELLO')
1836
1837    env.broadcast('SAVE')
1838    for x in range(10):
1839        env.broadcast('DEBUG RELOAD')
1840
1841
1842def normalize_row(row):
1843    return to_dict(row)
1844
1845
1846def assertAggrowsEqual(env, exp, got):
1847    env.assertEqual(exp[0], got[0])
1848    env.assertEqual(len(exp), len(got))
1849
1850    # and now, it's just free form:
1851    exp = sorted(to_dict(x) for x in exp[1:])
1852    got = sorted(to_dict(x) for x in got[1:])
1853    env.assertEqual(exp, got)
1854
1855def assertResultsEqual(env, exp, got, inorder=True):
1856    from pprint import pprint
1857    # pprint(exp)
1858    # pprint(got)
1859    env.assertEqual(exp[0], got[0])
1860    env.assertEqual(len(exp), len(got))
1861
1862    exp = list(grouper(exp[1:], 2))
1863    got = list(grouper(got[1:], 2))
1864
1865    for x in range(len(exp)):
1866        exp_did, exp_fields = exp[x]
1867        got_did, got_fields = got[x]
1868        env.assertEqual(exp_did, got_did, message="at position {}".format(x))
1869        got_fields = to_dict(got_fields)
1870        exp_fields = to_dict(exp_fields)
1871        env.assertEqual(exp_fields, got_fields, message="at position {}".format(x))
1872
1873def testAlterIndex(env):
1874    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'f1', 'TEXT')
1875    env.cmd('FT.ADD', 'idx', 'doc1', 1.0, 'FIELDS', 'f1', 'hello', 'f2', 'world')
1876    env.cmd('FT.ALTER', 'idx', 'SCHEMA', 'ADD', 'f2', 'TEXT')
1877    waitForIndex(env, 'idx')
1878    env.cmd('FT.ADD', 'idx', 'doc2', 1.0, 'FIELDS', 'f1', 'hello', 'f2', 'world')
1879
1880    # RS 2.0 reindex and after reload both documents are found
1881    # for _ in env.retry_with_reload():
1882    res = env.cmd('FT.SEARCH', 'idx', 'world')
1883    env.assertEqual(toSortedFlatList(res), toSortedFlatList([2L, 'doc2', ['f1', 'hello', 'f2', 'world'], 'doc1', ['f1', 'hello', 'f2', 'world']]))
1884    # env.assertEqual([1, 'doc2', ['f1', 'hello', 'f2', 'world']], ret)
1885
1886    env.cmd('FT.ALTER', 'idx', 'SCHEMA', 'ADD', 'f3', 'TEXT', 'SORTABLE')
1887    for x in range(10):
1888        env.cmd('FT.ADD', 'idx', 'doc{}'.format(x + 3), 1.0,
1889                 'FIELDS', 'f1', 'hello', 'f3', 'val{}'.format(x))
1890
1891    for _ in env.retry_with_reload():
1892        waitForIndex(env, 'idx')
1893        # Test that sortable works
1894        res = env.cmd('FT.SEARCH', 'idx', 'hello', 'SORTBY', 'f3', 'DESC')
1895        exp = [12, 'doc12', ['f1', 'hello', 'f3', 'val9'], 'doc11', ['f1', 'hello', 'f3', 'val8'],
1896                   'doc10', ['f1', 'hello', 'f3', 'val7'], 'doc9',  ['f1', 'hello', 'f3', 'val6'],
1897                   'doc8',  ['f1', 'hello', 'f3', 'val5'], 'doc7',  ['f1', 'hello', 'f3', 'val4'],
1898                   'doc6',  ['f1', 'hello', 'f3', 'val3'], 'doc5',  ['f1', 'hello', 'f3', 'val2'],
1899                   'doc4',  ['f1', 'hello', 'f3', 'val1'], 'doc3',  ['f1', 'hello', 'f3', 'val0']]
1900        assertResultsEqual(env, exp, res)
1901
1902    # Test that we can add a numeric field
1903    env.cmd('FT.ALTER', 'idx', 'SCHEMA', 'ADD', 'n1', 'NUMERIC')
1904    env.cmd('FT.ADD', 'idx', 'docN1', 1.0, 'FIELDS', 'n1', 50)
1905    env.cmd('FT.ADD', 'idx', 'docN2', 1.0, 'FIELDS', 'n1', 250)
1906    for _ in env.retry_with_reload():
1907        waitForIndex(env, 'idx')
1908        res = env.cmd('FT.SEARCH', 'idx', '@n1:[0 100]')
1909        env.assertEqual([1, 'docN1', ['n1', '50']], res)
1910
1911    env.expect('FT.ALTER', 'idx', 'SCHEMA', 'NOT_ADD', 'f2', 'TEXT').error()
1912    env.expect('FT.ALTER', 'idx', 'SCHEMA', 'ADD').error()
1913    env.expect('FT.ALTER', 'idx', 'SCHEMA', 'ADD', 'f2').error()
1914    env.expect('FT.ALTER', 'idx', 'ADD', 'SCHEMA', 'f2', 'TEXT').error()
1915    env.expect('FT.ALTER', 'idx', 'f2', 'TEXT').error()
1916
1917def testAlterValidation(env):
1918    # Test that constraints for ALTER comand
1919    env.cmd('FT.CREATE', 'idx1', 'ON', 'HASH', 'SCHEMA', 'f0', 'TEXT')
1920    for x in range(1, 32):
1921        env.cmd('FT.ALTER', 'idx1', 'SCHEMA', 'ADD', 'f{}'.format(x), 'TEXT')
1922    # OK for now.
1923
1924    # Should be too many indexes
1925    env.assertRaises(redis.ResponseError, env.cmd, 'FT.ALTER',
1926                      'idx1', 'SCHEMA', 'ADD', 'tooBig', 'TEXT')
1927
1928    env.cmd('FT.CREATE', 'idx2', 'MAXTEXTFIELDS', 'ON', 'HASH', 'SCHEMA', 'f0', 'TEXT')
1929    # print env.cmd('FT.INFO', 'idx2')
1930    for x in range(1, 50):
1931        env.cmd('FT.ALTER', 'idx2', 'SCHEMA', 'ADD', 'f{}'.format(x + 1), 'TEXT')
1932
1933    env.cmd('FT.ADD', 'idx2', 'doc1', 1.0, 'FIELDS', 'f50', 'hello')
1934    for _ in env.retry_with_reload():
1935        waitForIndex(env, 'idx2')
1936        ret = env.cmd('FT.SEARCH', 'idx2', '@f50:hello')
1937        env.assertEqual([1, 'doc1', ['f50', 'hello']], ret)
1938
1939    env.cmd('FT.CREATE', 'idx3', 'ON', 'HASH', 'SCHEMA', 'f0', 'text')
1940    # Try to alter the index with garbage
1941    env.assertRaises(redis.ResponseError, env.cmd, 'FT.ALTER', 'idx3',
1942                      'SCHEMA', 'ADD', 'f1', 'TEXT', 'f2', 'garbage')
1943    ret = to_dict(env.cmd('ft.info', 'idx3'))
1944    env.assertEqual(1, len(ret['fields']))
1945
1946    env.assertRaises(redis.ResponseError, env.cmd, 'FT.ALTER',
1947                      'nonExist', 'SCHEMA', 'ADD', 'f1', 'TEXT')
1948
1949    # test with no fields!
1950    env.assertRaises(redis.ResponseError, env.cmd, 'FT.ALTER', 'idx2', 'SCHEMA', 'ADD')
1951
1952    # test with no fields!
1953    env.assertRaises(redis.ResponseError, env.cmd, 'FT.ALTER', 'idx2', 'SCHEMA', 'ADD')
1954
1955def testIssue366_2(env):
1956    # FT.CREATE atest SCHEMA textfield TEXT numfield NUMERIC
1957    # FT.ADD atest anId 1 PAYLOAD '{"hello":"world"}' FIELDS textfield sometext numfield 1234
1958    # FT.ADD atest anId 1 PAYLOAD '{"hello":"world2"}' REPLACE PARTIAL FIELDS numfield 1111
1959    # shutdown
1960    env.cmd('FT.CREATE', 'idx1', 'ON', 'HASH',
1961            'SCHEMA', 'textfield', 'TEXT', 'numfield', 'NUMERIC')
1962    env.cmd('FT.ADD', 'idx1', 'doc1', 1, 'PAYLOAD', '{"hello":"world"}',
1963             'FIELDS', 'textfield', 'sometext', 'numfield', 1234)
1964    env.cmd('ft.add', 'idx1', 'doc1', 1,
1965             'PAYLOAD', '{"hello":"world2"}',
1966             'REPLACE', 'PARTIAL',
1967             'FIELDS', 'textfield', 'sometext', 'numfield', 1111)
1968    for _ in env.retry_with_reload():
1969        pass  #
1970
1971def testIssue654(env):
1972    # Crashes during FILTER
1973    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'id', 'numeric')
1974    env.cmd('ft.add', 'idx', 1, 1, 'fields', 'id', 1)
1975    env.cmd('ft.add', 'idx', 2, 1, 'fields', 'id', 2)
1976    res = env.cmd('ft.search', 'idx', '*', 'filter', '@version', 0, 2)
1977
1978def testReplaceReload(env):
1979    env.cmd('FT.CREATE', 'idx2', 'ON', 'HASH',
1980            'SCHEMA', 'textfield', 'TEXT', 'numfield', 'NUMERIC')
1981    # Create a document and then replace it.
1982    env.cmd('FT.ADD', 'idx2', 'doc2', 1.0, 'FIELDS', 'textfield', 's1', 'numfield', 99)
1983    env.cmd('FT.ADD', 'idx2', 'doc2', 1.0, 'REPLACE', 'PARTIAL',
1984             'FIELDS', 'textfield', 's100', 'numfield', 990)
1985    env.dump_and_reload()
1986    # RDB Should still be fine
1987
1988    env.cmd('FT.ADD', 'idx2', 'doc2', 1.0, 'REPLACE', 'PARTIAL',
1989             'FIELDS', 'textfield', 's200', 'numfield', 1090)
1990    doc = to_dict(env.cmd('FT.GET', 'idx2', 'doc2'))
1991    env.assertEqual('s200', doc['textfield'])
1992    env.assertEqual('1090', doc['numfield'])
1993
1994
1995# command = 'FT.CREATE idx SCHEMA '
1996# for i in range(255):
1997#     command += 't%d NUMERIC SORTABLE ' % i
1998# command = command[:-1]
1999# r.execute_command(command)
2000# r.execute_command('save')
2001# // reload from ...
2002# r.execute_command('FT.ADD idx doc1 1.0 FIELDS t0 1')
2003def testIssue417(env):
2004    command = ['ft.create', 'idx', 'ON', 'HASH', 'schema']
2005    for x in range(255):
2006        command += ['t{}'.format(x), 'numeric', 'sortable']
2007    command = command[:-1]
2008    env.cmd(*command)
2009    for _ in env.reloading_iterator():
2010        waitForIndex(env, 'idx')
2011        try:
2012            env.execute_command('FT.ADD', 'idx', 'doc1', '1.0', 'FIELDS', 't0', '1')
2013        except redis.ResponseError as e:
2014            env.assertTrue('already' in e.message.lower())
2015
2016# >FT.CREATE myIdx SCHEMA title TEXT WEIGHT 5.0 body TEXT url TEXT
2017# >FT.ADD myIdx doc1 1.0 FIELDS title "hello world" body "lorem ipsum" url "www.google.com"
2018# >FT.SEARCH myIdx "no-as"
2019# Could not connect to Redis at 127.0.0.1:6379: Connection refused
2020# >FT.SEARCH myIdx "no-as"
2021# (error) Unknown Index name
2022def testIssue422(env):
2023    env.cmd('ft.create', 'myIdx', 'ON', 'HASH', 'schema',
2024             'title', 'TEXT', 'WEIGHT', '5.0',
2025             'body', 'TEXT',
2026             'url', 'TEXT')
2027    env.cmd('ft.add', 'myIdx', 'doc1', '1.0', 'FIELDS', 'title', 'hello world', 'bod', 'lorem ipsum', 'url', 'www.google.com')
2028    rv = env.cmd('ft.search', 'myIdx', 'no-as')
2029    env.assertEqual([0], rv)
2030
2031def testIssue446(env):
2032    env.cmd('ft.create', 'myIdx', 'ON', 'HASH', 'schema',
2033             'title', 'TEXT', 'SORTABLE')
2034    env.cmd('ft.add', 'myIdx', 'doc1', '1.0', 'fields', 'title', 'hello world', 'body', 'lorem ipsum', 'url', '"www.google.com')
2035    rv = env.cmd('ft.search', 'myIdx', 'hello', 'limit', '0', '0')
2036    env.assertEqual([1], rv)
2037
2038    # Related - issue 635
2039    env.cmd('ft.add', 'myIdx', 'doc2', '1.0', 'fields', 'title', 'hello')
2040    rv = env.cmd('ft.search', 'myIdx', 'hello', 'limit', '0', '0')
2041    env.assertEqual([2], rv)
2042
2043
2044def testTimeoutSettings(env):
2045    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 't1', 'text')
2046    env.expect('ft.search', 'idx', '*', 'ON_TIMEOUT', 'BLAHBLAH').raiseError()
2047    env.expect('ft.search', 'idx', '*', 'ON_TIMEOUT', 'RETURN').notRaiseError()
2048    env.expect('ft.search', 'idx', '*', 'ON_TIMEOUT', 'FAIL').notRaiseError()
2049
2050def testAlias(env):
2051    conn = getConnectionByEnv(env)
2052    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'PREFIX', 1, 'doc1', 'schema', 't1', 'text')
2053    env.cmd('ft.create', 'idx2', 'ON', 'HASH', 'PREFIX', 1, 'doc2', 'schema', 't1', 'text')
2054
2055    env.expect('ft.aliasAdd', 'myIndex').raiseError()
2056    env.expect('ft.aliasupdate', 'fake_alias', 'imaginary_alias', 'Too_many_args').raiseError()
2057    env.cmd('ft.aliasAdd', 'myIndex', 'idx')
2058    env.cmd('ft.add', 'myIndex', 'doc1', 1.0, 'fields', 't1', 'hello')
2059    r = env.cmd('ft.search', 'idx', 'hello')
2060    env.assertEqual([1, 'doc1', ['t1', 'hello']], r)
2061    r2 = env.cmd('ft.search', 'myIndex', 'hello')
2062    env.assertEqual(r, r2)
2063
2064    # try to add the same alias again; should be an error
2065    env.expect('ft.aliasAdd', 'myIndex', 'idx2').raiseError()
2066    env.expect('ft.aliasAdd', 'alias2', 'idx').notRaiseError()
2067    # now delete the index
2068    env.cmd('ft.drop', 'myIndex')
2069    # RS2 does not delete doc on ft.drop
2070    conn.execute_command('DEL', 'doc1')
2071
2072    # index list should be cleared now. This can be tested by trying to alias
2073    # the old alias to different index
2074    env.cmd('ft.aliasAdd', 'myIndex', 'idx2')
2075    env.cmd('ft.aliasAdd', 'alias2', 'idx2')
2076    env.cmd('ft.add', 'myIndex', 'doc2', 1.0, 'fields', 't1', 'hello')
2077    r = env.cmd('ft.search', 'alias2', 'hello')
2078    env.assertEqual([1L, 'doc2', ['t1', 'hello']], r)
2079
2080    # check that aliasing one alias to another returns an error. This will
2081    # end up being confusing
2082    env.expect('ft.aliasAdd', 'alias3', 'myIndex').raiseError()
2083
2084    # check that deleting the alias works as expected
2085    env.expect('ft.aliasDel', 'myIndex').notRaiseError()
2086    env.expect('ft.search', 'myIndex', 'foo').raiseError()
2087
2088    # create a new index and see if we can use the old name
2089    env.cmd('ft.create', 'idx3', 'ON', 'HASH', 'PREFIX', 1, 'doc3', 'schema', 't1', 'text')
2090    env.cmd('ft.add', 'idx3', 'doc3', 1.0, 'fields', 't1', 'foo')
2091    env.cmd('ft.aliasAdd', 'myIndex', 'idx3')
2092    # also, check that this works in rdb save
2093    for _ in env.retry_with_rdb_reload():
2094        waitForIndex(env, 'myIndex')
2095        r = env.cmd('ft.search', 'myIndex', 'foo')
2096        env.assertEqual([1L, 'doc3', ['t1', 'foo']], r)
2097
2098    # Check that we can move an alias from one index to another
2099    env.cmd('ft.aliasUpdate', 'myIndex', 'idx2')
2100    r = env.cmd('ft.search', 'myIndex', "hello")
2101    env.assertEqual([1L, 'doc2', ['t1', 'hello']], r)
2102
2103    # Test that things like ft.get, ft.aggregate, etc. work
2104    r = env.cmd('ft.get', 'myIndex', 'doc2')
2105    env.assertEqual(['t1', 'hello'], r)
2106
2107    r = env.cmd('ft.aggregate', 'myIndex', 'hello', 'LOAD', '1', '@t1')
2108    env.assertEqual([1, ['t1', 'hello']], r)
2109
2110    # Test update
2111    env.expect('ft.aliasAdd', 'updateIndex', 'idx3')
2112    env.expect('ft.aliasUpdate', 'updateIndex', 'fake_idx')
2113
2114    r = env.cmd('ft.del', 'idx2', 'doc2')
2115    env.assertEqual(1, r)
2116    env.expect('ft.aliasdel').raiseError()
2117    env.expect('ft.aliasdel', 'myIndex', 'yourIndex').raiseError()
2118    env.expect('ft.aliasdel', 'non_existing_alias').raiseError()
2119
2120
2121def testNoCreate(env):
2122    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'f1', 'text')
2123    env.expect('ft.add', 'idx', 'schema', 'f1').raiseError()
2124    env.expect('ft.add', 'idx', 'doc1', 1, 'nocreate', 'fields', 'f1', 'hello').raiseError()
2125    env.expect('ft.add', 'idx', 'doc1', 1, 'replace', 'nocreate', 'fields', 'f1', 'hello').raiseError()
2126    env.expect('ft.add', 'idx', 'doc1', 1, 'replace', 'fields', 'f1', 'hello').notRaiseError()
2127    env.expect('ft.add', 'idx', 'doc1', 1, 'replace', 'nocreate', 'fields', 'f1', 'world').notRaiseError()
2128
2129def testSpellCheck(env):
2130    env.cmd('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'report', 'TEXT')
2131    env.cmd('FT.ADD', 'idx', 'doc1', 1.0, 'FIELDS', 'report', 'report content')
2132    rv = env.cmd('FT.SPELLCHECK', 'idx', '111111')
2133    env.assertEqual([['TERM', '111111', []]], rv)
2134    if not env.isCluster():
2135        rv = env.cmd('FT.SPELLCHECK', 'idx', '111111', 'FULLSCOREINFO')
2136        env.assertEqual([1L, ['TERM', '111111', []]], rv)
2137
2138# Standalone functionality
2139def testIssue484(env):
2140# Issue with split
2141# 127.0.0.1:6379> ft.drop productSearch1
2142# OK
2143# 127.0.0.1:6379> "FT.CREATE" "productSearch1" "NOSCOREIDX" "SCHEMA" "productid" "TEXT" "categoryid" "TEXT"  "color" "TEXT" "timestamp" "NUMERIC"
2144# OK
2145# 127.0.0.1:6379> "FT.ADD" "productSearch1" "GUID1" "1.0" "REPLACE" "FIELDS" "productid" "1" "categoryid" "cars" "color" "blue" "categoryType" 0
2146# OK
2147# 127.0.0.1:6379> "FT.ADD" "productSearch1" "GUID2" "1.0" "REPLACE" "FIELDS" "productid" "1" "categoryid" "small cars" "color" "white" "categoryType" 0
2148# OK
2149# 127.0.0.1:6379> "FT.ADD" "productSearch1" "GUID3" "1.0" "REPLACE" "FIELDS" "productid" "2" "categoryid" "Big cars" "color" "white" "categoryType" 0
2150# OK
2151# 127.0.0.1:6379> "FT.ADD" "productSearch1" "GUID4" "1.0" "REPLACE" "FIELDS" "productid" "2" "categoryid" "Big cars" "color" "green" "categoryType" 0
2152# OK
2153# 127.0.0.1:6379> "FT.ADD" "productSearch1" "GUID5" "1.0" "REPLACE" "FIELDS" "productid" "3" "categoryid" "cars" "color" "blue" "categoryType" 0
2154# OK
2155# 127.0.0.1:6379>  FT.AGGREGATE productSearch1 * load 2 @color @categoryid APPLY "split(format(\"%s-%s\",@color,@categoryid),\"-\")" as value GROUPBY 1 @value REDUCE COUNT 0 as value_count
2156    env.cmd('ft.create', 'productSearch1', 'noscoreidx', 'ON', 'HASH', 'schema', 'productid',
2157            'text', 'categoryid', 'text', 'color', 'text', 'timestamp', 'numeric')
2158    env.cmd('ft.add', 'productSearch1', 'GUID1', '1.0', 'REPLACE', 'FIELDS', 'productid', '1', 'categoryid', 'cars', 'color', 'blue', 'categoryType', 0)
2159    env.cmd('ft.add', 'productSearch1', 'GUID2', '1.0', 'REPLACE', 'FIELDS', 'productid', '1', 'categoryid', 'small cars', 'color', 'white', 'categoryType', 0)
2160    env.cmd('ft.add', 'productSearch1', 'GUID3', '1.0', 'REPLACE', 'FIELDS', 'productid', '2', 'categoryid', 'Big cars', 'color', 'white', 'categoryType', 0)
2161    env.cmd('ft.add', 'productSearch1', 'GUID4', '1.0', 'REPLACE', 'FIELDS', 'productid', '2', 'categoryid', 'Big cars', 'color', 'green', 'categoryType', 0)
2162    env.cmd('ft.add', 'productSearch1', 'GUID5', '1.0', 'REPLACE', 'FIELDS', 'productid', '3', 'categoryid', 'cars', 'color', 'blue', 'categoryType', 0)
2163    res = env.cmd('FT.AGGREGATE', 'productSearch1', '*',
2164        'load', '2', '@color', '@categoryid',
2165        'APPLY', 'split(format("%s-%s",@color,@categoryid),"-")', 'as', 'value',
2166        'GROUPBY', '1', '@value',
2167        'REDUCE', 'COUNT', '0', 'as', 'value_count',
2168        'SORTBY', '4', '@value_count', 'DESC', '@value', 'ASC')
2169    expected = [6, ['value', 'white', 'value_count', '2'], ['value', 'cars', 'value_count', '2'], ['value', 'small cars', 'value_count', '1'], ['value', 'blue', 'value_count', '2'], ['value', 'Big cars', 'value_count', '2'], ['value', 'green', 'value_count', '1']]
2170    assertAggrowsEqual(env, expected, res)
2171    for var in expected:
2172        env.assertIn(var, res)
2173
2174def testIssue501(env):
2175    env.cmd('FT.CREATE', 'incidents', 'ON', 'HASH', 'SCHEMA', 'report', 'TEXT')
2176    env.cmd('FT.ADD', 'incidents', 'doc1', 1.0, 'FIELDS', 'report', 'report content')
2177    env.cmd('FT.DICTADD', 'slang', 'timmies', 'toque', 'toonie', 'serviette', 'kerfuffle', 'chesterfield')
2178    rv = env.cmd('FT.SPELLCHECK', 'incidents', 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq',
2179        'TERMS', 'INCLUDE', 'slang', 'TERMS', 'EXCLUDE', 'slang')
2180    env.assertEqual("qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq", rv[0][1])
2181    env.assertEqual([], rv[0][2])
2182
2183    env.expect('FT.SPELLCHECK', 'incidents', 'qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq',
2184        'TERMS', 'FAKE_COMMAND', 'slang').error()
2185
2186def testIssue589(env):
2187    env.cmd('FT.CREATE', 'incidents', 'ON', 'HASH', 'SCHEMA', 'report', 'TEXT')
2188    env.cmd('FT.ADD', 'incidents', 'doc1', 1.0, 'FIELDS', 'report', 'report content')
2189    env.expect('FT.SPELLCHECK', 'incidents', 'report :').error().contains("Syntax error at offset")
2190
2191def testIssue621(env):
2192    env.expect('ft.create', 'test', 'ON', 'HASH', 'SCHEMA', 'uuid', 'TAG', 'title', 'TEXT').equal('OK')
2193    env.expect('ft.add', 'test', 'a', '1', 'REPLACE', 'PARTIAL', 'FIELDS', 'uuid', 'foo', 'title', 'bar').equal('OK')
2194    env.expect('ft.add', 'test', 'a', '1', 'REPLACE', 'PARTIAL', 'FIELDS', 'title', 'bar').equal('OK')
2195    res = env.cmd('ft.search', 'test', '@uuid:{foo}')
2196    env.assertEqual(toSortedFlatList(res), toSortedFlatList([1L, 'a', ['uuid', 'foo', 'title', 'bar']]))
2197
2198# Server crash on doc names that conflict with index keys #666
2199# again this test is not relevant cause index is out of key space
2200# def testIssue666(env):
2201#     # We cannot reliably determine that any error will occur in cluster mode
2202#     # because of the key name
2203#     env.skipOnCluster()
2204
2205#     env.cmd('ft.create', 'foo', 'schema', 'bar', 'text')
2206#     env.cmd('ft.add', 'foo', 'mydoc', 1, 'fields', 'bar', 'one two three')
2207
2208#     # crashes here
2209#     with env.assertResponseError():
2210#         env.cmd('ft.add', 'foo', 'ft:foo/two', '1', 'fields', 'bar', 'four five six')
2211#     # try with replace:
2212#     with env.assertResponseError():
2213#         env.cmd('ft.add', 'foo', 'ft:foo/two', '1', 'REPLACE',
2214#             'FIELDS', 'bar', 'four five six')
2215#     with env.assertResponseError():
2216#         env.cmd('ft.add', 'foo', 'idx:foo', '1', 'REPLACE',
2217#             'FIELDS', 'bar', 'four five six')
2218
2219#     env.cmd('ft.add', 'foo', 'mydoc1', 1, 'fields', 'bar', 'four five six')
2220
2221# 127.0.0.1:6379> flushdb
2222# OK
2223# 127.0.0.1:6379> ft.create foo SCHEMA bar text
2224# OK
2225# 127.0.0.1:6379> ft.add foo mydoc 1 FIELDS bar "one two three"
2226# OK
2227# 127.0.0.1:6379> keys *
2228# 1) "mydoc"
2229# 2) "ft:foo/one"
2230# 3) "idx:foo"
2231# 4) "ft:foo/two"
2232# 5) "ft:foo/three"
2233# 127.0.0.1:6379> ft.add foo "ft:foo/two" 1 FIELDS bar "four five six"
2234# Could not connect to Redis at 127.0.0.1:6379: Connection refused
2235
2236@unstable
2237def testPrefixDeletedExpansions(env):
2238    env.skipOnCluster()
2239
2240    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'txt1', 'text', 'tag1', 'tag')
2241    # get the number of maximum expansions
2242    maxexpansions = int(env.cmd('ft.config', 'get', 'MAXEXPANSIONS')[0][1])
2243
2244    for x in range(maxexpansions):
2245        env.cmd('ft.add', 'idx', 'doc{}'.format(x), 1, 'fields',
2246                'txt1', 'term{}'.format(x), 'tag1', 'tag{}'.format(x))
2247
2248    for x in range(maxexpansions):
2249        env.cmd('ft.del', 'idx', 'doc{}'.format(x))
2250
2251    env.cmd('ft.add', 'idx', 'doc_XXX', 1, 'fields', 'txt1', 'termZZZ', 'tag1', 'tagZZZ')
2252
2253    # r = env.cmd('ft.search', 'idx', 'term*')
2254    # print(r)
2255    # r = env.cmd('ft.search', 'idx', '@tag1:{tag*}')
2256    # print(r)
2257
2258    tmax = time.time() + 0.5  # 250ms max
2259    iters = 0
2260    while time.time() < tmax:
2261        iters += 1
2262        forceInvokeGC(env, 'idx')
2263        r = env.cmd('ft.search', 'idx', '@txt1:term* @tag1:{tag*}')
2264        if r[0]:
2265            break
2266
2267    # print 'did {} iterations'.format(iters)
2268    r = env.cmd('ft.search', 'idx', '@txt1:term* @tag1:{tag*}')
2269    env.assertEqual(toSortedFlatList([1, 'doc_XXX', ['txt1', 'termZZZ', 'tag1', 'tagZZZ']]), toSortedFlatList(r))
2270
2271
2272def testOptionalFilter(env):
2273    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 't1', 'text')
2274    for x in range(100):
2275        env.cmd('ft.add', 'idx', 'doc_{}'.format(x), 1, 'fields', 't1', 'hello world word{}'.format(x))
2276
2277    env.cmd('ft.explain', 'idx', '(~@t1:word20)')
2278    # print(r)
2279
2280    r = env.cmd('ft.search', 'idx', '~(word20 => {$weight: 2.0})')
2281
2282
2283def testIssue736(env):
2284    #for new RS 2.0 ft.add does not return certian errors
2285    env.skip()
2286    # 1. create the schema, we need a tag field
2287    env.cmd('ft.create', 'idx', 'ON', 'HASH',
2288            'schema', 't1', 'text', 'n2', 'numeric', 't2', 'tag')
2289    # 2. create a single document to initialize at least one RSAddDocumentCtx
2290    env.cmd('ft.add', 'idx', 'doc1', 1, 'fields', 't1', 'hello', 't2', 'foo, bar')
2291    # 3. create a second document with many filler fields to force a realloc:
2292    extra_fields = []
2293    for x in range(20):
2294        extra_fields += ['nidx_fld{}'.format(x), 'val{}'.format(x)]
2295    extra_fields += ['n2', 'not-a-number', 't2', 'random, junk']
2296    with env.assertResponseError():
2297        env.cmd('ft.add', 'idx', 'doc2', 1, 'fields', *extra_fields)
2298
2299def testCriteriaTesterDeactivated():
2300    env = Env(moduleArgs='_MAX_RESULTS_TO_UNSORTED_MODE 1')
2301    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 't1', 'text')
2302    env.cmd('ft.add', 'idx', 'doc1', 1, 'fields', 't1', 'hello1 hey hello2')
2303    env.cmd('ft.add', 'idx', 'doc2', 1, 'fields', 't1', 'hello2 hey')
2304    env.cmd('ft.add', 'idx', 'doc3', 1, 'fields', 't1', 'hey')
2305    res = env.execute_command('ft.search', 'idx', '(hey hello1)|(hello2 hey)')
2306    expected = [2L, 'doc1', ['t1', 'hello1 hey hello2'], 'doc2', ['t1', 'hello2 hey']]
2307    env.assertEqual(toSortedFlatList(res), toSortedFlatList(expected))
2308
2309def testIssue828(env):
2310    env.cmd('ft.create', 'beers', 'ON', 'HASH', 'SCHEMA',
2311        'name', 'TEXT', 'PHONETIC', 'dm:en',
2312        'style', 'TAG', 'SORTABLE',
2313        'abv', 'NUMERIC', 'SORTABLE')
2314    rv = env.cmd("FT.ADD", "beers", "802", "1.0",
2315        "FIELDS", "index", "25", "abv", "0.049",
2316        "name", "Hell or High Watermelon Wheat (2009)",
2317        "style", "Fruit / Vegetable Beer")
2318    env.assertEqual('OK', rv)
2319
2320def testIssue862(env):
2321    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE')
2322    rv = env.cmd("FT.ADD", "idx", "doc1", "1.0", "FIELDS", "test", "foo")
2323    env.assertEqual('OK', rv)
2324    env.cmd("FT.SEARCH", "idx", "foo", 'WITHSORTKEYS')
2325    env.assertTrue(env.isUp())
2326
2327def testIssue_884(env):
2328    env.expect('FT.create', 'idx', 'ON', 'HASH', 'STOPWORDS', '0', 'SCHEMA', 'title', 'text', 'weight',
2329               '50', 'subtitle', 'text', 'weight', '10', 'author', 'text', 'weight',
2330               '10', 'description', 'text', 'weight', '20').equal('OK')
2331
2332    env.expect('FT.ADD', 'idx', 'doc4', '1.0', 'FIELDS', 'title', 'mohsin conversation the conversation tahir').equal('OK')
2333    env.expect('FT.ADD', 'idx', 'doc3', '1.0', 'FIELDS', 'title', 'Fareham Civilization Church - Sermons and conversations mohsin conversation the').equal('OK')
2334    env.expect('FT.ADD', 'idx', 'doc2', '1.0', 'FIELDS', 'title', 'conversation the conversation - a drama about conversation, the science of conversation.').equal('OK')
2335    env.expect('FT.ADD', 'idx', 'doc1', '1.0', 'FIELDS', 'title', 'mohsin conversation with the mohsin').equal('OK')
2336
2337    expected = [2L, 'doc2', ['title', 'conversation the conversation - a drama about conversation, the science of conversation.'], 'doc4', ['title', 'mohsin conversation the conversation tahir']]
2338    res = env.cmd('FT.SEARCH', 'idx', '@title:(conversation) (@title:(conversation the conversation))=>{$inorder: true;$slop: 0}')
2339    env.assertEquals(len(expected), len(res))
2340    for v in expected:
2341        env.assertContains(v, res)
2342
2343def testIssue_848(env):
2344    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test1', 'TEXT', 'SORTABLE').equal('OK')
2345    env.expect('FT.ADD', 'idx', 'doc1', '1.0', 'FIELDS', 'test1', 'foo').equal('OK')
2346    env.expect('FT.ALTER', 'idx', 'SCHEMA', 'ADD', 'test2', 'TEXT', 'SORTABLE').equal('OK')
2347    env.expect('FT.ADD', 'idx', 'doc2', '1.0', 'FIELDS', 'test1', 'foo', 'test2', 'bar').equal('OK')
2348    env.expect('FT.SEARCH', 'idx', 'foo', 'SORTBY', 'test2', 'ASC').equal([2L, 'doc1', ['test1', 'foo'], 'doc2', ['test2', 'bar', 'test1', 'foo']])
2349
2350def testMod_309(env):
2351    n = 10000 if VALGRIND else 100000
2352    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2353    conn = getConnectionByEnv(env)
2354    for i in range(n):
2355        conn.execute_command('HSET', 'doc%d'%i, 'test', 'foo')
2356    res = env.cmd('FT.AGGREGATE', 'idx', 'foo')
2357    env.assertEqual(len(res), n + 1)
2358
2359    # test with cursor
2360    env.skipOnCluster()
2361    res = env.cmd('FT.AGGREGATE', 'idx', 'foo', 'WITHCURSOR')
2362    l = len(res[0]) - 1 # do not count the number of results (the first element in the results)
2363    cursor = res[1]
2364    while cursor != 0:
2365        r, cursor = env.cmd('FT.CURSOR', 'READ', 'idx', str(cursor))
2366        l += len(r) - 1
2367    env.assertEqual(l, n)
2368
2369def testIssue_865(env):
2370    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', '1', 'TEXT', 'SORTABLE').equal('OK')
2371    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', '1', 'foo1').equal('OK')
2372    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', '1', 'foo2').equal('OK')
2373    env.expect('ft.search', 'idx', 'foo*', 'SORTBY', '1', 'ASC').equal([2, 'doc1', ['1', 'foo1'], 'doc2', ['1', 'foo2']])
2374    env.expect('ft.search', 'idx', 'foo*', 'SORTBY', '1', 'DESC').equal([2, 'doc2', ['1', 'foo2'], 'doc1', ['1', 'foo1']])
2375    env.expect('ft.search', 'idx', 'foo*', 'SORTBY', '1', 'bad').error()
2376    env.expect('ft.search', 'idx', 'foo*', 'SORTBY', 'bad', 'bad').error()
2377    env.expect('ft.search', 'idx', 'foo*', 'SORTBY', 'bad').error()
2378    env.expect('ft.search', 'idx', 'foo*', 'SORTBY').error()
2379
2380def testIssue_779(env):
2381    # FT.ADD should return NOADD and not change the doc if value < same_value, but it returns OK and makes the change.
2382    # Note that "greater than" ">" does not have the same bug.
2383
2384    env.cmd('FT.CREATE idx2 ON HASH SCHEMA ot1 TAG')
2385    env.cmd('FT.ADD idx2 doc2 1.0 FIELDS newf CAT ot1 4001')
2386    res = env.cmd('FT.GET idx2 doc2')
2387    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "CAT", "ot1", "4001"]))
2388
2389    # NOADD is expected since 4001 is not < 4000, and no updates to the doc2 is expected as a result
2390    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<4000 FIELDS newf DOG ot1 4000', 'NOADD')
2391    res = env.cmd('FT.GET idx2 doc2')
2392    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "CAT", "ot1", "4001"]))
2393
2394    # OK is expected since 4001 < 4002 and the doc2 is updated
2395    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<4002 FIELDS newf DOG ot1 4002').equal('OK')
2396    res = env.cmd('FT.GET idx2 doc2')
2397    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "DOG", "ot1", "4002"]))
2398
2399    # OK is NOT expected since 4002 is not < 4002
2400    # We expect NOADD and doc2 update; however, we get OK and doc2 updated
2401    # After fix, @ot1 implicitly converted to a number, thus we expect NOADD
2402    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<4002 FIELDS newf FISH ot1 4002').equal('NOADD')
2403    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if to_number(@ot1)<4002 FIELDS newf FISH ot1 4002').equal('NOADD')
2404    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<to_str(4002) FIELDS newf FISH ot1 4002').equal('NOADD')
2405    res = env.cmd('FT.GET idx2 doc2')
2406    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "DOG", "ot1", "4002"]))
2407
2408    # OK and doc2 update is expected since 4002 < 4003
2409    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<4003 FIELDS newf HORSE ot1 4003').equal('OK')
2410    res = env.cmd('FT.GET idx2 doc2')
2411    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "HORSE", "ot1", "4003"]))
2412
2413    # Expect NOADD since 4003 is not > 4003
2414    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1>4003 FIELDS newf COW ot1 4003').equal('NOADD')
2415    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if 4003<@ot1 FIELDS newf COW ot1 4003').equal('NOADD')
2416
2417    # Expect OK and doc2 updated since 4003 > 4002
2418    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1>4002 FIELDS newf PIG ot1 4002').equal('OK')
2419    res = env.cmd('FT.GET idx2 doc2')
2420    env.assertEqual(toSortedFlatList(res), toSortedFlatList(["newf", "PIG", "ot1", "4002"]))
2421
2422    # Syntax errors
2423    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<4-002 FIELDS newf DOG ot1 4002').contains('Syntax error')
2424    env.expect('FT.ADD idx2 doc2 1.0 REPLACE PARTIAL if @ot1<to_number(4-002) FIELDS newf DOG ot1 4002').contains('Syntax error')
2425
2426def testUnknownSymbolErrorOnConditionalAdd(env):
2427    env.expect('FT.CREATE idx ON HASH SCHEMA f1 TAG f2 NUMERIC NOINDEX f3 TAG NOINDEX').ok()
2428    env.expect('ft.add idx doc1 1.0 REPLACE PARTIAL IF @f1<awfwaf FIELDS f1 foo f2 1 f3 boo').ok()
2429    env.expect('ft.add idx doc1 1.0 REPLACE PARTIAL IF @f1<awfwaf FIELDS f1 foo f2 1 f3 boo').error()
2430
2431def testWrongResultsReturnedBySkipOptimization(env):
2432    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'f1', 'TEXT', 'f2', 'TEXT').equal('OK')
2433    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'f1', 'foo', 'f2', 'bar').equal('OK')
2434    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'f1', 'moo', 'f2', 'foo').equal('OK')
2435    env.expect('ft.search', 'idx', 'foo @f2:moo').equal([0L])
2436
2437def testErrorWithApply(env):
2438    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2439    env.expect('FT.ADD', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo bar').equal('OK')
2440    err = env.cmd('FT.AGGREGATE', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'split()')[1]
2441    env.assertEqual(str(err[0]), 'Invalid number of arguments for split')
2442
2443def testSummerizeWithAggregateRaiseError(env):
2444    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2445    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2446    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2447    env.expect('ft.aggregate', 'idx', 'foo2', 'SUMMARIZE', 'FIELDS', '1', 'test',
2448               'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', '0').error()
2449
2450def testSummerizeHighlightParseError(env):
2451    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2452    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2453    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2454    env.expect('ft.search', 'idx', 'foo2', 'SUMMARIZE', 'FIELDS', 'WITHSCORES').error()
2455    env.expect('ft.search', 'idx', 'foo2', 'HIGHLIGHT', 'FIELDS', 'WITHSCORES').error()
2456
2457def testCursorBadArgument(env):
2458    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2459    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2460    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2461    env.expect('ft.aggregate', 'idx', '*',
2462               'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', '0',
2463               'WITHCURSOR', 'COUNT', 'BAD').error()
2464
2465def testLimitBadArgument(env):
2466    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2467    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2468    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2469    env.expect('ft.search', 'idx', '*', 'LIMIT', '1').error()
2470
2471def testOnTimeoutBadArgument(env):
2472    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2473    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2474    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2475    env.expect('ft.search', 'idx', '*', 'ON_TIMEOUT', 'bad').error()
2476
2477def testAggregateSortByWrongArgument(env):
2478    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2479    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2480    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2481    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', 'bad').error()
2482
2483def testAggregateSortByMaxNumberOfFields(env):
2484    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA',
2485               'test1', 'TEXT', 'SORTABLE',
2486               'test2', 'TEXT', 'SORTABLE',
2487               'test3', 'TEXT', 'SORTABLE',
2488               'test4', 'TEXT', 'SORTABLE',
2489               'test5', 'TEXT', 'SORTABLE',
2490               'test6', 'TEXT', 'SORTABLE',
2491               'test7', 'TEXT', 'SORTABLE',
2492               'test8', 'TEXT', 'SORTABLE',
2493               'test9', 'TEXT', 'SORTABLE'
2494               ).equal('OK')
2495    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo1').equal('OK')
2496    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', 'foo2').equal('OK')
2497    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', '9', *['@test%d' % (i + 1) for i in range(9)]).error()
2498    args = ['@test%d' % (i + 1) for i in range(8)] + ['bad']
2499    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', '9', *args).error()
2500    args = ['@test%d' % (i + 1) for i in range(8)] + ['ASC', 'MAX', 'bad']
2501    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', '9', *args).error()
2502    args = ['@test%d' % (i + 1) for i in range(8)] + ['ASC', 'MAX']
2503    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', '9', *args).error()
2504
2505def testNumericFilterError(env):
2506    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC', 'SORTABLE').equal('OK')
2507    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '1').equal('OK')
2508    env.expect('ft.search', 'idx', '*', 'FILTER', 'test', 'bad', '2').error()
2509    env.expect('ft.search', 'idx', '*', 'FILTER', 'test', '0', 'bad').error()
2510    env.expect('ft.search', 'idx', '*', 'FILTER', 'test', '0').error()
2511    env.expect('ft.search', 'idx', '*', 'FILTER', 'test', 'bad').error()
2512    env.expect('ft.search', 'idx', '*', 'FILTER', 'test', '0', '2', 'FILTER', 'test', '0', 'bla').error()
2513
2514def testGeoFilterError(env):
2515    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC', 'SORTABLE').equal('OK')
2516    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '1').equal('OK')
2517    env.expect('ft.search', 'idx', '*', 'GEOFILTER', 'test', '1').error()
2518    env.expect('ft.search', 'idx', '*', 'GEOFILTER', 'test', 'bad' , '2', '3', 'km').error()
2519    env.expect('ft.search', 'idx', '*', 'GEOFILTER', 'test', '1' , 'bad', '3', 'km').error()
2520    env.expect('ft.search', 'idx', '*', 'GEOFILTER', 'test', '1' , '2', 'bad', 'km').error()
2521    env.expect('ft.search', 'idx', '*', 'GEOFILTER', 'test', '1' , '2', '3', 'bad').error()
2522
2523def testReducerError(env):
2524    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC', 'SORTABLE').equal('OK')
2525    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '1').equal('OK')
2526    env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', 'bad').error()
2527    env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', '0', 'as').error()
2528
2529def testGroupbyError(env):
2530    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC', 'SORTABLE').equal('OK')
2531    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '1').equal('OK')
2532    env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test', 'REDUCE').error()
2533    if not env.isCluster(): # todo: remove once fix on coordinator
2534        env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test1').error()
2535    env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test', 'REDUCE', 'bad', '0').error()
2536    if not env.isCluster(): # todo: remove once fix on coordinator
2537        env.expect('ft.aggregate', 'idx', '*', 'GROUPBY', '1', '@test', 'REDUCE', 'SUM', '1', '@test1').error()
2538
2539def testGroupbyWithSort(env):
2540    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC', 'SORTABLE').equal('OK')
2541    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '1').equal('OK')
2542    env.expect('ft.add', 'idx', 'doc2', '1.0', 'FIELDS', 'test', '1').equal('OK')
2543    env.expect('ft.add', 'idx', 'doc3', '1.0', 'FIELDS', 'test', '2').equal('OK')
2544    env.expect('ft.aggregate', 'idx', '*', 'SORTBY', '2', '@test', 'ASC',
2545               'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', '0', 'as', 'count').equal([2L, ['test', '2', 'count', '1'], ['test', '1', 'count', '2']])
2546
2547def testApplyError(env):
2548    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2549    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2550    env.expect('ft.aggregate', 'idx', '*', 'APPLY', 'split(@test)', 'as').error()
2551
2552def testLoadError(env):
2553    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2554    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2555    env.expect('ft.aggregate', 'idx', '*', 'LOAD').error()
2556    env.expect('ft.aggregate', 'idx', '*', 'LOAD', 'bad').error()
2557    env.expect('ft.aggregate', 'idx', '*', 'LOAD', 'bad', 'test').error()
2558    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '2', 'test').error()
2559    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '2', '@test').error()
2560
2561def testMissingArgsError(env):
2562    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2563    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2564    env.expect('ft.aggregate', 'idx').error()
2565
2566def testUnexistsScorer(env):
2567    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT', 'SORTABLE').equal('OK')
2568    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2569    env.expect('ft.search', 'idx', '*', 'SCORER', 'bad').error()
2570
2571def testHighlightWithUnknowsProperty(env):
2572    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2573    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2574    env.expect('ft.aggregate', 'idx', '*', 'HIGHLIGHT', 'FIELDS', '1', 'test1').error()
2575
2576def testBadFilterExpression(env):
2577    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2578    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2579    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'FILTER', 'blabla').error()
2580    if not env.isCluster(): # todo: remove once fix on coordinator
2581        env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'FILTER', '@test1 > 1').error()
2582
2583def testWithSortKeysOnNoneSortableValue(env):
2584    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2585    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2586    env.expect('ft.search', 'idx', '*', 'WITHSORTKEYS', 'SORTBY', 'test').equal([1L, 'doc1', '$foo', ['test', 'foo']])
2587
2588def testWithWithRawIds(env):
2589    env.skipOnCluster() # todo: remove once fix on coordinator
2590    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2591    waitForIndex(env, 'idx')
2592    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2593    env.expect('ft.search', 'idx', '*', 'WITHRAWIDS').equal([1L, 'doc1', 1L, ['test', 'foo']])
2594
2595def testUnkownIndex(env):
2596    env.skipOnCluster() # todo: remove once fix on coordinator
2597    env.expect('ft.aggregate').error()
2598    env.expect('ft.aggregate', 'idx', '*').error()
2599    env.expect('ft.aggregate', 'idx', '*', 'WITHCURSOR').error()
2600
2601def testExplainError(env):
2602    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2603    env.expect('FT.EXPLAIN', 'idx', '(').error()
2604
2605def testBadCursor(env):
2606    env.expect('FT.CURSOR', 'READ', 'idx').error()
2607    env.expect('FT.CURSOR', 'READ', 'idx', '1111').error()
2608    env.expect('FT.CURSOR', 'READ', 'idx', 'bad').error()
2609    env.expect('FT.CURSOR', 'DROP', 'idx', '1111').error()
2610    env.expect('FT.CURSOR', 'bad', 'idx', '1111').error()
2611
2612def testGroupByWithApplyError(env):
2613    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2614    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2615    err = env.cmd('FT.AGGREGATE', 'idx', '*', 'APPLY', 'split()', 'GROUPBY', '1', '@test', 'REDUCE', 'COUNT', '0', 'AS', 'count')[1]
2616    assertEqualIgnoreCluster(env, str(err[0]), 'Invalid number of arguments for split')
2617
2618def testSubStrErrors(env):
2619    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2620    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2621
2622    err = env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'matched_terms()', 'as', 'a', 'APPLY', 'substr(@a,0,4)')[1]
2623    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2624
2625    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test",3,-2)', 'as', 'a')
2626    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test",3,1000)', 'as', 'a')
2627    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test",-1,2)', 'as', 'a')
2628    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test")', 'as', 'a')
2629    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr(1)', 'as', 'a')
2630    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test", "test")', 'as', 'a')
2631    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test", "test", "test")', 'as', 'a')
2632    env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test2', 'APPLY', 'substr("test", "-1", "-1")', 'as', 'a')
2633    env.assertTrue(env.isUp())
2634
2635def testToUpperLower(env):
2636    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2637    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2638    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'lower(@test)', 'as', 'a').equal([1L, ['test', 'foo', 'a', 'foo']])
2639    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'lower("FOO")', 'as', 'a').equal([1L, ['test', 'foo', 'a', 'foo']])
2640    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'upper(@test)', 'as', 'a').equal([1L, ['test', 'foo', 'a', 'FOO']])
2641    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'upper("foo")', 'as', 'a').equal([1L, ['test', 'foo', 'a', 'FOO']])
2642
2643    err = env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'upper()', 'as', 'a')[1]
2644    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2645    err = env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'lower()', 'as', 'a')[1]
2646    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2647
2648    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'upper(1)', 'as', 'a').equal([1L, ['test', 'foo', 'a', None]])
2649    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'lower(1)', 'as', 'a').equal([1L, ['test', 'foo', 'a', None]])
2650
2651    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2652    err = env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'upper(1,2)', 'as', 'a')[1]
2653    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2654    err = env.cmd('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'lower(1,2)', 'as', 'a')[1]
2655    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2656
2657def testMatchedTerms(env):
2658    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2659    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2660    env.expect('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'matched_terms()', 'as', 'a').equal([1L, ['test', 'foo', 'a', None]])
2661    env.expect('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'matched_terms()', 'as', 'a').equal([1L, ['test', 'foo', 'a', ['foo']]])
2662    env.expect('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'matched_terms(100)', 'as', 'a').equal([1L, ['test', 'foo', 'a', ['foo']]])
2663    env.expect('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'matched_terms(-100)', 'as', 'a').equal([1L, ['test', 'foo', 'a', ['foo']]])
2664    env.expect('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'matched_terms("test")', 'as', 'a').equal([1L, ['test', 'foo', 'a', ['foo']]])
2665
2666def testStrFormatError(env):
2667    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'TEXT').equal('OK')
2668    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', 'foo').equal('OK')
2669    err = env.cmd('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'format()', 'as', 'a')[1]
2670    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2671
2672    err = env.cmd('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'format("%s")', 'as', 'a')[1]
2673    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2674
2675    err = env.cmd('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'format("%", "test")', 'as', 'a')[1]
2676    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2677
2678    err = env.cmd('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'format("%b", "test")', 'as', 'a')[1]
2679    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2680
2681    err = env.cmd('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'format(5)', 'as', 'a')[1]
2682    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2683
2684    env.expect('ft.aggregate', 'idx', 'foo', 'LOAD', '1', '@test', 'APPLY', 'upper(1)', 'as', 'b', 'APPLY', 'format("%s", @b)', 'as', 'a').equal([1L, ['test', 'foo', 'b', None, 'a', '(null)']])
2685
2686    # working example
2687    env.expect('ft.aggregate', 'idx', 'foo', 'APPLY', 'format("%%s-test", "test")', 'as', 'a').equal([1L, ['a', '%s-test']])
2688    env.expect('ft.aggregate', 'idx', 'foo', 'APPLY', 'format("%s-test", "test")', 'as', 'a').equal([1L, ['a', 'test-test']])
2689
2690def testTimeFormatError(env):
2691    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC').equal('OK')
2692    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '12234556').equal('OK')
2693
2694    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt()', 'as', 'a')[1]
2695    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2696
2697    if not env.isCluster(): # todo: remove once fix on coordinator
2698        env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt(@test1)', 'as', 'a').error()
2699
2700    env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt(@test)', 'as', 'a')
2701
2702    env.assertTrue(env.isUp())
2703
2704    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt(@test, 4)', 'as', 'a')[1]
2705    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2706
2707    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt("awfawf")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2708
2709    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt(235325153152356426246246246254)', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2710
2711    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'timefmt(@test, "%s")' % ('d' * 2048), 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2712
2713    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'hour("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2714    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'minute("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2715    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'day("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2716    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'month("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2717    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'dayofweek("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2718    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'dayofmonth("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2719    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'dayofyear("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2720    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'year("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2721    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'monthofyear("not_number")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2722
2723def testMonthOfYear(env):
2724    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC').equal('OK')
2725    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '12234556').equal('OK')
2726
2727    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'monthofyear(@test)', 'as', 'a').equal([1L, ['test', '12234556', 'a', '4']])
2728
2729    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'monthofyear(@test, 112)', 'as', 'a')[1]
2730    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2731
2732    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'monthofyear()', 'as', 'a')[1]
2733    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2734
2735    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'monthofyear("bad")', 'as', 'a').equal([1L, ['test', '12234556', 'a', None]])
2736
2737def testParseTime(env):
2738    conn = getConnectionByEnv(env)
2739    conn.execute_command('FT.CREATE', 'idx', 'SCHEMA', 'test', 'TAG')
2740    conn.execute_command('HSET', 'doc1', 'test', '20210401')
2741
2742    # check for errors
2743    err = conn.execute_command('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'parsetime()', 'as', 'a')[1]
2744    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2745
2746    err = conn.execute_command('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'parsetime(11)', 'as', 'a')[1]
2747    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2748
2749    err = conn.execute_command('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'parsetime(11,22)', 'as', 'a')[1]
2750    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2751
2752    # valid test
2753    res = conn.execute_command('ft.aggregate', 'idx', '*', 'LOAD', '1', '@test', 'APPLY', 'parsetime(@test, "%Y%m%d")', 'as', 'a')
2754    assertEqualIgnoreCluster(env, res, [1L, ['test', '20210401', 'a', '1617235200']])
2755
2756def testMathFunctions(env):
2757    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC').equal('OK')
2758    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '12234556').equal('OK')
2759
2760    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'exp(@test)', 'as', 'a').equal([1L, ['test', '12234556', 'a', 'inf']])
2761    env.expect('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'ceil(@test)', 'as', 'a').equal([1L, ['test', '12234556', 'a', '12234556']])
2762
2763def testErrorOnOpperation(env):
2764    env.expect('FT.CREATE', 'idx', 'ON', 'HASH', 'SCHEMA', 'test', 'NUMERIC').equal('OK')
2765    env.expect('ft.add', 'idx', 'doc1', '1.0', 'FIELDS', 'test', '12234556').equal('OK')
2766
2767    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', '1 + split()', 'as', 'a')[1]
2768    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2769
2770    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'split() + 1', 'as', 'a')[1]
2771    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2772
2773    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', '"bad" + "bad"', 'as', 'a')[1]
2774    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2775
2776    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', 'split("bad" + "bad")', 'as', 'a')[1]
2777    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2778
2779    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'LOAD', '1', '@test', 'APPLY', '!(split("bad" + "bad"))', 'as', 'a')[1]
2780    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2781
2782    err = env.cmd('ft.aggregate', 'idx', '@test:[0..inf]', 'APPLY', '!@test', 'as', 'a')[1]
2783    assertEqualIgnoreCluster(env, type(err[0]), redis.exceptions.ResponseError)
2784
2785
2786def testSortkeyUnsortable(env):
2787    env.cmd('ft.create', 'idx', 'ON', 'HASH', 'schema', 'test', 'text')
2788    env.cmd('ft.add', 'idx', 'doc1', 1, 'fields', 'test', 'foo')
2789    rv = env.cmd('ft.aggregate', 'idx', 'foo', 'withsortkeys',
2790        'load', '1', '@test',
2791        'sortby', '1', '@test')
2792    env.assertEqual([1, '$foo', ['test', 'foo']], rv)
2793
2794
2795def testIssue919(env):
2796    # This only works if the missing field has a lower sortable index
2797    # than the present field..
2798    env.cmd('ft.create', 'idx', 'ON', 'HASH',
2799            'schema', 't1', 'text', 'sortable', 'n1', 'numeric', 'sortable')
2800    env.cmd('ft.add', 'idx', 'doc1', 1, 'fields', 'n1', 42)
2801    rv = env.cmd('ft.search', 'idx', '*', 'sortby', 't1', 'desc')
2802    env.assertEqual([1L, 'doc1', ['n1', '42']], rv)
2803
2804
2805def testIssue1074(env):
2806    # Ensure that sortable fields are returned in their string form from the
2807    # document
2808    env.cmd('ft.create', 'idx', 'ON', 'HASH',
2809            'schema', 't1', 'text', 'n1', 'numeric', 'sortable')
2810    env.cmd('ft.add', 'idx', 'doc1', 1, 'fields', 't1', 'hello', 'n1', 1581011976800)
2811    rv = env.cmd('ft.search', 'idx', '*', 'sortby', 'n1')
2812    env.assertEqual([1L, 'doc1', ['n1', '1581011976800', 't1', 'hello']], rv)
2813
2814def testIssue1085(env):
2815    env.skipOnCluster()
2816    env.cmd('FT.CREATE issue1085 ON HASH SCHEMA foo TEXT SORTABLE bar NUMERIC SORTABLE')
2817    for i in range(1, 10):
2818        env.cmd('FT.ADD issue1085 document_%d 1 REPLACE FIELDS foo foo%d bar %d' % (i, i, i))
2819    res = env.cmd('FT.SEARCH', 'issue1085', '@bar:[8 8]')
2820    env.assertEqual(toSortedFlatList(res), toSortedFlatList([1L, 'document_8', ['foo', 'foo8', 'bar', '8']]))
2821
2822    for i in range(1, 10):
2823        env.cmd('FT.ADD issue1085 document_8 1 REPLACE FIELDS foo foo8 bar 8')
2824
2825    forceInvokeGC(env, 'issue1085')
2826
2827    res = env.cmd('FT.SEARCH', 'issue1085', '@bar:[8 8]')
2828    env.assertEqual(toSortedFlatList(res), toSortedFlatList([1, 'document_8', ['foo', 'foo8', 'bar', '8']]))
2829
2830
2831def grouper(iterable, n, fillvalue=None):
2832    "Collect data into fixed-length chunks or blocks"
2833    from itertools import izip_longest
2834    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
2835    args = [iter(iterable)] * n
2836    return izip_longest(fillvalue=fillvalue, *args)
2837
2838
2839def to_dict(r):
2840    return {r[i]: r[i + 1] for i in range(0, len(r), 2)}
2841
2842def testInfoError(env):
2843    env.expect('ft.info', 'no_idx').error()
2844
2845def testIndexNotRemovedFromCursorListAfterRecreated(env):
2846    env.expect('FT.CREATE idx ON HASH SCHEMA f1 TEXT').ok()
2847    env.expect('FT.AGGREGATE idx * WITHCURSOR').equal([[0], 0])
2848    env.expect('FT.CREATE idx ON HASH SCHEMA f1 TEXT').error()
2849    env.expect('FT.AGGREGATE idx * WITHCURSOR').equal([[0], 0])
2850
2851def testHindiStemmer(env):
2852    env.cmd('FT.CREATE', 'idxTest', 'LANGUAGE_FIELD', '__language', 'SCHEMA', 'body', 'TEXT')
2853    env.cmd('FT.ADD', 'idxTest', 'doc1', 1.0, 'LANGUAGE', 'hindi', 'FIELDS', 'body', u'अँगरेजी अँगरेजों अँगरेज़')
2854    res = env.cmd('FT.SEARCH', 'idxTest', u'अँगरेज़')
2855    res1 = {res[2][i]:res[2][i + 1] for i in range(0, len(res[2]), 2)}
2856    env.assertEqual(u'अँगरेजी अँगरेजों अँगरेज़', unicode(res1['body'], 'utf-8'))
2857
2858def testMOD507(env):
2859    env.skipOnCluster()
2860    env.expect('ft.create idx ON HASH SCHEMA t1 TEXT').ok()
2861
2862    for i in range(50):
2863        env.expect('ft.add idx doc-%d 1.0 FIELDS t1 foo' % i).ok()
2864
2865    for i in range(50):
2866        env.expect('del doc-%d' % i).equal(1)
2867
2868    res = env.cmd('FT.SEARCH', 'idx', '*', 'WITHSCORES', 'SUMMARIZE', 'FRAGS', '1', 'LEN', '25', 'HIGHLIGHT', 'TAGS', "<span style='background-color:yellow'>", "</span>")
2869
2870    # from redisearch 2.0, docs are removed from index when `DEL` is called
2871    env.assertEqual(len(res), 1)
2872
2873def testUnseportedSortableTypeErrorOnTags(env):
2874    env.skipOnCluster()
2875    env.expect('FT.CREATE idx ON HASH SCHEMA f1 TEXT SORTABLE f2 NUMERIC SORTABLE NOINDEX f3 TAG SORTABLE NOINDEX f4 TEXT SORTABLE NOINDEX').ok()
2876    env.expect('FT.ADD idx doc1 1.0 FIELDS f1 foo1 f2 1 f3 foo1 f4 foo1').ok()
2877    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL FIELDS f2 2 f3 foo2 f4 foo2').ok()
2878    res = env.cmd('HGETALL doc1')
2879    env.assertEqual(toSortedFlatList(res), toSortedFlatList(['f1', 'foo1', 'f2', '2', 'f3', 'foo2', 'f4', 'foo2', '__score', '1.0']))
2880    res = env.cmd('FT.SEARCH idx *')
2881    env.assertEqual(toSortedFlatList(res), toSortedFlatList([1L, 'doc1', ['f1', 'foo1', 'f2', '2', 'f3', 'foo2', 'f4', 'foo2']]))
2882
2883
2884def testIssue1158(env):
2885    env.cmd('FT.CREATE idx ON HASH SCHEMA txt1 TEXT txt2 TEXT txt3 TEXT')
2886
2887    env.cmd('FT.ADD idx doc1 1.0 FIELDS txt1 10 txt2 num1')
2888    res = env.cmd('FT.GET idx doc1')
2889    env.assertEqual(toSortedFlatList(res), toSortedFlatList(['txt1', '10', 'txt2', 'num1']))
2890
2891    # only 1st checked (2nd returns an error)
2892    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if @txt1||to_number(@txt2)<5 FIELDS txt1 5').equal('OK')
2893    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if @txt3&&to_number(@txt2)<5 FIELDS txt1 5').equal('NOADD')
2894
2895    # both are checked
2896    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if to_number(@txt1)>11||to_number(@txt1)>42 FIELDS txt2 num2').equal('NOADD')
2897    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if to_number(@txt1)>11||to_number(@txt1)<42 FIELDS txt2 num2').equal('OK')
2898    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if to_number(@txt1)>11&&to_number(@txt1)>42 FIELDS txt2 num2').equal('NOADD')
2899    env.expect('FT.ADD idx doc1 1.0 REPLACE PARTIAL if to_number(@txt1)>11&&to_number(@txt1)<42 FIELDS txt2 num2').equal('NOADD')
2900    res = env.cmd('FT.GET idx doc1')
2901    env.assertEqual(toSortedFlatList(res), toSortedFlatList(['txt1', '5', 'txt2', 'num2']))
2902
2903def testIssue1159(env):
2904    env.cmd('FT.CREATE idx ON HASH SCHEMA f1 TAG')
2905    for i in range(1000):
2906        env.cmd('FT.add idx doc%d 1.0 FIELDS f1 foo' % i)
2907
2908def testIssue1169(env):
2909    env.cmd('FT.CREATE idx ON HASH SCHEMA txt1 TEXT txt2 TEXT')
2910    env.cmd('FT.ADD idx doc1 1.0 FIELDS txt1 foo')
2911
2912    env.expect('FT.AGGREGATE idx foo GROUPBY 1 @txt1 REDUCE FIRST_VALUE 1 @txt2 as test').equal([1L, ['txt1', 'foo', 'test', None]])
2913
2914def testIssue1184(env):
2915    env.skipOnCluster()
2916
2917    field_types = ['TEXT', 'NUMERIC', 'TAG']
2918    env.assertOk(env.execute_command('ft.config', 'set', 'FORK_GC_CLEAN_THRESHOLD', 0))
2919    for ft in field_types:
2920        env.assertOk(env.execute_command('FT.CREATE idx ON HASH SCHEMA  field ' + ft))
2921
2922        res = env.execute_command('ft.info', 'idx')
2923        d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
2924        env.assertEqual(d['inverted_sz_mb'], '0')
2925        env.assertEqual(d['num_records'], '0')
2926
2927
2928        value = '42'
2929        env.assertOk(env.execute_command('FT.ADD idx doc0 1 FIELD field ' + value))
2930        doc = env.cmd('FT.SEARCH idx *')
2931        env.assertEqual(doc, [1L, 'doc0', ['field', value]])
2932
2933        res = env.execute_command('ft.info', 'idx')
2934        d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
2935        env.assertGreater(d['inverted_sz_mb'], '0')
2936        env.assertEqual(d['num_records'], '1')
2937
2938        env.assertEqual(env.execute_command('FT.DEL idx doc0'), 1)
2939
2940        forceInvokeGC(env, 'idx')
2941
2942        res = env.execute_command('ft.info', 'idx')
2943        d = {res[i]: res[i + 1] for i in range(0, len(res), 2)}
2944        env.assertEqual(d['inverted_sz_mb'], '0')
2945        env.assertEqual(d['num_records'], '0')
2946
2947        env.cmd('FT.DROP idx')
2948        env.cmd('DEL doc0')
2949
2950def testIndexListCommand(env):
2951    env.expect('FT.CREATE idx1 ON HASH SCHEMA n NUMERIC').ok()
2952    env.expect('FT.CREATE idx2 ON HASH SCHEMA n NUMERIC').ok()
2953    res = env.cmd('FT._LIST')
2954    env.assertEqual(set(res), set(['idx1', 'idx2']))
2955    env.expect('FT.DROP idx1').ok()
2956    env.expect('FT._LIST').equal(['idx2'])
2957    env.expect('FT.CREATE idx3 ON HASH SCHEMA n NUMERIC').ok()
2958    res = env.cmd('FT._LIST')
2959    env.assertEqual(set(res), set(['idx2', 'idx3']))
2960
2961
2962def testIssue1208(env):
2963    env.cmd('FT.CREATE idx ON HASH SCHEMA n NUMERIC')
2964    env.cmd('FT.ADD idx doc1 1 FIELDS n 1.0321e5')
2965    env.cmd('FT.ADD idx doc2 1 FIELDS n 101.11')
2966    env.cmd('FT.ADD idx doc3 1 FIELDS n 0.0011')
2967    env.expect('FT.SEARCH', 'idx', '@n:[1.1432E3 inf]').equal([1L, 'doc1', ['n', '1.0321e5']])
2968    env.expect('FT.SEARCH', 'idx', '@n:[-1.12E-3 1.12E-1]').equal([1L, 'doc3', ['n', '0.0011']])
2969    res = [3L, 'doc3', ['n', '0.0011'], 'doc2', ['n', '101.11'], 'doc1', ['n', '1.0321e5']]
2970    env.expect('FT.SEARCH', 'idx', '@n:[-inf inf]').equal(res)
2971
2972    env.expect('FT.ADD idx doc3 1 REPLACE PARTIAL IF @n>42e3 FIELDS n 100').equal('NOADD')
2973    env.expect('FT.ADD idx doc3 1 REPLACE PARTIAL IF @n<42e3 FIELDS n 100').ok()
2974    # print env.cmd('FT.SEARCH', 'idx', '@n:[-inf inf]')
2975
2976def testFieldsCaseSensetive(env):
2977    # this test will not pass on coordinator coorently as if one shard return empty results coordinator
2978    # will not reflect the errors
2979    env.skipOnCluster()
2980    conn = getConnectionByEnv(env)
2981    env.cmd('FT.CREATE idx ON HASH SCHEMA n NUMERIC f TEXT t TAG g GEO')
2982
2983    # make sure text fields are case sesitive
2984    conn.execute_command('hset', 'doc1', 'F', 'test')
2985    conn.execute_command('hset', 'doc2', 'f', 'test')
2986    env.expect('ft.search idx @f:test').equal([1L, 'doc2', ['f', 'test']])
2987    env.expect('ft.search idx @F:test').equal([0])
2988
2989    # make sure numeric fields are case sesitive
2990    conn.execute_command('hset', 'doc3', 'N', '1.0')
2991    conn.execute_command('hset', 'doc4', 'n', '1.0')
2992    env.expect('ft.search', 'idx', '@n:[0 2]').equal([1L, 'doc4', ['n', '1.0']])
2993    env.expect('ft.search', 'idx', '@N:[0 2]').equal([0])
2994
2995    # make sure tag fields are case sesitive
2996    conn.execute_command('hset', 'doc5', 'T', 'tag')
2997    conn.execute_command('hset', 'doc6', 't', 'tag')
2998    env.expect('ft.search', 'idx', '@t:{tag}').equal([1L, 'doc6', ['t', 'tag']])
2999    env.expect('ft.search', 'idx', '@T:{tag}').equal([0])
3000
3001    # make sure geo fields are case sesitive
3002    conn.execute_command('hset', 'doc8', 'G', '-113.524,53.5244')
3003    conn.execute_command('hset', 'doc9', 'g', '-113.524,53.5244')
3004    env.expect('ft.search', 'idx', '@g:[-113.52 53.52 20 mi]').equal([1L, 'doc9', ['g', '-113.524,53.5244']])
3005    env.expect('ft.search', 'idx', '@G:[-113.52 53.52 20 mi]').equal([0])
3006
3007    # make sure search filter are case sensitive
3008    env.expect('ft.search', 'idx', '@n:[0 2]', 'FILTER', 'n', 0, 2).equal([1L, 'doc4', ['n', '1.0']])
3009    env.expect('ft.search', 'idx', '@n:[0 2]', 'FILTER', 'N', 0, 2).equal([0])
3010
3011    # make sure RETURN are case sensitive
3012    env.expect('ft.search', 'idx', '@n:[0 2]', 'RETURN', '1', 'n').equal([1L, 'doc4', ['n', '1']])
3013    env.expect('ft.search', 'idx', '@n:[0 2]', 'RETURN', '1', 'N').equal([1L, 'doc4', []])
3014
3015    # make sure SORTBY are case sensitive
3016    conn.execute_command('hset', 'doc7', 'n', '1.1')
3017    env.expect('ft.search', 'idx', '@n:[0 2]', 'SORTBY', 'n').equal([2L, 'doc4', ['n', '1.0'], 'doc7', ['n', '1.1']])
3018    env.expect('ft.search', 'idx', '@n:[0 2]', 'SORTBY', 'N').error().contains('not loaded nor in schema')
3019
3020    # make sure aggregation load are case sensitive
3021    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n').equal([1L, ['n', '1'], ['n', '1.1']])
3022    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@N').equal([1L, [], []])
3023
3024    # make sure aggregation apply are case sensitive
3025    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'apply', '@n', 'as', 'r').equal([1L, ['n', '1', 'r', '1'], ['n', '1.1', 'r', '1.1']])
3026    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'apply', '@N', 'as', 'r').error().contains('not loaded in pipeline')
3027
3028    # make sure aggregation filter are case sensitive
3029    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'filter', '@n==1.0').equal([1L, ['n', '1']])
3030    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'filter', '@N==1.0').error().contains('not loaded in pipeline')
3031
3032    # make sure aggregation groupby are case sensitive
3033    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'groupby', '1', '@n', 'reduce', 'count', 0, 'as', 'count').equal([2L, ['n', '1', 'count', '1'], ['n', '1.1', 'count', '1']])
3034    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'groupby', '1', '@N', 'reduce', 'count', 0, 'as', 'count').error().contains('No such property')
3035
3036    # make sure aggregation sortby are case sensitive
3037    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'sortby', '1', '@n').equal([2L, ['n', '1'], ['n', '1.1']])
3038    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'LOAD', '1', '@n', 'sortby', '1', '@N').error().contains('not loaded')
3039
3040def testSortedFieldsCaseSensetive(env):
3041    # this test will not pass on coordinator coorently as if one shard return empty results coordinator
3042    # will not reflect the errors
3043    env.skipOnCluster()
3044    conn = getConnectionByEnv(env)
3045    env.cmd('FT.CREATE idx ON HASH SCHEMA n NUMERIC SORTABLE f TEXT SORTABLE t TAG SORTABLE g GEO SORTABLE')
3046
3047    # make sure text fields are case sesitive
3048    conn.execute_command('hset', 'doc1', 'F', 'test')
3049    conn.execute_command('hset', 'doc2', 'f', 'test')
3050    env.expect('ft.search idx @f:test').equal([1L, 'doc2', ['f', 'test']])
3051    env.expect('ft.search idx @F:test').equal([0])
3052
3053    # make sure numeric fields are case sesitive
3054    conn.execute_command('hset', 'doc3', 'N', '1.0')
3055    conn.execute_command('hset', 'doc4', 'n', '1.0')
3056    env.expect('ft.search', 'idx', '@n:[0 2]').equal([1L, 'doc4', ['n', '1.0']])
3057    env.expect('ft.search', 'idx', '@N:[0 2]').equal([0])
3058
3059    # make sure tag fields are case sesitive
3060    conn.execute_command('hset', 'doc5', 'T', 'tag')
3061    conn.execute_command('hset', 'doc6', 't', 'tag')
3062    env.expect('ft.search', 'idx', '@t:{tag}').equal([1L, 'doc6', ['t', 'tag']])
3063    env.expect('ft.search', 'idx', '@T:{tag}').equal([0])
3064
3065    # make sure geo fields are case sesitive
3066    conn.execute_command('hset', 'doc8', 'G', '-113.524,53.5244')
3067    conn.execute_command('hset', 'doc9', 'g', '-113.524,53.5244')
3068    env.expect('ft.search', 'idx', '@g:[-113.52 53.52 20 mi]').equal([1L, 'doc9', ['g', '-113.524,53.5244']])
3069    env.expect('ft.search', 'idx', '@G:[-113.52 53.52 20 mi]').equal([0])
3070
3071    # make sure search filter are case sensitive
3072    env.expect('ft.search', 'idx', '@n:[0 2]', 'FILTER', 'n', 0, 2).equal([1L, 'doc4', ['n', '1.0']])
3073    env.expect('ft.search', 'idx', '@n:[0 2]', 'FILTER', 'N', 0, 2).equal([0])
3074
3075    # make sure RETURN are case sensitive
3076    env.expect('ft.search', 'idx', '@n:[0 2]', 'RETURN', '1', 'n').equal([1L, 'doc4', ['n', '1']])
3077    env.expect('ft.search', 'idx', '@n:[0 2]', 'RETURN', '1', 'N').equal([1L, 'doc4', []])
3078
3079    # make sure SORTBY are case sensitive
3080    conn.execute_command('hset', 'doc7', 'n', '1.1')
3081    env.expect('ft.search', 'idx', '@n:[0 2]', 'SORTBY', 'n').equal([2L, 'doc4', ['n', '1.0'], 'doc7', ['n', '1.1']])
3082    env.expect('ft.search', 'idx', '@n:[0 2]', 'SORTBY', 'N').error().contains('not loaded nor in schema')
3083
3084    # make sure aggregation apply are case sensitive
3085    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'apply', '@n', 'as', 'r').equal([1L, ['n', '1', 'r', '1'], ['n', '1.1', 'r', '1.1']])
3086    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'apply', '@N', 'as', 'r').error().contains('not loaded in pipeline')
3087
3088    # make sure aggregation filter are case sensitive
3089    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'filter', '@n==1.0').equal([1L, ['n', '1']])
3090    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'filter', '@N==1.0').error().contains('not loaded in pipeline')
3091
3092    # make sure aggregation groupby are case sensitive
3093    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'groupby', '1', '@n', 'reduce', 'count', 0, 'as', 'count').equal([2L, ['n', '1', 'count', '1'], ['n', '1.1', 'count', '1']])
3094    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'groupby', '1', '@N', 'reduce', 'count', 0, 'as', 'count').error().contains('No such property')
3095
3096    # make sure aggregation sortby are case sensitive
3097    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'sortby', '1', '@n').equal([2L, ['n', '1'], ['n', '1.1']])
3098    env.expect('ft.aggregate', 'idx', '@n:[0 2]', 'sortby', '1', '@N').error().contains('not loaded')
3099
3100def testScoreLangPayloadAreReturnedIfCaseNotMatchToSpecialFields(env):
3101    conn = getConnectionByEnv(env)
3102    env.cmd('FT.CREATE idx ON HASH SCHEMA n NUMERIC SORTABLE')
3103    conn.execute_command('hset', 'doc1', 'n', '1.0', '__Language', 'eng', '__Score', '1', '__Payload', '10')
3104    res = env.cmd('ft.search', 'idx', '@n:[0 2]')
3105    env.assertEqual(toSortedFlatList(res), toSortedFlatList([1L, 'doc1', ['n', '1.0', '__Language', 'eng', '__Score', '1', '__Payload', '10']]))
3106
3107def testReturnSameFieldDifferentCase(env):
3108    conn = getConnectionByEnv(env)
3109    env.cmd('FT.CREATE idx ON HASH SCHEMA n NUMERIC SORTABLE N NUMERIC SORTABLE')
3110    conn.execute_command('hset', 'doc1', 'n', '1.0', 'N', '2.0')
3111    env.expect('ft.search', 'idx', '@n:[0 2]', 'RETURN', '2', 'n', 'N').equal([1L, 'doc1', ['n', '1', 'N', '2']])
3112
3113def testCreateIfNX(env):
3114    env.expect('FT._CREATEIFNX idx ON HASH SCHEMA n NUMERIC SORTABLE N NUMERIC SORTABLE').ok()
3115    env.expect('FT._CREATEIFNX idx ON HASH SCHEMA n NUMERIC SORTABLE N NUMERIC SORTABLE').ok()
3116
3117def testDropIfX(env):
3118    env.expect('FT._DROPIFX idx').ok()
3119
3120def testDeleteIfX(env):
3121    env.expect('FT._DROPINDEXIFX idx').ok()
3122
3123def testAlterIfNX(env):
3124    env.expect('FT.CREATE idx ON HASH SCHEMA n NUMERIC').ok()
3125    env.expect('FT._ALTERIFNX idx SCHEMA ADD n1 NUMERIC').ok()
3126    env.expect('FT._ALTERIFNX idx SCHEMA ADD n1 NUMERIC').ok()
3127    res = env.cmd('ft.info idx')
3128    res = {res[i]: res[i + 1] for i in range(0, len(res), 2)}['fields']
3129    env.assertEqual(res, [['n', 'type', 'NUMERIC'], ['n1', 'type', 'NUMERIC']])
3130
3131def testAliasAddIfNX(env):
3132    env.expect('FT.CREATE idx ON HASH SCHEMA n NUMERIC').ok()
3133    env.expect('FT._ALIASADDIFNX a1 idx').ok()
3134    env.expect('FT._ALIASADDIFNX a1 idx').ok()
3135
3136def testAliasDelIfX(env):
3137    env.expect('FT._ALIASDELIFX a1').ok()
3138
3139def testEmptyDoc(env):
3140    conn = getConnectionByEnv(env)
3141    env.expect('FT.CREATE idx SCHEMA t TEXT').ok()
3142    env.expect('FT.ADD idx doc1 1 FIELDS t foo').ok()
3143    env.expect('FT.ADD idx doc2 1 FIELDS t foo').ok()
3144    env.expect('FT.ADD idx doc3 1 FIELDS t foo').ok()
3145    env.expect('FT.ADD idx doc4 1 FIELDS t foo').ok()
3146    env.expect('FT.SEARCH idx * limit 0 0').equal([4])
3147    conn.execute_command('DEL', 'doc1')
3148    conn.execute_command('DEL', 'doc3')
3149    env.expect('FT.SEARCH idx *').equal([2L, 'doc4', ['t', 'foo'], 'doc2', ['t', 'foo']])
3150
3151def testRED47209(env):
3152    conn = getConnectionByEnv(env)
3153    env.expect('FT.CREATE idx SCHEMA t TEXT').ok()
3154    conn.execute_command('hset', 'doc1', 't', 'foo')
3155    if env.isCluster():
3156        # on cluster we have WITHSCORES set unconditionally for FT.SEARCH
3157        res = [1L, 'doc1', ['t', 'foo']]
3158    else:
3159        res = [1L, 'doc1', None, ['t', 'foo']]
3160    env.expect('FT.SEARCH idx foo WITHSORTKEYS LIMIT 0 1').equal(res)
3161
3162def testInvertedIndexWasEntirelyDeletedDuringCursor():
3163    env = Env(moduleArgs='GC_POLICY FORK FORK_GC_CLEAN_THRESHOLD 1')
3164
3165    env.skipOnCluster()
3166
3167    env.expect('FT.CREATE idx SCHEMA t TEXT').ok()
3168    env.expect('HSET doc1 t foo').equal(1)
3169    env.expect('HSET doc2 t foo').equal(1)
3170
3171    res, cursor = env.cmd('FT.AGGREGATE idx foo WITHCURSOR COUNT 1')
3172    env.assertEqual(res, [1L, []])
3173
3174    # delete both documents and run the GC to clean 'foo' inverted index
3175    env.expect('DEL doc1').equal(1)
3176    env.expect('DEL doc2').equal(1)
3177
3178    forceInvokeGC(env, 'idx')
3179
3180    # make sure the inverted index was cleaned
3181    env.expect('FT.DEBUG DUMP_INVIDX idx foo').error().contains('not find the inverted index')
3182
3183    # read from the cursor
3184    res, cursor = env.cmd('FT.CURSOR READ idx %d' % cursor)
3185
3186    env.assertEqual(res, [0L])
3187    env.assertEqual(cursor, 0)
3188
3189def testNegativeOnly(env):
3190    conn = getConnectionByEnv(env)
3191    env.expect('FT.CREATE idx SCHEMA t TEXT').ok()
3192    conn.execute_command('HSET', 'doc1', 'not', 'foo')
3193
3194    env.expect('FT.SEARCH idx *').equal([1L, 'doc1', ['not', 'foo']])
3195    env.expect('FT.SEARCH', 'idx', '-bar').equal([1L, 'doc1', ['not', 'foo']])
3196
3197def testNotOnly(env):
3198  conn = getConnectionByEnv(env)
3199  conn.execute_command('FT.CREATE', 'idx', 'SCHEMA', 'txt1', 'TEXT')
3200  conn.execute_command('HSET', 'a', 'txt1', 'hello', 'txt2', 'world')
3201  conn.execute_command('HSET', 'b', 'txt1', 'world', 'txt2', 'hello')
3202  env.expect('ft.search idx !world').equal([1L, 'b', ['txt1', 'world', 'txt2', 'hello']])
3203
3204def testServerVersion(env):
3205    env.assertTrue(server_version_at_least(env, "6.0.0"))
3206
3207def testMod1407(env):
3208    conn = getConnectionByEnv(env)
3209
3210    env.expect('FT.CREATE', 'idx', 'SCHEMA', 'limit', 'TEXT', 'LimitationTypeID', 'TAG', 'LimitationTypeDesc', 'TEXT').ok()
3211
3212    conn.execute_command('HSET', 'doc1', 'limit', 'foo1', 'LimitationTypeID', 'boo1', 'LimitationTypeDesc', 'doo1')
3213    conn.execute_command('HSET', 'doc2', 'limit', 'foo2', 'LimitationTypeID', 'boo2', 'LimitationTypeDesc', 'doo2')
3214
3215    env.expect('FT.AGGREGATE', 'idx', '*', 'SORTBY', '3', '@limit', '@LimitationTypeID', 'ASC').equal([2L, ['limit', 'foo1', 'LimitationTypeID', 'boo1'], ['limit', 'foo2', 'LimitationTypeID', 'boo2']])
3216
3217    # make sure the crashed query is not crashing anymore
3218    env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '2', 'LLimitationTypeID', 'LLimitationTypeDesc', 'REDUCE', 'COUNT', '0')
3219
3220    # make sure correct query not crashing and return the right results
3221    env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '2', '@LimitationTypeID', '@LimitationTypeDesc', 'REDUCE', 'COUNT', '0').equal([2L, ['LimitationTypeID', 'boo2', 'LimitationTypeDesc', 'doo2', '__generated_aliascount', '1'], ['LimitationTypeID', 'boo1', 'LimitationTypeDesc', 'doo1', '__generated_aliascount', '1']])
3222
3223def testMod1452(env):
3224    if not env.isCluster():
3225        # this test is only relevant on cluster
3226        env.skip()
3227
3228    conn = getConnectionByEnv(env)
3229
3230    env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
3231
3232    conn.execute_command('HSET', 'doc1', 't', 'foo')
3233
3234    # here we only check that its not crashing
3235    env.expect('FT.AGGREGATE', 'idx', '*', 'GROUPBY', '1', 'foo', 'REDUCE', 'FIRST_VALUE', 3, '@not_exists', 'BY', '@foo')
3236
3237def test_empty_field_name(env):
3238    conn = getConnectionByEnv(env)
3239
3240    env.expect('FT.CREATE', 'idx', 'SCHEMA', '', 'TEXT').ok()
3241    conn.execute_command('hset', 'doc1', '', 'foo')
3242    env.expect('FT.SEARCH', 'idx', 'foo').equal([1L, 'doc1', ['', 'foo']])
3243