1 /* Copyright (c) 2000-2008 MySQL AB, 2009 Sun Microsystems, Inc.
2 Use is subject to license terms.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; version 2 of the License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 GNU General Public License for more details.
12
13 You should have received a copy of the GNU General Public License
14 along with this program; if not, write to the Free Software
15 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
16
17 /* Written by Sergei A. Golubchik, who has a shared copyright to this code */
18
19 #include "ftdefs.h"
20
21 ulong ft_min_word_len= 4;
22 ulong ft_max_word_len= HA_FT_MAXCHARLEN;
23 ulong ft_query_expansion_limit= 5;
24 const char *ft_boolean_syntax= DEFAULT_FTB_SYNTAX;
25
26 const HA_KEYSEG ft_keysegs[FT_SEGS]= {
27 {
28 0, /* charset */
29 HA_FT_WLEN, /* start */
30 0, /* null_pos */
31 0, /* Bit pos */
32 HA_VAR_LENGTH_PART | HA_PACK_KEY, /* flag */
33 HA_FT_MAXBYTELEN, /* length */
34 63, /* language (will be overwritten) */
35 HA_KEYTYPE_VARTEXT2, /* type */
36 0, /* null_bit */
37 2, 0, 0 /* bit_start, bit_end, bit_length */
38 },
39 {
40 /*
41 Note, this (and the last HA_KEYTYPE_END) segment should NOT
42 be packed in any way, otherwise w_search() won't be able to
43 update key entry 'in vivo'
44 */
45 0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0, 0
46 }
47 };
48
49 const struct _ft_vft _ft_vft_nlq= {
50 ft_nlq_read_next, ft_nlq_find_relevance, ft_nlq_close_search,
51 ft_nlq_get_relevance, ft_nlq_reinit_search
52 };
53 const struct _ft_vft _ft_vft_boolean= {
54 ft_boolean_read_next, ft_boolean_find_relevance, ft_boolean_close_search,
55 ft_boolean_get_relevance, ft_boolean_reinit_search
56 };
57
58
ft_init_search(uint flags,void * info,uint keynr,uchar * query,uint query_len,CHARSET_INFO * cs,uchar * record)59 FT_INFO *ft_init_search(uint flags, void *info, uint keynr,
60 uchar *query, uint query_len, CHARSET_INFO *cs,
61 uchar *record)
62 {
63 FT_INFO *res;
64 if (flags & FT_BOOL)
65 res= ft_init_boolean_search((MI_INFO *)info, keynr, query, query_len,cs);
66 else
67 res= ft_init_nlq_search((MI_INFO *)info, keynr, query, query_len, flags,
68 record);
69 return res;
70 }
71
72 const char *ft_stopword_file= 0;
73 const char *ft_precompiled_stopwords[]= {
74
75 #ifdef COMPILE_STOPWORDS_IN
76
77 /* This particular stopword list was taken from SMART distribution
78 ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z
79 it was slightly modified to my taste, though
80 */
81
82 "a's",
83 "able",
84 "about",
85 "above",
86 "according",
87 "accordingly",
88 "across",
89 "actually",
90 "after",
91 "afterwards",
92 "again",
93 "against",
94 "ain't",
95 "all",
96 "allow",
97 "allows",
98 "almost",
99 "alone",
100 "along",
101 "already",
102 "also",
103 "although",
104 "always",
105 "am",
106 "among",
107 "amongst",
108 "an",
109 "and",
110 "another",
111 "any",
112 "anybody",
113 "anyhow",
114 "anyone",
115 "anything",
116 "anyway",
117 "anyways",
118 "anywhere",
119 "apart",
120 "appear",
121 "appreciate",
122 "appropriate",
123 "are",
124 "aren't",
125 "around",
126 "as",
127 "aside",
128 "ask",
129 "asking",
130 "associated",
131 "at",
132 "available",
133 "away",
134 "awfully",
135 "be",
136 "became",
137 "because",
138 "become",
139 "becomes",
140 "becoming",
141 "been",
142 "before",
143 "beforehand",
144 "behind",
145 "being",
146 "believe",
147 "below",
148 "beside",
149 "besides",
150 "best",
151 "better",
152 "between",
153 "beyond",
154 "both",
155 "brief",
156 "but",
157 "by",
158 "c'mon",
159 "c's",
160 "came",
161 "can",
162 "can't",
163 "cannot",
164 "cant",
165 "cause",
166 "causes",
167 "certain",
168 "certainly",
169 "changes",
170 "clearly",
171 "co",
172 "com",
173 "come",
174 "comes",
175 "concerning",
176 "consequently",
177 "consider",
178 "considering",
179 "contain",
180 "containing",
181 "contains",
182 "corresponding",
183 "could",
184 "couldn't",
185 "course",
186 "currently",
187 "definitely",
188 "described",
189 "despite",
190 "did",
191 "didn't",
192 "different",
193 "do",
194 "does",
195 "doesn't",
196 "doing",
197 "don't",
198 "done",
199 "down",
200 "downwards",
201 "during",
202 "each",
203 "edu",
204 "eg",
205 "eight",
206 "either",
207 "else",
208 "elsewhere",
209 "enough",
210 "entirely",
211 "especially",
212 "et",
213 "etc",
214 "even",
215 "ever",
216 "every",
217 "everybody",
218 "everyone",
219 "everything",
220 "everywhere",
221 "ex",
222 "exactly",
223 "example",
224 "except",
225 "far",
226 "few",
227 "fifth",
228 "first",
229 "five",
230 "followed",
231 "following",
232 "follows",
233 "for",
234 "former",
235 "formerly",
236 "forth",
237 "four",
238 "from",
239 "further",
240 "furthermore",
241 "get",
242 "gets",
243 "getting",
244 "given",
245 "gives",
246 "go",
247 "goes",
248 "going",
249 "gone",
250 "got",
251 "gotten",
252 "greetings",
253 "had",
254 "hadn't",
255 "happens",
256 "hardly",
257 "has",
258 "hasn't",
259 "have",
260 "haven't",
261 "having",
262 "he",
263 "he's",
264 "hello",
265 "help",
266 "hence",
267 "her",
268 "here",
269 "here's",
270 "hereafter",
271 "hereby",
272 "herein",
273 "hereupon",
274 "hers",
275 "herself",
276 "hi",
277 "him",
278 "himself",
279 "his",
280 "hither",
281 "hopefully",
282 "how",
283 "howbeit",
284 "however",
285 "i'd",
286 "i'll",
287 "i'm",
288 "i've",
289 "ie",
290 "if",
291 "ignored",
292 "immediate",
293 "in",
294 "inasmuch",
295 "inc",
296 "indeed",
297 "indicate",
298 "indicated",
299 "indicates",
300 "inner",
301 "insofar",
302 "instead",
303 "into",
304 "inward",
305 "is",
306 "isn't",
307 "it",
308 "it'd",
309 "it'll",
310 "it's",
311 "its",
312 "itself",
313 "just",
314 "keep",
315 "keeps",
316 "kept",
317 "know",
318 "knows",
319 "known",
320 "last",
321 "lately",
322 "later",
323 "latter",
324 "latterly",
325 "least",
326 "less",
327 "lest",
328 "let",
329 "let's",
330 "like",
331 "liked",
332 "likely",
333 "little",
334 "look",
335 "looking",
336 "looks",
337 "ltd",
338 "mainly",
339 "many",
340 "may",
341 "maybe",
342 "me",
343 "mean",
344 "meanwhile",
345 "merely",
346 "might",
347 "more",
348 "moreover",
349 "most",
350 "mostly",
351 "much",
352 "must",
353 "my",
354 "myself",
355 "name",
356 "namely",
357 "nd",
358 "near",
359 "nearly",
360 "necessary",
361 "need",
362 "needs",
363 "neither",
364 "never",
365 "nevertheless",
366 "new",
367 "next",
368 "nine",
369 "no",
370 "nobody",
371 "non",
372 "none",
373 "noone",
374 "nor",
375 "normally",
376 "not",
377 "nothing",
378 "novel",
379 "now",
380 "nowhere",
381 "obviously",
382 "of",
383 "off",
384 "often",
385 "oh",
386 "ok",
387 "okay",
388 "old",
389 "on",
390 "once",
391 "one",
392 "ones",
393 "only",
394 "onto",
395 "or",
396 "other",
397 "others",
398 "otherwise",
399 "ought",
400 "our",
401 "ours",
402 "ourselves",
403 "out",
404 "outside",
405 "over",
406 "overall",
407 "own",
408 "particular",
409 "particularly",
410 "per",
411 "perhaps",
412 "placed",
413 "please",
414 "plus",
415 "possible",
416 "presumably",
417 "probably",
418 "provides",
419 "que",
420 "quite",
421 "qv",
422 "rather",
423 "rd",
424 "re",
425 "really",
426 "reasonably",
427 "regarding",
428 "regardless",
429 "regards",
430 "relatively",
431 "respectively",
432 "right",
433 "said",
434 "same",
435 "saw",
436 "say",
437 "saying",
438 "says",
439 "second",
440 "secondly",
441 "see",
442 "seeing",
443 "seem",
444 "seemed",
445 "seeming",
446 "seems",
447 "seen",
448 "self",
449 "selves",
450 "sensible",
451 "sent",
452 "serious",
453 "seriously",
454 "seven",
455 "several",
456 "shall",
457 "she",
458 "should",
459 "shouldn't",
460 "since",
461 "six",
462 "so",
463 "some",
464 "somebody",
465 "somehow",
466 "someone",
467 "something",
468 "sometime",
469 "sometimes",
470 "somewhat",
471 "somewhere",
472 "soon",
473 "sorry",
474 "specified",
475 "specify",
476 "specifying",
477 "still",
478 "sub",
479 "such",
480 "sup",
481 "sure",
482 "t's",
483 "take",
484 "taken",
485 "tell",
486 "tends",
487 "th",
488 "than",
489 "thank",
490 "thanks",
491 "thanx",
492 "that",
493 "that's",
494 "thats",
495 "the",
496 "their",
497 "theirs",
498 "them",
499 "themselves",
500 "then",
501 "thence",
502 "there",
503 "there's",
504 "thereafter",
505 "thereby",
506 "therefore",
507 "therein",
508 "theres",
509 "thereupon",
510 "these",
511 "they",
512 "they'd",
513 "they'll",
514 "they're",
515 "they've",
516 "think",
517 "third",
518 "this",
519 "thorough",
520 "thoroughly",
521 "those",
522 "though",
523 "three",
524 "through",
525 "throughout",
526 "thru",
527 "thus",
528 "to",
529 "together",
530 "too",
531 "took",
532 "toward",
533 "towards",
534 "tried",
535 "tries",
536 "truly",
537 "try",
538 "trying",
539 "twice",
540 "two",
541 "un",
542 "under",
543 "unfortunately",
544 "unless",
545 "unlikely",
546 "until",
547 "unto",
548 "up",
549 "upon",
550 "us",
551 "use",
552 "used",
553 "useful",
554 "uses",
555 "using",
556 "usually",
557 "value",
558 "various",
559 "very",
560 "via",
561 "viz",
562 "vs",
563 "want",
564 "wants",
565 "was",
566 "wasn't",
567 "way",
568 "we",
569 "we'd",
570 "we'll",
571 "we're",
572 "we've",
573 "welcome",
574 "well",
575 "went",
576 "were",
577 "weren't",
578 "what",
579 "what's",
580 "whatever",
581 "when",
582 "whence",
583 "whenever",
584 "where",
585 "where's",
586 "whereafter",
587 "whereas",
588 "whereby",
589 "wherein",
590 "whereupon",
591 "wherever",
592 "whether",
593 "which",
594 "while",
595 "whither",
596 "who",
597 "who's",
598 "whoever",
599 "whole",
600 "whom",
601 "whose",
602 "why",
603 "will",
604 "willing",
605 "wish",
606 "with",
607 "within",
608 "without",
609 "won't",
610 "wonder",
611 "would",
612 "wouldn't",
613 "yes",
614 "yet",
615 "you",
616 "you'd",
617 "you'll",
618 "you're",
619 "you've",
620 "your",
621 "yours",
622 "yourself",
623 "yourselves",
624 "zero",
625 #endif
626
627 NULL };
628
ft_default_parser_parse(MYSQL_FTPARSER_PARAM * param)629 static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param)
630 {
631 return param->mysql_parse(param, param->doc, param->length);
632 }
633
634 struct st_mysql_ftparser ft_default_parser=
635 {
636 MYSQL_FTPARSER_INTERFACE_VERSION, ft_default_parser_parse, 0, 0
637 };
638
639