1 #include "license.hunspell"
2 #include "license.myspell"
3
4 #include <stdlib.h>
5 #include <string.h>
6 #include <stdio.h>
7 #include <ctype.h>
8
9 #include "suggestmgr.hxx"
10 #include "htypes.hxx"
11 #include "csutil.hxx"
12
13 const w_char W_VLINE = { '\0', '|' };
14
SuggestMgr(const char * tryme,int maxn,AffixMgr * aptr)15 SuggestMgr::SuggestMgr(const char * tryme, int maxn,
16 AffixMgr * aptr)
17 {
18
19 // register affix manager and check in string of chars to
20 // try when building candidate suggestions
21 pAMgr = aptr;
22
23 csconv = NULL;
24
25 ckeyl = 0;
26 ckey = NULL;
27 ckey_utf = NULL;
28
29 ctryl = 0;
30 ctry = NULL;
31 ctry_utf = NULL;
32
33 utf8 = 0;
34 langnum = 0;
35 complexprefixes = 0;
36
37 maxSug = maxn;
38 nosplitsugs = 0;
39 maxngramsugs = MAXNGRAMSUGS;
40 maxcpdsugs = MAXCOMPOUNDSUGS;
41
42 if (pAMgr) {
43 langnum = pAMgr->get_langnum();
44 ckey = pAMgr->get_key_string();
45 nosplitsugs = pAMgr->get_nosplitsugs();
46 if (pAMgr->get_maxngramsugs() >= 0)
47 maxngramsugs = pAMgr->get_maxngramsugs();
48 utf8 = pAMgr->get_utf8();
49 if (pAMgr->get_maxcpdsugs() >= 0)
50 maxcpdsugs = pAMgr->get_maxcpdsugs();
51 if (!utf8)
52 {
53 char * enc = pAMgr->get_encoding();
54 csconv = get_current_cs(enc);
55 free(enc);
56 }
57 complexprefixes = pAMgr->get_complexprefixes();
58 }
59
60 if (ckey) {
61 if (utf8) {
62 w_char t[MAXSWL];
63 ckeyl = u8_u16(t, MAXSWL, ckey);
64 ckey_utf = (w_char *) malloc(ckeyl * sizeof(w_char));
65 if (ckey_utf) memcpy(ckey_utf, t, ckeyl * sizeof(w_char));
66 else ckeyl = 0;
67 } else {
68 ckeyl = strlen(ckey);
69 }
70 }
71
72 if (tryme) {
73 ctry = mystrdup(tryme);
74 if (ctry) ctryl = strlen(ctry);
75 if (ctry && utf8) {
76 w_char t[MAXSWL];
77 ctryl = u8_u16(t, MAXSWL, tryme);
78 ctry_utf = (w_char *) malloc(ctryl * sizeof(w_char));
79 if (ctry_utf) memcpy(ctry_utf, t, ctryl * sizeof(w_char));
80 else ctryl = 0;
81 }
82 }
83 }
84
85
~SuggestMgr()86 SuggestMgr::~SuggestMgr()
87 {
88 pAMgr = NULL;
89 if (ckey) free(ckey);
90 ckey = NULL;
91 if (ckey_utf) free(ckey_utf);
92 ckey_utf = NULL;
93 ckeyl = 0;
94 if (ctry) free(ctry);
95 ctry = NULL;
96 if (ctry_utf) free(ctry_utf);
97 ctry_utf = NULL;
98 ctryl = 0;
99 maxSug = 0;
100 #ifdef MOZILLA_CLIENT
101 delete [] csconv;
102 #endif
103 }
104
testsug(char ** wlst,const char * candidate,int wl,int ns,int cpdsuggest,int * timer,clock_t * timelimit)105 int SuggestMgr::testsug(char** wlst, const char * candidate, int wl, int ns, int cpdsuggest,
106 int * timer, clock_t * timelimit) {
107 int cwrd = 1;
108 if (ns == maxSug) return maxSug;
109 for (int k=0; k < ns; k++) {
110 if (strcmp(candidate,wlst[k]) == 0) {
111 cwrd = 0;
112 break;
113 }
114 }
115 if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) {
116 wlst[ns] = mystrdup(candidate);
117 if (wlst[ns] == NULL) {
118 for (int j=0; j<ns; j++) free(wlst[j]);
119 return -1;
120 }
121 ns++;
122 }
123 return ns;
124 }
125
126 // generate suggestions for a misspelled word
127 // pass in address of array of char * pointers
128 // onlycompoundsug: probably bad suggestions (need for ngram sugs, too)
129
suggest(char *** slst,const char * w,int nsug,int * onlycompoundsug)130 int SuggestMgr::suggest(char*** slst, const char * w, int nsug,
131 int * onlycompoundsug)
132 {
133 int nocompoundtwowords = 0;
134 char ** wlst;
135 w_char word_utf[MAXSWL];
136 int wl = 0;
137 int nsugorig = nsug;
138 char w2[MAXWORDUTF8LEN];
139 const char * word = w;
140 int oldSug = 0;
141
142 // word reversing wrapper for complex prefixes
143 if (complexprefixes) {
144 strcpy(w2, w);
145 if (utf8) reverseword_utf(w2); else reverseword(w2);
146 word = w2;
147 }
148
149 if (*slst) {
150 wlst = *slst;
151 } else {
152 wlst = (char **) malloc(maxSug * sizeof(char *));
153 if (wlst == NULL) return -1;
154 for (int i = 0; i < maxSug; i++) {
155 wlst[i] = NULL;
156 }
157 }
158
159 if (utf8) {
160 wl = u8_u16(word_utf, MAXSWL, word);
161 if (wl == -1) {
162 *slst = wlst;
163 return nsug;
164 }
165 }
166
167 for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) {
168
169 // limit compound suggestion
170 if (cpdsuggest > 0) oldSug = nsug;
171
172 // suggestions for an uppercase word (html -> HTML)
173 if ((nsug < maxSug) && (nsug > -1)) {
174 nsug = (utf8) ? capchars_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
175 capchars(wlst, word, nsug, cpdsuggest);
176 }
177
178 // perhaps we made a typical fault of spelling
179 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
180 nsug = replchars(wlst, word, nsug, cpdsuggest);
181 }
182
183 // perhaps we made chose the wrong char from a related set
184 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
185 nsug = mapchars(wlst, word, nsug, cpdsuggest);
186 }
187
188 // only suggest compound words when no other suggestion
189 if ((cpdsuggest == 0) && (nsug > nsugorig)) nocompoundtwowords=1;
190
191 // did we swap the order of chars by mistake
192 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
193 nsug = (utf8) ? swapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
194 swapchar(wlst, word, nsug, cpdsuggest);
195 }
196
197 // did we swap the order of non adjacent chars by mistake
198 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
199 nsug = (utf8) ? longswapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
200 longswapchar(wlst, word, nsug, cpdsuggest);
201 }
202
203 // did we just hit the wrong key in place of a good char (case and keyboard)
204 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
205 nsug = (utf8) ? badcharkey_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
206 badcharkey(wlst, word, nsug, cpdsuggest);
207 }
208
209 // did we add a char that should not be there
210 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
211 nsug = (utf8) ? extrachar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
212 extrachar(wlst, word, nsug, cpdsuggest);
213 }
214
215
216 // did we forgot a char
217 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
218 nsug = (utf8) ? forgotchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
219 forgotchar(wlst, word, nsug, cpdsuggest);
220 }
221
222 // did we move a char
223 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
224 nsug = (utf8) ? movechar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
225 movechar(wlst, word, nsug, cpdsuggest);
226 }
227
228 // did we just hit the wrong key in place of a good char
229 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
230 nsug = (utf8) ? badchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
231 badchar(wlst, word, nsug, cpdsuggest);
232 }
233
234 // did we double two characters
235 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
236 nsug = (utf8) ? doubletwochars_utf(wlst, word_utf, wl, nsug, cpdsuggest) :
237 doubletwochars(wlst, word, nsug, cpdsuggest);
238 }
239
240 // perhaps we forgot to hit space and two words ran together
241 if (!nosplitsugs && (nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) {
242 nsug = twowords(wlst, word, nsug, cpdsuggest);
243 }
244
245 } // repeating ``for'' statement compounding support
246
247 if (nsug < 0) {
248 // we ran out of memory - we should free up as much as possible
249 for (int i = 0; i < maxSug; i++)
250 if (wlst[i] != NULL) free(wlst[i]);
251 free(wlst);
252 wlst = NULL;
253 }
254
255 if (!nocompoundtwowords && (nsug > 0) && onlycompoundsug) *onlycompoundsug = 1;
256
257 *slst = wlst;
258 return nsug;
259 }
260
261 // generate suggestions for a word with typical mistake
262 // pass in address of array of char * pointers
263 #ifdef HUNSPELL_EXPERIMENTAL
suggest_auto(char *** slst,const char * w,int nsug)264 int SuggestMgr::suggest_auto(char*** slst, const char * w, int nsug)
265 {
266 int nocompoundtwowords = 0;
267 char ** wlst;
268 int oldSug;
269
270 char w2[MAXWORDUTF8LEN];
271 const char * word = w;
272
273 // word reversing wrapper for complex prefixes
274 if (complexprefixes) {
275 strcpy(w2, w);
276 if (utf8) reverseword_utf(w2); else reverseword(w2);
277 word = w2;
278 }
279
280 if (*slst) {
281 wlst = *slst;
282 } else {
283 wlst = (char **) malloc(maxSug * sizeof(char *));
284 if (wlst == NULL) return -1;
285 }
286
287 for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) {
288
289 // limit compound suggestion
290 if (cpdsuggest > 0) oldSug = nsug;
291
292 // perhaps we made a typical fault of spelling
293 if ((nsug < maxSug) && (nsug > -1))
294 nsug = replchars(wlst, word, nsug, cpdsuggest);
295
296 // perhaps we made chose the wrong char from a related set
297 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs)))
298 nsug = mapchars(wlst, word, nsug, cpdsuggest);
299
300 if ((cpdsuggest==0) && (nsug>0)) nocompoundtwowords=1;
301
302 // perhaps we forgot to hit space and two words ran together
303
304 if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs)) && check_forbidden(word, strlen(word))) {
305 nsug = twowords(wlst, word, nsug, cpdsuggest);
306 }
307
308 } // repeating ``for'' statement compounding support
309
310 if (nsug < 0) {
311 for (int i=0;i<maxSug; i++)
312 if (wlst[i] != NULL) free(wlst[i]);
313 free(wlst);
314 return -1;
315 }
316
317 *slst = wlst;
318 return nsug;
319 }
320 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
321
322 // suggestions for an uppercase word (html -> HTML)
capchars_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)323 int SuggestMgr::capchars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
324 {
325 char candidate[MAXSWUTF8L];
326 w_char candidate_utf[MAXSWL];
327 memcpy(candidate_utf, word, wl * sizeof(w_char));
328 mkallcap_utf(candidate_utf, wl, langnum);
329 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
330 return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
331 }
332
333 // suggestions for an uppercase word (html -> HTML)
capchars(char ** wlst,const char * word,int ns,int cpdsuggest)334 int SuggestMgr::capchars(char** wlst, const char * word, int ns, int cpdsuggest)
335 {
336 char candidate[MAXSWUTF8L];
337 strcpy(candidate, word);
338 mkallcap(candidate, csconv);
339 return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
340 }
341
342 // suggestions for when chose the wrong char out of a related set
mapchars(char ** wlst,const char * word,int ns,int cpdsuggest)343 int SuggestMgr::mapchars(char** wlst, const char * word, int ns, int cpdsuggest)
344 {
345 char candidate[MAXSWUTF8L];
346 clock_t timelimit;
347 int timer;
348 candidate[0] = '\0';
349
350 int wl = strlen(word);
351 if (wl < 2 || ! pAMgr) return ns;
352
353 int nummap = pAMgr->get_nummap();
354 struct mapentry* maptable = pAMgr->get_maptable();
355 if (maptable==NULL) return ns;
356
357 timelimit = clock();
358 timer = MINTIMER;
359 return map_related(word, (char *) &candidate, 0, 0, wlst, cpdsuggest, ns, maptable, nummap, &timer, &timelimit);
360 }
361
map_related(const char * word,char * candidate,int wn,int cn,char ** wlst,int cpdsuggest,int ns,const mapentry * maptable,int nummap,int * timer,clock_t * timelimit)362 int SuggestMgr::map_related(const char * word, char * candidate, int wn, int cn,
363 char** wlst, int cpdsuggest, int ns,
364 const mapentry* maptable, int nummap, int * timer, clock_t * timelimit)
365 {
366 if (*(word + wn) == '\0') {
367 int cwrd = 1;
368 *(candidate + cn) = '\0';
369 int wl = strlen(candidate);
370 for (int m=0; m < ns; m++) {
371 if (strcmp(candidate, wlst[m]) == 0) {
372 cwrd = 0;
373 break;
374 }
375 }
376 if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) {
377 if (ns < maxSug) {
378 wlst[ns] = mystrdup(candidate);
379 if (wlst[ns] == NULL) return -1;
380 ns++;
381 }
382 }
383 return ns;
384 }
385 int in_map = 0;
386 for (int j = 0; j < nummap; j++) {
387 for (int k = 0; k < maptable[j].len; k++) {
388 int len = strlen(maptable[j].set[k]);
389 if (strncmp(maptable[j].set[k], word + wn, len) == 0) {
390 in_map = 1;
391 for (int l = 0; l < maptable[j].len; l++) {
392 strcpy(candidate + cn, maptable[j].set[l]);
393 ns = map_related(word, candidate, wn + len, strlen(candidate), wlst,
394 cpdsuggest, ns, maptable, nummap, timer, timelimit);
395 if (!(*timer)) return ns;
396 }
397 }
398 }
399 }
400 if (!in_map) {
401 *(candidate + cn) = *(word + wn);
402 ns = map_related(word, candidate, wn + 1, cn + 1, wlst, cpdsuggest,
403 ns, maptable, nummap, timer, timelimit);
404 }
405 return ns;
406 }
407
408 // suggestions for a typical fault of spelling, that
409 // differs with more, than 1 letter from the right form.
replchars(char ** wlst,const char * word,int ns,int cpdsuggest)410 int SuggestMgr::replchars(char** wlst, const char * word, int ns, int cpdsuggest)
411 {
412 char candidate[MAXSWUTF8L];
413 const char * r;
414 int lenr, lenp;
415 int wl = strlen(word);
416 if (wl < 2 || ! pAMgr) return ns;
417 int numrep = pAMgr->get_numrep();
418 struct replentry* reptable = pAMgr->get_reptable();
419 if (reptable==NULL) return ns;
420 for (int i=0; i < numrep; i++ ) {
421 r = word;
422 lenr = strlen(reptable[i].pattern2);
423 lenp = strlen(reptable[i].pattern);
424 // search every occurence of the pattern in the word
425 while ((r=strstr(r, reptable[i].pattern)) != NULL && (!reptable[i].end || strlen(r) == strlen(reptable[i].pattern)) &&
426 (!reptable[i].start || r == word)) {
427 strcpy(candidate, word);
428 if (r-word + lenr + strlen(r+lenp) >= MAXSWUTF8L) break;
429 strcpy(candidate+(r-word),reptable[i].pattern2);
430 strcpy(candidate+(r-word)+lenr, r+lenp);
431 ns = testsug(wlst, candidate, wl-lenp+lenr, ns, cpdsuggest, NULL, NULL);
432 if (ns == -1) return -1;
433 // check REP suggestions with space
434 char * sp = strchr(candidate, ' ');
435 if (sp) {
436 char * prev = candidate;
437 while (sp) {
438 *sp = '\0';
439 if (checkword(prev, strlen(prev), 0, NULL, NULL)) {
440 int oldns = ns;
441 *sp = ' ';
442 ns = testsug(wlst, sp + 1, strlen(sp + 1), ns, cpdsuggest, NULL, NULL);
443 if (ns == -1) return -1;
444 if (oldns < ns) {
445 free(wlst[ns - 1]);
446 wlst[ns - 1] = mystrdup(candidate);
447 if (!wlst[ns - 1]) return -1;
448 }
449 }
450 *sp = ' ';
451 prev = sp + 1;
452 sp = strchr(prev, ' ');
453 }
454 }
455 r++; // search for the next letter
456 }
457 }
458 return ns;
459 }
460
461 // perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation)
doubletwochars(char ** wlst,const char * word,int ns,int cpdsuggest)462 int SuggestMgr::doubletwochars(char** wlst, const char * word, int ns, int cpdsuggest)
463 {
464 char candidate[MAXSWUTF8L];
465 int state=0;
466 int wl = strlen(word);
467 if (wl < 5 || ! pAMgr) return ns;
468 for (int i=2; i < wl; i++ ) {
469 if (word[i]==word[i-2]) {
470 state++;
471 if (state==3) {
472 strcpy(candidate,word);
473 strcpy(candidate+i-1,word+i+1);
474 ns = testsug(wlst, candidate, wl-2, ns, cpdsuggest, NULL, NULL);
475 if (ns == -1) return -1;
476 state=0;
477 }
478 } else {
479 state=0;
480 }
481 }
482 return ns;
483 }
484
485 // perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation)
doubletwochars_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)486 int SuggestMgr::doubletwochars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
487 {
488 w_char candidate_utf[MAXSWL];
489 char candidate[MAXSWUTF8L];
490 int state=0;
491 if (wl < 5 || ! pAMgr) return ns;
492 for (int i=2; i < wl; i++) {
493 if (w_char_eq(word[i], word[i-2])) {
494 state++;
495 if (state==3) {
496 memcpy(candidate_utf, word, (i - 1) * sizeof(w_char));
497 memcpy(candidate_utf+i-1, word+i+1, (wl-i-1) * sizeof(w_char));
498 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl-2);
499 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
500 if (ns == -1) return -1;
501 state=0;
502 }
503 } else {
504 state=0;
505 }
506 }
507 return ns;
508 }
509
510 // error is wrong char in place of correct one (case and keyboard related version)
badcharkey(char ** wlst,const char * word,int ns,int cpdsuggest)511 int SuggestMgr::badcharkey(char ** wlst, const char * word, int ns, int cpdsuggest)
512 {
513 char tmpc;
514 char candidate[MAXSWUTF8L];
515 int wl = strlen(word);
516 strcpy(candidate, word);
517 // swap out each char one by one and try uppercase and neighbor
518 // keyboard chars in its place to see if that makes a good word
519
520 for (int i=0; i < wl; i++) {
521 tmpc = candidate[i];
522 // check with uppercase letters
523 candidate[i] = csconv[((unsigned char)tmpc)].cupper;
524 if (tmpc != candidate[i]) {
525 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
526 if (ns == -1) return -1;
527 candidate[i] = tmpc;
528 }
529 // check neighbor characters in keyboard string
530 if (!ckey) continue;
531 char * loc = strchr(ckey, tmpc);
532 while (loc) {
533 if ((loc > ckey) && (*(loc - 1) != '|')) {
534 candidate[i] = *(loc - 1);
535 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
536 if (ns == -1) return -1;
537 }
538 if ((*(loc + 1) != '|') && (*(loc + 1) != '\0')) {
539 candidate[i] = *(loc + 1);
540 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
541 if (ns == -1) return -1;
542 }
543 loc = strchr(loc + 1, tmpc);
544 }
545 candidate[i] = tmpc;
546 }
547 return ns;
548 }
549
550 // error is wrong char in place of correct one (case and keyboard related version)
badcharkey_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)551 int SuggestMgr::badcharkey_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
552 {
553 w_char tmpc;
554 w_char candidate_utf[MAXSWL];
555 char candidate[MAXSWUTF8L];
556 memcpy(candidate_utf, word, wl * sizeof(w_char));
557 // swap out each char one by one and try all the tryme
558 // chars in its place to see if that makes a good word
559 for (int i=0; i < wl; i++) {
560 tmpc = candidate_utf[i];
561 // check with uppercase letters
562 mkallcap_utf(candidate_utf + i, 1, langnum);
563 if (!w_char_eq(tmpc, candidate_utf[i])) {
564 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
565 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
566 if (ns == -1) return -1;
567 candidate_utf[i] = tmpc;
568 }
569 // check neighbor characters in keyboard string
570 if (!ckey) continue;
571 w_char * loc = ckey_utf;
572 while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc)) loc++;
573 while (loc < (ckey_utf + ckeyl)) {
574 if ((loc > ckey_utf) && !w_char_eq(*(loc - 1), W_VLINE)) {
575 candidate_utf[i] = *(loc - 1);
576 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
577 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
578 if (ns == -1) return -1;
579 }
580 if (((loc + 1) < (ckey_utf + ckeyl)) && !w_char_eq(*(loc + 1), W_VLINE)) {
581 candidate_utf[i] = *(loc + 1);
582 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
583 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
584 if (ns == -1) return -1;
585 }
586 do { loc++; } while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc));
587 }
588 candidate_utf[i] = tmpc;
589 }
590 return ns;
591 }
592
593 // error is wrong char in place of correct one
badchar(char ** wlst,const char * word,int ns,int cpdsuggest)594 int SuggestMgr::badchar(char ** wlst, const char * word, int ns, int cpdsuggest)
595 {
596 char tmpc;
597 char candidate[MAXSWUTF8L];
598 clock_t timelimit = clock();
599 int timer = MINTIMER;
600 int wl = strlen(word);
601 strcpy(candidate, word);
602 // swap out each char one by one and try all the tryme
603 // chars in its place to see if that makes a good word
604 for (int j=0; j < ctryl; j++) {
605 for (int i=wl-1; i >= 0; i--) {
606 tmpc = candidate[i];
607 if (ctry[j] == tmpc) continue;
608 candidate[i] = ctry[j];
609 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, &timer, &timelimit);
610 if (ns == -1) return -1;
611 if (!timer) return ns;
612 candidate[i] = tmpc;
613 }
614 }
615 return ns;
616 }
617
618 // error is wrong char in place of correct one
badchar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)619 int SuggestMgr::badchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
620 {
621 w_char tmpc;
622 w_char candidate_utf[MAXSWL];
623 char candidate[MAXSWUTF8L];
624 clock_t timelimit = clock();
625 int timer = MINTIMER;
626 memcpy(candidate_utf, word, wl * sizeof(w_char));
627 // swap out each char one by one and try all the tryme
628 // chars in its place to see if that makes a good word
629 for (int j=0; j < ctryl; j++) {
630 for (int i=wl-1; i >= 0; i--) {
631 tmpc = candidate_utf[i];
632 if (w_char_eq(tmpc, ctry_utf[j])) continue;
633 candidate_utf[i] = ctry_utf[j];
634 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
635 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit);
636 if (ns == -1) return -1;
637 if (!timer) return ns;
638 candidate_utf[i] = tmpc;
639 }
640 }
641 return ns;
642 }
643
644 // error is word has an extra letter it does not need
extrachar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)645 int SuggestMgr::extrachar_utf(char** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
646 {
647 char candidate[MAXSWUTF8L];
648 w_char candidate_utf[MAXSWL];
649 w_char * p;
650 w_char tmpc = W_VLINE; // not used value, only for VCC warning message
651 if (wl < 2) return ns;
652 // try omitting one char of word at a time
653 memcpy(candidate_utf, word, wl * sizeof(w_char));
654 for (p = candidate_utf + wl - 1; p >= candidate_utf; p--) {
655 w_char tmpc2 = *p;
656 if (p < candidate_utf + wl - 1) *p = tmpc;
657 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl - 1);
658 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
659 if (ns == -1) return -1;
660 tmpc = tmpc2;
661 }
662 return ns;
663 }
664
665 // error is word has an extra letter it does not need
extrachar(char ** wlst,const char * word,int ns,int cpdsuggest)666 int SuggestMgr::extrachar(char** wlst, const char * word, int ns, int cpdsuggest)
667 {
668 char tmpc = '\0';
669 char candidate[MAXSWUTF8L];
670 char * p;
671 int wl = strlen(word);
672 if (wl < 2) return ns;
673 // try omitting one char of word at a time
674 strcpy (candidate, word);
675 for (p = candidate + wl - 1; p >=candidate; p--) {
676 char tmpc2 = *p;
677 *p = tmpc;
678 ns = testsug(wlst, candidate, wl-1, ns, cpdsuggest, NULL, NULL);
679 if (ns == -1) return -1;
680 tmpc = tmpc2;
681 }
682 return ns;
683 }
684
685 // error is missing a letter it needs
forgotchar(char ** wlst,const char * word,int ns,int cpdsuggest)686 int SuggestMgr::forgotchar(char ** wlst, const char * word, int ns, int cpdsuggest)
687 {
688 char candidate[MAXSWUTF8L + 4];
689 char * p;
690 clock_t timelimit = clock();
691 int timer = MINTIMER;
692 int wl = strlen(word);
693 // try inserting a tryme character before every letter (and the null terminator)
694 for (int i = 0; i < ctryl; i++) {
695 strcpy(candidate, word);
696 for (p = candidate + wl; p >= candidate; p--) {
697 *(p+1) = *p;
698 *p = ctry[i];
699 ns = testsug(wlst, candidate, wl+1, ns, cpdsuggest, &timer, &timelimit);
700 if (ns == -1) return -1;
701 if (!timer) return ns;
702 }
703 }
704 return ns;
705 }
706
707 // error is missing a letter it needs
forgotchar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)708 int SuggestMgr::forgotchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
709 {
710 w_char candidate_utf[MAXSWL + 1];
711 char candidate[MAXSWUTF8L + 4];
712 w_char * p;
713 clock_t timelimit = clock();
714 int timer = MINTIMER;
715 // try inserting a tryme character at the end of the word and before every letter
716 for (int i = 0; i < ctryl; i++) {
717 memcpy (candidate_utf, word, wl * sizeof(w_char));
718 for (p = candidate_utf + wl; p >= candidate_utf; p--) {
719 *(p + 1) = *p;
720 *p = ctry_utf[i];
721 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl + 1);
722 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit);
723 if (ns == -1) return -1;
724 if (!timer) return ns;
725 }
726 }
727 return ns;
728 }
729
730
731 /* error is should have been two words */
twowords(char ** wlst,const char * word,int ns,int cpdsuggest)732 int SuggestMgr::twowords(char ** wlst, const char * word, int ns, int cpdsuggest)
733 {
734 char candidate[MAXSWUTF8L];
735 char * p;
736 int c1, c2;
737 int forbidden = 0;
738 int cwrd;
739
740 int wl=strlen(word);
741 if (wl < 3) return ns;
742
743 if (langnum == LANG_hu) forbidden = check_forbidden(word, wl);
744
745 strcpy(candidate + 1, word);
746 // split the string into two pieces after every char
747 // if both pieces are good words make them a suggestion
748 for (p = candidate + 1; p[1] != '\0'; p++) {
749 p[-1] = *p;
750 // go to end of the UTF-8 character
751 while (utf8 && ((p[1] & 0xc0) == 0x80)) {
752 *p = p[1];
753 p++;
754 }
755 if (utf8 && p[1] == '\0') break; // last UTF-8 character
756 *p = '\0';
757 c1 = checkword(candidate,strlen(candidate), cpdsuggest, NULL, NULL);
758 if (c1) {
759 c2 = checkword((p+1),strlen(p+1), cpdsuggest, NULL, NULL);
760 if (c2) {
761 *p = ' ';
762
763 // spec. Hungarian code (need a better compound word support)
764 if ((langnum == LANG_hu) && !forbidden &&
765 // if 3 repeating letter, use - instead of space
766 (((p[-1] == p[1]) && (((p>candidate+1) && (p[-1] == p[-2])) || (p[-1] == p[2]))) ||
767 // or multiple compounding, with more, than 6 syllables
768 ((c1 == 3) && (c2 >= 2)))) *p = '-';
769
770 cwrd = 1;
771 for (int k=0; k < ns; k++) {
772 if (strcmp(candidate,wlst[k]) == 0) {
773 cwrd = 0;
774 break;
775 }
776 }
777 if (ns < maxSug) {
778 if (cwrd) {
779 wlst[ns] = mystrdup(candidate);
780 if (wlst[ns] == NULL) return -1;
781 ns++;
782 }
783 } else return ns;
784 // add two word suggestion with dash, if TRY string contains
785 // "a" or "-"
786 // NOTE: cwrd doesn't modified for REP twoword sugg.
787 if (ctry && (strchr(ctry, 'a') || strchr(ctry, '-')) &&
788 mystrlen(p + 1) > 1 &&
789 mystrlen(candidate) - mystrlen(p) > 1) {
790 *p = '-';
791 for (int k=0; k < ns; k++) {
792 if (strcmp(candidate,wlst[k]) == 0) {
793 cwrd = 0;
794 break;
795 }
796 }
797 if (ns < maxSug) {
798 if (cwrd) {
799 wlst[ns] = mystrdup(candidate);
800 if (wlst[ns] == NULL) return -1;
801 ns++;
802 }
803 } else return ns;
804 }
805 }
806 }
807 }
808 return ns;
809 }
810
811
812 // error is adjacent letter were swapped
swapchar(char ** wlst,const char * word,int ns,int cpdsuggest)813 int SuggestMgr::swapchar(char ** wlst, const char * word, int ns, int cpdsuggest)
814 {
815 char candidate[MAXSWUTF8L];
816 char * p;
817 char tmpc;
818 int wl=strlen(word);
819 // try swapping adjacent chars one by one
820 strcpy(candidate, word);
821 for (p = candidate; p[1] != 0; p++) {
822 tmpc = *p;
823 *p = p[1];
824 p[1] = tmpc;
825 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
826 if (ns == -1) return -1;
827 p[1] = *p;
828 *p = tmpc;
829 }
830 // try double swaps for short words
831 // ahev -> have, owudl -> would
832 if (wl == 4 || wl == 5) {
833 candidate[0] = word[1];
834 candidate[1] = word[0];
835 candidate[2] = word[2];
836 candidate[wl - 2] = word[wl - 1];
837 candidate[wl - 1] = word[wl - 2];
838 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
839 if (ns == -1) return -1;
840 if (wl == 5) {
841 candidate[0] = word[0];
842 candidate[1] = word[2];
843 candidate[2] = word[1];
844 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
845 if (ns == -1) return -1;
846 }
847 }
848 return ns;
849 }
850
851 // error is adjacent letter were swapped
swapchar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)852 int SuggestMgr::swapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
853 {
854 w_char candidate_utf[MAXSWL];
855 char candidate[MAXSWUTF8L];
856 w_char * p;
857 w_char tmpc;
858 int len = 0;
859 // try swapping adjacent chars one by one
860 memcpy (candidate_utf, word, wl * sizeof(w_char));
861 for (p = candidate_utf; p < (candidate_utf + wl - 1); p++) {
862 tmpc = *p;
863 *p = p[1];
864 p[1] = tmpc;
865 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
866 if (len == 0) len = strlen(candidate);
867 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
868 if (ns == -1) return -1;
869 p[1] = *p;
870 *p = tmpc;
871 }
872 // try double swaps for short words
873 // ahev -> have, owudl -> would, suodn -> sound
874 if (wl == 4 || wl == 5) {
875 candidate_utf[0] = word[1];
876 candidate_utf[1] = word[0];
877 candidate_utf[2] = word[2];
878 candidate_utf[wl - 2] = word[wl - 1];
879 candidate_utf[wl - 1] = word[wl - 2];
880 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
881 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
882 if (ns == -1) return -1;
883 if (wl == 5) {
884 candidate_utf[0] = word[0];
885 candidate_utf[1] = word[2];
886 candidate_utf[2] = word[1];
887 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
888 ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL);
889 if (ns == -1) return -1;
890 }
891 }
892 return ns;
893 }
894
895 // error is not adjacent letter were swapped
longswapchar(char ** wlst,const char * word,int ns,int cpdsuggest)896 int SuggestMgr::longswapchar(char ** wlst, const char * word, int ns, int cpdsuggest)
897 {
898 char candidate[MAXSWUTF8L];
899 char * p;
900 char * q;
901 char tmpc;
902 int wl=strlen(word);
903 // try swapping not adjacent chars one by one
904 strcpy(candidate, word);
905 for (p = candidate; *p != 0; p++) {
906 for (q = candidate; *q != 0; q++) {
907 if (abs((int)(p-q)) > 1) {
908 tmpc = *p;
909 *p = *q;
910 *q = tmpc;
911 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
912 if (ns == -1) return -1;
913 *q = *p;
914 *p = tmpc;
915 }
916 }
917 }
918 return ns;
919 }
920
921
922 // error is adjacent letter were swapped
longswapchar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)923 int SuggestMgr::longswapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
924 {
925 w_char candidate_utf[MAXSWL];
926 char candidate[MAXSWUTF8L];
927 w_char * p;
928 w_char * q;
929 w_char tmpc;
930 // try swapping not adjacent chars
931 memcpy (candidate_utf, word, wl * sizeof(w_char));
932 for (p = candidate_utf; p < (candidate_utf + wl); p++) {
933 for (q = candidate_utf; q < (candidate_utf + wl); q++) {
934 if (abs((int)(p-q)) > 1) {
935 tmpc = *p;
936 *p = *q;
937 *q = tmpc;
938 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
939 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
940 if (ns == -1) return -1;
941 *q = *p;
942 *p = tmpc;
943 }
944 }
945 }
946 return ns;
947 }
948
949 // error is a letter was moved
movechar(char ** wlst,const char * word,int ns,int cpdsuggest)950 int SuggestMgr::movechar(char ** wlst, const char * word, int ns, int cpdsuggest)
951 {
952 char candidate[MAXSWUTF8L];
953 char * p;
954 char * q;
955 char tmpc;
956
957 int wl=strlen(word);
958 // try moving a char
959 strcpy(candidate, word);
960 for (p = candidate; *p != 0; p++) {
961 for (q = p + 1; (*q != 0) && ((q - p) < 10); q++) {
962 tmpc = *(q-1);
963 *(q-1) = *q;
964 *q = tmpc;
965 if ((q-p) < 2) continue; // omit swap char
966 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
967 if (ns == -1) return -1;
968 }
969 strcpy(candidate, word);
970 }
971 for (p = candidate + wl - 1; p > candidate; p--) {
972 for (q = p - 1; (q >= candidate) && ((p - q) < 10); q--) {
973 tmpc = *(q+1);
974 *(q+1) = *q;
975 *q = tmpc;
976 if ((p-q) < 2) continue; // omit swap char
977 ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL);
978 if (ns == -1) return -1;
979 }
980 strcpy(candidate, word);
981 }
982 return ns;
983 }
984
985 // error is a letter was moved
movechar_utf(char ** wlst,const w_char * word,int wl,int ns,int cpdsuggest)986 int SuggestMgr::movechar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest)
987 {
988 w_char candidate_utf[MAXSWL];
989 char candidate[MAXSWUTF8L];
990 w_char * p;
991 w_char * q;
992 w_char tmpc;
993 // try moving a char
994 memcpy (candidate_utf, word, wl * sizeof(w_char));
995 for (p = candidate_utf; p < (candidate_utf + wl); p++) {
996 for (q = p + 1; (q < (candidate_utf + wl)) && ((q - p) < 10); q++) {
997 tmpc = *(q-1);
998 *(q-1) = *q;
999 *q = tmpc;
1000 if ((q-p) < 2) continue; // omit swap char
1001 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
1002 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
1003 if (ns == -1) return -1;
1004 }
1005 memcpy (candidate_utf, word, wl * sizeof(w_char));
1006 }
1007 for (p = candidate_utf + wl - 1; p > candidate_utf; p--) {
1008 for (q = p - 1; (q >= candidate_utf) && ((p - q) < 10); q--) {
1009 tmpc = *(q+1);
1010 *(q+1) = *q;
1011 *q = tmpc;
1012 if ((p-q) < 2) continue; // omit swap char
1013 u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl);
1014 ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL);
1015 if (ns == -1) return -1;
1016 }
1017 memcpy (candidate_utf, word, wl * sizeof(w_char));
1018 }
1019 return ns;
1020 }
1021
1022 // generate a set of suggestions for very poorly spelled words
ngsuggest(char ** wlst,char * w,int ns,HashMgr ** pHMgr,int md)1023 int SuggestMgr::ngsuggest(char** wlst, char * w, int ns, HashMgr** pHMgr, int md)
1024 {
1025
1026 int i, j;
1027 int lval;
1028 int sc, scphon;
1029 int lp, lpphon;
1030 int nonbmp = 0;
1031
1032 // exhaustively search through all root words
1033 // keeping track of the MAX_ROOTS most similar root words
1034 struct hentry * roots[MAX_ROOTS];
1035 char * rootsphon[MAX_ROOTS];
1036 int scores[MAX_ROOTS];
1037 int scoresphon[MAX_ROOTS];
1038 for (i = 0; i < MAX_ROOTS; i++) {
1039 roots[i] = NULL;
1040 scores[i] = -100 * i;
1041 rootsphon[i] = NULL;
1042 scoresphon[i] = -100 * i;
1043 }
1044 lp = MAX_ROOTS - 1;
1045 lpphon = MAX_ROOTS - 1;
1046 scphon = -20000;
1047 int low = NGRAM_LOWERING;
1048
1049 char w2[MAXWORDUTF8LEN];
1050 char f[MAXSWUTF8L];
1051 char * word = w;
1052
1053 // word reversing wrapper for complex prefixes
1054 if (complexprefixes) {
1055 strcpy(w2, w);
1056 if (utf8) reverseword_utf(w2); else reverseword(w2);
1057 word = w2;
1058 }
1059
1060 char mw[MAXSWUTF8L];
1061 w_char u8[MAXSWL];
1062 int nc = strlen(word);
1063 int n = (utf8) ? u8_u16(u8, MAXSWL, word) : nc;
1064
1065 // set character based ngram suggestion for words with non-BMP Unicode characters
1066 if (n == -1) {
1067 utf8 = 0; // XXX not state-free
1068 n = nc;
1069 nonbmp = 1;
1070 low = 0;
1071 }
1072
1073 struct hentry* hp = NULL;
1074 int col = -1;
1075 phonetable * ph = (pAMgr) ? pAMgr->get_phonetable() : NULL;
1076 char target[MAXSWUTF8L];
1077 char candidate[MAXSWUTF8L];
1078 if (ph) {
1079 if (utf8) {
1080 w_char _w[MAXSWL];
1081 int _wl = u8_u16(_w, MAXSWL, word);
1082 mkallcap_utf(_w, _wl, langnum);
1083 u16_u8(candidate, MAXSWUTF8L, _w, _wl);
1084 } else {
1085 strcpy(candidate, word);
1086 if (!nonbmp) mkallcap(candidate, csconv);
1087 }
1088 phonet(candidate, target, nc, *ph); // XXX phonet() is 8-bit (nc, not n)
1089 }
1090
1091 FLAG forbiddenword = pAMgr ? pAMgr->get_forbiddenword() : FLAG_NULL;
1092 FLAG nosuggest = pAMgr ? pAMgr->get_nosuggest() : FLAG_NULL;
1093 FLAG nongramsuggest = pAMgr ? pAMgr->get_nongramsuggest() : FLAG_NULL;
1094 FLAG onlyincompound = pAMgr ? pAMgr->get_onlyincompound() : FLAG_NULL;
1095
1096 for (i = 0; i < md; i++) {
1097 while (0 != (hp = (pHMgr[i])->walk_hashtable(col, hp))) {
1098 if ((hp->astr) && (pAMgr) &&
1099 (TESTAFF(hp->astr, forbiddenword, hp->alen) ||
1100 TESTAFF(hp->astr, ONLYUPCASEFLAG, hp->alen) ||
1101 TESTAFF(hp->astr, nosuggest, hp->alen) ||
1102 TESTAFF(hp->astr, nongramsuggest, hp->alen) ||
1103 TESTAFF(hp->astr, onlyincompound, hp->alen))) continue;
1104
1105 sc = ngram(3, word, HENTRY_WORD(hp), NGRAM_LONGER_WORSE + low) +
1106 leftcommonsubstring(word, HENTRY_WORD(hp));
1107
1108 // check special pronounciation
1109 if ((hp->var & H_OPT_PHON) && copy_field(f, HENTRY_DATA(hp), MORPH_PHON)) {
1110 int sc2 = ngram(3, word, f, NGRAM_LONGER_WORSE + low) +
1111 + leftcommonsubstring(word, f);
1112 if (sc2 > sc) sc = sc2;
1113 }
1114
1115 scphon = -20000;
1116 if (ph && (sc > 2) && (abs(n - (int) hp->clen) <= 3)) {
1117 char target2[MAXSWUTF8L];
1118 if (utf8) {
1119 w_char _w[MAXSWL];
1120 int _wl = u8_u16(_w, MAXSWL, HENTRY_WORD(hp));
1121 mkallcap_utf(_w, _wl, langnum);
1122 u16_u8(candidate, MAXSWUTF8L, _w, _wl);
1123 } else {
1124 strcpy(candidate, HENTRY_WORD(hp));
1125 mkallcap(candidate, csconv);
1126 }
1127 phonet(candidate, target2, -1, *ph);
1128 scphon = 2 * ngram(3, target, target2, NGRAM_LONGER_WORSE);
1129 }
1130
1131 if (sc > scores[lp]) {
1132 scores[lp] = sc;
1133 roots[lp] = hp;
1134 lval = sc;
1135 for (j=0; j < MAX_ROOTS; j++)
1136 if (scores[j] < lval) {
1137 lp = j;
1138 lval = scores[j];
1139 }
1140 }
1141
1142
1143 if (scphon > scoresphon[lpphon]) {
1144 scoresphon[lpphon] = scphon;
1145 rootsphon[lpphon] = HENTRY_WORD(hp);
1146 lval = scphon;
1147 for (j=0; j < MAX_ROOTS; j++)
1148 if (scoresphon[j] < lval) {
1149 lpphon = j;
1150 lval = scoresphon[j];
1151 }
1152 }
1153 }}
1154
1155 // find minimum threshold for a passable suggestion
1156 // mangle original word three differnt ways
1157 // and score them to generate a minimum acceptable score
1158 int thresh = 0;
1159 for (int sp = 1; sp < 4; sp++) {
1160 if (utf8) {
1161 for (int k=sp; k < n; k+=4) *((unsigned short *) u8 + k) = '*';
1162 u16_u8(mw, MAXSWUTF8L, u8, n);
1163 thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
1164 } else {
1165 strcpy(mw, word);
1166 for (int k=sp; k < n; k+=4) *(mw + k) = '*';
1167 thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low);
1168 }
1169 }
1170 thresh = thresh / 3;
1171 thresh--;
1172
1173 // now expand affixes on each of these root words and
1174 // and use length adjusted ngram scores to select
1175 // possible suggestions
1176 char * guess[MAX_GUESS];
1177 char * guessorig[MAX_GUESS];
1178 int gscore[MAX_GUESS];
1179 for(i=0;i<MAX_GUESS;i++) {
1180 guess[i] = NULL;
1181 guessorig[i] = NULL;
1182 gscore[i] = -100 * i;
1183 }
1184
1185 lp = MAX_GUESS - 1;
1186
1187 struct guessword * glst;
1188 glst = (struct guessword *) calloc(MAX_WORDS,sizeof(struct guessword));
1189 if (! glst) {
1190 if (nonbmp) utf8 = 1;
1191 return ns;
1192 }
1193
1194 for (i = 0; i < MAX_ROOTS; i++) {
1195 if (roots[i]) {
1196 struct hentry * rp = roots[i];
1197 int nw = pAMgr->expand_rootword(glst, MAX_WORDS, HENTRY_WORD(rp), rp->blen,
1198 rp->astr, rp->alen, word, nc,
1199 ((rp->var & H_OPT_PHON) ? copy_field(f, HENTRY_DATA(rp), MORPH_PHON) : NULL));
1200
1201 for (int k = 0; k < nw ; k++) {
1202 sc = ngram(n, word, glst[k].word, NGRAM_ANY_MISMATCH + low) +
1203 leftcommonsubstring(word, glst[k].word);
1204
1205 if (sc > thresh) {
1206 if (sc > gscore[lp]) {
1207 if (guess[lp]) {
1208 free (guess[lp]);
1209 if (guessorig[lp]) {
1210 free(guessorig[lp]);
1211 guessorig[lp] = NULL;
1212 }
1213 }
1214 gscore[lp] = sc;
1215 guess[lp] = glst[k].word;
1216 guessorig[lp] = glst[k].orig;
1217 lval = sc;
1218 for (j=0; j < MAX_GUESS; j++)
1219 if (gscore[j] < lval) {
1220 lp = j;
1221 lval = gscore[j];
1222 }
1223 } else {
1224 free(glst[k].word);
1225 if (glst[k].orig) free(glst[k].orig);
1226 }
1227 } else {
1228 free(glst[k].word);
1229 if (glst[k].orig) free(glst[k].orig);
1230 }
1231 }
1232 }
1233 }
1234 free(glst);
1235
1236 // now we are done generating guesses
1237 // sort in order of decreasing score
1238
1239
1240 bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
1241 if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
1242
1243 // weight suggestions with a similarity index, based on
1244 // the longest common subsequent algorithm and resort
1245
1246 int is_swap = 0;
1247 int re = 0;
1248 double fact = 1.0;
1249 if (pAMgr) {
1250 int maxd = pAMgr->get_maxdiff();
1251 if (maxd >= 0) fact = (10.0 - maxd)/5.0;
1252 }
1253
1254 for (i=0; i < MAX_GUESS; i++) {
1255 if (guess[i]) {
1256 // lowering guess[i]
1257 char gl[MAXSWUTF8L];
1258 int len;
1259 if (utf8) {
1260 w_char _w[MAXSWL];
1261 len = u8_u16(_w, MAXSWL, guess[i]);
1262 mkallsmall_utf(_w, len, langnum);
1263 u16_u8(gl, MAXSWUTF8L, _w, len);
1264 } else {
1265 strcpy(gl, guess[i]);
1266 if (!nonbmp) mkallsmall(gl, csconv);
1267 len = strlen(guess[i]);
1268 }
1269
1270 int _lcs = lcslen(word, gl);
1271
1272 // same characters with different casing
1273 if ((n == len) && (n == _lcs)) {
1274 gscore[i] += 2000;
1275 break;
1276 }
1277 // using 2-gram instead of 3, and other weightening
1278
1279 re = ngram(2, word, gl, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED) +
1280 ngram(2, gl, word, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED);
1281
1282 gscore[i] =
1283 // length of longest common subsequent minus length difference
1284 2 * _lcs - abs((int) (n - len)) +
1285 // weight length of the left common substring
1286 leftcommonsubstring(word, gl) +
1287 // weight equal character positions
1288 (!nonbmp && commoncharacterpositions(word, gl, &is_swap) ? 1: 0) +
1289 // swap character (not neighboring)
1290 ((is_swap) ? 10 : 0) +
1291 // ngram
1292 ngram(4, word, gl, NGRAM_ANY_MISMATCH + low) +
1293 // weighted ngrams
1294 re +
1295 // different limit for dictionaries with PHONE rules
1296 (ph ? (re < len * fact ? -1000 : 0) : (re < (n + len)*fact? -1000 : 0));
1297 }
1298 }
1299
1300 bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
1301
1302 // phonetic version
1303 if (ph) for (i=0; i < MAX_ROOTS; i++) {
1304 if (rootsphon[i]) {
1305 // lowering rootphon[i]
1306 char gl[MAXSWUTF8L];
1307 int len;
1308 if (utf8) {
1309 w_char _w[MAXSWL];
1310 len = u8_u16(_w, MAXSWL, rootsphon[i]);
1311 mkallsmall_utf(_w, len, langnum);
1312 u16_u8(gl, MAXSWUTF8L, _w, len);
1313 } else {
1314 strcpy(gl, rootsphon[i]);
1315 if (!nonbmp) mkallsmall(gl, csconv);
1316 len = strlen(rootsphon[i]);
1317 }
1318
1319 // heuristic weigthing of ngram scores
1320 scoresphon[i] += 2 * lcslen(word, gl) - abs((int) (n - len)) +
1321 // weight length of the left common substring
1322 leftcommonsubstring(word, gl);
1323 }
1324 }
1325
1326 if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
1327
1328 // copy over
1329 int oldns = ns;
1330
1331 int same = 0;
1332 for (i=0; i < MAX_GUESS; i++) {
1333 if (guess[i]) {
1334 if ((ns < oldns + maxngramsugs) && (ns < maxSug) && (!same || (gscore[i] > 1000))) {
1335 int unique = 1;
1336 // leave only excellent suggestions, if exists
1337 if (gscore[i] > 1000) same = 1; else if (gscore[i] < -100) {
1338 same = 1;
1339 // keep the best ngram suggestions, unless in ONLYMAXDIFF mode
1340 if (ns > oldns || (pAMgr && pAMgr->get_onlymaxdiff())) {
1341 free(guess[i]);
1342 if (guessorig[i]) free(guessorig[i]);
1343 continue;
1344 }
1345 }
1346 for (j = 0; j < ns; j++) {
1347 // don't suggest previous suggestions or a previous suggestion with prefixes or affixes
1348 if ((!guessorig[i] && strstr(guess[i], wlst[j])) ||
1349 (guessorig[i] && strstr(guessorig[i], wlst[j])) ||
1350 // check forbidden words
1351 !checkword(guess[i], strlen(guess[i]), 0, NULL, NULL)) {
1352 unique = 0;
1353 break;
1354 }
1355 }
1356 if (unique) {
1357 wlst[ns++] = guess[i];
1358 if (guessorig[i]) {
1359 free(guess[i]);
1360 wlst[ns-1] = guessorig[i];
1361 }
1362 } else {
1363 free(guess[i]);
1364 if (guessorig[i]) free(guessorig[i]);
1365 }
1366 } else {
1367 free(guess[i]);
1368 if (guessorig[i]) free(guessorig[i]);
1369 }
1370 }
1371 }
1372
1373 oldns = ns;
1374 if (ph) for (i=0; i < MAX_ROOTS; i++) {
1375 if (rootsphon[i]) {
1376 if ((ns < oldns + MAXPHONSUGS) && (ns < maxSug)) {
1377 int unique = 1;
1378 for (j = 0; j < ns; j++) {
1379 // don't suggest previous suggestions or a previous suggestion with prefixes or affixes
1380 if (strstr(rootsphon[i], wlst[j]) ||
1381 // check forbidden words
1382 !checkword(rootsphon[i], strlen(rootsphon[i]), 0, NULL, NULL)) {
1383 unique = 0;
1384 break;
1385 }
1386 }
1387 if (unique) {
1388 wlst[ns++] = mystrdup(rootsphon[i]);
1389 if (!wlst[ns - 1]) return ns - 1;
1390 }
1391 }
1392 }
1393 }
1394
1395 if (nonbmp) utf8 = 1;
1396 return ns;
1397 }
1398
1399
1400 // see if a candidate suggestion is spelled correctly
1401 // needs to check both root words and words with affixes
1402
1403 // obsolote MySpell-HU modifications:
1404 // return value 2 and 3 marks compounding with hyphen (-)
1405 // `3' marks roots without suffix
checkword(const char * word,int len,int cpdsuggest,int * timer,clock_t * timelimit)1406 int SuggestMgr::checkword(const char * word, int len, int cpdsuggest, int * timer, clock_t * timelimit)
1407 {
1408 struct hentry * rv=NULL;
1409 struct hentry * rv2=NULL;
1410 int nosuffix = 0;
1411
1412 // check time limit
1413 if (timer) {
1414 (*timer)--;
1415 if (!(*timer) && timelimit) {
1416 if ((clock() - *timelimit) > TIMELIMIT) return 0;
1417 *timer = MAXPLUSTIMER;
1418 }
1419 }
1420
1421 if (pAMgr) {
1422 if (cpdsuggest==1) {
1423 if (pAMgr->get_compound()) {
1424 rv = pAMgr->compound_check(word, len, 0, 0, 100, 0, NULL, 0, 1, 0); //EXT
1425 if (rv && (!(rv2 = pAMgr->lookup(word)) || !rv2->astr ||
1426 !(TESTAFF(rv2->astr,pAMgr->get_forbiddenword(),rv2->alen) ||
1427 TESTAFF(rv2->astr,pAMgr->get_nosuggest(),rv2->alen)))) return 3; // XXX obsolote categorisation + only ICONV needs affix flag check?
1428 }
1429 return 0;
1430 }
1431
1432 rv = pAMgr->lookup(word);
1433
1434 if (rv) {
1435 if ((rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen)
1436 || TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen))) return 0;
1437 while (rv) {
1438 if (rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) ||
1439 TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1440 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) {
1441 rv = rv->next_homonym;
1442 } else break;
1443 }
1444 } else rv = pAMgr->prefix_check(word, len, 0); // only prefix, and prefix + suffix XXX
1445
1446 if (rv) {
1447 nosuffix=1;
1448 } else {
1449 rv = pAMgr->suffix_check(word, len, 0, NULL, NULL, 0, NULL); // only suffix
1450 }
1451
1452 if (!rv && pAMgr->have_contclass()) {
1453 rv = pAMgr->suffix_check_twosfx(word, len, 0, NULL, FLAG_NULL);
1454 if (!rv) rv = pAMgr->prefix_check_twosfx(word, len, 1, FLAG_NULL);
1455 }
1456
1457 // check forbidden words
1458 if ((rv) && (rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen) ||
1459 TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
1460 TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen) ||
1461 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) return 0;
1462
1463 if (rv) { // XXX obsolote
1464 if ((pAMgr->get_compoundflag()) &&
1465 TESTAFF(rv->astr, pAMgr->get_compoundflag(), rv->alen)) return 2 + nosuffix;
1466 return 1;
1467 }
1468 }
1469 return 0;
1470 }
1471
check_forbidden(const char * word,int len)1472 int SuggestMgr::check_forbidden(const char * word, int len)
1473 {
1474 struct hentry * rv = NULL;
1475
1476 if (pAMgr) {
1477 rv = pAMgr->lookup(word);
1478 if (rv && rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) ||
1479 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) rv = NULL;
1480 if (!(pAMgr->prefix_check(word,len,1)))
1481 rv = pAMgr->suffix_check(word,len, 0, NULL, NULL, 0, NULL); // prefix+suffix, suffix
1482 // check forbidden words
1483 if ((rv) && (rv->astr) && TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen)) return 1;
1484 }
1485 return 0;
1486 }
1487
1488 #ifdef HUNSPELL_EXPERIMENTAL
1489 // suggest possible stems
suggest_pos_stems(char *** slst,const char * w,int nsug)1490 int SuggestMgr::suggest_pos_stems(char*** slst, const char * w, int nsug)
1491 {
1492 char ** wlst;
1493
1494 struct hentry * rv = NULL;
1495
1496 char w2[MAXSWUTF8L];
1497 const char * word = w;
1498
1499 // word reversing wrapper for complex prefixes
1500 if (complexprefixes) {
1501 strcpy(w2, w);
1502 if (utf8) reverseword_utf(w2); else reverseword(w2);
1503 word = w2;
1504 }
1505
1506 int wl = strlen(word);
1507
1508
1509 if (*slst) {
1510 wlst = *slst;
1511 } else {
1512 wlst = (char **) calloc(maxSug, sizeof(char *));
1513 if (wlst == NULL) return -1;
1514 }
1515
1516 rv = pAMgr->suffix_check(word, wl, 0, NULL, wlst, maxSug, &nsug);
1517
1518 // delete dash from end of word
1519 if (nsug > 0) {
1520 for (int j=0; j < nsug; j++) {
1521 if (wlst[j][strlen(wlst[j]) - 1] == '-') wlst[j][strlen(wlst[j]) - 1] = '\0';
1522 }
1523 }
1524
1525 *slst = wlst;
1526 return nsug;
1527 }
1528 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
1529
1530
suggest_morph(const char * w)1531 char * SuggestMgr::suggest_morph(const char * w)
1532 {
1533 char result[MAXLNLEN];
1534 char * r = (char *) result;
1535 char * st;
1536
1537 struct hentry * rv = NULL;
1538
1539 *result = '\0';
1540
1541 if (! pAMgr) return NULL;
1542
1543 char w2[MAXSWUTF8L];
1544 const char * word = w;
1545
1546 // word reversing wrapper for complex prefixes
1547 if (complexprefixes) {
1548 strcpy(w2, w);
1549 if (utf8) reverseword_utf(w2); else reverseword(w2);
1550 word = w2;
1551 }
1552
1553 rv = pAMgr->lookup(word);
1554
1555 while (rv) {
1556 if ((!rv->astr) || !(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
1557 TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
1558 TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) {
1559 if (!HENTRY_FIND(rv, MORPH_STEM)) {
1560 mystrcat(result, " ", MAXLNLEN);
1561 mystrcat(result, MORPH_STEM, MAXLNLEN);
1562 mystrcat(result, word, MAXLNLEN);
1563 }
1564 if (HENTRY_DATA(rv)) {
1565 mystrcat(result, " ", MAXLNLEN);
1566 mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN);
1567 }
1568 mystrcat(result, "\n", MAXLNLEN);
1569 }
1570 rv = rv->next_homonym;
1571 }
1572
1573 st = pAMgr->affix_check_morph(word,strlen(word));
1574 if (st) {
1575 mystrcat(result, st, MAXLNLEN);
1576 free(st);
1577 }
1578
1579 if (pAMgr->get_compound() && (*result == '\0'))
1580 pAMgr->compound_check_morph(word, strlen(word),
1581 0, 0, 100, 0,NULL, 0, &r, NULL);
1582
1583 return (*result) ? mystrdup(line_uniq(result, MSEP_REC)) : NULL;
1584 }
1585
1586 #ifdef HUNSPELL_EXPERIMENTAL
suggest_morph_for_spelling_error(const char * word)1587 char * SuggestMgr::suggest_morph_for_spelling_error(const char * word)
1588 {
1589 char * p = NULL;
1590 char ** wlst = (char **) calloc(maxSug, sizeof(char *));
1591 if (!**wlst) return NULL;
1592 // we will use only the first suggestion
1593 for (int i = 0; i < maxSug - 1; i++) wlst[i] = "";
1594 int ns = suggest(&wlst, word, maxSug - 1, NULL);
1595 if (ns == maxSug) {
1596 p = suggest_morph(wlst[maxSug - 1]);
1597 free(wlst[maxSug - 1]);
1598 }
1599 if (wlst) free(wlst);
1600 return p;
1601 }
1602 #endif // END OF HUNSPELL_EXPERIMENTAL CODE
1603
1604 /* affixation */
suggest_hentry_gen(hentry * rv,char * pattern)1605 char * SuggestMgr::suggest_hentry_gen(hentry * rv, char * pattern)
1606 {
1607 char result[MAXLNLEN];
1608 *result = '\0';
1609 int sfxcount = get_sfxcount(pattern);
1610
1611 if (get_sfxcount(HENTRY_DATA(rv)) > sfxcount) return NULL;
1612
1613 if (HENTRY_DATA(rv)) {
1614 char * aff = pAMgr->morphgen(HENTRY_WORD(rv), rv->blen, rv->astr, rv->alen,
1615 HENTRY_DATA(rv), pattern, 0);
1616 if (aff) {
1617 mystrcat(result, aff, MAXLNLEN);
1618 mystrcat(result, "\n", MAXLNLEN);
1619 free(aff);
1620 }
1621 }
1622
1623 // check all allomorphs
1624 char allomorph[MAXLNLEN];
1625 char * p = NULL;
1626 if (HENTRY_DATA(rv)) p = (char *) strstr(HENTRY_DATA2(rv), MORPH_ALLOMORPH);
1627 while (p) {
1628 struct hentry * rv2 = NULL;
1629 p += MORPH_TAG_LEN;
1630 int plen = fieldlen(p);
1631 strncpy(allomorph, p, plen);
1632 allomorph[plen] = '\0';
1633 rv2 = pAMgr->lookup(allomorph);
1634 while (rv2) {
1635 // if (HENTRY_DATA(rv2) && get_sfxcount(HENTRY_DATA(rv2)) <= sfxcount) {
1636 if (HENTRY_DATA(rv2)) {
1637 char * st = (char *) strstr(HENTRY_DATA2(rv2), MORPH_STEM);
1638 if (st && (strncmp(st + MORPH_TAG_LEN,
1639 HENTRY_WORD(rv), fieldlen(st + MORPH_TAG_LEN)) == 0)) {
1640 char * aff = pAMgr->morphgen(HENTRY_WORD(rv2), rv2->blen, rv2->astr, rv2->alen,
1641 HENTRY_DATA(rv2), pattern, 0);
1642 if (aff) {
1643 mystrcat(result, aff, MAXLNLEN);
1644 mystrcat(result, "\n", MAXLNLEN);
1645 free(aff);
1646 }
1647 }
1648 }
1649 rv2 = rv2->next_homonym;
1650 }
1651 p = strstr(p + plen, MORPH_ALLOMORPH);
1652 }
1653
1654 return (*result) ? mystrdup(result) : NULL;
1655 }
1656
suggest_gen(char ** desc,int n,char * pattern)1657 char * SuggestMgr::suggest_gen(char ** desc, int n, char * pattern) {
1658 char result[MAXLNLEN];
1659 char result2[MAXLNLEN];
1660 char newpattern[MAXLNLEN];
1661 *newpattern = '\0';
1662 if (n == 0) return 0;
1663 *result2 = '\0';
1664 struct hentry * rv = NULL;
1665 if (!pAMgr) return NULL;
1666
1667 // search affixed forms with and without derivational suffixes
1668 while(1) {
1669
1670 for (int k = 0; k < n; k++) {
1671 *result = '\0';
1672 // add compound word parts (except the last one)
1673 char * s = (char *) desc[k];
1674 char * part = strstr(s, MORPH_PART);
1675 if (part) {
1676 char * nextpart = strstr(part + 1, MORPH_PART);
1677 while (nextpart) {
1678 copy_field(result + strlen(result), part, MORPH_PART);
1679 part = nextpart;
1680 nextpart = strstr(part + 1, MORPH_PART);
1681 }
1682 s = part;
1683 }
1684
1685 char **pl;
1686 char tok[MAXLNLEN];
1687 strcpy(tok, s);
1688 char * alt = strstr(tok, " | ");
1689 while (alt) {
1690 alt[1] = MSEP_ALT;
1691 alt = strstr(alt, " | ");
1692 }
1693 int pln = line_tok(tok, &pl, MSEP_ALT);
1694 for (int i = 0; i < pln; i++) {
1695 // remove inflectional and terminal suffixes
1696 char * is = strstr(pl[i], MORPH_INFL_SFX);
1697 if (is) *is = '\0';
1698 char * ts = strstr(pl[i], MORPH_TERM_SFX);
1699 while (ts) {
1700 *ts = '_';
1701 ts = strstr(pl[i], MORPH_TERM_SFX);
1702 }
1703 char * st = strstr(s, MORPH_STEM);
1704 if (st) {
1705 copy_field(tok, st, MORPH_STEM);
1706 rv = pAMgr->lookup(tok);
1707 while (rv) {
1708 char newpat[MAXLNLEN];
1709 strcpy(newpat, pl[i]);
1710 strcat(newpat, pattern);
1711 char * sg = suggest_hentry_gen(rv, newpat);
1712 if (!sg) sg = suggest_hentry_gen(rv, pattern);
1713 if (sg) {
1714 char ** gen;
1715 int genl = line_tok(sg, &gen, MSEP_REC);
1716 free(sg);
1717 sg = NULL;
1718 for (int j = 0; j < genl; j++) {
1719 if (strstr(pl[i], MORPH_SURF_PFX)) {
1720 int r2l = strlen(result2);
1721 result2[r2l] = MSEP_REC;
1722 strcpy(result2 + r2l + 1, result);
1723 copy_field(result2 + strlen(result2), pl[i], MORPH_SURF_PFX);
1724 mystrcat(result2, gen[j], MAXLNLEN);
1725 } else {
1726 sprintf(result2 + strlen(result2), "%c%s%s",
1727 MSEP_REC, result, gen[j]);
1728 }
1729 }
1730 freelist(&gen, genl);
1731 }
1732 rv = rv->next_homonym;
1733 }
1734 }
1735 }
1736 freelist(&pl, pln);
1737 }
1738
1739 if (*result2 || !strstr(pattern, MORPH_DERI_SFX)) break;
1740 strcpy(newpattern, pattern);
1741 pattern = newpattern;
1742 char * ds = strstr(pattern, MORPH_DERI_SFX);
1743 while (ds) {
1744 strncpy(ds, MORPH_TERM_SFX, MORPH_TAG_LEN);
1745 ds = strstr(pattern, MORPH_DERI_SFX);
1746 }
1747 }
1748 return (*result2 ? mystrdup(result2) : NULL);
1749 }
1750
1751
1752 // generate an n-gram score comparing s1 and s2
ngram(int n,char * s1,const char * s2,int opt)1753 int SuggestMgr::ngram(int n, char * s1, const char * s2, int opt)
1754 {
1755 int nscore = 0;
1756 int ns;
1757 int l1;
1758 int l2;
1759 int test = 0;
1760
1761 if (utf8) {
1762 w_char su1[MAXSWL];
1763 w_char su2[MAXSWL];
1764 l1 = u8_u16(su1, MAXSWL, s1);
1765 l2 = u8_u16(su2, MAXSWL, s2);
1766 if ((l2 <= 0) || (l1 == -1)) return 0;
1767 // lowering dictionary word
1768 if (opt & NGRAM_LOWERING) mkallsmall_utf(su2, l2, langnum);
1769 for (int j = 1; j <= n; j++) {
1770 ns = 0;
1771 for (int i = 0; i <= (l1-j); i++) {
1772 int k = 0;
1773 for (int l = 0; l <= (l2-j); l++) {
1774 for (k = 0; k < j; k++) {
1775 w_char * c1 = su1 + i + k;
1776 w_char * c2 = su2 + l + k;
1777 if ((c1->l != c2->l) || (c1->h != c2->h)) break;
1778 }
1779 if (k == j) {
1780 ns++;
1781 break;
1782 }
1783 }
1784 if (k != j && opt & NGRAM_WEIGHTED) {
1785 ns--;
1786 test++;
1787 if (i == 0 || i == l1-j) ns--; // side weight
1788 }
1789 }
1790 nscore = nscore + ns;
1791 if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break;
1792 }
1793 } else {
1794 l2 = strlen(s2);
1795 if (l2 == 0) return 0;
1796 l1 = strlen(s1);
1797 char *t = mystrdup(s2);
1798 if (opt & NGRAM_LOWERING) mkallsmall(t, csconv);
1799 for (int j = 1; j <= n; j++) {
1800 ns = 0;
1801 for (int i = 0; i <= (l1-j); i++) {
1802 char c = *(s1 + i + j);
1803 *(s1 + i + j) = '\0';
1804 if (strstr(t,(s1+i))) {
1805 ns++;
1806 } else if (opt & NGRAM_WEIGHTED) {
1807 ns--;
1808 test++;
1809 if (i == 0 || i == l1-j) ns--; // side weight
1810 }
1811 *(s1 + i + j ) = c;
1812 }
1813 nscore = nscore + ns;
1814 if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break;
1815 }
1816 free(t);
1817 }
1818
1819 ns = 0;
1820 if (opt & NGRAM_LONGER_WORSE) ns = (l2-l1)-2;
1821 if (opt & NGRAM_ANY_MISMATCH) ns = abs(l2-l1)-2;
1822 ns = (nscore - ((ns > 0) ? ns : 0));
1823 return ns;
1824 }
1825
1826 // length of the left common substring of s1 and (decapitalised) s2
leftcommonsubstring(char * s1,const char * s2)1827 int SuggestMgr::leftcommonsubstring(char * s1, const char * s2) {
1828 if (utf8) {
1829 w_char su1[MAXSWL];
1830 w_char su2[MAXSWL];
1831 su1[0].l = su2[0].l = su1[0].h = su2[0].h = 0;
1832 // decapitalize dictionary word
1833 if (complexprefixes) {
1834 int l1 = u8_u16(su1, MAXSWL, s1);
1835 int l2 = u8_u16(su2, MAXSWL, s2);
1836 if (*((short *)su1+l1-1) == *((short *)su2+l2-1)) return 1;
1837 } else {
1838 int i;
1839 u8_u16(su1, 1, s1);
1840 u8_u16(su2, 1, s2);
1841 unsigned short idx = (su2->h << 8) + su2->l;
1842 unsigned short otheridx = (su1->h << 8) + su1->l;
1843 if (otheridx != idx &&
1844 (otheridx != unicodetolower(idx, langnum))) return 0;
1845 int l1 = u8_u16(su1, MAXSWL, s1);
1846 int l2 = u8_u16(su2, MAXSWL, s2);
1847 for(i = 1; (i < l1) && (i < l2) &&
1848 (su1[i].l == su2[i].l) && (su1[i].h == su2[i].h); i++);
1849 return i;
1850 }
1851 } else {
1852 if (complexprefixes) {
1853 int l1 = strlen(s1);
1854 int l2 = strlen(s2);
1855 if (*(s2+l1-1) == *(s2+l2-1)) return 1;
1856 } else {
1857 char * olds = s1;
1858 // decapitalise dictionary word
1859 if ((*s1 != *s2) && (*s1 != csconv[((unsigned char)*s2)].clower)) return 0;
1860 do {
1861 s1++; s2++;
1862 } while ((*s1 == *s2) && (*s1 != '\0'));
1863 return (int)(s1 - olds);
1864 }
1865 }
1866 return 0;
1867 }
1868
commoncharacterpositions(char * s1,const char * s2,int * is_swap)1869 int SuggestMgr::commoncharacterpositions(char * s1, const char * s2, int * is_swap) {
1870 int num = 0;
1871 int diff = 0;
1872 int diffpos[2];
1873 *is_swap = 0;
1874 if (utf8) {
1875 w_char su1[MAXSWL];
1876 w_char su2[MAXSWL];
1877 int l1 = u8_u16(su1, MAXSWL, s1);
1878 int l2 = u8_u16(su2, MAXSWL, s2);
1879
1880 if (l1 <= 0 || l2 <= 0)
1881 return 0;
1882
1883 // decapitalize dictionary word
1884 if (complexprefixes) {
1885 mkallsmall_utf(su2+l2-1, 1, langnum);
1886 } else {
1887 mkallsmall_utf(su2, 1, langnum);
1888 }
1889 for (int i = 0; (i < l1) && (i < l2); i++) {
1890 if (((short *) su1)[i] == ((short *) su2)[i]) {
1891 num++;
1892 } else {
1893 if (diff < 2) diffpos[diff] = i;
1894 diff++;
1895 }
1896 }
1897 if ((diff == 2) && (l1 == l2) &&
1898 (((short *) su1)[diffpos[0]] == ((short *) su2)[diffpos[1]]) &&
1899 (((short *) su1)[diffpos[1]] == ((short *) su2)[diffpos[0]])) *is_swap = 1;
1900 } else {
1901 int i;
1902 char t[MAXSWUTF8L];
1903 strcpy(t, s2);
1904 // decapitalize dictionary word
1905 if (complexprefixes) {
1906 int l2 = strlen(t);
1907 *(t+l2-1) = csconv[((unsigned char)*(t+l2-1))].clower;
1908 } else {
1909 mkallsmall(t, csconv);
1910 }
1911 for (i = 0; (*(s1+i) != 0) && (*(t+i) != 0); i++) {
1912 if (*(s1+i) == *(t+i)) {
1913 num++;
1914 } else {
1915 if (diff < 2) diffpos[diff] = i;
1916 diff++;
1917 }
1918 }
1919 if ((diff == 2) && (*(s1+i) == 0) && (*(t+i) == 0) &&
1920 (*(s1+diffpos[0]) == *(t+diffpos[1])) &&
1921 (*(s1+diffpos[1]) == *(t+diffpos[0]))) *is_swap = 1;
1922 }
1923 return num;
1924 }
1925
mystrlen(const char * word)1926 int SuggestMgr::mystrlen(const char * word) {
1927 if (utf8) {
1928 w_char w[MAXSWL];
1929 return u8_u16(w, MAXSWL, word);
1930 } else return strlen(word);
1931 }
1932
1933 // sort in decreasing order of score
bubblesort(char ** rword,char ** rword2,int * rsc,int n)1934 void SuggestMgr::bubblesort(char** rword, char** rword2, int* rsc, int n )
1935 {
1936 int m = 1;
1937 while (m < n) {
1938 int j = m;
1939 while (j > 0) {
1940 if (rsc[j-1] < rsc[j]) {
1941 int sctmp = rsc[j-1];
1942 char * wdtmp = rword[j-1];
1943 rsc[j-1] = rsc[j];
1944 rword[j-1] = rword[j];
1945 rsc[j] = sctmp;
1946 rword[j] = wdtmp;
1947 if (rword2) {
1948 wdtmp = rword2[j-1];
1949 rword2[j-1] = rword2[j];
1950 rword2[j] = wdtmp;
1951 }
1952 j--;
1953 } else break;
1954 }
1955 m++;
1956 }
1957 return;
1958 }
1959
1960 // longest common subsequence
lcs(const char * s,const char * s2,int * l1,int * l2,char ** result)1961 void SuggestMgr::lcs(const char * s, const char * s2, int * l1, int * l2, char ** result) {
1962 int n, m;
1963 w_char su[MAXSWL];
1964 w_char su2[MAXSWL];
1965 char * b;
1966 char * c;
1967 int i;
1968 int j;
1969 if (utf8) {
1970 m = u8_u16(su, MAXSWL, s);
1971 n = u8_u16(su2, MAXSWL, s2);
1972 } else {
1973 m = strlen(s);
1974 n = strlen(s2);
1975 }
1976 c = (char *) malloc((m + 1) * (n + 1));
1977 b = (char *) malloc((m + 1) * (n + 1));
1978 if (!c || !b) {
1979 if (c) free(c);
1980 if (b) free(b);
1981 *result = NULL;
1982 return;
1983 }
1984 for (i = 1; i <= m; i++) c[i*(n+1)] = 0;
1985 for (j = 0; j <= n; j++) c[j] = 0;
1986 for (i = 1; i <= m; i++) {
1987 for (j = 1; j <= n; j++) {
1988 if ( ((utf8) && (*((short *) su+i-1) == *((short *)su2+j-1)))
1989 || ((!utf8) && ((*(s+i-1)) == (*(s2+j-1))))) {
1990 c[i*(n+1) + j] = c[(i-1)*(n+1) + j-1]+1;
1991 b[i*(n+1) + j] = LCS_UPLEFT;
1992 } else if (c[(i-1)*(n+1) + j] >= c[i*(n+1) + j-1]) {
1993 c[i*(n+1) + j] = c[(i-1)*(n+1) + j];
1994 b[i*(n+1) + j] = LCS_UP;
1995 } else {
1996 c[i*(n+1) + j] = c[i*(n+1) + j-1];
1997 b[i*(n+1) + j] = LCS_LEFT;
1998 }
1999 }
2000 }
2001 *result = b;
2002 free(c);
2003 *l1 = m;
2004 *l2 = n;
2005 }
2006
lcslen(const char * s,const char * s2)2007 int SuggestMgr::lcslen(const char * s, const char* s2) {
2008 int m;
2009 int n;
2010 int i;
2011 int j;
2012 char * result;
2013 int len = 0;
2014 lcs(s, s2, &m, &n, &result);
2015 if (!result) return 0;
2016 i = m;
2017 j = n;
2018 while ((i != 0) && (j != 0)) {
2019 if (result[i*(n+1) + j] == LCS_UPLEFT) {
2020 len++;
2021 i--;
2022 j--;
2023 } else if (result[i*(n+1) + j] == LCS_UP) {
2024 i--;
2025 } else j--;
2026 }
2027 free(result);
2028 return len;
2029 }
2030