1 /* vi:set ts=8 sts=4 sw=4 noet:
2 *
3 * VIM - Vi IMproved by Bram Moolenaar
4 *
5 * Do ":help uganda" in Vim to read copying and usage conditions.
6 * Do ":help credits" in Vim to see a list of people who contributed.
7 * See README.txt for an overview of the Vim source code.
8 */
9
10 /*
11 * spellfile.c: code for reading and writing spell files.
12 *
13 * See spell.c for information about spell checking.
14 */
15
16 /*
17 * Vim spell file format: <HEADER>
18 * <SECTIONS>
19 * <LWORDTREE>
20 * <KWORDTREE>
21 * <PREFIXTREE>
22 *
23 * <HEADER>: <fileID> <versionnr>
24 *
25 * <fileID> 8 bytes "VIMspell"
26 * <versionnr> 1 byte VIMSPELLVERSION
27 *
28 *
29 * Sections make it possible to add information to the .spl file without
30 * making it incompatible with previous versions. There are two kinds of
31 * sections:
32 * 1. Not essential for correct spell checking. E.g. for making suggestions.
33 * These are skipped when not supported.
34 * 2. Optional information, but essential for spell checking when present.
35 * E.g. conditions for affixes. When this section is present but not
36 * supported an error message is given.
37 *
38 * <SECTIONS>: <section> ... <sectionend>
39 *
40 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents)
41 *
42 * <sectionID> 1 byte number from 0 to 254 identifying the section
43 *
44 * <sectionflags> 1 byte SNF_REQUIRED: this section is required for correct
45 * spell checking
46 *
47 * <sectionlen> 4 bytes length of section contents, MSB first
48 *
49 * <sectionend> 1 byte SN_END
50 *
51 *
52 * sectionID == SN_INFO: <infotext>
53 * <infotext> N bytes free format text with spell file info (version,
54 * website, etc)
55 *
56 * sectionID == SN_REGION: <regionname> ...
57 * <regionname> 2 bytes Up to MAXREGIONS region names: ca, au, etc. Lower
58 * case. First <regionname> is region 1.
59 *
60 * sectionID == SN_CHARFLAGS: <charflagslen> <charflags>
61 * <folcharslen> <folchars>
62 * <charflagslen> 1 byte Number of bytes in <charflags> (should be 128).
63 * <charflags> N bytes List of flags (first one is for character 128):
64 * 0x01 word character CF_WORD
65 * 0x02 upper-case character CF_UPPER
66 * <folcharslen> 2 bytes Number of bytes in <folchars>.
67 * <folchars> N bytes Folded characters, first one is for character 128.
68 *
69 * sectionID == SN_MIDWORD: <midword>
70 * <midword> N bytes Characters that are word characters only when used
71 * in the middle of a word.
72 *
73 * sectionID == SN_PREFCOND: <prefcondcnt> <prefcond> ...
74 * <prefcondcnt> 2 bytes Number of <prefcond> items following.
75 * <prefcond> : <condlen> <condstr>
76 * <condlen> 1 byte Length of <condstr>.
77 * <condstr> N bytes Condition for the prefix.
78 *
79 * sectionID == SN_REP: <repcount> <rep> ...
80 * <repcount> 2 bytes number of <rep> items, MSB first.
81 * <rep> : <repfromlen> <repfrom> <reptolen> <repto>
82 * <repfromlen> 1 byte length of <repfrom>
83 * <repfrom> N bytes "from" part of replacement
84 * <reptolen> 1 byte length of <repto>
85 * <repto> N bytes "to" part of replacement
86 *
87 * sectionID == SN_REPSAL: <repcount> <rep> ...
88 * just like SN_REP but for soundfolded words
89 *
90 * sectionID == SN_SAL: <salflags> <salcount> <sal> ...
91 * <salflags> 1 byte flags for soundsalike conversion:
92 * SAL_F0LLOWUP
93 * SAL_COLLAPSE
94 * SAL_REM_ACCENTS
95 * <salcount> 2 bytes number of <sal> items following
96 * <sal> : <salfromlen> <salfrom> <saltolen> <salto>
97 * <salfromlen> 1 byte length of <salfrom>
98 * <salfrom> N bytes "from" part of soundsalike
99 * <saltolen> 1 byte length of <salto>
100 * <salto> N bytes "to" part of soundsalike
101 *
102 * sectionID == SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
103 * <sofofromlen> 2 bytes length of <sofofrom>
104 * <sofofrom> N bytes "from" part of soundfold
105 * <sofotolen> 2 bytes length of <sofoto>
106 * <sofoto> N bytes "to" part of soundfold
107 *
108 * sectionID == SN_SUGFILE: <timestamp>
109 * <timestamp> 8 bytes time in seconds that must match with .sug file
110 *
111 * sectionID == SN_NOSPLITSUGS: nothing
112 *
113 * sectionID == SN_NOCOMPOUNDSUGS: nothing
114 *
115 * sectionID == SN_WORDS: <word> ...
116 * <word> N bytes NUL terminated common word
117 *
118 * sectionID == SN_MAP: <mapstr>
119 * <mapstr> N bytes String with sequences of similar characters,
120 * separated by slashes.
121 *
122 * sectionID == SN_COMPOUND: <compmax> <compminlen> <compsylmax> <compoptions>
123 * <comppatcount> <comppattern> ... <compflags>
124 * <compmax> 1 byte Maximum nr of words in compound word.
125 * <compminlen> 1 byte Minimal word length for compounding.
126 * <compsylmax> 1 byte Maximum nr of syllables in compound word.
127 * <compoptions> 2 bytes COMP_ flags.
128 * <comppatcount> 2 bytes number of <comppattern> following
129 * <compflags> N bytes Flags from COMPOUNDRULE items, separated by
130 * slashes.
131 *
132 * <comppattern>: <comppatlen> <comppattext>
133 * <comppatlen> 1 byte length of <comppattext>
134 * <comppattext> N bytes end or begin chars from CHECKCOMPOUNDPATTERN
135 *
136 * sectionID == SN_NOBREAK: (empty, its presence is what matters)
137 *
138 * sectionID == SN_SYLLABLE: <syllable>
139 * <syllable> N bytes String from SYLLABLE item.
140 *
141 * <LWORDTREE>: <wordtree>
142 *
143 * <KWORDTREE>: <wordtree>
144 *
145 * <PREFIXTREE>: <wordtree>
146 *
147 *
148 * <wordtree>: <nodecount> <nodedata> ...
149 *
150 * <nodecount> 4 bytes Number of nodes following. MSB first.
151 *
152 * <nodedata>: <siblingcount> <sibling> ...
153 *
154 * <siblingcount> 1 byte Number of siblings in this node. The siblings
155 * follow in sorted order.
156 *
157 * <sibling>: <byte> [ <nodeidx> <xbyte>
158 * | <flags> [<flags2>] [<region>] [<affixID>]
159 * | [<pflags>] <affixID> <prefcondnr> ]
160 *
161 * <byte> 1 byte Byte value of the sibling. Special cases:
162 * BY_NOFLAGS: End of word without flags and for all
163 * regions.
164 * For PREFIXTREE <affixID> and
165 * <prefcondnr> follow.
166 * BY_FLAGS: End of word, <flags> follow.
167 * For PREFIXTREE <pflags>, <affixID>
168 * and <prefcondnr> follow.
169 * BY_FLAGS2: End of word, <flags> and <flags2>
170 * follow. Not used in PREFIXTREE.
171 * BY_INDEX: Child of sibling is shared, <nodeidx>
172 * and <xbyte> follow.
173 *
174 * <nodeidx> 3 bytes Index of child for this sibling, MSB first.
175 *
176 * <xbyte> 1 byte byte value of the sibling.
177 *
178 * <flags> 1 byte bitmask of:
179 * WF_ALLCAP word must have only capitals
180 * WF_ONECAP first char of word must be capital
181 * WF_KEEPCAP keep-case word
182 * WF_FIXCAP keep-case word, all caps not allowed
183 * WF_RARE rare word
184 * WF_BANNED bad word
185 * WF_REGION <region> follows
186 * WF_AFX <affixID> follows
187 *
188 * <flags2> 1 byte Bitmask of:
189 * WF_HAS_AFF >> 8 word includes affix
190 * WF_NEEDCOMP >> 8 word only valid in compound
191 * WF_NOSUGGEST >> 8 word not used for suggestions
192 * WF_COMPROOT >> 8 word already a compound
193 * WF_NOCOMPBEF >> 8 no compounding before this word
194 * WF_NOCOMPAFT >> 8 no compounding after this word
195 *
196 * <pflags> 1 byte bitmask of:
197 * WFP_RARE rare prefix
198 * WFP_NC non-combining prefix
199 * WFP_UP letter after prefix made upper case
200 *
201 * <region> 1 byte Bitmask for regions in which word is valid. When
202 * omitted it's valid in all regions.
203 * Lowest bit is for region 1.
204 *
205 * <affixID> 1 byte ID of affix that can be used with this word. In
206 * PREFIXTREE used for the required prefix ID.
207 *
208 * <prefcondnr> 2 bytes Prefix condition number, index in <prefcond> list
209 * from HEADER.
210 *
211 * All text characters are in 'encoding', but stored as single bytes.
212 */
213
214 /*
215 * Vim .sug file format: <SUGHEADER>
216 * <SUGWORDTREE>
217 * <SUGTABLE>
218 *
219 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
220 *
221 * <fileID> 6 bytes "VIMsug"
222 * <versionnr> 1 byte VIMSUGVERSION
223 * <timestamp> 8 bytes timestamp that must match with .spl file
224 *
225 *
226 * <SUGWORDTREE>: <wordtree> (see above, no flags or region used)
227 *
228 *
229 * <SUGTABLE>: <sugwcount> <sugline> ...
230 *
231 * <sugwcount> 4 bytes number of <sugline> following
232 *
233 * <sugline>: <sugnr> ... NUL
234 *
235 * <sugnr>: X bytes word number that results in this soundfolded word,
236 * stored as an offset to the previous number in as
237 * few bytes as possible, see offset2bytes())
238 */
239
240 #include "vim.h"
241
242 #if defined(FEAT_SPELL) || defined(PROTO)
243
244 #ifndef UNIX // it's in os_unix.h for Unix
245 # include <time.h> // for time_t
246 #endif
247
248 #ifndef UNIX // it's in os_unix.h for Unix
249 # include <time.h> // for time_t
250 #endif
251
252 // Special byte values for <byte>. Some are only used in the tree for
253 // postponed prefixes, some only in the other trees. This is a bit messy...
254 #define BY_NOFLAGS 0 // end of word without flags or region; for
255 // postponed prefix: no <pflags>
256 #define BY_INDEX 1 // child is shared, index follows
257 #define BY_FLAGS 2 // end of word, <flags> byte follows; for
258 // postponed prefix: <pflags> follows
259 #define BY_FLAGS2 3 // end of word, <flags> and <flags2> bytes
260 // follow; never used in prefix tree
261 #define BY_SPECIAL BY_FLAGS2 // highest special byte value
262
263 #define ZERO_FLAG 65009 // used when flag is zero: "0"
264
265 // Flags used in .spl file for soundsalike flags.
266 #define SAL_F0LLOWUP 1
267 #define SAL_COLLAPSE 2
268 #define SAL_REM_ACCENTS 4
269
270 #define VIMSPELLMAGIC "VIMspell" // string at start of Vim spell file
271 #define VIMSPELLMAGICL 8
272 #define VIMSPELLVERSION 50
273
274 // Section IDs. Only renumber them when VIMSPELLVERSION changes!
275 #define SN_REGION 0 // <regionname> section
276 #define SN_CHARFLAGS 1 // charflags section
277 #define SN_MIDWORD 2 // <midword> section
278 #define SN_PREFCOND 3 // <prefcond> section
279 #define SN_REP 4 // REP items section
280 #define SN_SAL 5 // SAL items section
281 #define SN_SOFO 6 // soundfolding section
282 #define SN_MAP 7 // MAP items section
283 #define SN_COMPOUND 8 // compound words section
284 #define SN_SYLLABLE 9 // syllable section
285 #define SN_NOBREAK 10 // NOBREAK section
286 #define SN_SUGFILE 11 // timestamp for .sug file
287 #define SN_REPSAL 12 // REPSAL items section
288 #define SN_WORDS 13 // common words
289 #define SN_NOSPLITSUGS 14 // don't split word for suggestions
290 #define SN_INFO 15 // info section
291 #define SN_NOCOMPOUNDSUGS 16 // don't compound for suggestions
292 #define SN_END 255 // end of sections
293
294 #define SNF_REQUIRED 1 // <sectionflags>: required section
295
296 #define CF_WORD 0x01
297 #define CF_UPPER 0x02
298
299 /*
300 * Loop through all the siblings of a node (including the node)
301 */
302 #define FOR_ALL_NODE_SIBLINGS(node, np) \
303 for ((np) = (node); (np) != NULL; (np) = (np)->wn_sibling)
304
305 static int set_spell_finish(spelltab_T *new_st);
306 static int write_spell_prefcond(FILE *fd, garray_T *gap);
307 static int read_region_section(FILE *fd, slang_T *slang, int len);
308 static int read_charflags_section(FILE *fd);
309 static int read_prefcond_section(FILE *fd, slang_T *lp);
310 static int read_rep_section(FILE *fd, garray_T *gap, short *first);
311 static int read_sal_section(FILE *fd, slang_T *slang);
312 static int read_words_section(FILE *fd, slang_T *lp, int len);
313 static int read_sofo_section(FILE *fd, slang_T *slang);
314 static int read_compound(FILE *fd, slang_T *slang, int len);
315 static int set_sofo(slang_T *lp, char_u *from, char_u *to);
316 static void set_sal_first(slang_T *lp);
317 static int *mb_str2wide(char_u *s);
318 static int spell_read_tree(FILE *fd, char_u **bytsp, long *bytsp_len, idx_T **idxsp, int prefixtree, int prefixcnt);
319 static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);
320 static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);
321 static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);
322 static void set_map_str(slang_T *lp, char_u *map);
323
324
325 static char *e_spell_trunc = N_("E758: Truncated spell file");
326 static char *e_afftrailing = N_("Trailing text in %s line %d: %s");
327 static char *e_affname = N_("Affix name too long in %s line %d: %s");
328 static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");
329 static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");
330 static char *msg_compressing = N_("Compressing word tree...");
331
332 /*
333 * Load one spell file and store the info into a slang_T.
334 *
335 * This is invoked in three ways:
336 * - From spell_load_cb() to load a spell file for the first time. "lang" is
337 * the language name, "old_lp" is NULL. Will allocate an slang_T.
338 * - To reload a spell file that was changed. "lang" is NULL and "old_lp"
339 * points to the existing slang_T.
340 * - Just after writing a .spl file; it's read back to produce the .sug file.
341 * "old_lp" is NULL and "lang" is NULL. Will allocate an slang_T.
342 *
343 * Returns the slang_T the spell file was loaded into. NULL for error.
344 */
345 slang_T *
spell_load_file(char_u * fname,char_u * lang,slang_T * old_lp,int silent)346 spell_load_file(
347 char_u *fname,
348 char_u *lang,
349 slang_T *old_lp,
350 int silent) // no error if file doesn't exist
351 {
352 FILE *fd;
353 char_u buf[VIMSPELLMAGICL];
354 char_u *p;
355 int i;
356 int n;
357 int len;
358 slang_T *lp = NULL;
359 int c = 0;
360 int res;
361 int did_estack_push = FALSE;
362 ESTACK_CHECK_DECLARATION
363
364 fd = mch_fopen((char *)fname, "r");
365 if (fd == NULL)
366 {
367 if (!silent)
368 semsg(_(e_notopen), fname);
369 else if (p_verbose > 2)
370 {
371 verbose_enter();
372 smsg((const char *)e_notopen, fname);
373 verbose_leave();
374 }
375 goto endFAIL;
376 }
377 if (p_verbose > 2)
378 {
379 verbose_enter();
380 smsg(_("Reading spell file \"%s\""), fname);
381 verbose_leave();
382 }
383
384 if (old_lp == NULL)
385 {
386 lp = slang_alloc(lang);
387 if (lp == NULL)
388 goto endFAIL;
389
390 // Remember the file name, used to reload the file when it's updated.
391 lp->sl_fname = vim_strsave(fname);
392 if (lp->sl_fname == NULL)
393 goto endFAIL;
394
395 // Check for .add.spl (_add.spl for VMS).
396 lp->sl_add = strstr((char *)gettail(fname), SPL_FNAME_ADD) != NULL;
397 }
398 else
399 lp = old_lp;
400
401 // Set sourcing_name, so that error messages mention the file name.
402 estack_push(ETYPE_SPELL, fname, 0);
403 ESTACK_CHECK_SETUP
404 did_estack_push = TRUE;
405
406 /*
407 * <HEADER>: <fileID>
408 */
409 for (i = 0; i < VIMSPELLMAGICL; ++i)
410 buf[i] = getc(fd); // <fileID>
411 if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0)
412 {
413 emsg(_("E757: This does not look like a spell file"));
414 goto endFAIL;
415 }
416 c = getc(fd); // <versionnr>
417 if (c < VIMSPELLVERSION)
418 {
419 emsg(_("E771: Old spell file, needs to be updated"));
420 goto endFAIL;
421 }
422 else if (c > VIMSPELLVERSION)
423 {
424 emsg(_("E772: Spell file is for newer version of Vim"));
425 goto endFAIL;
426 }
427
428
429 /*
430 * <SECTIONS>: <section> ... <sectionend>
431 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents)
432 */
433 for (;;)
434 {
435 n = getc(fd); // <sectionID> or <sectionend>
436 if (n == SN_END)
437 break;
438 c = getc(fd); // <sectionflags>
439 len = get4c(fd); // <sectionlen>
440 if (len < 0)
441 goto truncerr;
442
443 res = 0;
444 switch (n)
445 {
446 case SN_INFO:
447 lp->sl_info = read_string(fd, len); // <infotext>
448 if (lp->sl_info == NULL)
449 goto endFAIL;
450 break;
451
452 case SN_REGION:
453 res = read_region_section(fd, lp, len);
454 break;
455
456 case SN_CHARFLAGS:
457 res = read_charflags_section(fd);
458 break;
459
460 case SN_MIDWORD:
461 lp->sl_midword = read_string(fd, len); // <midword>
462 if (lp->sl_midword == NULL)
463 goto endFAIL;
464 break;
465
466 case SN_PREFCOND:
467 res = read_prefcond_section(fd, lp);
468 break;
469
470 case SN_REP:
471 res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first);
472 break;
473
474 case SN_REPSAL:
475 res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first);
476 break;
477
478 case SN_SAL:
479 res = read_sal_section(fd, lp);
480 break;
481
482 case SN_SOFO:
483 res = read_sofo_section(fd, lp);
484 break;
485
486 case SN_MAP:
487 p = read_string(fd, len); // <mapstr>
488 if (p == NULL)
489 goto endFAIL;
490 set_map_str(lp, p);
491 vim_free(p);
492 break;
493
494 case SN_WORDS:
495 res = read_words_section(fd, lp, len);
496 break;
497
498 case SN_SUGFILE:
499 lp->sl_sugtime = get8ctime(fd); // <timestamp>
500 break;
501
502 case SN_NOSPLITSUGS:
503 lp->sl_nosplitsugs = TRUE;
504 break;
505
506 case SN_NOCOMPOUNDSUGS:
507 lp->sl_nocompoundsugs = TRUE;
508 break;
509
510 case SN_COMPOUND:
511 res = read_compound(fd, lp, len);
512 break;
513
514 case SN_NOBREAK:
515 lp->sl_nobreak = TRUE;
516 break;
517
518 case SN_SYLLABLE:
519 lp->sl_syllable = read_string(fd, len); // <syllable>
520 if (lp->sl_syllable == NULL)
521 goto endFAIL;
522 if (init_syl_tab(lp) != OK)
523 goto endFAIL;
524 break;
525
526 default:
527 // Unsupported section. When it's required give an error
528 // message. When it's not required skip the contents.
529 if (c & SNF_REQUIRED)
530 {
531 emsg(_("E770: Unsupported section in spell file"));
532 goto endFAIL;
533 }
534 while (--len >= 0)
535 if (getc(fd) < 0)
536 goto truncerr;
537 break;
538 }
539 someerror:
540 if (res == SP_FORMERROR)
541 {
542 emsg(_(e_format));
543 goto endFAIL;
544 }
545 if (res == SP_TRUNCERROR)
546 {
547 truncerr:
548 emsg(_(e_spell_trunc));
549 goto endFAIL;
550 }
551 if (res == SP_OTHERERROR)
552 goto endFAIL;
553 }
554
555 // <LWORDTREE>
556 res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fbyts_len,
557 &lp->sl_fidxs, FALSE, 0);
558 if (res != 0)
559 goto someerror;
560
561 // <KWORDTREE>
562 res = spell_read_tree(fd, &lp->sl_kbyts, NULL, &lp->sl_kidxs, FALSE, 0);
563 if (res != 0)
564 goto someerror;
565
566 // <PREFIXTREE>
567 res = spell_read_tree(fd, &lp->sl_pbyts, NULL, &lp->sl_pidxs, TRUE,
568 lp->sl_prefixcnt);
569 if (res != 0)
570 goto someerror;
571
572 // For a new file link it in the list of spell files.
573 if (old_lp == NULL && lang != NULL)
574 {
575 lp->sl_next = first_lang;
576 first_lang = lp;
577 }
578
579 goto endOK;
580
581 endFAIL:
582 if (lang != NULL)
583 // truncating the name signals the error to spell_load_lang()
584 *lang = NUL;
585 if (lp != NULL && old_lp == NULL)
586 slang_free(lp);
587 lp = NULL;
588
589 endOK:
590 if (fd != NULL)
591 fclose(fd);
592 if (did_estack_push)
593 {
594 ESTACK_CHECK_NOW
595 estack_pop();
596 }
597
598 return lp;
599 }
600
601 /*
602 * Fill in the wordcount fields for a trie.
603 * Returns the total number of words.
604 */
605 static void
tree_count_words(char_u * byts,idx_T * idxs)606 tree_count_words(char_u *byts, idx_T *idxs)
607 {
608 int depth;
609 idx_T arridx[MAXWLEN];
610 int curi[MAXWLEN];
611 int c;
612 idx_T n;
613 int wordcount[MAXWLEN];
614
615 arridx[0] = 0;
616 curi[0] = 1;
617 wordcount[0] = 0;
618 depth = 0;
619 while (depth >= 0 && !got_int)
620 {
621 if (curi[depth] > byts[arridx[depth]])
622 {
623 // Done all bytes at this node, go up one level.
624 idxs[arridx[depth]] = wordcount[depth];
625 if (depth > 0)
626 wordcount[depth - 1] += wordcount[depth];
627
628 --depth;
629 fast_breakcheck();
630 }
631 else
632 {
633 // Do one more byte at this node.
634 n = arridx[depth] + curi[depth];
635 ++curi[depth];
636
637 c = byts[n];
638 if (c == 0)
639 {
640 // End of word, count it.
641 ++wordcount[depth];
642
643 // Skip over any other NUL bytes (same word with different
644 // flags).
645 while (byts[n + 1] == 0)
646 {
647 ++n;
648 ++curi[depth];
649 }
650 }
651 else
652 {
653 // Normal char, go one level deeper to count the words.
654 ++depth;
655 arridx[depth] = idxs[n];
656 curi[depth] = 1;
657 wordcount[depth] = 0;
658 }
659 }
660 }
661 }
662
663 /*
664 * Load the .sug files for languages that have one and weren't loaded yet.
665 */
666 void
suggest_load_files(void)667 suggest_load_files(void)
668 {
669 langp_T *lp;
670 int lpi;
671 slang_T *slang;
672 char_u *dotp;
673 FILE *fd;
674 char_u buf[MAXWLEN];
675 int i;
676 time_t timestamp;
677 int wcount;
678 int wordnr;
679 garray_T ga;
680 int c;
681
682 // Do this for all languages that support sound folding.
683 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
684 {
685 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
686 slang = lp->lp_slang;
687 if (slang->sl_sugtime != 0 && !slang->sl_sugloaded)
688 {
689 // Change ".spl" to ".sug" and open the file. When the file isn't
690 // found silently skip it. Do set "sl_sugloaded" so that we
691 // don't try again and again.
692 slang->sl_sugloaded = TRUE;
693
694 dotp = vim_strrchr(slang->sl_fname, '.');
695 if (dotp == NULL || fnamecmp(dotp, ".spl") != 0)
696 continue;
697 STRCPY(dotp, ".sug");
698 fd = mch_fopen((char *)slang->sl_fname, "r");
699 if (fd == NULL)
700 goto nextone;
701
702 /*
703 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
704 */
705 for (i = 0; i < VIMSUGMAGICL; ++i)
706 buf[i] = getc(fd); // <fileID>
707 if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0)
708 {
709 semsg(_("E778: This does not look like a .sug file: %s"),
710 slang->sl_fname);
711 goto nextone;
712 }
713 c = getc(fd); // <versionnr>
714 if (c < VIMSUGVERSION)
715 {
716 semsg(_("E779: Old .sug file, needs to be updated: %s"),
717 slang->sl_fname);
718 goto nextone;
719 }
720 else if (c > VIMSUGVERSION)
721 {
722 semsg(_("E780: .sug file is for newer version of Vim: %s"),
723 slang->sl_fname);
724 goto nextone;
725 }
726
727 // Check the timestamp, it must be exactly the same as the one in
728 // the .spl file. Otherwise the word numbers won't match.
729 timestamp = get8ctime(fd); // <timestamp>
730 if (timestamp != slang->sl_sugtime)
731 {
732 semsg(_("E781: .sug file doesn't match .spl file: %s"),
733 slang->sl_fname);
734 goto nextone;
735 }
736
737 /*
738 * <SUGWORDTREE>: <wordtree>
739 * Read the trie with the soundfolded words.
740 */
741 if (spell_read_tree(fd, &slang->sl_sbyts, NULL, &slang->sl_sidxs,
742 FALSE, 0) != 0)
743 {
744 someerror:
745 semsg(_("E782: error while reading .sug file: %s"),
746 slang->sl_fname);
747 slang_clear_sug(slang);
748 goto nextone;
749 }
750
751 /*
752 * <SUGTABLE>: <sugwcount> <sugline> ...
753 *
754 * Read the table with word numbers. We use a file buffer for
755 * this, because it's so much like a file with lines. Makes it
756 * possible to swap the info and save on memory use.
757 */
758 slang->sl_sugbuf = open_spellbuf();
759 if (slang->sl_sugbuf == NULL)
760 goto someerror;
761 // <sugwcount>
762 wcount = get4c(fd);
763 if (wcount < 0)
764 goto someerror;
765
766 // Read all the wordnr lists into the buffer, one NUL terminated
767 // list per line.
768 ga_init2(&ga, 1, 100);
769 for (wordnr = 0; wordnr < wcount; ++wordnr)
770 {
771 ga.ga_len = 0;
772 for (;;)
773 {
774 c = getc(fd); // <sugline>
775 if (c < 0 || ga_grow(&ga, 1) == FAIL)
776 goto someerror;
777 ((char_u *)ga.ga_data)[ga.ga_len++] = c;
778 if (c == NUL)
779 break;
780 }
781 if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr,
782 ga.ga_data, ga.ga_len, TRUE) == FAIL)
783 goto someerror;
784 }
785 ga_clear(&ga);
786
787 /*
788 * Need to put word counts in the word tries, so that we can find
789 * a word by its number.
790 */
791 tree_count_words(slang->sl_fbyts, slang->sl_fidxs);
792 tree_count_words(slang->sl_sbyts, slang->sl_sidxs);
793
794 nextone:
795 if (fd != NULL)
796 fclose(fd);
797 STRCPY(dotp, ".spl");
798 }
799 }
800 }
801
802
803 /*
804 * Read a length field from "fd" in "cnt_bytes" bytes.
805 * Allocate memory, read the string into it and add a NUL at the end.
806 * Returns NULL when the count is zero.
807 * Sets "*cntp" to SP_*ERROR when there is an error, length of the result
808 * otherwise.
809 */
810 static char_u *
read_cnt_string(FILE * fd,int cnt_bytes,int * cntp)811 read_cnt_string(FILE *fd, int cnt_bytes, int *cntp)
812 {
813 int cnt = 0;
814 int i;
815 char_u *str;
816
817 // read the length bytes, MSB first
818 for (i = 0; i < cnt_bytes; ++i)
819 {
820 int c = getc(fd);
821
822 if (c == EOF)
823 {
824 *cntp = SP_TRUNCERROR;
825 return NULL;
826 }
827 cnt = (cnt << 8) + (unsigned)c;
828 }
829 *cntp = cnt;
830 if (cnt == 0)
831 return NULL; // nothing to read, return NULL
832
833 str = read_string(fd, cnt);
834 if (str == NULL)
835 *cntp = SP_OTHERERROR;
836 return str;
837 }
838
839 /*
840 * Read SN_REGION: <regionname> ...
841 * Return SP_*ERROR flags.
842 */
843 static int
read_region_section(FILE * fd,slang_T * lp,int len)844 read_region_section(FILE *fd, slang_T *lp, int len)
845 {
846 int i;
847
848 if (len > MAXREGIONS * 2)
849 return SP_FORMERROR;
850 for (i = 0; i < len; ++i)
851 lp->sl_regions[i] = getc(fd); // <regionname>
852 lp->sl_regions[len] = NUL;
853 return 0;
854 }
855
856 /*
857 * Read SN_CHARFLAGS section: <charflagslen> <charflags>
858 * <folcharslen> <folchars>
859 * Return SP_*ERROR flags.
860 */
861 static int
read_charflags_section(FILE * fd)862 read_charflags_section(FILE *fd)
863 {
864 char_u *flags;
865 char_u *fol;
866 int flagslen, follen;
867
868 // <charflagslen> <charflags>
869 flags = read_cnt_string(fd, 1, &flagslen);
870 if (flagslen < 0)
871 return flagslen;
872
873 // <folcharslen> <folchars>
874 fol = read_cnt_string(fd, 2, &follen);
875 if (follen < 0)
876 {
877 vim_free(flags);
878 return follen;
879 }
880
881 // Set the word-char flags and fill SPELL_ISUPPER() table.
882 if (flags != NULL && fol != NULL)
883 set_spell_charflags(flags, flagslen, fol);
884
885 vim_free(flags);
886 vim_free(fol);
887
888 // When <charflagslen> is zero then <fcharlen> must also be zero.
889 if ((flags == NULL) != (fol == NULL))
890 return SP_FORMERROR;
891 return 0;
892 }
893
894 /*
895 * Read SN_PREFCOND section.
896 * Return SP_*ERROR flags.
897 */
898 static int
read_prefcond_section(FILE * fd,slang_T * lp)899 read_prefcond_section(FILE *fd, slang_T *lp)
900 {
901 int cnt;
902 int i;
903 int n;
904 char_u *p;
905 char_u buf[MAXWLEN + 1];
906
907 // <prefcondcnt> <prefcond> ...
908 cnt = get2c(fd); // <prefcondcnt>
909 if (cnt <= 0)
910 return SP_FORMERROR;
911
912 lp->sl_prefprog = ALLOC_CLEAR_MULT(regprog_T *, cnt);
913 if (lp->sl_prefprog == NULL)
914 return SP_OTHERERROR;
915 lp->sl_prefixcnt = cnt;
916
917 for (i = 0; i < cnt; ++i)
918 {
919 // <prefcond> : <condlen> <condstr>
920 n = getc(fd); // <condlen>
921 if (n < 0 || n >= MAXWLEN)
922 return SP_FORMERROR;
923
924 // When <condlen> is zero we have an empty condition. Otherwise
925 // compile the regexp program used to check for the condition.
926 if (n > 0)
927 {
928 buf[0] = '^'; // always match at one position only
929 p = buf + 1;
930 while (n-- > 0)
931 *p++ = getc(fd); // <condstr>
932 *p = NUL;
933 lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING);
934 }
935 }
936 return 0;
937 }
938
939 /*
940 * Read REP or REPSAL items section from "fd": <repcount> <rep> ...
941 * Return SP_*ERROR flags.
942 */
943 static int
read_rep_section(FILE * fd,garray_T * gap,short * first)944 read_rep_section(FILE *fd, garray_T *gap, short *first)
945 {
946 int cnt;
947 fromto_T *ftp;
948 int i;
949
950 cnt = get2c(fd); // <repcount>
951 if (cnt < 0)
952 return SP_TRUNCERROR;
953
954 if (ga_grow(gap, cnt) == FAIL)
955 return SP_OTHERERROR;
956
957 // <rep> : <repfromlen> <repfrom> <reptolen> <repto>
958 for (; gap->ga_len < cnt; ++gap->ga_len)
959 {
960 ftp = &((fromto_T *)gap->ga_data)[gap->ga_len];
961 ftp->ft_from = read_cnt_string(fd, 1, &i);
962 if (i < 0)
963 return i;
964 if (i == 0)
965 return SP_FORMERROR;
966 ftp->ft_to = read_cnt_string(fd, 1, &i);
967 if (i <= 0)
968 {
969 vim_free(ftp->ft_from);
970 if (i < 0)
971 return i;
972 return SP_FORMERROR;
973 }
974 }
975
976 // Fill the first-index table.
977 for (i = 0; i < 256; ++i)
978 first[i] = -1;
979 for (i = 0; i < gap->ga_len; ++i)
980 {
981 ftp = &((fromto_T *)gap->ga_data)[i];
982 if (first[*ftp->ft_from] == -1)
983 first[*ftp->ft_from] = i;
984 }
985 return 0;
986 }
987
988 /*
989 * Read SN_SAL section: <salflags> <salcount> <sal> ...
990 * Return SP_*ERROR flags.
991 */
992 static int
read_sal_section(FILE * fd,slang_T * slang)993 read_sal_section(FILE *fd, slang_T *slang)
994 {
995 int i;
996 int cnt;
997 garray_T *gap;
998 salitem_T *smp;
999 int ccnt;
1000 char_u *p;
1001
1002 slang->sl_sofo = FALSE;
1003
1004 i = getc(fd); // <salflags>
1005 if (i & SAL_F0LLOWUP)
1006 slang->sl_followup = TRUE;
1007 if (i & SAL_COLLAPSE)
1008 slang->sl_collapse = TRUE;
1009 if (i & SAL_REM_ACCENTS)
1010 slang->sl_rem_accents = TRUE;
1011
1012 cnt = get2c(fd); // <salcount>
1013 if (cnt < 0)
1014 return SP_TRUNCERROR;
1015
1016 gap = &slang->sl_sal;
1017 ga_init2(gap, sizeof(salitem_T), 10);
1018 if (ga_grow(gap, cnt + 1) == FAIL)
1019 return SP_OTHERERROR;
1020
1021 // <sal> : <salfromlen> <salfrom> <saltolen> <salto>
1022 for (; gap->ga_len < cnt; ++gap->ga_len)
1023 {
1024 int c = NUL;
1025
1026 smp = &((salitem_T *)gap->ga_data)[gap->ga_len];
1027 ccnt = getc(fd); // <salfromlen>
1028 if (ccnt < 0)
1029 return SP_TRUNCERROR;
1030 if ((p = alloc(ccnt + 2)) == NULL)
1031 return SP_OTHERERROR;
1032 smp->sm_lead = p;
1033
1034 // Read up to the first special char into sm_lead.
1035 for (i = 0; i < ccnt; ++i)
1036 {
1037 c = getc(fd); // <salfrom>
1038 if (vim_strchr((char_u *)"0123456789(-<^$", c) != NULL)
1039 break;
1040 *p++ = c;
1041 }
1042 smp->sm_leadlen = (int)(p - smp->sm_lead);
1043 *p++ = NUL;
1044
1045 // Put (abc) chars in sm_oneof, if any.
1046 if (c == '(')
1047 {
1048 smp->sm_oneof = p;
1049 for (++i; i < ccnt; ++i)
1050 {
1051 c = getc(fd); // <salfrom>
1052 if (c == ')')
1053 break;
1054 *p++ = c;
1055 }
1056 *p++ = NUL;
1057 if (++i < ccnt)
1058 c = getc(fd);
1059 }
1060 else
1061 smp->sm_oneof = NULL;
1062
1063 // Any following chars go in sm_rules.
1064 smp->sm_rules = p;
1065 if (i < ccnt)
1066 // store the char we got while checking for end of sm_lead
1067 *p++ = c;
1068 for (++i; i < ccnt; ++i)
1069 *p++ = getc(fd); // <salfrom>
1070 *p++ = NUL;
1071
1072 // <saltolen> <salto>
1073 smp->sm_to = read_cnt_string(fd, 1, &ccnt);
1074 if (ccnt < 0)
1075 {
1076 vim_free(smp->sm_lead);
1077 return ccnt;
1078 }
1079
1080 if (has_mbyte)
1081 {
1082 // convert the multi-byte strings to wide char strings
1083 smp->sm_lead_w = mb_str2wide(smp->sm_lead);
1084 smp->sm_leadlen = mb_charlen(smp->sm_lead);
1085 if (smp->sm_oneof == NULL)
1086 smp->sm_oneof_w = NULL;
1087 else
1088 smp->sm_oneof_w = mb_str2wide(smp->sm_oneof);
1089 if (smp->sm_to == NULL)
1090 smp->sm_to_w = NULL;
1091 else
1092 smp->sm_to_w = mb_str2wide(smp->sm_to);
1093 if (smp->sm_lead_w == NULL
1094 || (smp->sm_oneof_w == NULL && smp->sm_oneof != NULL)
1095 || (smp->sm_to_w == NULL && smp->sm_to != NULL))
1096 {
1097 vim_free(smp->sm_lead);
1098 vim_free(smp->sm_to);
1099 vim_free(smp->sm_lead_w);
1100 vim_free(smp->sm_oneof_w);
1101 vim_free(smp->sm_to_w);
1102 return SP_OTHERERROR;
1103 }
1104 }
1105 }
1106
1107 if (gap->ga_len > 0)
1108 {
1109 // Add one extra entry to mark the end with an empty sm_lead. Avoids
1110 // that we need to check the index every time.
1111 smp = &((salitem_T *)gap->ga_data)[gap->ga_len];
1112 if ((p = alloc(1)) == NULL)
1113 return SP_OTHERERROR;
1114 p[0] = NUL;
1115 smp->sm_lead = p;
1116 smp->sm_leadlen = 0;
1117 smp->sm_oneof = NULL;
1118 smp->sm_rules = p;
1119 smp->sm_to = NULL;
1120 if (has_mbyte)
1121 {
1122 smp->sm_lead_w = mb_str2wide(smp->sm_lead);
1123 smp->sm_leadlen = 0;
1124 smp->sm_oneof_w = NULL;
1125 smp->sm_to_w = NULL;
1126 }
1127 ++gap->ga_len;
1128 }
1129
1130 // Fill the first-index table.
1131 set_sal_first(slang);
1132
1133 return 0;
1134 }
1135
1136 /*
1137 * Read SN_WORDS: <word> ...
1138 * Return SP_*ERROR flags.
1139 */
1140 static int
read_words_section(FILE * fd,slang_T * lp,int len)1141 read_words_section(FILE *fd, slang_T *lp, int len)
1142 {
1143 int done = 0;
1144 int i;
1145 int c;
1146 char_u word[MAXWLEN];
1147
1148 while (done < len)
1149 {
1150 // Read one word at a time.
1151 for (i = 0; ; ++i)
1152 {
1153 c = getc(fd);
1154 if (c == EOF)
1155 return SP_TRUNCERROR;
1156 word[i] = c;
1157 if (word[i] == NUL)
1158 break;
1159 if (i == MAXWLEN - 1)
1160 return SP_FORMERROR;
1161 }
1162
1163 // Init the count to 10.
1164 count_common_word(lp, word, -1, 10);
1165 done += i + 1;
1166 }
1167 return 0;
1168 }
1169
1170 /*
1171 * SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
1172 * Return SP_*ERROR flags.
1173 */
1174 static int
read_sofo_section(FILE * fd,slang_T * slang)1175 read_sofo_section(FILE *fd, slang_T *slang)
1176 {
1177 int cnt;
1178 char_u *from, *to;
1179 int res;
1180
1181 slang->sl_sofo = TRUE;
1182
1183 // <sofofromlen> <sofofrom>
1184 from = read_cnt_string(fd, 2, &cnt);
1185 if (cnt < 0)
1186 return cnt;
1187
1188 // <sofotolen> <sofoto>
1189 to = read_cnt_string(fd, 2, &cnt);
1190 if (cnt < 0)
1191 {
1192 vim_free(from);
1193 return cnt;
1194 }
1195
1196 // Store the info in slang->sl_sal and/or slang->sl_sal_first.
1197 if (from != NULL && to != NULL)
1198 res = set_sofo(slang, from, to);
1199 else if (from != NULL || to != NULL)
1200 res = SP_FORMERROR; // only one of two strings is an error
1201 else
1202 res = 0;
1203
1204 vim_free(from);
1205 vim_free(to);
1206 return res;
1207 }
1208
1209 /*
1210 * Read the compound section from the .spl file:
1211 * <compmax> <compminlen> <compsylmax> <compoptions> <compflags>
1212 * Returns SP_*ERROR flags.
1213 */
1214 static int
read_compound(FILE * fd,slang_T * slang,int len)1215 read_compound(FILE *fd, slang_T *slang, int len)
1216 {
1217 int todo = len;
1218 int c;
1219 int atstart;
1220 char_u *pat;
1221 char_u *pp;
1222 char_u *cp;
1223 char_u *ap;
1224 char_u *crp;
1225 int cnt;
1226 garray_T *gap;
1227
1228 if (todo < 2)
1229 return SP_FORMERROR; // need at least two bytes
1230
1231 --todo;
1232 c = getc(fd); // <compmax>
1233 if (c < 2)
1234 c = MAXWLEN;
1235 slang->sl_compmax = c;
1236
1237 --todo;
1238 c = getc(fd); // <compminlen>
1239 if (c < 1)
1240 c = 0;
1241 slang->sl_compminlen = c;
1242
1243 --todo;
1244 c = getc(fd); // <compsylmax>
1245 if (c < 1)
1246 c = MAXWLEN;
1247 slang->sl_compsylmax = c;
1248
1249 c = getc(fd); // <compoptions>
1250 if (c != 0)
1251 ungetc(c, fd); // be backwards compatible with Vim 7.0b
1252 else
1253 {
1254 --todo;
1255 c = getc(fd); // only use the lower byte for now
1256 --todo;
1257 slang->sl_compoptions = c;
1258
1259 gap = &slang->sl_comppat;
1260 c = get2c(fd); // <comppatcount>
1261 if (c < 0)
1262 return SP_TRUNCERROR;
1263 todo -= 2;
1264 ga_init2(gap, sizeof(char_u *), c);
1265 if (ga_grow(gap, c) == OK)
1266 while (--c >= 0)
1267 {
1268 ((char_u **)(gap->ga_data))[gap->ga_len++] =
1269 read_cnt_string(fd, 1, &cnt);
1270 // <comppatlen> <comppattext>
1271 if (cnt < 0)
1272 return cnt;
1273 todo -= cnt + 1;
1274 }
1275 }
1276 if (todo < 0)
1277 return SP_FORMERROR;
1278
1279 // Turn the COMPOUNDRULE items into a regexp pattern:
1280 // "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$".
1281 // Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes.
1282 // Conversion to utf-8 may double the size.
1283 c = todo * 2 + 7;
1284 if (enc_utf8)
1285 c += todo * 2;
1286 pat = alloc(c);
1287 if (pat == NULL)
1288 return SP_OTHERERROR;
1289
1290 // We also need a list of all flags that can appear at the start and one
1291 // for all flags.
1292 cp = alloc(todo + 1);
1293 if (cp == NULL)
1294 {
1295 vim_free(pat);
1296 return SP_OTHERERROR;
1297 }
1298 slang->sl_compstartflags = cp;
1299 *cp = NUL;
1300
1301 ap = alloc(todo + 1);
1302 if (ap == NULL)
1303 {
1304 vim_free(pat);
1305 return SP_OTHERERROR;
1306 }
1307 slang->sl_compallflags = ap;
1308 *ap = NUL;
1309
1310 // And a list of all patterns in their original form, for checking whether
1311 // compounding may work in match_compoundrule(). This is freed when we
1312 // encounter a wildcard, the check doesn't work then.
1313 crp = alloc(todo + 1);
1314 slang->sl_comprules = crp;
1315
1316 pp = pat;
1317 *pp++ = '^';
1318 *pp++ = '\\';
1319 *pp++ = '(';
1320
1321 atstart = 1;
1322 while (todo-- > 0)
1323 {
1324 c = getc(fd); // <compflags>
1325 if (c == EOF)
1326 {
1327 vim_free(pat);
1328 return SP_TRUNCERROR;
1329 }
1330
1331 // Add all flags to "sl_compallflags".
1332 if (vim_strchr((char_u *)"?*+[]/", c) == NULL
1333 && !byte_in_str(slang->sl_compallflags, c))
1334 {
1335 *ap++ = c;
1336 *ap = NUL;
1337 }
1338
1339 if (atstart != 0)
1340 {
1341 // At start of item: copy flags to "sl_compstartflags". For a
1342 // [abc] item set "atstart" to 2 and copy up to the ']'.
1343 if (c == '[')
1344 atstart = 2;
1345 else if (c == ']')
1346 atstart = 0;
1347 else
1348 {
1349 if (!byte_in_str(slang->sl_compstartflags, c))
1350 {
1351 *cp++ = c;
1352 *cp = NUL;
1353 }
1354 if (atstart == 1)
1355 atstart = 0;
1356 }
1357 }
1358
1359 // Copy flag to "sl_comprules", unless we run into a wildcard.
1360 if (crp != NULL)
1361 {
1362 if (c == '?' || c == '+' || c == '*')
1363 {
1364 VIM_CLEAR(slang->sl_comprules);
1365 crp = NULL;
1366 }
1367 else
1368 *crp++ = c;
1369 }
1370
1371 if (c == '/') // slash separates two items
1372 {
1373 *pp++ = '\\';
1374 *pp++ = '|';
1375 atstart = 1;
1376 }
1377 else // normal char, "[abc]" and '*' are copied as-is
1378 {
1379 if (c == '?' || c == '+' || c == '~')
1380 *pp++ = '\\'; // "a?" becomes "a\?", "a+" becomes "a\+"
1381 if (enc_utf8)
1382 pp += mb_char2bytes(c, pp);
1383 else
1384 *pp++ = c;
1385 }
1386 }
1387
1388 *pp++ = '\\';
1389 *pp++ = ')';
1390 *pp++ = '$';
1391 *pp = NUL;
1392
1393 if (crp != NULL)
1394 *crp = NUL;
1395
1396 slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT);
1397 vim_free(pat);
1398 if (slang->sl_compprog == NULL)
1399 return SP_FORMERROR;
1400
1401 return 0;
1402 }
1403
1404 /*
1405 * Set the SOFOFROM and SOFOTO items in language "lp".
1406 * Returns SP_*ERROR flags when there is something wrong.
1407 */
1408 static int
set_sofo(slang_T * lp,char_u * from,char_u * to)1409 set_sofo(slang_T *lp, char_u *from, char_u *to)
1410 {
1411 int i;
1412
1413 garray_T *gap;
1414 char_u *s;
1415 char_u *p;
1416 int c;
1417 int *inp;
1418
1419 if (has_mbyte)
1420 {
1421 // Use "sl_sal" as an array with 256 pointers to a list of wide
1422 // characters. The index is the low byte of the character.
1423 // The list contains from-to pairs with a terminating NUL.
1424 // sl_sal_first[] is used for latin1 "from" characters.
1425 gap = &lp->sl_sal;
1426 ga_init2(gap, sizeof(int *), 1);
1427 if (ga_grow(gap, 256) == FAIL)
1428 return SP_OTHERERROR;
1429 vim_memset(gap->ga_data, 0, sizeof(int *) * 256);
1430 gap->ga_len = 256;
1431
1432 // First count the number of items for each list. Temporarily use
1433 // sl_sal_first[] for this.
1434 for (p = from, s = to; *p != NUL && *s != NUL; )
1435 {
1436 c = mb_cptr2char_adv(&p);
1437 MB_CPTR_ADV(s);
1438 if (c >= 256)
1439 ++lp->sl_sal_first[c & 0xff];
1440 }
1441 if (*p != NUL || *s != NUL) // lengths differ
1442 return SP_FORMERROR;
1443
1444 // Allocate the lists.
1445 for (i = 0; i < 256; ++i)
1446 if (lp->sl_sal_first[i] > 0)
1447 {
1448 p = alloc(sizeof(int) * (lp->sl_sal_first[i] * 2 + 1));
1449 if (p == NULL)
1450 return SP_OTHERERROR;
1451 ((int **)gap->ga_data)[i] = (int *)p;
1452 *(int *)p = 0;
1453 }
1454
1455 // Put the characters up to 255 in sl_sal_first[] the rest in a sl_sal
1456 // list.
1457 vim_memset(lp->sl_sal_first, 0, sizeof(salfirst_T) * 256);
1458 for (p = from, s = to; *p != NUL && *s != NUL; )
1459 {
1460 c = mb_cptr2char_adv(&p);
1461 i = mb_cptr2char_adv(&s);
1462 if (c >= 256)
1463 {
1464 // Append the from-to chars at the end of the list with
1465 // the low byte.
1466 inp = ((int **)gap->ga_data)[c & 0xff];
1467 while (*inp != 0)
1468 ++inp;
1469 *inp++ = c; // from char
1470 *inp++ = i; // to char
1471 *inp++ = NUL; // NUL at the end
1472 }
1473 else
1474 // mapping byte to char is done in sl_sal_first[]
1475 lp->sl_sal_first[c] = i;
1476 }
1477 }
1478 else
1479 {
1480 // mapping bytes to bytes is done in sl_sal_first[]
1481 if (STRLEN(from) != STRLEN(to))
1482 return SP_FORMERROR;
1483
1484 for (i = 0; to[i] != NUL; ++i)
1485 lp->sl_sal_first[from[i]] = to[i];
1486 lp->sl_sal.ga_len = 1; // indicates we have soundfolding
1487 }
1488
1489 return 0;
1490 }
1491
1492 /*
1493 * Fill the first-index table for "lp".
1494 */
1495 static void
set_sal_first(slang_T * lp)1496 set_sal_first(slang_T *lp)
1497 {
1498 salfirst_T *sfirst;
1499 int i;
1500 salitem_T *smp;
1501 int c;
1502 garray_T *gap = &lp->sl_sal;
1503
1504 sfirst = lp->sl_sal_first;
1505 for (i = 0; i < 256; ++i)
1506 sfirst[i] = -1;
1507 smp = (salitem_T *)gap->ga_data;
1508 for (i = 0; i < gap->ga_len; ++i)
1509 {
1510 if (has_mbyte)
1511 // Use the lowest byte of the first character. For latin1 it's
1512 // the character, for other encodings it should differ for most
1513 // characters.
1514 c = *smp[i].sm_lead_w & 0xff;
1515 else
1516 c = *smp[i].sm_lead;
1517 if (sfirst[c] == -1)
1518 {
1519 sfirst[c] = i;
1520 if (has_mbyte)
1521 {
1522 int n;
1523
1524 // Make sure all entries with this byte are following each
1525 // other. Move the ones that are in the wrong position. Do
1526 // keep the same ordering!
1527 while (i + 1 < gap->ga_len
1528 && (*smp[i + 1].sm_lead_w & 0xff) == c)
1529 // Skip over entry with same index byte.
1530 ++i;
1531
1532 for (n = 1; i + n < gap->ga_len; ++n)
1533 if ((*smp[i + n].sm_lead_w & 0xff) == c)
1534 {
1535 salitem_T tsal;
1536
1537 // Move entry with same index byte after the entries
1538 // we already found.
1539 ++i;
1540 --n;
1541 tsal = smp[i + n];
1542 mch_memmove(smp + i + 1, smp + i,
1543 sizeof(salitem_T) * n);
1544 smp[i] = tsal;
1545 }
1546 }
1547 }
1548 }
1549 }
1550
1551 /*
1552 * Turn a multi-byte string into a wide character string.
1553 * Return it in allocated memory (NULL for out-of-memory)
1554 */
1555 static int *
mb_str2wide(char_u * s)1556 mb_str2wide(char_u *s)
1557 {
1558 int *res;
1559 char_u *p;
1560 int i = 0;
1561
1562 res = ALLOC_MULT(int, mb_charlen(s) + 1);
1563 if (res != NULL)
1564 {
1565 for (p = s; *p != NUL; )
1566 res[i++] = mb_ptr2char_adv(&p);
1567 res[i] = NUL;
1568 }
1569 return res;
1570 }
1571
1572 /*
1573 * Read a tree from the .spl or .sug file.
1574 * Allocates the memory and stores pointers in "bytsp" and "idxsp".
1575 * This is skipped when the tree has zero length.
1576 * Returns zero when OK, SP_ value for an error.
1577 */
1578 static int
spell_read_tree(FILE * fd,char_u ** bytsp,long * bytsp_len,idx_T ** idxsp,int prefixtree,int prefixcnt)1579 spell_read_tree(
1580 FILE *fd,
1581 char_u **bytsp,
1582 long *bytsp_len,
1583 idx_T **idxsp,
1584 int prefixtree, // TRUE for the prefix tree
1585 int prefixcnt) // when "prefixtree" is TRUE: prefix count
1586 {
1587 long len;
1588 int idx;
1589 char_u *bp;
1590 idx_T *ip;
1591
1592 // The tree size was computed when writing the file, so that we can
1593 // allocate it as one long block. <nodecount>
1594 len = get4c(fd);
1595 if (len < 0)
1596 return SP_TRUNCERROR;
1597 if (len >= LONG_MAX / (long)sizeof(int))
1598 // Invalid length, multiply with sizeof(int) would overflow.
1599 return SP_FORMERROR;
1600 if (len > 0)
1601 {
1602 // Allocate the byte array.
1603 bp = alloc(len);
1604 if (bp == NULL)
1605 return SP_OTHERERROR;
1606 *bytsp = bp;
1607 if (bytsp_len != NULL)
1608 *bytsp_len = len;
1609
1610 // Allocate the index array.
1611 ip = lalloc_clear(len * sizeof(int), TRUE);
1612 if (ip == NULL)
1613 return SP_OTHERERROR;
1614 *idxsp = ip;
1615
1616 // Recursively read the tree and store it in the array.
1617 idx = read_tree_node(fd, bp, ip, len, 0, prefixtree, prefixcnt);
1618 if (idx < 0)
1619 return idx;
1620 }
1621 return 0;
1622 }
1623
1624 /*
1625 * Read one row of siblings from the spell file and store it in the byte array
1626 * "byts" and index array "idxs". Recursively read the children.
1627 *
1628 * NOTE: The code here must match put_node()!
1629 *
1630 * Returns the index (>= 0) following the siblings.
1631 * Returns SP_TRUNCERROR if the file is shorter than expected.
1632 * Returns SP_FORMERROR if there is a format error.
1633 */
1634 static idx_T
read_tree_node(FILE * fd,char_u * byts,idx_T * idxs,int maxidx,idx_T startidx,int prefixtree,int maxprefcondnr)1635 read_tree_node(
1636 FILE *fd,
1637 char_u *byts,
1638 idx_T *idxs,
1639 int maxidx, // size of arrays
1640 idx_T startidx, // current index in "byts" and "idxs"
1641 int prefixtree, // TRUE for reading PREFIXTREE
1642 int maxprefcondnr) // maximum for <prefcondnr>
1643 {
1644 int len;
1645 int i;
1646 int n;
1647 idx_T idx = startidx;
1648 int c;
1649 int c2;
1650 #define SHARED_MASK 0x8000000
1651
1652 len = getc(fd); // <siblingcount>
1653 if (len <= 0)
1654 return SP_TRUNCERROR;
1655
1656 if (startidx + len >= maxidx)
1657 return SP_FORMERROR;
1658 byts[idx++] = len;
1659
1660 // Read the byte values, flag/region bytes and shared indexes.
1661 for (i = 1; i <= len; ++i)
1662 {
1663 c = getc(fd); // <byte>
1664 if (c < 0)
1665 return SP_TRUNCERROR;
1666 if (c <= BY_SPECIAL)
1667 {
1668 if (c == BY_NOFLAGS && !prefixtree)
1669 {
1670 // No flags, all regions.
1671 idxs[idx] = 0;
1672 c = 0;
1673 }
1674 else if (c != BY_INDEX)
1675 {
1676 if (prefixtree)
1677 {
1678 // Read the optional pflags byte, the prefix ID and the
1679 // condition nr. In idxs[] store the prefix ID in the low
1680 // byte, the condition index shifted up 8 bits, the flags
1681 // shifted up 24 bits.
1682 if (c == BY_FLAGS)
1683 c = getc(fd) << 24; // <pflags>
1684 else
1685 c = 0;
1686
1687 c |= getc(fd); // <affixID>
1688
1689 n = get2c(fd); // <prefcondnr>
1690 if (n >= maxprefcondnr)
1691 return SP_FORMERROR;
1692 c |= (n << 8);
1693 }
1694 else // c must be BY_FLAGS or BY_FLAGS2
1695 {
1696 // Read flags and optional region and prefix ID. In
1697 // idxs[] the flags go in the low two bytes, region above
1698 // that and prefix ID above the region.
1699 c2 = c;
1700 c = getc(fd); // <flags>
1701 if (c2 == BY_FLAGS2)
1702 c = (getc(fd) << 8) + c; // <flags2>
1703 if (c & WF_REGION)
1704 c = (getc(fd) << 16) + c; // <region>
1705 if (c & WF_AFX)
1706 c = (getc(fd) << 24) + c; // <affixID>
1707 }
1708
1709 idxs[idx] = c;
1710 c = 0;
1711 }
1712 else // c == BY_INDEX
1713 {
1714 // <nodeidx>
1715 n = get3c(fd);
1716 if (n < 0 || n >= maxidx)
1717 return SP_FORMERROR;
1718 idxs[idx] = n + SHARED_MASK;
1719 c = getc(fd); // <xbyte>
1720 }
1721 }
1722 byts[idx++] = c;
1723 }
1724
1725 // Recursively read the children for non-shared siblings.
1726 // Skip the end-of-word ones (zero byte value) and the shared ones (and
1727 // remove SHARED_MASK)
1728 for (i = 1; i <= len; ++i)
1729 if (byts[startidx + i] != 0)
1730 {
1731 if (idxs[startidx + i] & SHARED_MASK)
1732 idxs[startidx + i] &= ~SHARED_MASK;
1733 else
1734 {
1735 idxs[startidx + i] = idx;
1736 idx = read_tree_node(fd, byts, idxs, maxidx, idx,
1737 prefixtree, maxprefcondnr);
1738 if (idx < 0)
1739 break;
1740 }
1741 }
1742
1743 return idx;
1744 }
1745
1746 /*
1747 * Reload the spell file "fname" if it's loaded.
1748 */
1749 static void
spell_reload_one(char_u * fname,int added_word)1750 spell_reload_one(
1751 char_u *fname,
1752 int added_word) // invoked through "zg"
1753 {
1754 slang_T *slang;
1755 int didit = FALSE;
1756
1757 FOR_ALL_SPELL_LANGS(slang)
1758 {
1759 if (fullpathcmp(fname, slang->sl_fname, FALSE, TRUE) == FPC_SAME)
1760 {
1761 slang_clear(slang);
1762 if (spell_load_file(fname, NULL, slang, FALSE) == NULL)
1763 // reloading failed, clear the language
1764 slang_clear(slang);
1765 redraw_all_later(SOME_VALID);
1766 didit = TRUE;
1767 }
1768 }
1769
1770 // When "zg" was used and the file wasn't loaded yet, should redo
1771 // 'spelllang' to load it now.
1772 if (added_word && !didit)
1773 did_set_spelllang(curwin);
1774 }
1775
1776
1777 /*
1778 * Functions for ":mkspell".
1779 */
1780
1781 #define MAXLINELEN 500 // Maximum length in bytes of a line in a .aff
1782 // and .dic file.
1783 /*
1784 * Main structure to store the contents of a ".aff" file.
1785 */
1786 typedef struct afffile_S
1787 {
1788 char_u *af_enc; // "SET", normalized, alloc'ed string or NULL
1789 int af_flagtype; // AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG
1790 unsigned af_rare; // RARE ID for rare word
1791 unsigned af_keepcase; // KEEPCASE ID for keep-case word
1792 unsigned af_bad; // BAD ID for banned word
1793 unsigned af_needaffix; // NEEDAFFIX ID
1794 unsigned af_circumfix; // CIRCUMFIX ID
1795 unsigned af_needcomp; // NEEDCOMPOUND ID
1796 unsigned af_comproot; // COMPOUNDROOT ID
1797 unsigned af_compforbid; // COMPOUNDFORBIDFLAG ID
1798 unsigned af_comppermit; // COMPOUNDPERMITFLAG ID
1799 unsigned af_nosuggest; // NOSUGGEST ID
1800 int af_pfxpostpone; // postpone prefixes without chop string and
1801 // without flags
1802 int af_ignoreextra; // IGNOREEXTRA present
1803 hashtab_T af_pref; // hashtable for prefixes, affheader_T
1804 hashtab_T af_suff; // hashtable for suffixes, affheader_T
1805 hashtab_T af_comp; // hashtable for compound flags, compitem_T
1806 } afffile_T;
1807
1808 #define AFT_CHAR 0 // flags are one character
1809 #define AFT_LONG 1 // flags are two characters
1810 #define AFT_CAPLONG 2 // flags are one or two characters
1811 #define AFT_NUM 3 // flags are numbers, comma separated
1812
1813 typedef struct affentry_S affentry_T;
1814 // Affix entry from ".aff" file. Used for prefixes and suffixes.
1815 struct affentry_S
1816 {
1817 affentry_T *ae_next; // next affix with same name/number
1818 char_u *ae_chop; // text to chop off basic word (can be NULL)
1819 char_u *ae_add; // text to add to basic word (can be NULL)
1820 char_u *ae_flags; // flags on the affix (can be NULL)
1821 char_u *ae_cond; // condition (NULL for ".")
1822 regprog_T *ae_prog; // regexp program for ae_cond or NULL
1823 char ae_compforbid; // COMPOUNDFORBIDFLAG found
1824 char ae_comppermit; // COMPOUNDPERMITFLAG found
1825 };
1826
1827 #define AH_KEY_LEN 17 // 2 x 8 bytes + NUL
1828
1829 // Affix header from ".aff" file. Used for af_pref and af_suff.
1830 typedef struct affheader_S
1831 {
1832 char_u ah_key[AH_KEY_LEN]; // key for hashtab == name of affix
1833 unsigned ah_flag; // affix name as number, uses "af_flagtype"
1834 int ah_newID; // prefix ID after renumbering; 0 if not used
1835 int ah_combine; // suffix may combine with prefix
1836 int ah_follows; // another affix block should be following
1837 affentry_T *ah_first; // first affix entry
1838 } affheader_T;
1839
1840 #define HI2AH(hi) ((affheader_T *)(hi)->hi_key)
1841
1842 // Flag used in compound items.
1843 typedef struct compitem_S
1844 {
1845 char_u ci_key[AH_KEY_LEN]; // key for hashtab == name of compound
1846 unsigned ci_flag; // affix name as number, uses "af_flagtype"
1847 int ci_newID; // affix ID after renumbering.
1848 } compitem_T;
1849
1850 #define HI2CI(hi) ((compitem_T *)(hi)->hi_key)
1851
1852 /*
1853 * Structure that is used to store the items in the word tree. This avoids
1854 * the need to keep track of each allocated thing, everything is freed all at
1855 * once after ":mkspell" is done.
1856 * Note: "sb_next" must be just before "sb_data" to make sure the alignment of
1857 * "sb_data" is correct for systems where pointers must be aligned on
1858 * pointer-size boundaries and sizeof(pointer) > sizeof(int) (e.g., Sparc).
1859 */
1860 #define SBLOCKSIZE 16000 // size of sb_data
1861 typedef struct sblock_S sblock_T;
1862 struct sblock_S
1863 {
1864 int sb_used; // nr of bytes already in use
1865 sblock_T *sb_next; // next block in list
1866 char_u sb_data[1]; // data, actually longer
1867 };
1868
1869 /*
1870 * A node in the tree.
1871 */
1872 typedef struct wordnode_S wordnode_T;
1873 struct wordnode_S
1874 {
1875 union // shared to save space
1876 {
1877 char_u hashkey[6]; // the hash key, only used while compressing
1878 int index; // index in written nodes (valid after first
1879 // round)
1880 } wn_u1;
1881 union // shared to save space
1882 {
1883 wordnode_T *next; // next node with same hash key
1884 wordnode_T *wnode; // parent node that will write this node
1885 } wn_u2;
1886 wordnode_T *wn_child; // child (next byte in word)
1887 wordnode_T *wn_sibling; // next sibling (alternate byte in word,
1888 // always sorted)
1889 int wn_refs; // Nr. of references to this node. Only
1890 // relevant for first node in a list of
1891 // siblings, in following siblings it is
1892 // always one.
1893 char_u wn_byte; // Byte for this node. NUL for word end
1894
1895 // Info for when "wn_byte" is NUL.
1896 // In PREFIXTREE "wn_region" is used for the prefcondnr.
1897 // In the soundfolded word tree "wn_flags" has the MSW of the wordnr and
1898 // "wn_region" the LSW of the wordnr.
1899 char_u wn_affixID; // supported/required prefix ID or 0
1900 short_u wn_flags; // WF_ flags
1901 short wn_region; // region mask
1902
1903 #ifdef SPELL_PRINTTREE
1904 int wn_nr; // sequence nr for printing
1905 #endif
1906 };
1907
1908 #define WN_MASK 0xffff // mask relevant bits of "wn_flags"
1909
1910 #define HI2WN(hi) (wordnode_T *)((hi)->hi_key)
1911
1912 /*
1913 * Info used while reading the spell files.
1914 */
1915 typedef struct spellinfo_S
1916 {
1917 wordnode_T *si_foldroot; // tree with case-folded words
1918 long si_foldwcount; // nr of words in si_foldroot
1919
1920 wordnode_T *si_keeproot; // tree with keep-case words
1921 long si_keepwcount; // nr of words in si_keeproot
1922
1923 wordnode_T *si_prefroot; // tree with postponed prefixes
1924
1925 long si_sugtree; // creating the soundfolding trie
1926
1927 sblock_T *si_blocks; // memory blocks used
1928 long si_blocks_cnt; // memory blocks allocated
1929 int si_did_emsg; // TRUE when ran out of memory
1930
1931 long si_compress_cnt; // words to add before lowering
1932 // compression limit
1933 wordnode_T *si_first_free; // List of nodes that have been freed during
1934 // compression, linked by "wn_child" field.
1935 long si_free_count; // number of nodes in si_first_free
1936 #ifdef SPELL_PRINTTREE
1937 int si_wordnode_nr; // sequence nr for nodes
1938 #endif
1939 buf_T *si_spellbuf; // buffer used to store soundfold word table
1940
1941 int si_ascii; // handling only ASCII words
1942 int si_add; // addition file
1943 int si_clear_chartab; // when TRUE clear char tables
1944 int si_region; // region mask
1945 vimconv_T si_conv; // for conversion to 'encoding'
1946 int si_memtot; // runtime memory used
1947 int si_verbose; // verbose messages
1948 int si_msg_count; // number of words added since last message
1949 char_u *si_info; // info text chars or NULL
1950 int si_region_count; // number of regions supported (1 when there
1951 // are no regions)
1952 char_u si_region_name[MAXREGIONS * 2 + 1];
1953 // region names; used only if
1954 // si_region_count > 1)
1955
1956 garray_T si_rep; // list of fromto_T entries from REP lines
1957 garray_T si_repsal; // list of fromto_T entries from REPSAL lines
1958 garray_T si_sal; // list of fromto_T entries from SAL lines
1959 char_u *si_sofofr; // SOFOFROM text
1960 char_u *si_sofoto; // SOFOTO text
1961 int si_nosugfile; // NOSUGFILE item found
1962 int si_nosplitsugs; // NOSPLITSUGS item found
1963 int si_nocompoundsugs; // NOCOMPOUNDSUGS item found
1964 int si_followup; // soundsalike: ?
1965 int si_collapse; // soundsalike: ?
1966 hashtab_T si_commonwords; // hashtable for common words
1967 time_t si_sugtime; // timestamp for .sug file
1968 int si_rem_accents; // soundsalike: remove accents
1969 garray_T si_map; // MAP info concatenated
1970 char_u *si_midword; // MIDWORD chars or NULL
1971 int si_compmax; // max nr of words for compounding
1972 int si_compminlen; // minimal length for compounding
1973 int si_compsylmax; // max nr of syllables for compounding
1974 int si_compoptions; // COMP_ flags
1975 garray_T si_comppat; // CHECKCOMPOUNDPATTERN items, each stored as
1976 // a string
1977 char_u *si_compflags; // flags used for compounding
1978 char_u si_nobreak; // NOBREAK
1979 char_u *si_syllable; // syllable string
1980 garray_T si_prefcond; // table with conditions for postponed
1981 // prefixes, each stored as a string
1982 int si_newprefID; // current value for ah_newID
1983 int si_newcompID; // current value for compound ID
1984 } spellinfo_T;
1985
1986 static int is_aff_rule(char_u **items, int itemcnt, char *rulename, int mincount);
1987 static void aff_process_flags(afffile_T *affile, affentry_T *entry);
1988 static int spell_info_item(char_u *s);
1989 static unsigned affitem2flag(int flagtype, char_u *item, char_u *fname, int lnum);
1990 static unsigned get_affitem(int flagtype, char_u **pp);
1991 static void process_compflags(spellinfo_T *spin, afffile_T *aff, char_u *compflags);
1992 static void check_renumber(spellinfo_T *spin);
1993 static void aff_check_number(int spinval, int affval, char *name);
1994 static void aff_check_string(char_u *spinval, char_u *affval, char *name);
1995 static int str_equal(char_u *s1, char_u *s2);
1996 static void add_fromto(spellinfo_T *spin, garray_T *gap, char_u *from, char_u *to);
1997 static int sal_to_bool(char_u *s);
1998 static int get_affix_flags(afffile_T *affile, char_u *afflist);
1999 static int get_pfxlist(afffile_T *affile, char_u *afflist, char_u *store_afflist);
2000 static void get_compflags(afffile_T *affile, char_u *afflist, char_u *store_afflist);
2001 static int store_aff_word(spellinfo_T *spin, char_u *word, char_u *afflist, afffile_T *affile, hashtab_T *ht, hashtab_T *xht, int condit, int flags, char_u *pfxlist, int pfxlen);
2002 static void *getroom(spellinfo_T *spin, size_t len, int align);
2003 static char_u *getroom_save(spellinfo_T *spin, char_u *s);
2004 static int store_word(spellinfo_T *spin, char_u *word, int flags, int region, char_u *pfxlist, int need_affix);
2005 static int tree_add_word(spellinfo_T *spin, char_u *word, wordnode_T *tree, int flags, int region, int affixID);
2006 static wordnode_T *get_wordnode(spellinfo_T *spin);
2007 static void free_wordnode(spellinfo_T *spin, wordnode_T *n);
2008 static void wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name);
2009 static long node_compress(spellinfo_T *spin, wordnode_T *node, hashtab_T *ht, long *tot);
2010 static int node_equal(wordnode_T *n1, wordnode_T *n2);
2011 static void clear_node(wordnode_T *node);
2012 static int put_node(FILE *fd, wordnode_T *node, int idx, int regionmask, int prefixtree);
2013 static int sug_filltree(spellinfo_T *spin, slang_T *slang);
2014 static int sug_maketable(spellinfo_T *spin);
2015 static int sug_filltable(spellinfo_T *spin, wordnode_T *node, int startwordnr, garray_T *gap);
2016 static int offset2bytes(int nr, char_u *buf);
2017 static void sug_write(spellinfo_T *spin, char_u *fname);
2018 static void spell_message(spellinfo_T *spin, char_u *str);
2019 static void init_spellfile(void);
2020
2021 // In the postponed prefixes tree wn_flags is used to store the WFP_ flags,
2022 // but it must be negative to indicate the prefix tree to tree_add_word().
2023 // Use a negative number with the lower 8 bits zero.
2024 #define PFX_FLAGS -256
2025
2026 // flags for "condit" argument of store_aff_word()
2027 #define CONDIT_COMB 1 // affix must combine
2028 #define CONDIT_CFIX 2 // affix must have CIRCUMFIX flag
2029 #define CONDIT_SUF 4 // add a suffix for matching flags
2030 #define CONDIT_AFF 8 // word already has an affix
2031
2032 /*
2033 * Tunable parameters for when the tree is compressed. Filled from the
2034 * 'mkspellmem' option.
2035 */
2036 static long compress_start = 30000; // memory / SBLOCKSIZE
2037 static long compress_inc = 100; // memory / SBLOCKSIZE
2038 static long compress_added = 500000; // word count
2039
2040 /*
2041 * Check the 'mkspellmem' option. Return FAIL if it's wrong.
2042 * Sets "sps_flags".
2043 */
2044 int
spell_check_msm(void)2045 spell_check_msm(void)
2046 {
2047 char_u *p = p_msm;
2048 long start = 0;
2049 long incr = 0;
2050 long added = 0;
2051
2052 if (!VIM_ISDIGIT(*p))
2053 return FAIL;
2054 // block count = (value * 1024) / SBLOCKSIZE (but avoid overflow)
2055 start = (getdigits(&p) * 10) / (SBLOCKSIZE / 102);
2056 if (*p != ',')
2057 return FAIL;
2058 ++p;
2059 if (!VIM_ISDIGIT(*p))
2060 return FAIL;
2061 incr = (getdigits(&p) * 102) / (SBLOCKSIZE / 10);
2062 if (*p != ',')
2063 return FAIL;
2064 ++p;
2065 if (!VIM_ISDIGIT(*p))
2066 return FAIL;
2067 added = getdigits(&p) * 1024;
2068 if (*p != NUL)
2069 return FAIL;
2070
2071 if (start == 0 || incr == 0 || added == 0 || incr > start)
2072 return FAIL;
2073
2074 compress_start = start;
2075 compress_inc = incr;
2076 compress_added = added;
2077 return OK;
2078 }
2079
2080 #ifdef SPELL_PRINTTREE
2081 /*
2082 * For debugging the tree code: print the current tree in a (more or less)
2083 * readable format, so that we can see what happens when adding a word and/or
2084 * compressing the tree.
2085 * Based on code from Olaf Seibert.
2086 */
2087 #define PRINTLINESIZE 1000
2088 #define PRINTWIDTH 6
2089
2090 #define PRINTSOME(l, depth, fmt, a1, a2) vim_snprintf(l + depth * PRINTWIDTH, \
2091 PRINTLINESIZE - PRINTWIDTH * depth, fmt, a1, a2)
2092
2093 static char line1[PRINTLINESIZE];
2094 static char line2[PRINTLINESIZE];
2095 static char line3[PRINTLINESIZE];
2096
2097 static void
spell_clear_flags(wordnode_T * node)2098 spell_clear_flags(wordnode_T *node)
2099 {
2100 wordnode_T *np;
2101
2102 FOR_ALL_NODE_SIBLINGS(node, np)
2103 {
2104 np->wn_u1.index = FALSE;
2105 spell_clear_flags(np->wn_child);
2106 }
2107 }
2108
2109 static void
spell_print_node(wordnode_T * node,int depth)2110 spell_print_node(wordnode_T *node, int depth)
2111 {
2112 if (node->wn_u1.index)
2113 {
2114 // Done this node before, print the reference.
2115 PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0);
2116 PRINTSOME(line2, depth, " ", 0, 0);
2117 PRINTSOME(line3, depth, " ", 0, 0);
2118 msg(line1);
2119 msg(line2);
2120 msg(line3);
2121 }
2122 else
2123 {
2124 node->wn_u1.index = TRUE;
2125
2126 if (node->wn_byte != NUL)
2127 {
2128 if (node->wn_child != NULL)
2129 PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0);
2130 else
2131 // Cannot happen?
2132 PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0);
2133 }
2134 else
2135 PRINTSOME(line1, depth, " $ ", 0, 0);
2136
2137 PRINTSOME(line2, depth, "%d/%d ", node->wn_nr, node->wn_refs);
2138
2139 if (node->wn_sibling != NULL)
2140 PRINTSOME(line3, depth, " | ", 0, 0);
2141 else
2142 PRINTSOME(line3, depth, " ", 0, 0);
2143
2144 if (node->wn_byte == NUL)
2145 {
2146 msg(line1);
2147 msg(line2);
2148 msg(line3);
2149 }
2150
2151 // do the children
2152 if (node->wn_byte != NUL && node->wn_child != NULL)
2153 spell_print_node(node->wn_child, depth + 1);
2154
2155 // do the siblings
2156 if (node->wn_sibling != NULL)
2157 {
2158 // get rid of all parent details except |
2159 STRCPY(line1, line3);
2160 STRCPY(line2, line3);
2161 spell_print_node(node->wn_sibling, depth);
2162 }
2163 }
2164 }
2165
2166 static void
spell_print_tree(wordnode_T * root)2167 spell_print_tree(wordnode_T *root)
2168 {
2169 if (root != NULL)
2170 {
2171 // Clear the "wn_u1.index" fields, used to remember what has been
2172 // done.
2173 spell_clear_flags(root);
2174
2175 // Recursively print the tree.
2176 spell_print_node(root, 0);
2177 }
2178 }
2179 #endif // SPELL_PRINTTREE
2180
2181 /*
2182 * Read the affix file "fname".
2183 * Returns an afffile_T, NULL for complete failure.
2184 */
2185 static afffile_T *
spell_read_aff(spellinfo_T * spin,char_u * fname)2186 spell_read_aff(spellinfo_T *spin, char_u *fname)
2187 {
2188 FILE *fd;
2189 afffile_T *aff;
2190 char_u rline[MAXLINELEN];
2191 char_u *line;
2192 char_u *pc = NULL;
2193 #define MAXITEMCNT 30
2194 char_u *(items[MAXITEMCNT]);
2195 int itemcnt;
2196 char_u *p;
2197 int lnum = 0;
2198 affheader_T *cur_aff = NULL;
2199 int did_postpone_prefix = FALSE;
2200 int aff_todo = 0;
2201 hashtab_T *tp;
2202 char_u *low = NULL;
2203 char_u *fol = NULL;
2204 char_u *upp = NULL;
2205 int do_rep;
2206 int do_repsal;
2207 int do_sal;
2208 int do_mapline;
2209 int found_map = FALSE;
2210 hashitem_T *hi;
2211 int l;
2212 int compminlen = 0; // COMPOUNDMIN value
2213 int compsylmax = 0; // COMPOUNDSYLMAX value
2214 int compoptions = 0; // COMP_ flags
2215 int compmax = 0; // COMPOUNDWORDMAX value
2216 char_u *compflags = NULL; // COMPOUNDFLAG and COMPOUNDRULE
2217 // concatenated
2218 char_u *midword = NULL; // MIDWORD value
2219 char_u *syllable = NULL; // SYLLABLE value
2220 char_u *sofofrom = NULL; // SOFOFROM value
2221 char_u *sofoto = NULL; // SOFOTO value
2222
2223 /*
2224 * Open the file.
2225 */
2226 fd = mch_fopen((char *)fname, "r");
2227 if (fd == NULL)
2228 {
2229 semsg(_(e_notopen), fname);
2230 return NULL;
2231 }
2232
2233 vim_snprintf((char *)IObuff, IOSIZE, _("Reading affix file %s..."), fname);
2234 spell_message(spin, IObuff);
2235
2236 // Only do REP lines when not done in another .aff file already.
2237 do_rep = spin->si_rep.ga_len == 0;
2238
2239 // Only do REPSAL lines when not done in another .aff file already.
2240 do_repsal = spin->si_repsal.ga_len == 0;
2241
2242 // Only do SAL lines when not done in another .aff file already.
2243 do_sal = spin->si_sal.ga_len == 0;
2244
2245 // Only do MAP lines when not done in another .aff file already.
2246 do_mapline = spin->si_map.ga_len == 0;
2247
2248 /*
2249 * Allocate and init the afffile_T structure.
2250 */
2251 aff = (afffile_T *)getroom(spin, sizeof(afffile_T), TRUE);
2252 if (aff == NULL)
2253 {
2254 fclose(fd);
2255 return NULL;
2256 }
2257 hash_init(&aff->af_pref);
2258 hash_init(&aff->af_suff);
2259 hash_init(&aff->af_comp);
2260
2261 /*
2262 * Read all the lines in the file one by one.
2263 */
2264 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int)
2265 {
2266 line_breakcheck();
2267 ++lnum;
2268
2269 // Skip comment lines.
2270 if (*rline == '#')
2271 continue;
2272
2273 // Convert from "SET" to 'encoding' when needed.
2274 vim_free(pc);
2275 if (spin->si_conv.vc_type != CONV_NONE)
2276 {
2277 pc = string_convert(&spin->si_conv, rline, NULL);
2278 if (pc == NULL)
2279 {
2280 smsg(_("Conversion failure for word in %s line %d: %s"),
2281 fname, lnum, rline);
2282 continue;
2283 }
2284 line = pc;
2285 }
2286 else
2287 {
2288 pc = NULL;
2289 line = rline;
2290 }
2291
2292 // Split the line up in white separated items. Put a NUL after each
2293 // item.
2294 itemcnt = 0;
2295 for (p = line; ; )
2296 {
2297 while (*p != NUL && *p <= ' ') // skip white space and CR/NL
2298 ++p;
2299 if (*p == NUL)
2300 break;
2301 if (itemcnt == MAXITEMCNT) // too many items
2302 break;
2303 items[itemcnt++] = p;
2304 // A few items have arbitrary text argument, don't split them.
2305 if (itemcnt == 2 && spell_info_item(items[0]))
2306 while (*p >= ' ' || *p == TAB) // skip until CR/NL
2307 ++p;
2308 else
2309 while (*p > ' ') // skip until white space or CR/NL
2310 ++p;
2311 if (*p == NUL)
2312 break;
2313 *p++ = NUL;
2314 }
2315
2316 // Handle non-empty lines.
2317 if (itemcnt > 0)
2318 {
2319 if (is_aff_rule(items, itemcnt, "SET", 2) && aff->af_enc == NULL)
2320 {
2321 // Setup for conversion from "ENC" to 'encoding'.
2322 aff->af_enc = enc_canonize(items[1]);
2323 if (aff->af_enc != NULL && !spin->si_ascii
2324 && convert_setup(&spin->si_conv, aff->af_enc,
2325 p_enc) == FAIL)
2326 smsg(_("Conversion in %s not supported: from %s to %s"),
2327 fname, aff->af_enc, p_enc);
2328 spin->si_conv.vc_fail = TRUE;
2329 }
2330 else if (is_aff_rule(items, itemcnt, "FLAG", 2)
2331 && aff->af_flagtype == AFT_CHAR)
2332 {
2333 if (STRCMP(items[1], "long") == 0)
2334 aff->af_flagtype = AFT_LONG;
2335 else if (STRCMP(items[1], "num") == 0)
2336 aff->af_flagtype = AFT_NUM;
2337 else if (STRCMP(items[1], "caplong") == 0)
2338 aff->af_flagtype = AFT_CAPLONG;
2339 else
2340 smsg(_("Invalid value for FLAG in %s line %d: %s"),
2341 fname, lnum, items[1]);
2342 if (aff->af_rare != 0
2343 || aff->af_keepcase != 0
2344 || aff->af_bad != 0
2345 || aff->af_needaffix != 0
2346 || aff->af_circumfix != 0
2347 || aff->af_needcomp != 0
2348 || aff->af_comproot != 0
2349 || aff->af_nosuggest != 0
2350 || compflags != NULL
2351 || aff->af_suff.ht_used > 0
2352 || aff->af_pref.ht_used > 0)
2353 smsg(_("FLAG after using flags in %s line %d: %s"),
2354 fname, lnum, items[1]);
2355 }
2356 else if (spell_info_item(items[0]))
2357 {
2358 p = (char_u *)getroom(spin,
2359 (spin->si_info == NULL ? 0 : STRLEN(spin->si_info))
2360 + STRLEN(items[0])
2361 + STRLEN(items[1]) + 3, FALSE);
2362 if (p != NULL)
2363 {
2364 if (spin->si_info != NULL)
2365 {
2366 STRCPY(p, spin->si_info);
2367 STRCAT(p, "\n");
2368 }
2369 STRCAT(p, items[0]);
2370 STRCAT(p, " ");
2371 STRCAT(p, items[1]);
2372 spin->si_info = p;
2373 }
2374 }
2375 else if (is_aff_rule(items, itemcnt, "MIDWORD", 2)
2376 && midword == NULL)
2377 {
2378 midword = getroom_save(spin, items[1]);
2379 }
2380 else if (is_aff_rule(items, itemcnt, "TRY", 2))
2381 {
2382 // ignored, we look in the tree for what chars may appear
2383 }
2384 // TODO: remove "RAR" later
2385 else if ((is_aff_rule(items, itemcnt, "RAR", 2)
2386 || is_aff_rule(items, itemcnt, "RARE", 2))
2387 && aff->af_rare == 0)
2388 {
2389 aff->af_rare = affitem2flag(aff->af_flagtype, items[1],
2390 fname, lnum);
2391 }
2392 // TODO: remove "KEP" later
2393 else if ((is_aff_rule(items, itemcnt, "KEP", 2)
2394 || is_aff_rule(items, itemcnt, "KEEPCASE", 2))
2395 && aff->af_keepcase == 0)
2396 {
2397 aff->af_keepcase = affitem2flag(aff->af_flagtype, items[1],
2398 fname, lnum);
2399 }
2400 else if ((is_aff_rule(items, itemcnt, "BAD", 2)
2401 || is_aff_rule(items, itemcnt, "FORBIDDENWORD", 2))
2402 && aff->af_bad == 0)
2403 {
2404 aff->af_bad = affitem2flag(aff->af_flagtype, items[1],
2405 fname, lnum);
2406 }
2407 else if (is_aff_rule(items, itemcnt, "NEEDAFFIX", 2)
2408 && aff->af_needaffix == 0)
2409 {
2410 aff->af_needaffix = affitem2flag(aff->af_flagtype, items[1],
2411 fname, lnum);
2412 }
2413 else if (is_aff_rule(items, itemcnt, "CIRCUMFIX", 2)
2414 && aff->af_circumfix == 0)
2415 {
2416 aff->af_circumfix = affitem2flag(aff->af_flagtype, items[1],
2417 fname, lnum);
2418 }
2419 else if (is_aff_rule(items, itemcnt, "NOSUGGEST", 2)
2420 && aff->af_nosuggest == 0)
2421 {
2422 aff->af_nosuggest = affitem2flag(aff->af_flagtype, items[1],
2423 fname, lnum);
2424 }
2425 else if ((is_aff_rule(items, itemcnt, "NEEDCOMPOUND", 2)
2426 || is_aff_rule(items, itemcnt, "ONLYINCOMPOUND", 2))
2427 && aff->af_needcomp == 0)
2428 {
2429 aff->af_needcomp = affitem2flag(aff->af_flagtype, items[1],
2430 fname, lnum);
2431 }
2432 else if (is_aff_rule(items, itemcnt, "COMPOUNDROOT", 2)
2433 && aff->af_comproot == 0)
2434 {
2435 aff->af_comproot = affitem2flag(aff->af_flagtype, items[1],
2436 fname, lnum);
2437 }
2438 else if (is_aff_rule(items, itemcnt, "COMPOUNDFORBIDFLAG", 2)
2439 && aff->af_compforbid == 0)
2440 {
2441 aff->af_compforbid = affitem2flag(aff->af_flagtype, items[1],
2442 fname, lnum);
2443 if (aff->af_pref.ht_used > 0)
2444 smsg(_("Defining COMPOUNDFORBIDFLAG after PFX item may give wrong results in %s line %d"),
2445 fname, lnum);
2446 }
2447 else if (is_aff_rule(items, itemcnt, "COMPOUNDPERMITFLAG", 2)
2448 && aff->af_comppermit == 0)
2449 {
2450 aff->af_comppermit = affitem2flag(aff->af_flagtype, items[1],
2451 fname, lnum);
2452 if (aff->af_pref.ht_used > 0)
2453 smsg(_("Defining COMPOUNDPERMITFLAG after PFX item may give wrong results in %s line %d"),
2454 fname, lnum);
2455 }
2456 else if (is_aff_rule(items, itemcnt, "COMPOUNDFLAG", 2)
2457 && compflags == NULL)
2458 {
2459 // Turn flag "c" into COMPOUNDRULE compatible string "c+",
2460 // "Na" into "Na+", "1234" into "1234+".
2461 p = getroom(spin, STRLEN(items[1]) + 2, FALSE);
2462 if (p != NULL)
2463 {
2464 STRCPY(p, items[1]);
2465 STRCAT(p, "+");
2466 compflags = p;
2467 }
2468 }
2469 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULES", 2))
2470 {
2471 // We don't use the count, but do check that it's a number and
2472 // not COMPOUNDRULE mistyped.
2473 if (atoi((char *)items[1]) == 0)
2474 smsg(_("Wrong COMPOUNDRULES value in %s line %d: %s"),
2475 fname, lnum, items[1]);
2476 }
2477 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULE", 2))
2478 {
2479 // Don't use the first rule if it is a number.
2480 if (compflags != NULL || *skipdigits(items[1]) != NUL)
2481 {
2482 // Concatenate this string to previously defined ones,
2483 // using a slash to separate them.
2484 l = (int)STRLEN(items[1]) + 1;
2485 if (compflags != NULL)
2486 l += (int)STRLEN(compflags) + 1;
2487 p = getroom(spin, l, FALSE);
2488 if (p != NULL)
2489 {
2490 if (compflags != NULL)
2491 {
2492 STRCPY(p, compflags);
2493 STRCAT(p, "/");
2494 }
2495 STRCAT(p, items[1]);
2496 compflags = p;
2497 }
2498 }
2499 }
2500 else if (is_aff_rule(items, itemcnt, "COMPOUNDWORDMAX", 2)
2501 && compmax == 0)
2502 {
2503 compmax = atoi((char *)items[1]);
2504 if (compmax == 0)
2505 smsg(_("Wrong COMPOUNDWORDMAX value in %s line %d: %s"),
2506 fname, lnum, items[1]);
2507 }
2508 else if (is_aff_rule(items, itemcnt, "COMPOUNDMIN", 2)
2509 && compminlen == 0)
2510 {
2511 compminlen = atoi((char *)items[1]);
2512 if (compminlen == 0)
2513 smsg(_("Wrong COMPOUNDMIN value in %s line %d: %s"),
2514 fname, lnum, items[1]);
2515 }
2516 else if (is_aff_rule(items, itemcnt, "COMPOUNDSYLMAX", 2)
2517 && compsylmax == 0)
2518 {
2519 compsylmax = atoi((char *)items[1]);
2520 if (compsylmax == 0)
2521 smsg(_("Wrong COMPOUNDSYLMAX value in %s line %d: %s"),
2522 fname, lnum, items[1]);
2523 }
2524 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDDUP", 1))
2525 {
2526 compoptions |= COMP_CHECKDUP;
2527 }
2528 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDREP", 1))
2529 {
2530 compoptions |= COMP_CHECKREP;
2531 }
2532 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDCASE", 1))
2533 {
2534 compoptions |= COMP_CHECKCASE;
2535 }
2536 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDTRIPLE", 1))
2537 {
2538 compoptions |= COMP_CHECKTRIPLE;
2539 }
2540 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 2))
2541 {
2542 if (atoi((char *)items[1]) == 0)
2543 smsg(_("Wrong CHECKCOMPOUNDPATTERN value in %s line %d: %s"),
2544 fname, lnum, items[1]);
2545 }
2546 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 3))
2547 {
2548 garray_T *gap = &spin->si_comppat;
2549 int i;
2550
2551 // Only add the couple if it isn't already there.
2552 for (i = 0; i < gap->ga_len - 1; i += 2)
2553 if (STRCMP(((char_u **)(gap->ga_data))[i], items[1]) == 0
2554 && STRCMP(((char_u **)(gap->ga_data))[i + 1],
2555 items[2]) == 0)
2556 break;
2557 if (i >= gap->ga_len && ga_grow(gap, 2) == OK)
2558 {
2559 ((char_u **)(gap->ga_data))[gap->ga_len++]
2560 = getroom_save(spin, items[1]);
2561 ((char_u **)(gap->ga_data))[gap->ga_len++]
2562 = getroom_save(spin, items[2]);
2563 }
2564 }
2565 else if (is_aff_rule(items, itemcnt, "SYLLABLE", 2)
2566 && syllable == NULL)
2567 {
2568 syllable = getroom_save(spin, items[1]);
2569 }
2570 else if (is_aff_rule(items, itemcnt, "NOBREAK", 1))
2571 {
2572 spin->si_nobreak = TRUE;
2573 }
2574 else if (is_aff_rule(items, itemcnt, "NOSPLITSUGS", 1))
2575 {
2576 spin->si_nosplitsugs = TRUE;
2577 }
2578 else if (is_aff_rule(items, itemcnt, "NOCOMPOUNDSUGS", 1))
2579 {
2580 spin->si_nocompoundsugs = TRUE;
2581 }
2582 else if (is_aff_rule(items, itemcnt, "NOSUGFILE", 1))
2583 {
2584 spin->si_nosugfile = TRUE;
2585 }
2586 else if (is_aff_rule(items, itemcnt, "PFXPOSTPONE", 1))
2587 {
2588 aff->af_pfxpostpone = TRUE;
2589 }
2590 else if (is_aff_rule(items, itemcnt, "IGNOREEXTRA", 1))
2591 {
2592 aff->af_ignoreextra = TRUE;
2593 }
2594 else if ((STRCMP(items[0], "PFX") == 0
2595 || STRCMP(items[0], "SFX") == 0)
2596 && aff_todo == 0
2597 && itemcnt >= 4)
2598 {
2599 int lasti = 4;
2600 char_u key[AH_KEY_LEN];
2601
2602 if (*items[0] == 'P')
2603 tp = &aff->af_pref;
2604 else
2605 tp = &aff->af_suff;
2606
2607 // Myspell allows the same affix name to be used multiple
2608 // times. The affix files that do this have an undocumented
2609 // "S" flag on all but the last block, thus we check for that
2610 // and store it in ah_follows.
2611 vim_strncpy(key, items[1], AH_KEY_LEN - 1);
2612 hi = hash_find(tp, key);
2613 if (!HASHITEM_EMPTY(hi))
2614 {
2615 cur_aff = HI2AH(hi);
2616 if (cur_aff->ah_combine != (*items[2] == 'Y'))
2617 smsg(_("Different combining flag in continued affix block in %s line %d: %s"),
2618 fname, lnum, items[1]);
2619 if (!cur_aff->ah_follows)
2620 smsg(_("Duplicate affix in %s line %d: %s"),
2621 fname, lnum, items[1]);
2622 }
2623 else
2624 {
2625 // New affix letter.
2626 cur_aff = (affheader_T *)getroom(spin,
2627 sizeof(affheader_T), TRUE);
2628 if (cur_aff == NULL)
2629 break;
2630 cur_aff->ah_flag = affitem2flag(aff->af_flagtype, items[1],
2631 fname, lnum);
2632 if (cur_aff->ah_flag == 0 || STRLEN(items[1]) >= AH_KEY_LEN)
2633 break;
2634 if (cur_aff->ah_flag == aff->af_bad
2635 || cur_aff->ah_flag == aff->af_rare
2636 || cur_aff->ah_flag == aff->af_keepcase
2637 || cur_aff->ah_flag == aff->af_needaffix
2638 || cur_aff->ah_flag == aff->af_circumfix
2639 || cur_aff->ah_flag == aff->af_nosuggest
2640 || cur_aff->ah_flag == aff->af_needcomp
2641 || cur_aff->ah_flag == aff->af_comproot)
2642 smsg(_("Affix also used for BAD/RARE/KEEPCASE/NEEDAFFIX/NEEDCOMPOUND/NOSUGGEST in %s line %d: %s"),
2643 fname, lnum, items[1]);
2644 STRCPY(cur_aff->ah_key, items[1]);
2645 hash_add(tp, cur_aff->ah_key);
2646
2647 cur_aff->ah_combine = (*items[2] == 'Y');
2648 }
2649
2650 // Check for the "S" flag, which apparently means that another
2651 // block with the same affix name is following.
2652 if (itemcnt > lasti && STRCMP(items[lasti], "S") == 0)
2653 {
2654 ++lasti;
2655 cur_aff->ah_follows = TRUE;
2656 }
2657 else
2658 cur_aff->ah_follows = FALSE;
2659
2660 // Myspell allows extra text after the item, but that might
2661 // mean mistakes go unnoticed. Require a comment-starter.
2662 if (itemcnt > lasti && *items[lasti] != '#')
2663 smsg(_(e_afftrailing), fname, lnum, items[lasti]);
2664
2665 if (STRCMP(items[2], "Y") != 0 && STRCMP(items[2], "N") != 0)
2666 smsg(_("Expected Y or N in %s line %d: %s"),
2667 fname, lnum, items[2]);
2668
2669 if (*items[0] == 'P' && aff->af_pfxpostpone)
2670 {
2671 if (cur_aff->ah_newID == 0)
2672 {
2673 // Use a new number in the .spl file later, to be able
2674 // to handle multiple .aff files.
2675 check_renumber(spin);
2676 cur_aff->ah_newID = ++spin->si_newprefID;
2677
2678 // We only really use ah_newID if the prefix is
2679 // postponed. We know that only after handling all
2680 // the items.
2681 did_postpone_prefix = FALSE;
2682 }
2683 else
2684 // Did use the ID in a previous block.
2685 did_postpone_prefix = TRUE;
2686 }
2687
2688 aff_todo = atoi((char *)items[3]);
2689 }
2690 else if ((STRCMP(items[0], "PFX") == 0
2691 || STRCMP(items[0], "SFX") == 0)
2692 && aff_todo > 0
2693 && STRCMP(cur_aff->ah_key, items[1]) == 0
2694 && itemcnt >= 5)
2695 {
2696 affentry_T *aff_entry;
2697 int upper = FALSE;
2698 int lasti = 5;
2699
2700 // Myspell allows extra text after the item, but that might
2701 // mean mistakes go unnoticed. Require a comment-starter,
2702 // unless IGNOREEXTRA is used. Hunspell uses a "-" item.
2703 if (itemcnt > lasti
2704 && !aff->af_ignoreextra
2705 && *items[lasti] != '#'
2706 && (STRCMP(items[lasti], "-") != 0
2707 || itemcnt != lasti + 1))
2708 smsg(_(e_afftrailing), fname, lnum, items[lasti]);
2709
2710 // New item for an affix letter.
2711 --aff_todo;
2712 aff_entry = (affentry_T *)getroom(spin,
2713 sizeof(affentry_T), TRUE);
2714 if (aff_entry == NULL)
2715 break;
2716
2717 if (STRCMP(items[2], "0") != 0)
2718 aff_entry->ae_chop = getroom_save(spin, items[2]);
2719 if (STRCMP(items[3], "0") != 0)
2720 {
2721 aff_entry->ae_add = getroom_save(spin, items[3]);
2722
2723 // Recognize flags on the affix: abcd/XYZ
2724 aff_entry->ae_flags = vim_strchr(aff_entry->ae_add, '/');
2725 if (aff_entry->ae_flags != NULL)
2726 {
2727 *aff_entry->ae_flags++ = NUL;
2728 aff_process_flags(aff, aff_entry);
2729 }
2730 }
2731
2732 // Don't use an affix entry with non-ASCII characters when
2733 // "spin->si_ascii" is TRUE.
2734 if (!spin->si_ascii || !(has_non_ascii(aff_entry->ae_chop)
2735 || has_non_ascii(aff_entry->ae_add)))
2736 {
2737 aff_entry->ae_next = cur_aff->ah_first;
2738 cur_aff->ah_first = aff_entry;
2739
2740 if (STRCMP(items[4], ".") != 0)
2741 {
2742 char_u buf[MAXLINELEN];
2743
2744 aff_entry->ae_cond = getroom_save(spin, items[4]);
2745 if (*items[0] == 'P')
2746 sprintf((char *)buf, "^%s", items[4]);
2747 else
2748 sprintf((char *)buf, "%s$", items[4]);
2749 aff_entry->ae_prog = vim_regcomp(buf,
2750 RE_MAGIC + RE_STRING + RE_STRICT);
2751 if (aff_entry->ae_prog == NULL)
2752 smsg(_("Broken condition in %s line %d: %s"),
2753 fname, lnum, items[4]);
2754 }
2755
2756 // For postponed prefixes we need an entry in si_prefcond
2757 // for the condition. Use an existing one if possible.
2758 // Can't be done for an affix with flags, ignoring
2759 // COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG.
2760 if (*items[0] == 'P' && aff->af_pfxpostpone
2761 && aff_entry->ae_flags == NULL)
2762 {
2763 // When the chop string is one lower-case letter and
2764 // the add string ends in the upper-case letter we set
2765 // the "upper" flag, clear "ae_chop" and remove the
2766 // letters from "ae_add". The condition must either
2767 // be empty or start with the same letter.
2768 if (aff_entry->ae_chop != NULL
2769 && aff_entry->ae_add != NULL
2770 && aff_entry->ae_chop[(*mb_ptr2len)(
2771 aff_entry->ae_chop)] == NUL)
2772 {
2773 int c, c_up;
2774
2775 c = PTR2CHAR(aff_entry->ae_chop);
2776 c_up = SPELL_TOUPPER(c);
2777 if (c_up != c
2778 && (aff_entry->ae_cond == NULL
2779 || PTR2CHAR(aff_entry->ae_cond) == c))
2780 {
2781 p = aff_entry->ae_add
2782 + STRLEN(aff_entry->ae_add);
2783 MB_PTR_BACK(aff_entry->ae_add, p);
2784 if (PTR2CHAR(p) == c_up)
2785 {
2786 upper = TRUE;
2787 aff_entry->ae_chop = NULL;
2788 *p = NUL;
2789
2790 // The condition is matched with the
2791 // actual word, thus must check for the
2792 // upper-case letter.
2793 if (aff_entry->ae_cond != NULL)
2794 {
2795 char_u buf[MAXLINELEN];
2796
2797 if (has_mbyte)
2798 {
2799 onecap_copy(items[4], buf, TRUE);
2800 aff_entry->ae_cond = getroom_save(
2801 spin, buf);
2802 }
2803 else
2804 *aff_entry->ae_cond = c_up;
2805 if (aff_entry->ae_cond != NULL)
2806 {
2807 sprintf((char *)buf, "^%s",
2808 aff_entry->ae_cond);
2809 vim_regfree(aff_entry->ae_prog);
2810 aff_entry->ae_prog = vim_regcomp(
2811 buf, RE_MAGIC + RE_STRING);
2812 }
2813 }
2814 }
2815 }
2816 }
2817
2818 if (aff_entry->ae_chop == NULL
2819 && aff_entry->ae_flags == NULL)
2820 {
2821 int idx;
2822 char_u **pp;
2823 int n;
2824
2825 // Find a previously used condition.
2826 for (idx = spin->si_prefcond.ga_len - 1; idx >= 0;
2827 --idx)
2828 {
2829 p = ((char_u **)spin->si_prefcond.ga_data)[idx];
2830 if (str_equal(p, aff_entry->ae_cond))
2831 break;
2832 }
2833 if (idx < 0 && ga_grow(&spin->si_prefcond, 1) == OK)
2834 {
2835 // Not found, add a new condition.
2836 idx = spin->si_prefcond.ga_len++;
2837 pp = ((char_u **)spin->si_prefcond.ga_data)
2838 + idx;
2839 if (aff_entry->ae_cond == NULL)
2840 *pp = NULL;
2841 else
2842 *pp = getroom_save(spin,
2843 aff_entry->ae_cond);
2844 }
2845
2846 // Add the prefix to the prefix tree.
2847 if (aff_entry->ae_add == NULL)
2848 p = (char_u *)"";
2849 else
2850 p = aff_entry->ae_add;
2851
2852 // PFX_FLAGS is a negative number, so that
2853 // tree_add_word() knows this is the prefix tree.
2854 n = PFX_FLAGS;
2855 if (!cur_aff->ah_combine)
2856 n |= WFP_NC;
2857 if (upper)
2858 n |= WFP_UP;
2859 if (aff_entry->ae_comppermit)
2860 n |= WFP_COMPPERMIT;
2861 if (aff_entry->ae_compforbid)
2862 n |= WFP_COMPFORBID;
2863 tree_add_word(spin, p, spin->si_prefroot, n,
2864 idx, cur_aff->ah_newID);
2865 did_postpone_prefix = TRUE;
2866 }
2867
2868 // Didn't actually use ah_newID, backup si_newprefID.
2869 if (aff_todo == 0 && !did_postpone_prefix)
2870 {
2871 --spin->si_newprefID;
2872 cur_aff->ah_newID = 0;
2873 }
2874 }
2875 }
2876 }
2877 else if (is_aff_rule(items, itemcnt, "FOL", 2) && fol == NULL)
2878 {
2879 fol = vim_strsave(items[1]);
2880 }
2881 else if (is_aff_rule(items, itemcnt, "LOW", 2) && low == NULL)
2882 {
2883 low = vim_strsave(items[1]);
2884 }
2885 else if (is_aff_rule(items, itemcnt, "UPP", 2) && upp == NULL)
2886 {
2887 upp = vim_strsave(items[1]);
2888 }
2889 else if (is_aff_rule(items, itemcnt, "REP", 2)
2890 || is_aff_rule(items, itemcnt, "REPSAL", 2))
2891 {
2892 // Ignore REP/REPSAL count
2893 if (!isdigit(*items[1]))
2894 smsg(_("Expected REP(SAL) count in %s line %d"),
2895 fname, lnum);
2896 }
2897 else if ((STRCMP(items[0], "REP") == 0
2898 || STRCMP(items[0], "REPSAL") == 0)
2899 && itemcnt >= 3)
2900 {
2901 // REP/REPSAL item
2902 // Myspell ignores extra arguments, we require it starts with
2903 // # to detect mistakes.
2904 if (itemcnt > 3 && items[3][0] != '#')
2905 smsg(_(e_afftrailing), fname, lnum, items[3]);
2906 if (items[0][3] == 'S' ? do_repsal : do_rep)
2907 {
2908 // Replace underscore with space (can't include a space
2909 // directly).
2910 for (p = items[1]; *p != NUL; MB_PTR_ADV(p))
2911 if (*p == '_')
2912 *p = ' ';
2913 for (p = items[2]; *p != NUL; MB_PTR_ADV(p))
2914 if (*p == '_')
2915 *p = ' ';
2916 add_fromto(spin, items[0][3] == 'S'
2917 ? &spin->si_repsal
2918 : &spin->si_rep, items[1], items[2]);
2919 }
2920 }
2921 else if (is_aff_rule(items, itemcnt, "MAP", 2))
2922 {
2923 // MAP item or count
2924 if (!found_map)
2925 {
2926 // First line contains the count.
2927 found_map = TRUE;
2928 if (!isdigit(*items[1]))
2929 smsg(_("Expected MAP count in %s line %d"),
2930 fname, lnum);
2931 }
2932 else if (do_mapline)
2933 {
2934 int c;
2935
2936 // Check that every character appears only once.
2937 for (p = items[1]; *p != NUL; )
2938 {
2939 c = mb_ptr2char_adv(&p);
2940 if ((spin->si_map.ga_len > 0
2941 && vim_strchr(spin->si_map.ga_data, c)
2942 != NULL)
2943 || vim_strchr(p, c) != NULL)
2944 smsg(_("Duplicate character in MAP in %s line %d"),
2945 fname, lnum);
2946 }
2947
2948 // We simply concatenate all the MAP strings, separated by
2949 // slashes.
2950 ga_concat(&spin->si_map, items[1]);
2951 ga_append(&spin->si_map, '/');
2952 }
2953 }
2954 // Accept "SAL from to" and "SAL from to #comment".
2955 else if (is_aff_rule(items, itemcnt, "SAL", 3))
2956 {
2957 if (do_sal)
2958 {
2959 // SAL item (sounds-a-like)
2960 // Either one of the known keys or a from-to pair.
2961 if (STRCMP(items[1], "followup") == 0)
2962 spin->si_followup = sal_to_bool(items[2]);
2963 else if (STRCMP(items[1], "collapse_result") == 0)
2964 spin->si_collapse = sal_to_bool(items[2]);
2965 else if (STRCMP(items[1], "remove_accents") == 0)
2966 spin->si_rem_accents = sal_to_bool(items[2]);
2967 else
2968 // when "to" is "_" it means empty
2969 add_fromto(spin, &spin->si_sal, items[1],
2970 STRCMP(items[2], "_") == 0 ? (char_u *)""
2971 : items[2]);
2972 }
2973 }
2974 else if (is_aff_rule(items, itemcnt, "SOFOFROM", 2)
2975 && sofofrom == NULL)
2976 {
2977 sofofrom = getroom_save(spin, items[1]);
2978 }
2979 else if (is_aff_rule(items, itemcnt, "SOFOTO", 2)
2980 && sofoto == NULL)
2981 {
2982 sofoto = getroom_save(spin, items[1]);
2983 }
2984 else if (STRCMP(items[0], "COMMON") == 0)
2985 {
2986 int i;
2987
2988 for (i = 1; i < itemcnt; ++i)
2989 {
2990 if (HASHITEM_EMPTY(hash_find(&spin->si_commonwords,
2991 items[i])))
2992 {
2993 p = vim_strsave(items[i]);
2994 if (p == NULL)
2995 break;
2996 hash_add(&spin->si_commonwords, p);
2997 }
2998 }
2999 }
3000 else
3001 smsg(_("Unrecognized or duplicate item in %s line %d: %s"),
3002 fname, lnum, items[0]);
3003 }
3004 }
3005
3006 if (fol != NULL || low != NULL || upp != NULL)
3007 {
3008 if (spin->si_clear_chartab)
3009 {
3010 // Clear the char type tables, don't want to use any of the
3011 // currently used spell properties.
3012 init_spell_chartab();
3013 spin->si_clear_chartab = FALSE;
3014 }
3015
3016 /*
3017 * Don't write a word table for an ASCII file, so that we don't check
3018 * for conflicts with a word table that matches 'encoding'.
3019 * Don't write one for utf-8 either, we use utf_*() and
3020 * mb_get_class(), the list of chars in the file will be incomplete.
3021 */
3022 if (!spin->si_ascii && !enc_utf8)
3023 {
3024 if (fol == NULL || low == NULL || upp == NULL)
3025 smsg(_("Missing FOL/LOW/UPP line in %s"), fname);
3026 else
3027 (void)set_spell_chartab(fol, low, upp);
3028 }
3029
3030 vim_free(fol);
3031 vim_free(low);
3032 vim_free(upp);
3033 }
3034
3035 // Use compound specifications of the .aff file for the spell info.
3036 if (compmax != 0)
3037 {
3038 aff_check_number(spin->si_compmax, compmax, "COMPOUNDWORDMAX");
3039 spin->si_compmax = compmax;
3040 }
3041
3042 if (compminlen != 0)
3043 {
3044 aff_check_number(spin->si_compminlen, compminlen, "COMPOUNDMIN");
3045 spin->si_compminlen = compminlen;
3046 }
3047
3048 if (compsylmax != 0)
3049 {
3050 if (syllable == NULL)
3051 smsg(_("COMPOUNDSYLMAX used without SYLLABLE"));
3052 aff_check_number(spin->si_compsylmax, compsylmax, "COMPOUNDSYLMAX");
3053 spin->si_compsylmax = compsylmax;
3054 }
3055
3056 if (compoptions != 0)
3057 {
3058 aff_check_number(spin->si_compoptions, compoptions, "COMPOUND options");
3059 spin->si_compoptions |= compoptions;
3060 }
3061
3062 if (compflags != NULL)
3063 process_compflags(spin, aff, compflags);
3064
3065 // Check that we didn't use too many renumbered flags.
3066 if (spin->si_newcompID < spin->si_newprefID)
3067 {
3068 if (spin->si_newcompID == 127 || spin->si_newcompID == 255)
3069 msg(_("Too many postponed prefixes"));
3070 else if (spin->si_newprefID == 0 || spin->si_newprefID == 127)
3071 msg(_("Too many compound flags"));
3072 else
3073 msg(_("Too many postponed prefixes and/or compound flags"));
3074 }
3075
3076 if (syllable != NULL)
3077 {
3078 aff_check_string(spin->si_syllable, syllable, "SYLLABLE");
3079 spin->si_syllable = syllable;
3080 }
3081
3082 if (sofofrom != NULL || sofoto != NULL)
3083 {
3084 if (sofofrom == NULL || sofoto == NULL)
3085 smsg(_("Missing SOFO%s line in %s"),
3086 sofofrom == NULL ? "FROM" : "TO", fname);
3087 else if (spin->si_sal.ga_len > 0)
3088 smsg(_("Both SAL and SOFO lines in %s"), fname);
3089 else
3090 {
3091 aff_check_string(spin->si_sofofr, sofofrom, "SOFOFROM");
3092 aff_check_string(spin->si_sofoto, sofoto, "SOFOTO");
3093 spin->si_sofofr = sofofrom;
3094 spin->si_sofoto = sofoto;
3095 }
3096 }
3097
3098 if (midword != NULL)
3099 {
3100 aff_check_string(spin->si_midword, midword, "MIDWORD");
3101 spin->si_midword = midword;
3102 }
3103
3104 vim_free(pc);
3105 fclose(fd);
3106 return aff;
3107 }
3108
3109 /*
3110 * Return TRUE when items[0] equals "rulename", there are "mincount" items or
3111 * a comment is following after item "mincount".
3112 */
3113 static int
is_aff_rule(char_u ** items,int itemcnt,char * rulename,int mincount)3114 is_aff_rule(
3115 char_u **items,
3116 int itemcnt,
3117 char *rulename,
3118 int mincount)
3119 {
3120 return (STRCMP(items[0], rulename) == 0
3121 && (itemcnt == mincount
3122 || (itemcnt > mincount && items[mincount][0] == '#')));
3123 }
3124
3125 /*
3126 * For affix "entry" move COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG from
3127 * ae_flags to ae_comppermit and ae_compforbid.
3128 */
3129 static void
aff_process_flags(afffile_T * affile,affentry_T * entry)3130 aff_process_flags(afffile_T *affile, affentry_T *entry)
3131 {
3132 char_u *p;
3133 char_u *prevp;
3134 unsigned flag;
3135
3136 if (entry->ae_flags != NULL
3137 && (affile->af_compforbid != 0 || affile->af_comppermit != 0))
3138 {
3139 for (p = entry->ae_flags; *p != NUL; )
3140 {
3141 prevp = p;
3142 flag = get_affitem(affile->af_flagtype, &p);
3143 if (flag == affile->af_comppermit || flag == affile->af_compforbid)
3144 {
3145 STRMOVE(prevp, p);
3146 p = prevp;
3147 if (flag == affile->af_comppermit)
3148 entry->ae_comppermit = TRUE;
3149 else
3150 entry->ae_compforbid = TRUE;
3151 }
3152 if (affile->af_flagtype == AFT_NUM && *p == ',')
3153 ++p;
3154 }
3155 if (*entry->ae_flags == NUL)
3156 entry->ae_flags = NULL; // nothing left
3157 }
3158 }
3159
3160 /*
3161 * Return TRUE if "s" is the name of an info item in the affix file.
3162 */
3163 static int
spell_info_item(char_u * s)3164 spell_info_item(char_u *s)
3165 {
3166 return STRCMP(s, "NAME") == 0
3167 || STRCMP(s, "HOME") == 0
3168 || STRCMP(s, "VERSION") == 0
3169 || STRCMP(s, "AUTHOR") == 0
3170 || STRCMP(s, "EMAIL") == 0
3171 || STRCMP(s, "COPYRIGHT") == 0;
3172 }
3173
3174 /*
3175 * Turn an affix flag name into a number, according to the FLAG type.
3176 * returns zero for failure.
3177 */
3178 static unsigned
affitem2flag(int flagtype,char_u * item,char_u * fname,int lnum)3179 affitem2flag(
3180 int flagtype,
3181 char_u *item,
3182 char_u *fname,
3183 int lnum)
3184 {
3185 unsigned res;
3186 char_u *p = item;
3187
3188 res = get_affitem(flagtype, &p);
3189 if (res == 0)
3190 {
3191 if (flagtype == AFT_NUM)
3192 smsg(_("Flag is not a number in %s line %d: %s"),
3193 fname, lnum, item);
3194 else
3195 smsg(_("Illegal flag in %s line %d: %s"),
3196 fname, lnum, item);
3197 }
3198 if (*p != NUL)
3199 {
3200 smsg(_(e_affname), fname, lnum, item);
3201 return 0;
3202 }
3203
3204 return res;
3205 }
3206
3207 /*
3208 * Get one affix name from "*pp" and advance the pointer.
3209 * Returns ZERO_FLAG for "0".
3210 * Returns zero for an error, still advances the pointer then.
3211 */
3212 static unsigned
get_affitem(int flagtype,char_u ** pp)3213 get_affitem(int flagtype, char_u **pp)
3214 {
3215 int res;
3216
3217 if (flagtype == AFT_NUM)
3218 {
3219 if (!VIM_ISDIGIT(**pp))
3220 {
3221 ++*pp; // always advance, avoid getting stuck
3222 return 0;
3223 }
3224 res = getdigits(pp);
3225 if (res == 0)
3226 res = ZERO_FLAG;
3227 }
3228 else
3229 {
3230 res = mb_ptr2char_adv(pp);
3231 if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG
3232 && res >= 'A' && res <= 'Z'))
3233 {
3234 if (**pp == NUL)
3235 return 0;
3236 res = mb_ptr2char_adv(pp) + (res << 16);
3237 }
3238 }
3239 return res;
3240 }
3241
3242 /*
3243 * Process the "compflags" string used in an affix file and append it to
3244 * spin->si_compflags.
3245 * The processing involves changing the affix names to ID numbers, so that
3246 * they fit in one byte.
3247 */
3248 static void
process_compflags(spellinfo_T * spin,afffile_T * aff,char_u * compflags)3249 process_compflags(
3250 spellinfo_T *spin,
3251 afffile_T *aff,
3252 char_u *compflags)
3253 {
3254 char_u *p;
3255 char_u *prevp;
3256 unsigned flag;
3257 compitem_T *ci;
3258 int id;
3259 int len;
3260 char_u *tp;
3261 char_u key[AH_KEY_LEN];
3262 hashitem_T *hi;
3263
3264 // Make room for the old and the new compflags, concatenated with a / in
3265 // between. Processing it makes it shorter, but we don't know by how
3266 // much, thus allocate the maximum.
3267 len = (int)STRLEN(compflags) + 1;
3268 if (spin->si_compflags != NULL)
3269 len += (int)STRLEN(spin->si_compflags) + 1;
3270 p = getroom(spin, len, FALSE);
3271 if (p == NULL)
3272 return;
3273 if (spin->si_compflags != NULL)
3274 {
3275 STRCPY(p, spin->si_compflags);
3276 STRCAT(p, "/");
3277 }
3278 spin->si_compflags = p;
3279 tp = p + STRLEN(p);
3280
3281 for (p = compflags; *p != NUL; )
3282 {
3283 if (vim_strchr((char_u *)"/?*+[]", *p) != NULL)
3284 // Copy non-flag characters directly.
3285 *tp++ = *p++;
3286 else
3287 {
3288 // First get the flag number, also checks validity.
3289 prevp = p;
3290 flag = get_affitem(aff->af_flagtype, &p);
3291 if (flag != 0)
3292 {
3293 // Find the flag in the hashtable. If it was used before, use
3294 // the existing ID. Otherwise add a new entry.
3295 vim_strncpy(key, prevp, p - prevp);
3296 hi = hash_find(&aff->af_comp, key);
3297 if (!HASHITEM_EMPTY(hi))
3298 id = HI2CI(hi)->ci_newID;
3299 else
3300 {
3301 ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE);
3302 if (ci == NULL)
3303 break;
3304 STRCPY(ci->ci_key, key);
3305 ci->ci_flag = flag;
3306 // Avoid using a flag ID that has a special meaning in a
3307 // regexp (also inside []).
3308 do
3309 {
3310 check_renumber(spin);
3311 id = spin->si_newcompID--;
3312 } while (vim_strchr((char_u *)"/?*+[]\\-^", id) != NULL);
3313 ci->ci_newID = id;
3314 hash_add(&aff->af_comp, ci->ci_key);
3315 }
3316 *tp++ = id;
3317 }
3318 if (aff->af_flagtype == AFT_NUM && *p == ',')
3319 ++p;
3320 }
3321 }
3322
3323 *tp = NUL;
3324 }
3325
3326 /*
3327 * Check that the new IDs for postponed affixes and compounding don't overrun
3328 * each other. We have almost 255 available, but start at 0-127 to avoid
3329 * using two bytes for utf-8. When the 0-127 range is used up go to 128-255.
3330 * When that is used up an error message is given.
3331 */
3332 static void
check_renumber(spellinfo_T * spin)3333 check_renumber(spellinfo_T *spin)
3334 {
3335 if (spin->si_newprefID == spin->si_newcompID && spin->si_newcompID < 128)
3336 {
3337 spin->si_newprefID = 127;
3338 spin->si_newcompID = 255;
3339 }
3340 }
3341
3342 /*
3343 * Return TRUE if flag "flag" appears in affix list "afflist".
3344 */
3345 static int
flag_in_afflist(int flagtype,char_u * afflist,unsigned flag)3346 flag_in_afflist(int flagtype, char_u *afflist, unsigned flag)
3347 {
3348 char_u *p;
3349 unsigned n;
3350
3351 switch (flagtype)
3352 {
3353 case AFT_CHAR:
3354 return vim_strchr(afflist, flag) != NULL;
3355
3356 case AFT_CAPLONG:
3357 case AFT_LONG:
3358 for (p = afflist; *p != NUL; )
3359 {
3360 n = mb_ptr2char_adv(&p);
3361 if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z'))
3362 && *p != NUL)
3363 n = mb_ptr2char_adv(&p) + (n << 16);
3364 if (n == flag)
3365 return TRUE;
3366 }
3367 break;
3368
3369 case AFT_NUM:
3370 for (p = afflist; *p != NUL; )
3371 {
3372 n = getdigits(&p);
3373 if (n == 0)
3374 n = ZERO_FLAG;
3375 if (n == flag)
3376 return TRUE;
3377 if (*p != NUL) // skip over comma
3378 ++p;
3379 }
3380 break;
3381 }
3382 return FALSE;
3383 }
3384
3385 /*
3386 * Give a warning when "spinval" and "affval" numbers are set and not the same.
3387 */
3388 static void
aff_check_number(int spinval,int affval,char * name)3389 aff_check_number(int spinval, int affval, char *name)
3390 {
3391 if (spinval != 0 && spinval != affval)
3392 smsg(_("%s value differs from what is used in another .aff file"), name);
3393 }
3394
3395 /*
3396 * Give a warning when "spinval" and "affval" strings are set and not the same.
3397 */
3398 static void
aff_check_string(char_u * spinval,char_u * affval,char * name)3399 aff_check_string(char_u *spinval, char_u *affval, char *name)
3400 {
3401 if (spinval != NULL && STRCMP(spinval, affval) != 0)
3402 smsg(_("%s value differs from what is used in another .aff file"), name);
3403 }
3404
3405 /*
3406 * Return TRUE if strings "s1" and "s2" are equal. Also consider both being
3407 * NULL as equal.
3408 */
3409 static int
str_equal(char_u * s1,char_u * s2)3410 str_equal(char_u *s1, char_u *s2)
3411 {
3412 if (s1 == NULL || s2 == NULL)
3413 return s1 == s2;
3414 return STRCMP(s1, s2) == 0;
3415 }
3416
3417 /*
3418 * Add a from-to item to "gap". Used for REP and SAL items.
3419 * They are stored case-folded.
3420 */
3421 static void
add_fromto(spellinfo_T * spin,garray_T * gap,char_u * from,char_u * to)3422 add_fromto(
3423 spellinfo_T *spin,
3424 garray_T *gap,
3425 char_u *from,
3426 char_u *to)
3427 {
3428 fromto_T *ftp;
3429 char_u word[MAXWLEN];
3430
3431 if (ga_grow(gap, 1) == OK)
3432 {
3433 ftp = ((fromto_T *)gap->ga_data) + gap->ga_len;
3434 (void)spell_casefold(curwin, from, (int)STRLEN(from), word, MAXWLEN);
3435 ftp->ft_from = getroom_save(spin, word);
3436 (void)spell_casefold(curwin, to, (int)STRLEN(to), word, MAXWLEN);
3437 ftp->ft_to = getroom_save(spin, word);
3438 ++gap->ga_len;
3439 }
3440 }
3441
3442 /*
3443 * Convert a boolean argument in a SAL line to TRUE or FALSE;
3444 */
3445 static int
sal_to_bool(char_u * s)3446 sal_to_bool(char_u *s)
3447 {
3448 return STRCMP(s, "1") == 0 || STRCMP(s, "true") == 0;
3449 }
3450
3451 /*
3452 * Free the structure filled by spell_read_aff().
3453 */
3454 static void
spell_free_aff(afffile_T * aff)3455 spell_free_aff(afffile_T *aff)
3456 {
3457 hashtab_T *ht;
3458 hashitem_T *hi;
3459 int todo;
3460 affheader_T *ah;
3461 affentry_T *ae;
3462
3463 vim_free(aff->af_enc);
3464
3465 // All this trouble to free the "ae_prog" items...
3466 for (ht = &aff->af_pref; ; ht = &aff->af_suff)
3467 {
3468 todo = (int)ht->ht_used;
3469 for (hi = ht->ht_array; todo > 0; ++hi)
3470 {
3471 if (!HASHITEM_EMPTY(hi))
3472 {
3473 --todo;
3474 ah = HI2AH(hi);
3475 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next)
3476 vim_regfree(ae->ae_prog);
3477 }
3478 }
3479 if (ht == &aff->af_suff)
3480 break;
3481 }
3482
3483 hash_clear(&aff->af_pref);
3484 hash_clear(&aff->af_suff);
3485 hash_clear(&aff->af_comp);
3486 }
3487
3488 /*
3489 * Read dictionary file "fname".
3490 * Returns OK or FAIL;
3491 */
3492 static int
spell_read_dic(spellinfo_T * spin,char_u * fname,afffile_T * affile)3493 spell_read_dic(spellinfo_T *spin, char_u *fname, afffile_T *affile)
3494 {
3495 hashtab_T ht;
3496 char_u line[MAXLINELEN];
3497 char_u *p;
3498 char_u *afflist;
3499 char_u store_afflist[MAXWLEN];
3500 int pfxlen;
3501 int need_affix;
3502 char_u *dw;
3503 char_u *pc;
3504 char_u *w;
3505 int l;
3506 hash_T hash;
3507 hashitem_T *hi;
3508 FILE *fd;
3509 int lnum = 1;
3510 int non_ascii = 0;
3511 int retval = OK;
3512 char_u message[MAXLINELEN + MAXWLEN];
3513 int flags;
3514 int duplicate = 0;
3515 time_T last_msg_time = 0;
3516
3517 /*
3518 * Open the file.
3519 */
3520 fd = mch_fopen((char *)fname, "r");
3521 if (fd == NULL)
3522 {
3523 semsg(_(e_notopen), fname);
3524 return FAIL;
3525 }
3526
3527 // The hashtable is only used to detect duplicated words.
3528 hash_init(&ht);
3529
3530 vim_snprintf((char *)IObuff, IOSIZE,
3531 _("Reading dictionary file %s..."), fname);
3532 spell_message(spin, IObuff);
3533
3534 // start with a message for the first line
3535 spin->si_msg_count = 999999;
3536
3537 // Read and ignore the first line: word count.
3538 if (vim_fgets(line, MAXLINELEN, fd) || !vim_isdigit(*skipwhite(line)))
3539 semsg(_("E760: No word count in %s"), fname);
3540
3541 /*
3542 * Read all the lines in the file one by one.
3543 * The words are converted to 'encoding' here, before being added to
3544 * the hashtable.
3545 */
3546 while (!vim_fgets(line, MAXLINELEN, fd) && !got_int)
3547 {
3548 line_breakcheck();
3549 ++lnum;
3550 if (line[0] == '#' || line[0] == '/')
3551 continue; // comment line
3552
3553 // Remove CR, LF and white space from the end. White space halfway
3554 // the word is kept to allow e.g., "et al.".
3555 l = (int)STRLEN(line);
3556 while (l > 0 && line[l - 1] <= ' ')
3557 --l;
3558 if (l == 0)
3559 continue; // empty line
3560 line[l] = NUL;
3561
3562 // Convert from "SET" to 'encoding' when needed.
3563 if (spin->si_conv.vc_type != CONV_NONE)
3564 {
3565 pc = string_convert(&spin->si_conv, line, NULL);
3566 if (pc == NULL)
3567 {
3568 smsg(_("Conversion failure for word in %s line %d: %s"),
3569 fname, lnum, line);
3570 continue;
3571 }
3572 w = pc;
3573 }
3574 else
3575 {
3576 pc = NULL;
3577 w = line;
3578 }
3579
3580 // Truncate the word at the "/", set "afflist" to what follows.
3581 // Replace "\/" by "/" and "\\" by "\".
3582 afflist = NULL;
3583 for (p = w; *p != NUL; MB_PTR_ADV(p))
3584 {
3585 if (*p == '\\' && (p[1] == '\\' || p[1] == '/'))
3586 STRMOVE(p, p + 1);
3587 else if (*p == '/')
3588 {
3589 *p = NUL;
3590 afflist = p + 1;
3591 break;
3592 }
3593 }
3594
3595 // Skip non-ASCII words when "spin->si_ascii" is TRUE.
3596 if (spin->si_ascii && has_non_ascii(w))
3597 {
3598 ++non_ascii;
3599 vim_free(pc);
3600 continue;
3601 }
3602
3603 // This takes time, print a message every 10000 words, but not more
3604 // often than once per second.
3605 if (spin->si_verbose && spin->si_msg_count > 10000)
3606 {
3607 spin->si_msg_count = 0;
3608 if (vim_time() > last_msg_time)
3609 {
3610 last_msg_time = vim_time();
3611 vim_snprintf((char *)message, sizeof(message),
3612 _("line %6d, word %6ld - %s"),
3613 lnum, spin->si_foldwcount + spin->si_keepwcount, w);
3614 msg_start();
3615 msg_outtrans_long_attr(message, 0);
3616 msg_clr_eos();
3617 msg_didout = FALSE;
3618 msg_col = 0;
3619 out_flush();
3620 }
3621 }
3622
3623 // Store the word in the hashtable to be able to find duplicates.
3624 dw = (char_u *)getroom_save(spin, w);
3625 if (dw == NULL)
3626 {
3627 retval = FAIL;
3628 vim_free(pc);
3629 break;
3630 }
3631
3632 hash = hash_hash(dw);
3633 hi = hash_lookup(&ht, dw, hash);
3634 if (!HASHITEM_EMPTY(hi))
3635 {
3636 if (p_verbose > 0)
3637 smsg(_("Duplicate word in %s line %d: %s"),
3638 fname, lnum, dw);
3639 else if (duplicate == 0)
3640 smsg(_("First duplicate word in %s line %d: %s"),
3641 fname, lnum, dw);
3642 ++duplicate;
3643 }
3644 else
3645 hash_add_item(&ht, hi, dw, hash);
3646
3647 flags = 0;
3648 store_afflist[0] = NUL;
3649 pfxlen = 0;
3650 need_affix = FALSE;
3651 if (afflist != NULL)
3652 {
3653 // Extract flags from the affix list.
3654 flags |= get_affix_flags(affile, afflist);
3655
3656 if (affile->af_needaffix != 0 && flag_in_afflist(
3657 affile->af_flagtype, afflist, affile->af_needaffix))
3658 need_affix = TRUE;
3659
3660 if (affile->af_pfxpostpone)
3661 // Need to store the list of prefix IDs with the word.
3662 pfxlen = get_pfxlist(affile, afflist, store_afflist);
3663
3664 if (spin->si_compflags != NULL)
3665 // Need to store the list of compound flags with the word.
3666 // Concatenate them to the list of prefix IDs.
3667 get_compflags(affile, afflist, store_afflist + pfxlen);
3668 }
3669
3670 // Add the word to the word tree(s).
3671 if (store_word(spin, dw, flags, spin->si_region,
3672 store_afflist, need_affix) == FAIL)
3673 retval = FAIL;
3674
3675 if (afflist != NULL)
3676 {
3677 // Find all matching suffixes and add the resulting words.
3678 // Additionally do matching prefixes that combine.
3679 if (store_aff_word(spin, dw, afflist, affile,
3680 &affile->af_suff, &affile->af_pref,
3681 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL)
3682 retval = FAIL;
3683
3684 // Find all matching prefixes and add the resulting words.
3685 if (store_aff_word(spin, dw, afflist, affile,
3686 &affile->af_pref, NULL,
3687 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL)
3688 retval = FAIL;
3689 }
3690
3691 vim_free(pc);
3692 }
3693
3694 if (duplicate > 0)
3695 smsg(_("%d duplicate word(s) in %s"), duplicate, fname);
3696 if (spin->si_ascii && non_ascii > 0)
3697 smsg(_("Ignored %d word(s) with non-ASCII characters in %s"),
3698 non_ascii, fname);
3699 hash_clear(&ht);
3700
3701 fclose(fd);
3702 return retval;
3703 }
3704
3705 /*
3706 * Check for affix flags in "afflist" that are turned into word flags.
3707 * Return WF_ flags.
3708 */
3709 static int
get_affix_flags(afffile_T * affile,char_u * afflist)3710 get_affix_flags(afffile_T *affile, char_u *afflist)
3711 {
3712 int flags = 0;
3713
3714 if (affile->af_keepcase != 0 && flag_in_afflist(
3715 affile->af_flagtype, afflist, affile->af_keepcase))
3716 flags |= WF_KEEPCAP | WF_FIXCAP;
3717 if (affile->af_rare != 0 && flag_in_afflist(
3718 affile->af_flagtype, afflist, affile->af_rare))
3719 flags |= WF_RARE;
3720 if (affile->af_bad != 0 && flag_in_afflist(
3721 affile->af_flagtype, afflist, affile->af_bad))
3722 flags |= WF_BANNED;
3723 if (affile->af_needcomp != 0 && flag_in_afflist(
3724 affile->af_flagtype, afflist, affile->af_needcomp))
3725 flags |= WF_NEEDCOMP;
3726 if (affile->af_comproot != 0 && flag_in_afflist(
3727 affile->af_flagtype, afflist, affile->af_comproot))
3728 flags |= WF_COMPROOT;
3729 if (affile->af_nosuggest != 0 && flag_in_afflist(
3730 affile->af_flagtype, afflist, affile->af_nosuggest))
3731 flags |= WF_NOSUGGEST;
3732 return flags;
3733 }
3734
3735 /*
3736 * Get the list of prefix IDs from the affix list "afflist".
3737 * Used for PFXPOSTPONE.
3738 * Put the resulting flags in "store_afflist[MAXWLEN]" with a terminating NUL
3739 * and return the number of affixes.
3740 */
3741 static int
get_pfxlist(afffile_T * affile,char_u * afflist,char_u * store_afflist)3742 get_pfxlist(
3743 afffile_T *affile,
3744 char_u *afflist,
3745 char_u *store_afflist)
3746 {
3747 char_u *p;
3748 char_u *prevp;
3749 int cnt = 0;
3750 int id;
3751 char_u key[AH_KEY_LEN];
3752 hashitem_T *hi;
3753
3754 for (p = afflist; *p != NUL; )
3755 {
3756 prevp = p;
3757 if (get_affitem(affile->af_flagtype, &p) != 0)
3758 {
3759 // A flag is a postponed prefix flag if it appears in "af_pref"
3760 // and its ID is not zero.
3761 vim_strncpy(key, prevp, p - prevp);
3762 hi = hash_find(&affile->af_pref, key);
3763 if (!HASHITEM_EMPTY(hi))
3764 {
3765 id = HI2AH(hi)->ah_newID;
3766 if (id != 0)
3767 store_afflist[cnt++] = id;
3768 }
3769 }
3770 if (affile->af_flagtype == AFT_NUM && *p == ',')
3771 ++p;
3772 }
3773
3774 store_afflist[cnt] = NUL;
3775 return cnt;
3776 }
3777
3778 /*
3779 * Get the list of compound IDs from the affix list "afflist" that are used
3780 * for compound words.
3781 * Puts the flags in "store_afflist[]".
3782 */
3783 static void
get_compflags(afffile_T * affile,char_u * afflist,char_u * store_afflist)3784 get_compflags(
3785 afffile_T *affile,
3786 char_u *afflist,
3787 char_u *store_afflist)
3788 {
3789 char_u *p;
3790 char_u *prevp;
3791 int cnt = 0;
3792 char_u key[AH_KEY_LEN];
3793 hashitem_T *hi;
3794
3795 for (p = afflist; *p != NUL; )
3796 {
3797 prevp = p;
3798 if (get_affitem(affile->af_flagtype, &p) != 0)
3799 {
3800 // A flag is a compound flag if it appears in "af_comp".
3801 vim_strncpy(key, prevp, p - prevp);
3802 hi = hash_find(&affile->af_comp, key);
3803 if (!HASHITEM_EMPTY(hi))
3804 store_afflist[cnt++] = HI2CI(hi)->ci_newID;
3805 }
3806 if (affile->af_flagtype == AFT_NUM && *p == ',')
3807 ++p;
3808 }
3809
3810 store_afflist[cnt] = NUL;
3811 }
3812
3813 /*
3814 * Apply affixes to a word and store the resulting words.
3815 * "ht" is the hashtable with affentry_T that need to be applied, either
3816 * prefixes or suffixes.
3817 * "xht", when not NULL, is the prefix hashtable, to be used additionally on
3818 * the resulting words for combining affixes.
3819 *
3820 * Returns FAIL when out of memory.
3821 */
3822 static int
store_aff_word(spellinfo_T * spin,char_u * word,char_u * afflist,afffile_T * affile,hashtab_T * ht,hashtab_T * xht,int condit,int flags,char_u * pfxlist,int pfxlen)3823 store_aff_word(
3824 spellinfo_T *spin, // spell info
3825 char_u *word, // basic word start
3826 char_u *afflist, // list of names of supported affixes
3827 afffile_T *affile,
3828 hashtab_T *ht,
3829 hashtab_T *xht,
3830 int condit, // CONDIT_SUF et al.
3831 int flags, // flags for the word
3832 char_u *pfxlist, // list of prefix IDs
3833 int pfxlen) // nr of flags in "pfxlist" for prefixes, rest
3834 // is compound flags
3835 {
3836 int todo;
3837 hashitem_T *hi;
3838 affheader_T *ah;
3839 affentry_T *ae;
3840 char_u newword[MAXWLEN];
3841 int retval = OK;
3842 int i, j;
3843 char_u *p;
3844 int use_flags;
3845 char_u *use_pfxlist;
3846 int use_pfxlen;
3847 int need_affix;
3848 char_u store_afflist[MAXWLEN];
3849 char_u pfx_pfxlist[MAXWLEN];
3850 size_t wordlen = STRLEN(word);
3851 int use_condit;
3852
3853 todo = (int)ht->ht_used;
3854 for (hi = ht->ht_array; todo > 0 && retval == OK; ++hi)
3855 {
3856 if (!HASHITEM_EMPTY(hi))
3857 {
3858 --todo;
3859 ah = HI2AH(hi);
3860
3861 // Check that the affix combines, if required, and that the word
3862 // supports this affix.
3863 if (((condit & CONDIT_COMB) == 0 || ah->ah_combine)
3864 && flag_in_afflist(affile->af_flagtype, afflist,
3865 ah->ah_flag))
3866 {
3867 // Loop over all affix entries with this name.
3868 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next)
3869 {
3870 // Check the condition. It's not logical to match case
3871 // here, but it is required for compatibility with
3872 // Myspell.
3873 // Another requirement from Myspell is that the chop
3874 // string is shorter than the word itself.
3875 // For prefixes, when "PFXPOSTPONE" was used, only do
3876 // prefixes with a chop string and/or flags.
3877 // When a previously added affix had CIRCUMFIX this one
3878 // must have it too, if it had not then this one must not
3879 // have one either.
3880 if ((xht != NULL || !affile->af_pfxpostpone
3881 || ae->ae_chop != NULL
3882 || ae->ae_flags != NULL)
3883 && (ae->ae_chop == NULL
3884 || STRLEN(ae->ae_chop) < wordlen)
3885 && (ae->ae_prog == NULL
3886 || vim_regexec_prog(&ae->ae_prog, FALSE,
3887 word, (colnr_T)0))
3888 && (((condit & CONDIT_CFIX) == 0)
3889 == ((condit & CONDIT_AFF) == 0
3890 || ae->ae_flags == NULL
3891 || !flag_in_afflist(affile->af_flagtype,
3892 ae->ae_flags, affile->af_circumfix))))
3893 {
3894 // Match. Remove the chop and add the affix.
3895 if (xht == NULL)
3896 {
3897 // prefix: chop/add at the start of the word
3898 if (ae->ae_add == NULL)
3899 *newword = NUL;
3900 else
3901 vim_strncpy(newword, ae->ae_add, MAXWLEN - 1);
3902 p = word;
3903 if (ae->ae_chop != NULL)
3904 {
3905 // Skip chop string.
3906 if (has_mbyte)
3907 {
3908 i = mb_charlen(ae->ae_chop);
3909 for ( ; i > 0; --i)
3910 MB_PTR_ADV(p);
3911 }
3912 else
3913 p += STRLEN(ae->ae_chop);
3914 }
3915 STRCAT(newword, p);
3916 }
3917 else
3918 {
3919 // suffix: chop/add at the end of the word
3920 vim_strncpy(newword, word, MAXWLEN - 1);
3921 if (ae->ae_chop != NULL)
3922 {
3923 // Remove chop string.
3924 p = newword + STRLEN(newword);
3925 i = (int)MB_CHARLEN(ae->ae_chop);
3926 for ( ; i > 0; --i)
3927 MB_PTR_BACK(newword, p);
3928 *p = NUL;
3929 }
3930 if (ae->ae_add != NULL)
3931 STRCAT(newword, ae->ae_add);
3932 }
3933
3934 use_flags = flags;
3935 use_pfxlist = pfxlist;
3936 use_pfxlen = pfxlen;
3937 need_affix = FALSE;
3938 use_condit = condit | CONDIT_COMB | CONDIT_AFF;
3939 if (ae->ae_flags != NULL)
3940 {
3941 // Extract flags from the affix list.
3942 use_flags |= get_affix_flags(affile, ae->ae_flags);
3943
3944 if (affile->af_needaffix != 0 && flag_in_afflist(
3945 affile->af_flagtype, ae->ae_flags,
3946 affile->af_needaffix))
3947 need_affix = TRUE;
3948
3949 // When there is a CIRCUMFIX flag the other affix
3950 // must also have it and we don't add the word
3951 // with one affix.
3952 if (affile->af_circumfix != 0 && flag_in_afflist(
3953 affile->af_flagtype, ae->ae_flags,
3954 affile->af_circumfix))
3955 {
3956 use_condit |= CONDIT_CFIX;
3957 if ((condit & CONDIT_CFIX) == 0)
3958 need_affix = TRUE;
3959 }
3960
3961 if (affile->af_pfxpostpone
3962 || spin->si_compflags != NULL)
3963 {
3964 if (affile->af_pfxpostpone)
3965 // Get prefix IDS from the affix list.
3966 use_pfxlen = get_pfxlist(affile,
3967 ae->ae_flags, store_afflist);
3968 else
3969 use_pfxlen = 0;
3970 use_pfxlist = store_afflist;
3971
3972 // Combine the prefix IDs. Avoid adding the
3973 // same ID twice.
3974 for (i = 0; i < pfxlen; ++i)
3975 {
3976 for (j = 0; j < use_pfxlen; ++j)
3977 if (pfxlist[i] == use_pfxlist[j])
3978 break;
3979 if (j == use_pfxlen)
3980 use_pfxlist[use_pfxlen++] = pfxlist[i];
3981 }
3982
3983 if (spin->si_compflags != NULL)
3984 // Get compound IDS from the affix list.
3985 get_compflags(affile, ae->ae_flags,
3986 use_pfxlist + use_pfxlen);
3987
3988 // Combine the list of compound flags.
3989 // Concatenate them to the prefix IDs list.
3990 // Avoid adding the same ID twice.
3991 for (i = pfxlen; pfxlist[i] != NUL; ++i)
3992 {
3993 for (j = use_pfxlen;
3994 use_pfxlist[j] != NUL; ++j)
3995 if (pfxlist[i] == use_pfxlist[j])
3996 break;
3997 if (use_pfxlist[j] == NUL)
3998 {
3999 use_pfxlist[j++] = pfxlist[i];
4000 use_pfxlist[j] = NUL;
4001 }
4002 }
4003 }
4004 }
4005
4006 // Obey a "COMPOUNDFORBIDFLAG" of the affix: don't
4007 // use the compound flags.
4008 if (use_pfxlist != NULL && ae->ae_compforbid)
4009 {
4010 vim_strncpy(pfx_pfxlist, use_pfxlist, use_pfxlen);
4011 use_pfxlist = pfx_pfxlist;
4012 }
4013
4014 // When there are postponed prefixes...
4015 if (spin->si_prefroot != NULL
4016 && spin->si_prefroot->wn_sibling != NULL)
4017 {
4018 // ... add a flag to indicate an affix was used.
4019 use_flags |= WF_HAS_AFF;
4020
4021 // ... don't use a prefix list if combining
4022 // affixes is not allowed. But do use the
4023 // compound flags after them.
4024 if (!ah->ah_combine && use_pfxlist != NULL)
4025 use_pfxlist += use_pfxlen;
4026 }
4027
4028 // When compounding is supported and there is no
4029 // "COMPOUNDPERMITFLAG" then forbid compounding on the
4030 // side where the affix is applied.
4031 if (spin->si_compflags != NULL && !ae->ae_comppermit)
4032 {
4033 if (xht != NULL)
4034 use_flags |= WF_NOCOMPAFT;
4035 else
4036 use_flags |= WF_NOCOMPBEF;
4037 }
4038
4039 // Store the modified word.
4040 if (store_word(spin, newword, use_flags,
4041 spin->si_region, use_pfxlist,
4042 need_affix) == FAIL)
4043 retval = FAIL;
4044
4045 // When added a prefix or a first suffix and the affix
4046 // has flags may add a(nother) suffix. RECURSIVE!
4047 if ((condit & CONDIT_SUF) && ae->ae_flags != NULL)
4048 if (store_aff_word(spin, newword, ae->ae_flags,
4049 affile, &affile->af_suff, xht,
4050 use_condit & (xht == NULL
4051 ? ~0 : ~CONDIT_SUF),
4052 use_flags, use_pfxlist, pfxlen) == FAIL)
4053 retval = FAIL;
4054
4055 // When added a suffix and combining is allowed also
4056 // try adding a prefix additionally. Both for the
4057 // word flags and for the affix flags. RECURSIVE!
4058 if (xht != NULL && ah->ah_combine)
4059 {
4060 if (store_aff_word(spin, newword,
4061 afflist, affile,
4062 xht, NULL, use_condit,
4063 use_flags, use_pfxlist,
4064 pfxlen) == FAIL
4065 || (ae->ae_flags != NULL
4066 && store_aff_word(spin, newword,
4067 ae->ae_flags, affile,
4068 xht, NULL, use_condit,
4069 use_flags, use_pfxlist,
4070 pfxlen) == FAIL))
4071 retval = FAIL;
4072 }
4073 }
4074 }
4075 }
4076 }
4077 }
4078
4079 return retval;
4080 }
4081
4082 /*
4083 * Read a file with a list of words.
4084 */
4085 static int
spell_read_wordfile(spellinfo_T * spin,char_u * fname)4086 spell_read_wordfile(spellinfo_T *spin, char_u *fname)
4087 {
4088 FILE *fd;
4089 long lnum = 0;
4090 char_u rline[MAXLINELEN];
4091 char_u *line;
4092 char_u *pc = NULL;
4093 char_u *p;
4094 int l;
4095 int retval = OK;
4096 int did_word = FALSE;
4097 int non_ascii = 0;
4098 int flags;
4099 int regionmask;
4100
4101 /*
4102 * Open the file.
4103 */
4104 fd = mch_fopen((char *)fname, "r");
4105 if (fd == NULL)
4106 {
4107 semsg(_(e_notopen), fname);
4108 return FAIL;
4109 }
4110
4111 vim_snprintf((char *)IObuff, IOSIZE, _("Reading word file %s..."), fname);
4112 spell_message(spin, IObuff);
4113
4114 /*
4115 * Read all the lines in the file one by one.
4116 */
4117 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int)
4118 {
4119 line_breakcheck();
4120 ++lnum;
4121
4122 // Skip comment lines.
4123 if (*rline == '#')
4124 continue;
4125
4126 // Remove CR, LF and white space from the end.
4127 l = (int)STRLEN(rline);
4128 while (l > 0 && rline[l - 1] <= ' ')
4129 --l;
4130 if (l == 0)
4131 continue; // empty or blank line
4132 rline[l] = NUL;
4133
4134 // Convert from "/encoding={encoding}" to 'encoding' when needed.
4135 vim_free(pc);
4136 if (spin->si_conv.vc_type != CONV_NONE)
4137 {
4138 pc = string_convert(&spin->si_conv, rline, NULL);
4139 if (pc == NULL)
4140 {
4141 smsg(_("Conversion failure for word in %s line %ld: %s"),
4142 fname, lnum, rline);
4143 continue;
4144 }
4145 line = pc;
4146 }
4147 else
4148 {
4149 pc = NULL;
4150 line = rline;
4151 }
4152
4153 if (*line == '/')
4154 {
4155 ++line;
4156 if (STRNCMP(line, "encoding=", 9) == 0)
4157 {
4158 if (spin->si_conv.vc_type != CONV_NONE)
4159 smsg(_("Duplicate /encoding= line ignored in %s line %ld: %s"),
4160 fname, lnum, line - 1);
4161 else if (did_word)
4162 smsg(_("/encoding= line after word ignored in %s line %ld: %s"),
4163 fname, lnum, line - 1);
4164 else
4165 {
4166 char_u *enc;
4167
4168 // Setup for conversion to 'encoding'.
4169 line += 9;
4170 enc = enc_canonize(line);
4171 if (enc != NULL && !spin->si_ascii
4172 && convert_setup(&spin->si_conv, enc,
4173 p_enc) == FAIL)
4174 smsg(_("Conversion in %s not supported: from %s to %s"),
4175 fname, line, p_enc);
4176 vim_free(enc);
4177 spin->si_conv.vc_fail = TRUE;
4178 }
4179 continue;
4180 }
4181
4182 if (STRNCMP(line, "regions=", 8) == 0)
4183 {
4184 if (spin->si_region_count > 1)
4185 smsg(_("Duplicate /regions= line ignored in %s line %ld: %s"),
4186 fname, lnum, line);
4187 else
4188 {
4189 line += 8;
4190 if (STRLEN(line) > MAXREGIONS * 2)
4191 smsg(_("Too many regions in %s line %ld: %s"),
4192 fname, lnum, line);
4193 else
4194 {
4195 spin->si_region_count = (int)STRLEN(line) / 2;
4196 STRCPY(spin->si_region_name, line);
4197
4198 // Adjust the mask for a word valid in all regions.
4199 spin->si_region = (1 << spin->si_region_count) - 1;
4200 }
4201 }
4202 continue;
4203 }
4204
4205 smsg(_("/ line ignored in %s line %ld: %s"),
4206 fname, lnum, line - 1);
4207 continue;
4208 }
4209
4210 flags = 0;
4211 regionmask = spin->si_region;
4212
4213 // Check for flags and region after a slash.
4214 p = vim_strchr(line, '/');
4215 if (p != NULL)
4216 {
4217 *p++ = NUL;
4218 while (*p != NUL)
4219 {
4220 if (*p == '=') // keep-case word
4221 flags |= WF_KEEPCAP | WF_FIXCAP;
4222 else if (*p == '!') // Bad, bad, wicked word.
4223 flags |= WF_BANNED;
4224 else if (*p == '?') // Rare word.
4225 flags |= WF_RARE;
4226 else if (VIM_ISDIGIT(*p)) // region number(s)
4227 {
4228 if ((flags & WF_REGION) == 0) // first one
4229 regionmask = 0;
4230 flags |= WF_REGION;
4231
4232 l = *p - '0';
4233 if (l == 0 || l > spin->si_region_count)
4234 {
4235 smsg(_("Invalid region nr in %s line %ld: %s"),
4236 fname, lnum, p);
4237 break;
4238 }
4239 regionmask |= 1 << (l - 1);
4240 }
4241 else
4242 {
4243 smsg(_("Unrecognized flags in %s line %ld: %s"),
4244 fname, lnum, p);
4245 break;
4246 }
4247 ++p;
4248 }
4249 }
4250
4251 // Skip non-ASCII words when "spin->si_ascii" is TRUE.
4252 if (spin->si_ascii && has_non_ascii(line))
4253 {
4254 ++non_ascii;
4255 continue;
4256 }
4257
4258 // Normal word: store it.
4259 if (store_word(spin, line, flags, regionmask, NULL, FALSE) == FAIL)
4260 {
4261 retval = FAIL;
4262 break;
4263 }
4264 did_word = TRUE;
4265 }
4266
4267 vim_free(pc);
4268 fclose(fd);
4269
4270 if (spin->si_ascii && non_ascii > 0)
4271 {
4272 vim_snprintf((char *)IObuff, IOSIZE,
4273 _("Ignored %d words with non-ASCII characters"), non_ascii);
4274 spell_message(spin, IObuff);
4275 }
4276
4277 return retval;
4278 }
4279
4280 /*
4281 * Get part of an sblock_T, "len" bytes long.
4282 * This avoids calling free() for every little struct we use (and keeping
4283 * track of them).
4284 * The memory is cleared to all zeros.
4285 * Returns NULL when out of memory.
4286 */
4287 static void *
getroom(spellinfo_T * spin,size_t len,int align)4288 getroom(
4289 spellinfo_T *spin,
4290 size_t len, // length needed
4291 int align) // align for pointer
4292 {
4293 char_u *p;
4294 sblock_T *bl = spin->si_blocks;
4295
4296 if (align && bl != NULL)
4297 // Round size up for alignment. On some systems structures need to be
4298 // aligned to the size of a pointer (e.g., SPARC).
4299 bl->sb_used = (bl->sb_used + sizeof(char *) - 1)
4300 & ~(sizeof(char *) - 1);
4301
4302 if (bl == NULL || bl->sb_used + len > SBLOCKSIZE)
4303 {
4304 if (len >= SBLOCKSIZE)
4305 bl = NULL;
4306 else
4307 // Allocate a block of memory. It is not freed until much later.
4308 bl = alloc_clear(sizeof(sblock_T) + SBLOCKSIZE);
4309 if (bl == NULL)
4310 {
4311 if (!spin->si_did_emsg)
4312 {
4313 emsg(_("E845: Insufficient memory, word list will be incomplete"));
4314 spin->si_did_emsg = TRUE;
4315 }
4316 return NULL;
4317 }
4318 bl->sb_next = spin->si_blocks;
4319 spin->si_blocks = bl;
4320 bl->sb_used = 0;
4321 ++spin->si_blocks_cnt;
4322 }
4323
4324 p = bl->sb_data + bl->sb_used;
4325 bl->sb_used += (int)len;
4326
4327 return p;
4328 }
4329
4330 /*
4331 * Make a copy of a string into memory allocated with getroom().
4332 * Returns NULL when out of memory.
4333 */
4334 static char_u *
getroom_save(spellinfo_T * spin,char_u * s)4335 getroom_save(spellinfo_T *spin, char_u *s)
4336 {
4337 char_u *sc;
4338
4339 sc = (char_u *)getroom(spin, STRLEN(s) + 1, FALSE);
4340 if (sc != NULL)
4341 STRCPY(sc, s);
4342 return sc;
4343 }
4344
4345
4346 /*
4347 * Free the list of allocated sblock_T.
4348 */
4349 static void
free_blocks(sblock_T * bl)4350 free_blocks(sblock_T *bl)
4351 {
4352 sblock_T *next;
4353
4354 while (bl != NULL)
4355 {
4356 next = bl->sb_next;
4357 vim_free(bl);
4358 bl = next;
4359 }
4360 }
4361
4362 /*
4363 * Allocate the root of a word tree.
4364 * Returns NULL when out of memory.
4365 */
4366 static wordnode_T *
wordtree_alloc(spellinfo_T * spin)4367 wordtree_alloc(spellinfo_T *spin)
4368 {
4369 return (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE);
4370 }
4371
4372 /*
4373 * Store a word in the tree(s).
4374 * Always store it in the case-folded tree. For a keep-case word this is
4375 * useful when the word can also be used with all caps (no WF_FIXCAP flag) and
4376 * used to find suggestions.
4377 * For a keep-case word also store it in the keep-case tree.
4378 * When "pfxlist" is not NULL store the word for each postponed prefix ID and
4379 * compound flag.
4380 */
4381 static int
store_word(spellinfo_T * spin,char_u * word,int flags,int region,char_u * pfxlist,int need_affix)4382 store_word(
4383 spellinfo_T *spin,
4384 char_u *word,
4385 int flags, // extra flags, WF_BANNED
4386 int region, // supported region(s)
4387 char_u *pfxlist, // list of prefix IDs or NULL
4388 int need_affix) // only store word with affix ID
4389 {
4390 int len = (int)STRLEN(word);
4391 int ct = captype(word, word + len);
4392 char_u foldword[MAXWLEN];
4393 int res = OK;
4394 char_u *p;
4395
4396 (void)spell_casefold(curwin, word, len, foldword, MAXWLEN);
4397 for (p = pfxlist; res == OK; ++p)
4398 {
4399 if (!need_affix || (p != NULL && *p != NUL))
4400 res = tree_add_word(spin, foldword, spin->si_foldroot, ct | flags,
4401 region, p == NULL ? 0 : *p);
4402 if (p == NULL || *p == NUL)
4403 break;
4404 }
4405 ++spin->si_foldwcount;
4406
4407 if (res == OK && (ct == WF_KEEPCAP || (flags & WF_KEEPCAP)))
4408 {
4409 for (p = pfxlist; res == OK; ++p)
4410 {
4411 if (!need_affix || (p != NULL && *p != NUL))
4412 res = tree_add_word(spin, word, spin->si_keeproot, flags,
4413 region, p == NULL ? 0 : *p);
4414 if (p == NULL || *p == NUL)
4415 break;
4416 }
4417 ++spin->si_keepwcount;
4418 }
4419 return res;
4420 }
4421
4422 /*
4423 * Add word "word" to a word tree at "root".
4424 * When "flags" < 0 we are adding to the prefix tree where "flags" is used for
4425 * "rare" and "region" is the condition nr.
4426 * Returns FAIL when out of memory.
4427 */
4428 static int
tree_add_word(spellinfo_T * spin,char_u * word,wordnode_T * root,int flags,int region,int affixID)4429 tree_add_word(
4430 spellinfo_T *spin,
4431 char_u *word,
4432 wordnode_T *root,
4433 int flags,
4434 int region,
4435 int affixID)
4436 {
4437 wordnode_T *node = root;
4438 wordnode_T *np;
4439 wordnode_T *copyp, **copyprev;
4440 wordnode_T **prev = NULL;
4441 int i;
4442
4443 // Add each byte of the word to the tree, including the NUL at the end.
4444 for (i = 0; ; ++i)
4445 {
4446 // When there is more than one reference to this node we need to make
4447 // a copy, so that we can modify it. Copy the whole list of siblings
4448 // (we don't optimize for a partly shared list of siblings).
4449 if (node != NULL && node->wn_refs > 1)
4450 {
4451 --node->wn_refs;
4452 copyprev = prev;
4453 FOR_ALL_NODE_SIBLINGS(node, copyp)
4454 {
4455 // Allocate a new node and copy the info.
4456 np = get_wordnode(spin);
4457 if (np == NULL)
4458 return FAIL;
4459 np->wn_child = copyp->wn_child;
4460 if (np->wn_child != NULL)
4461 ++np->wn_child->wn_refs; // child gets extra ref
4462 np->wn_byte = copyp->wn_byte;
4463 if (np->wn_byte == NUL)
4464 {
4465 np->wn_flags = copyp->wn_flags;
4466 np->wn_region = copyp->wn_region;
4467 np->wn_affixID = copyp->wn_affixID;
4468 }
4469
4470 // Link the new node in the list, there will be one ref.
4471 np->wn_refs = 1;
4472 if (copyprev != NULL)
4473 *copyprev = np;
4474 copyprev = &np->wn_sibling;
4475
4476 // Let "node" point to the head of the copied list.
4477 if (copyp == node)
4478 node = np;
4479 }
4480 }
4481
4482 // Look for the sibling that has the same character. They are sorted
4483 // on byte value, thus stop searching when a sibling is found with a
4484 // higher byte value. For zero bytes (end of word) the sorting is
4485 // done on flags and then on affixID.
4486 while (node != NULL
4487 && (node->wn_byte < word[i]
4488 || (node->wn_byte == NUL
4489 && (flags < 0
4490 ? node->wn_affixID < (unsigned)affixID
4491 : (node->wn_flags < (unsigned)(flags & WN_MASK)
4492 || (node->wn_flags == (flags & WN_MASK)
4493 && (spin->si_sugtree
4494 ? (node->wn_region & 0xffff) < region
4495 : node->wn_affixID
4496 < (unsigned)affixID)))))))
4497 {
4498 prev = &node->wn_sibling;
4499 node = *prev;
4500 }
4501 if (node == NULL
4502 || node->wn_byte != word[i]
4503 || (word[i] == NUL
4504 && (flags < 0
4505 || spin->si_sugtree
4506 || node->wn_flags != (flags & WN_MASK)
4507 || node->wn_affixID != affixID)))
4508 {
4509 // Allocate a new node.
4510 np = get_wordnode(spin);
4511 if (np == NULL)
4512 return FAIL;
4513 np->wn_byte = word[i];
4514
4515 // If "node" is NULL this is a new child or the end of the sibling
4516 // list: ref count is one. Otherwise use ref count of sibling and
4517 // make ref count of sibling one (matters when inserting in front
4518 // of the list of siblings).
4519 if (node == NULL)
4520 np->wn_refs = 1;
4521 else
4522 {
4523 np->wn_refs = node->wn_refs;
4524 node->wn_refs = 1;
4525 }
4526 if (prev != NULL)
4527 *prev = np;
4528 np->wn_sibling = node;
4529 node = np;
4530 }
4531
4532 if (word[i] == NUL)
4533 {
4534 node->wn_flags = flags;
4535 node->wn_region |= region;
4536 node->wn_affixID = affixID;
4537 break;
4538 }
4539 prev = &node->wn_child;
4540 node = *prev;
4541 }
4542 #ifdef SPELL_PRINTTREE
4543 smsg("Added \"%s\"", word);
4544 spell_print_tree(root->wn_sibling);
4545 #endif
4546
4547 // count nr of words added since last message
4548 ++spin->si_msg_count;
4549
4550 if (spin->si_compress_cnt > 1)
4551 {
4552 if (--spin->si_compress_cnt == 1)
4553 // Did enough words to lower the block count limit.
4554 spin->si_blocks_cnt += compress_inc;
4555 }
4556
4557 /*
4558 * When we have allocated lots of memory we need to compress the word tree
4559 * to free up some room. But compression is slow, and we might actually
4560 * need that room, thus only compress in the following situations:
4561 * 1. When not compressed before (si_compress_cnt == 0): when using
4562 * "compress_start" blocks.
4563 * 2. When compressed before and used "compress_inc" blocks before
4564 * adding "compress_added" words (si_compress_cnt > 1).
4565 * 3. When compressed before, added "compress_added" words
4566 * (si_compress_cnt == 1) and the number of free nodes drops below the
4567 * maximum word length.
4568 */
4569 #ifndef SPELL_COMPRESS_ALLWAYS
4570 if (spin->si_compress_cnt == 1
4571 ? spin->si_free_count < MAXWLEN
4572 : spin->si_blocks_cnt >= compress_start)
4573 #endif
4574 {
4575 // Decrement the block counter. The effect is that we compress again
4576 // when the freed up room has been used and another "compress_inc"
4577 // blocks have been allocated. Unless "compress_added" words have
4578 // been added, then the limit is put back again.
4579 spin->si_blocks_cnt -= compress_inc;
4580 spin->si_compress_cnt = compress_added;
4581
4582 if (spin->si_verbose)
4583 {
4584 msg_start();
4585 msg_puts(_(msg_compressing));
4586 msg_clr_eos();
4587 msg_didout = FALSE;
4588 msg_col = 0;
4589 out_flush();
4590 }
4591
4592 // Compress both trees. Either they both have many nodes, which makes
4593 // compression useful, or one of them is small, which means
4594 // compression goes fast. But when filling the soundfold word tree
4595 // there is no keep-case tree.
4596 wordtree_compress(spin, spin->si_foldroot, "case-folded");
4597 if (affixID >= 0)
4598 wordtree_compress(spin, spin->si_keeproot, "keep-case");
4599 }
4600
4601 return OK;
4602 }
4603
4604 /*
4605 * Get a wordnode_T, either from the list of previously freed nodes or
4606 * allocate a new one.
4607 * Returns NULL when out of memory.
4608 */
4609 static wordnode_T *
get_wordnode(spellinfo_T * spin)4610 get_wordnode(spellinfo_T *spin)
4611 {
4612 wordnode_T *n;
4613
4614 if (spin->si_first_free == NULL)
4615 n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE);
4616 else
4617 {
4618 n = spin->si_first_free;
4619 spin->si_first_free = n->wn_child;
4620 CLEAR_POINTER(n);
4621 --spin->si_free_count;
4622 }
4623 #ifdef SPELL_PRINTTREE
4624 if (n != NULL)
4625 n->wn_nr = ++spin->si_wordnode_nr;
4626 #endif
4627 return n;
4628 }
4629
4630 /*
4631 * Decrement the reference count on a node (which is the head of a list of
4632 * siblings). If the reference count becomes zero free the node and its
4633 * siblings.
4634 * Returns the number of nodes actually freed.
4635 */
4636 static int
deref_wordnode(spellinfo_T * spin,wordnode_T * node)4637 deref_wordnode(spellinfo_T *spin, wordnode_T *node)
4638 {
4639 wordnode_T *np;
4640 int cnt = 0;
4641
4642 if (--node->wn_refs == 0)
4643 {
4644 FOR_ALL_NODE_SIBLINGS(node, np)
4645 {
4646 if (np->wn_child != NULL)
4647 cnt += deref_wordnode(spin, np->wn_child);
4648 free_wordnode(spin, np);
4649 ++cnt;
4650 }
4651 ++cnt; // length field
4652 }
4653 return cnt;
4654 }
4655
4656 /*
4657 * Free a wordnode_T for re-use later.
4658 * Only the "wn_child" field becomes invalid.
4659 */
4660 static void
free_wordnode(spellinfo_T * spin,wordnode_T * n)4661 free_wordnode(spellinfo_T *spin, wordnode_T *n)
4662 {
4663 n->wn_child = spin->si_first_free;
4664 spin->si_first_free = n;
4665 ++spin->si_free_count;
4666 }
4667
4668 /*
4669 * Compress a tree: find tails that are identical and can be shared.
4670 */
4671 static void
wordtree_compress(spellinfo_T * spin,wordnode_T * root,char * name)4672 wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name)
4673 {
4674 hashtab_T ht;
4675 long n;
4676 long tot = 0;
4677 long perc;
4678
4679 // Skip the root itself, it's not actually used. The first sibling is the
4680 // start of the tree.
4681 if (root->wn_sibling != NULL)
4682 {
4683 hash_init(&ht);
4684 n = node_compress(spin, root->wn_sibling, &ht, &tot);
4685
4686 #ifndef SPELL_PRINTTREE
4687 if (spin->si_verbose || p_verbose > 2)
4688 #endif
4689 {
4690 if (tot > 1000000)
4691 perc = (tot - n) / (tot / 100);
4692 else if (tot == 0)
4693 perc = 0;
4694 else
4695 perc = (tot - n) * 100 / tot;
4696 vim_snprintf((char *)IObuff, IOSIZE,
4697 _("Compressed %s: %ld of %ld nodes; %ld (%ld%%) remaining"),
4698 name, n, tot, tot - n, perc);
4699 spell_message(spin, IObuff);
4700 }
4701 #ifdef SPELL_PRINTTREE
4702 spell_print_tree(root->wn_sibling);
4703 #endif
4704 hash_clear(&ht);
4705 }
4706 }
4707
4708 /*
4709 * Compress a node, its siblings and its children, depth first.
4710 * Returns the number of compressed nodes.
4711 */
4712 static long
node_compress(spellinfo_T * spin,wordnode_T * node,hashtab_T * ht,long * tot)4713 node_compress(
4714 spellinfo_T *spin,
4715 wordnode_T *node,
4716 hashtab_T *ht,
4717 long *tot) // total count of nodes before compressing,
4718 // incremented while going through the tree
4719 {
4720 wordnode_T *np;
4721 wordnode_T *tp;
4722 wordnode_T *child;
4723 hash_T hash;
4724 hashitem_T *hi;
4725 long len = 0;
4726 unsigned nr, n;
4727 long compressed = 0;
4728
4729 /*
4730 * Go through the list of siblings. Compress each child and then try
4731 * finding an identical child to replace it.
4732 * Note that with "child" we mean not just the node that is pointed to,
4733 * but the whole list of siblings of which the child node is the first.
4734 */
4735 for (np = node; np != NULL && !got_int; np = np->wn_sibling)
4736 {
4737 ++len;
4738 if ((child = np->wn_child) != NULL)
4739 {
4740 // Compress the child first. This fills hashkey.
4741 compressed += node_compress(spin, child, ht, tot);
4742
4743 // Try to find an identical child.
4744 hash = hash_hash(child->wn_u1.hashkey);
4745 hi = hash_lookup(ht, child->wn_u1.hashkey, hash);
4746 if (!HASHITEM_EMPTY(hi))
4747 {
4748 // There are children we encountered before with a hash value
4749 // identical to the current child. Now check if there is one
4750 // that is really identical.
4751 for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next)
4752 if (node_equal(child, tp))
4753 {
4754 // Found one! Now use that child in place of the
4755 // current one. This means the current child and all
4756 // its siblings is unlinked from the tree.
4757 ++tp->wn_refs;
4758 compressed += deref_wordnode(spin, child);
4759 np->wn_child = tp;
4760 break;
4761 }
4762 if (tp == NULL)
4763 {
4764 // No other child with this hash value equals the child of
4765 // the node, add it to the linked list after the first
4766 // item.
4767 tp = HI2WN(hi);
4768 child->wn_u2.next = tp->wn_u2.next;
4769 tp->wn_u2.next = child;
4770 }
4771 }
4772 else
4773 // No other child has this hash value, add it to the
4774 // hashtable.
4775 hash_add_item(ht, hi, child->wn_u1.hashkey, hash);
4776 }
4777 }
4778 *tot += len + 1; // add one for the node that stores the length
4779
4780 /*
4781 * Make a hash key for the node and its siblings, so that we can quickly
4782 * find a lookalike node. This must be done after compressing the sibling
4783 * list, otherwise the hash key would become invalid by the compression.
4784 */
4785 node->wn_u1.hashkey[0] = len;
4786 nr = 0;
4787 FOR_ALL_NODE_SIBLINGS(node, np)
4788 {
4789 if (np->wn_byte == NUL)
4790 // end node: use wn_flags, wn_region and wn_affixID
4791 n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16);
4792 else
4793 // byte node: use the byte value and the child pointer
4794 n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8));
4795 nr = nr * 101 + n;
4796 }
4797
4798 // Avoid NUL bytes, it terminates the hash key.
4799 n = nr & 0xff;
4800 node->wn_u1.hashkey[1] = n == 0 ? 1 : n;
4801 n = (nr >> 8) & 0xff;
4802 node->wn_u1.hashkey[2] = n == 0 ? 1 : n;
4803 n = (nr >> 16) & 0xff;
4804 node->wn_u1.hashkey[3] = n == 0 ? 1 : n;
4805 n = (nr >> 24) & 0xff;
4806 node->wn_u1.hashkey[4] = n == 0 ? 1 : n;
4807 node->wn_u1.hashkey[5] = NUL;
4808
4809 // Check for CTRL-C pressed now and then.
4810 veryfast_breakcheck();
4811
4812 return compressed;
4813 }
4814
4815 /*
4816 * Return TRUE when two nodes have identical siblings and children.
4817 */
4818 static int
node_equal(wordnode_T * n1,wordnode_T * n2)4819 node_equal(wordnode_T *n1, wordnode_T *n2)
4820 {
4821 wordnode_T *p1;
4822 wordnode_T *p2;
4823
4824 for (p1 = n1, p2 = n2; p1 != NULL && p2 != NULL;
4825 p1 = p1->wn_sibling, p2 = p2->wn_sibling)
4826 if (p1->wn_byte != p2->wn_byte
4827 || (p1->wn_byte == NUL
4828 ? (p1->wn_flags != p2->wn_flags
4829 || p1->wn_region != p2->wn_region
4830 || p1->wn_affixID != p2->wn_affixID)
4831 : (p1->wn_child != p2->wn_child)))
4832 break;
4833
4834 return p1 == NULL && p2 == NULL;
4835 }
4836
4837 static int rep_compare(const void *s1, const void *s2);
4838
4839 /*
4840 * Function given to qsort() to sort the REP items on "from" string.
4841 */
4842 static int
rep_compare(const void * s1,const void * s2)4843 rep_compare(const void *s1, const void *s2)
4844 {
4845 fromto_T *p1 = (fromto_T *)s1;
4846 fromto_T *p2 = (fromto_T *)s2;
4847
4848 return STRCMP(p1->ft_from, p2->ft_from);
4849 }
4850
4851 /*
4852 * Write the Vim .spl file "fname".
4853 * Return FAIL or OK;
4854 */
4855 static int
write_vim_spell(spellinfo_T * spin,char_u * fname)4856 write_vim_spell(spellinfo_T *spin, char_u *fname)
4857 {
4858 FILE *fd;
4859 int regionmask;
4860 int round;
4861 wordnode_T *tree;
4862 int nodecount;
4863 int i;
4864 int l;
4865 garray_T *gap;
4866 fromto_T *ftp;
4867 char_u *p;
4868 int rr;
4869 int retval = OK;
4870 size_t fwv = 1; // collect return value of fwrite() to avoid
4871 // warnings from picky compiler
4872
4873 fd = mch_fopen((char *)fname, "w");
4874 if (fd == NULL)
4875 {
4876 semsg(_(e_notopen), fname);
4877 return FAIL;
4878 }
4879
4880 // <HEADER>: <fileID> <versionnr>
4881 // <fileID>
4882 fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd);
4883 if (fwv != (size_t)1)
4884 // Catch first write error, don't try writing more.
4885 goto theend;
4886
4887 putc(VIMSPELLVERSION, fd); // <versionnr>
4888
4889 /*
4890 * <SECTIONS>: <section> ... <sectionend>
4891 */
4892
4893 // SN_INFO: <infotext>
4894 if (spin->si_info != NULL)
4895 {
4896 putc(SN_INFO, fd); // <sectionID>
4897 putc(0, fd); // <sectionflags>
4898
4899 i = (int)STRLEN(spin->si_info);
4900 put_bytes(fd, (long_u)i, 4); // <sectionlen>
4901 fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); // <infotext>
4902 }
4903
4904 // SN_REGION: <regionname> ...
4905 // Write the region names only if there is more than one.
4906 if (spin->si_region_count > 1)
4907 {
4908 putc(SN_REGION, fd); // <sectionID>
4909 putc(SNF_REQUIRED, fd); // <sectionflags>
4910 l = spin->si_region_count * 2;
4911 put_bytes(fd, (long_u)l, 4); // <sectionlen>
4912 fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd);
4913 // <regionname> ...
4914 regionmask = (1 << spin->si_region_count) - 1;
4915 }
4916 else
4917 regionmask = 0;
4918
4919 // SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars>
4920 //
4921 // The table with character flags and the table for case folding.
4922 // This makes sure the same characters are recognized as word characters
4923 // when generating an when using a spell file.
4924 // Skip this for ASCII, the table may conflict with the one used for
4925 // 'encoding'.
4926 // Also skip this for an .add.spl file, the main spell file must contain
4927 // the table (avoids that it conflicts). File is shorter too.
4928 if (!spin->si_ascii && !spin->si_add)
4929 {
4930 char_u folchars[128 * 8];
4931 int flags;
4932
4933 putc(SN_CHARFLAGS, fd); // <sectionID>
4934 putc(SNF_REQUIRED, fd); // <sectionflags>
4935
4936 // Form the <folchars> string first, we need to know its length.
4937 l = 0;
4938 for (i = 128; i < 256; ++i)
4939 {
4940 if (has_mbyte)
4941 l += mb_char2bytes(spelltab.st_fold[i], folchars + l);
4942 else
4943 folchars[l++] = spelltab.st_fold[i];
4944 }
4945 put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4); // <sectionlen>
4946
4947 fputc(128, fd); // <charflagslen>
4948 for (i = 128; i < 256; ++i)
4949 {
4950 flags = 0;
4951 if (spelltab.st_isw[i])
4952 flags |= CF_WORD;
4953 if (spelltab.st_isu[i])
4954 flags |= CF_UPPER;
4955 fputc(flags, fd); // <charflags>
4956 }
4957
4958 put_bytes(fd, (long_u)l, 2); // <folcharslen>
4959 fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); // <folchars>
4960 }
4961
4962 // SN_MIDWORD: <midword>
4963 if (spin->si_midword != NULL)
4964 {
4965 putc(SN_MIDWORD, fd); // <sectionID>
4966 putc(SNF_REQUIRED, fd); // <sectionflags>
4967
4968 i = (int)STRLEN(spin->si_midword);
4969 put_bytes(fd, (long_u)i, 4); // <sectionlen>
4970 fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd);
4971 // <midword>
4972 }
4973
4974 // SN_PREFCOND: <prefcondcnt> <prefcond> ...
4975 if (spin->si_prefcond.ga_len > 0)
4976 {
4977 putc(SN_PREFCOND, fd); // <sectionID>
4978 putc(SNF_REQUIRED, fd); // <sectionflags>
4979
4980 l = write_spell_prefcond(NULL, &spin->si_prefcond);
4981 put_bytes(fd, (long_u)l, 4); // <sectionlen>
4982
4983 write_spell_prefcond(fd, &spin->si_prefcond);
4984 }
4985
4986 // SN_REP: <repcount> <rep> ...
4987 // SN_SAL: <salflags> <salcount> <sal> ...
4988 // SN_REPSAL: <repcount> <rep> ...
4989
4990 // round 1: SN_REP section
4991 // round 2: SN_SAL section (unless SN_SOFO is used)
4992 // round 3: SN_REPSAL section
4993 for (round = 1; round <= 3; ++round)
4994 {
4995 if (round == 1)
4996 gap = &spin->si_rep;
4997 else if (round == 2)
4998 {
4999 // Don't write SN_SAL when using a SN_SOFO section
5000 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
5001 continue;
5002 gap = &spin->si_sal;
5003 }
5004 else
5005 gap = &spin->si_repsal;
5006
5007 // Don't write the section if there are no items.
5008 if (gap->ga_len == 0)
5009 continue;
5010
5011 // Sort the REP/REPSAL items.
5012 if (round != 2)
5013 qsort(gap->ga_data, (size_t)gap->ga_len,
5014 sizeof(fromto_T), rep_compare);
5015
5016 i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL);
5017 putc(i, fd); // <sectionID>
5018
5019 // This is for making suggestions, section is not required.
5020 putc(0, fd); // <sectionflags>
5021
5022 // Compute the length of what follows.
5023 l = 2; // count <repcount> or <salcount>
5024 for (i = 0; i < gap->ga_len; ++i)
5025 {
5026 ftp = &((fromto_T *)gap->ga_data)[i];
5027 l += 1 + (int)STRLEN(ftp->ft_from); // count <*fromlen> and <*from>
5028 l += 1 + (int)STRLEN(ftp->ft_to); // count <*tolen> and <*to>
5029 }
5030 if (round == 2)
5031 ++l; // count <salflags>
5032 put_bytes(fd, (long_u)l, 4); // <sectionlen>
5033
5034 if (round == 2)
5035 {
5036 i = 0;
5037 if (spin->si_followup)
5038 i |= SAL_F0LLOWUP;
5039 if (spin->si_collapse)
5040 i |= SAL_COLLAPSE;
5041 if (spin->si_rem_accents)
5042 i |= SAL_REM_ACCENTS;
5043 putc(i, fd); // <salflags>
5044 }
5045
5046 put_bytes(fd, (long_u)gap->ga_len, 2); // <repcount> or <salcount>
5047 for (i = 0; i < gap->ga_len; ++i)
5048 {
5049 // <rep> : <repfromlen> <repfrom> <reptolen> <repto>
5050 // <sal> : <salfromlen> <salfrom> <saltolen> <salto>
5051 ftp = &((fromto_T *)gap->ga_data)[i];
5052 for (rr = 1; rr <= 2; ++rr)
5053 {
5054 p = rr == 1 ? ftp->ft_from : ftp->ft_to;
5055 l = (int)STRLEN(p);
5056 putc(l, fd);
5057 if (l > 0)
5058 fwv &= fwrite(p, l, (size_t)1, fd);
5059 }
5060 }
5061
5062 }
5063
5064 // SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
5065 // This is for making suggestions, section is not required.
5066 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
5067 {
5068 putc(SN_SOFO, fd); // <sectionID>
5069 putc(0, fd); // <sectionflags>
5070
5071 l = (int)STRLEN(spin->si_sofofr);
5072 put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4);
5073 // <sectionlen>
5074
5075 put_bytes(fd, (long_u)l, 2); // <sofofromlen>
5076 fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); // <sofofrom>
5077
5078 l = (int)STRLEN(spin->si_sofoto);
5079 put_bytes(fd, (long_u)l, 2); // <sofotolen>
5080 fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); // <sofoto>
5081 }
5082
5083 // SN_WORDS: <word> ...
5084 // This is for making suggestions, section is not required.
5085 if (spin->si_commonwords.ht_used > 0)
5086 {
5087 putc(SN_WORDS, fd); // <sectionID>
5088 putc(0, fd); // <sectionflags>
5089
5090 // round 1: count the bytes
5091 // round 2: write the bytes
5092 for (round = 1; round <= 2; ++round)
5093 {
5094 int todo;
5095 int len = 0;
5096 hashitem_T *hi;
5097
5098 todo = (int)spin->si_commonwords.ht_used;
5099 for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi)
5100 if (!HASHITEM_EMPTY(hi))
5101 {
5102 l = (int)STRLEN(hi->hi_key) + 1;
5103 len += l;
5104 if (round == 2) // <word>
5105 fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd);
5106 --todo;
5107 }
5108 if (round == 1)
5109 put_bytes(fd, (long_u)len, 4); // <sectionlen>
5110 }
5111 }
5112
5113 // SN_MAP: <mapstr>
5114 // This is for making suggestions, section is not required.
5115 if (spin->si_map.ga_len > 0)
5116 {
5117 putc(SN_MAP, fd); // <sectionID>
5118 putc(0, fd); // <sectionflags>
5119 l = spin->si_map.ga_len;
5120 put_bytes(fd, (long_u)l, 4); // <sectionlen>
5121 fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd);
5122 // <mapstr>
5123 }
5124
5125 // SN_SUGFILE: <timestamp>
5126 // This is used to notify that a .sug file may be available and at the
5127 // same time allows for checking that a .sug file that is found matches
5128 // with this .spl file. That's because the word numbers must be exactly
5129 // right.
5130 if (!spin->si_nosugfile
5131 && (spin->si_sal.ga_len > 0
5132 || (spin->si_sofofr != NULL && spin->si_sofoto != NULL)))
5133 {
5134 putc(SN_SUGFILE, fd); // <sectionID>
5135 putc(0, fd); // <sectionflags>
5136 put_bytes(fd, (long_u)8, 4); // <sectionlen>
5137
5138 // Set si_sugtime and write it to the file.
5139 spin->si_sugtime = time(NULL);
5140 put_time(fd, spin->si_sugtime); // <timestamp>
5141 }
5142
5143 // SN_NOSPLITSUGS: nothing
5144 // This is used to notify that no suggestions with word splits are to be
5145 // made.
5146 if (spin->si_nosplitsugs)
5147 {
5148 putc(SN_NOSPLITSUGS, fd); // <sectionID>
5149 putc(0, fd); // <sectionflags>
5150 put_bytes(fd, (long_u)0, 4); // <sectionlen>
5151 }
5152
5153 // SN_NOCOMPUNDSUGS: nothing
5154 // This is used to notify that no suggestions with compounds are to be
5155 // made.
5156 if (spin->si_nocompoundsugs)
5157 {
5158 putc(SN_NOCOMPOUNDSUGS, fd); // <sectionID>
5159 putc(0, fd); // <sectionflags>
5160 put_bytes(fd, (long_u)0, 4); // <sectionlen>
5161 }
5162
5163 // SN_COMPOUND: compound info.
5164 // We don't mark it required, when not supported all compound words will
5165 // be bad words.
5166 if (spin->si_compflags != NULL)
5167 {
5168 putc(SN_COMPOUND, fd); // <sectionID>
5169 putc(0, fd); // <sectionflags>
5170
5171 l = (int)STRLEN(spin->si_compflags);
5172 for (i = 0; i < spin->si_comppat.ga_len; ++i)
5173 l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1;
5174 put_bytes(fd, (long_u)(l + 7), 4); // <sectionlen>
5175
5176 putc(spin->si_compmax, fd); // <compmax>
5177 putc(spin->si_compminlen, fd); // <compminlen>
5178 putc(spin->si_compsylmax, fd); // <compsylmax>
5179 putc(0, fd); // for Vim 7.0b compatibility
5180 putc(spin->si_compoptions, fd); // <compoptions>
5181 put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2);
5182 // <comppatcount>
5183 for (i = 0; i < spin->si_comppat.ga_len; ++i)
5184 {
5185 p = ((char_u **)(spin->si_comppat.ga_data))[i];
5186 putc((int)STRLEN(p), fd); // <comppatlen>
5187 fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd);
5188 // <comppattext>
5189 }
5190 // <compflags>
5191 fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags),
5192 (size_t)1, fd);
5193 }
5194
5195 // SN_NOBREAK: NOBREAK flag
5196 if (spin->si_nobreak)
5197 {
5198 putc(SN_NOBREAK, fd); // <sectionID>
5199 putc(0, fd); // <sectionflags>
5200
5201 // It's empty, the presence of the section flags the feature.
5202 put_bytes(fd, (long_u)0, 4); // <sectionlen>
5203 }
5204
5205 // SN_SYLLABLE: syllable info.
5206 // We don't mark it required, when not supported syllables will not be
5207 // counted.
5208 if (spin->si_syllable != NULL)
5209 {
5210 putc(SN_SYLLABLE, fd); // <sectionID>
5211 putc(0, fd); // <sectionflags>
5212
5213 l = (int)STRLEN(spin->si_syllable);
5214 put_bytes(fd, (long_u)l, 4); // <sectionlen>
5215 fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd);
5216 // <syllable>
5217 }
5218
5219 // end of <SECTIONS>
5220 putc(SN_END, fd); // <sectionend>
5221
5222
5223 /*
5224 * <LWORDTREE> <KWORDTREE> <PREFIXTREE>
5225 */
5226 spin->si_memtot = 0;
5227 for (round = 1; round <= 3; ++round)
5228 {
5229 if (round == 1)
5230 tree = spin->si_foldroot->wn_sibling;
5231 else if (round == 2)
5232 tree = spin->si_keeproot->wn_sibling;
5233 else
5234 tree = spin->si_prefroot->wn_sibling;
5235
5236 // Clear the index and wnode fields in the tree.
5237 clear_node(tree);
5238
5239 // Count the number of nodes. Needed to be able to allocate the
5240 // memory when reading the nodes. Also fills in index for shared
5241 // nodes.
5242 nodecount = put_node(NULL, tree, 0, regionmask, round == 3);
5243
5244 // number of nodes in 4 bytes
5245 put_bytes(fd, (long_u)nodecount, 4); // <nodecount>
5246 spin->si_memtot += nodecount + nodecount * sizeof(int);
5247
5248 // Write the nodes.
5249 (void)put_node(fd, tree, 0, regionmask, round == 3);
5250 }
5251
5252 // Write another byte to check for errors (file system full).
5253 if (putc(0, fd) == EOF)
5254 retval = FAIL;
5255 theend:
5256 if (fclose(fd) == EOF)
5257 retval = FAIL;
5258
5259 if (fwv != (size_t)1)
5260 retval = FAIL;
5261 if (retval == FAIL)
5262 emsg(_(e_write));
5263
5264 return retval;
5265 }
5266
5267 /*
5268 * Clear the index and wnode fields of "node", it siblings and its
5269 * children. This is needed because they are a union with other items to save
5270 * space.
5271 */
5272 static void
clear_node(wordnode_T * node)5273 clear_node(wordnode_T *node)
5274 {
5275 wordnode_T *np;
5276
5277 if (node != NULL)
5278 FOR_ALL_NODE_SIBLINGS(node, np)
5279 {
5280 np->wn_u1.index = 0;
5281 np->wn_u2.wnode = NULL;
5282
5283 if (np->wn_byte != NUL)
5284 clear_node(np->wn_child);
5285 }
5286 }
5287
5288
5289 /*
5290 * Dump a word tree at node "node".
5291 *
5292 * This first writes the list of possible bytes (siblings). Then for each
5293 * byte recursively write the children.
5294 *
5295 * NOTE: The code here must match the code in read_tree_node(), since
5296 * assumptions are made about the indexes (so that we don't have to write them
5297 * in the file).
5298 *
5299 * Returns the number of nodes used.
5300 */
5301 static int
put_node(FILE * fd,wordnode_T * node,int idx,int regionmask,int prefixtree)5302 put_node(
5303 FILE *fd, // NULL when only counting
5304 wordnode_T *node,
5305 int idx,
5306 int regionmask,
5307 int prefixtree) // TRUE for PREFIXTREE
5308 {
5309 int newindex = idx;
5310 int siblingcount = 0;
5311 wordnode_T *np;
5312 int flags;
5313
5314 // If "node" is zero the tree is empty.
5315 if (node == NULL)
5316 return 0;
5317
5318 // Store the index where this node is written.
5319 node->wn_u1.index = idx;
5320
5321 // Count the number of siblings.
5322 FOR_ALL_NODE_SIBLINGS(node, np)
5323 ++siblingcount;
5324
5325 // Write the sibling count.
5326 if (fd != NULL)
5327 putc(siblingcount, fd); // <siblingcount>
5328
5329 // Write each sibling byte and optionally extra info.
5330 FOR_ALL_NODE_SIBLINGS(node, np)
5331 {
5332 if (np->wn_byte == 0)
5333 {
5334 if (fd != NULL)
5335 {
5336 // For a NUL byte (end of word) write the flags etc.
5337 if (prefixtree)
5338 {
5339 // In PREFIXTREE write the required affixID and the
5340 // associated condition nr (stored in wn_region). The
5341 // byte value is misused to store the "rare" and "not
5342 // combining" flags
5343 if (np->wn_flags == (short_u)PFX_FLAGS)
5344 putc(BY_NOFLAGS, fd); // <byte>
5345 else
5346 {
5347 putc(BY_FLAGS, fd); // <byte>
5348 putc(np->wn_flags, fd); // <pflags>
5349 }
5350 putc(np->wn_affixID, fd); // <affixID>
5351 put_bytes(fd, (long_u)np->wn_region, 2); // <prefcondnr>
5352 }
5353 else
5354 {
5355 // For word trees we write the flag/region items.
5356 flags = np->wn_flags;
5357 if (regionmask != 0 && np->wn_region != regionmask)
5358 flags |= WF_REGION;
5359 if (np->wn_affixID != 0)
5360 flags |= WF_AFX;
5361 if (flags == 0)
5362 {
5363 // word without flags or region
5364 putc(BY_NOFLAGS, fd); // <byte>
5365 }
5366 else
5367 {
5368 if (np->wn_flags >= 0x100)
5369 {
5370 putc(BY_FLAGS2, fd); // <byte>
5371 putc(flags, fd); // <flags>
5372 putc((unsigned)flags >> 8, fd); // <flags2>
5373 }
5374 else
5375 {
5376 putc(BY_FLAGS, fd); // <byte>
5377 putc(flags, fd); // <flags>
5378 }
5379 if (flags & WF_REGION)
5380 putc(np->wn_region, fd); // <region>
5381 if (flags & WF_AFX)
5382 putc(np->wn_affixID, fd); // <affixID>
5383 }
5384 }
5385 }
5386 }
5387 else
5388 {
5389 if (np->wn_child->wn_u1.index != 0
5390 && np->wn_child->wn_u2.wnode != node)
5391 {
5392 // The child is written elsewhere, write the reference.
5393 if (fd != NULL)
5394 {
5395 putc(BY_INDEX, fd); // <byte>
5396 // <nodeidx>
5397 put_bytes(fd, (long_u)np->wn_child->wn_u1.index, 3);
5398 }
5399 }
5400 else if (np->wn_child->wn_u2.wnode == NULL)
5401 // We will write the child below and give it an index.
5402 np->wn_child->wn_u2.wnode = node;
5403
5404 if (fd != NULL)
5405 if (putc(np->wn_byte, fd) == EOF) // <byte> or <xbyte>
5406 {
5407 emsg(_(e_write));
5408 return 0;
5409 }
5410 }
5411 }
5412
5413 // Space used in the array when reading: one for each sibling and one for
5414 // the count.
5415 newindex += siblingcount + 1;
5416
5417 // Recursively dump the children of each sibling.
5418 FOR_ALL_NODE_SIBLINGS(node, np)
5419 if (np->wn_byte != 0 && np->wn_child->wn_u2.wnode == node)
5420 newindex = put_node(fd, np->wn_child, newindex, regionmask,
5421 prefixtree);
5422
5423 return newindex;
5424 }
5425
5426
5427 /*
5428 * ":mkspell [-ascii] outfile infile ..."
5429 * ":mkspell [-ascii] addfile"
5430 */
5431 void
ex_mkspell(exarg_T * eap)5432 ex_mkspell(exarg_T *eap)
5433 {
5434 int fcount;
5435 char_u **fnames;
5436 char_u *arg = eap->arg;
5437 int ascii = FALSE;
5438
5439 if (STRNCMP(arg, "-ascii", 6) == 0)
5440 {
5441 ascii = TRUE;
5442 arg = skipwhite(arg + 6);
5443 }
5444
5445 // Expand all the remaining arguments (e.g., $VIMRUNTIME).
5446 if (get_arglist_exp(arg, &fcount, &fnames, FALSE) == OK)
5447 {
5448 mkspell(fcount, fnames, ascii, eap->forceit, FALSE);
5449 FreeWild(fcount, fnames);
5450 }
5451 }
5452
5453 /*
5454 * Create the .sug file.
5455 * Uses the soundfold info in "spin".
5456 * Writes the file with the name "wfname", with ".spl" changed to ".sug".
5457 */
5458 static void
spell_make_sugfile(spellinfo_T * spin,char_u * wfname)5459 spell_make_sugfile(spellinfo_T *spin, char_u *wfname)
5460 {
5461 char_u *fname = NULL;
5462 int len;
5463 slang_T *slang;
5464 int free_slang = FALSE;
5465
5466 /*
5467 * Read back the .spl file that was written. This fills the required
5468 * info for soundfolding. This also uses less memory than the
5469 * pointer-linked version of the trie. And it avoids having two versions
5470 * of the code for the soundfolding stuff.
5471 * It might have been done already by spell_reload_one().
5472 */
5473 FOR_ALL_SPELL_LANGS(slang)
5474 if (fullpathcmp(wfname, slang->sl_fname, FALSE, TRUE) == FPC_SAME)
5475 break;
5476 if (slang == NULL)
5477 {
5478 spell_message(spin, (char_u *)_("Reading back spell file..."));
5479 slang = spell_load_file(wfname, NULL, NULL, FALSE);
5480 if (slang == NULL)
5481 return;
5482 free_slang = TRUE;
5483 }
5484
5485 /*
5486 * Clear the info in "spin" that is used.
5487 */
5488 spin->si_blocks = NULL;
5489 spin->si_blocks_cnt = 0;
5490 spin->si_compress_cnt = 0; // will stay at 0 all the time
5491 spin->si_free_count = 0;
5492 spin->si_first_free = NULL;
5493 spin->si_foldwcount = 0;
5494
5495 /*
5496 * Go through the trie of good words, soundfold each word and add it to
5497 * the soundfold trie.
5498 */
5499 spell_message(spin, (char_u *)_("Performing soundfolding..."));
5500 if (sug_filltree(spin, slang) == FAIL)
5501 goto theend;
5502
5503 /*
5504 * Create the table which links each soundfold word with a list of the
5505 * good words it may come from. Creates buffer "spin->si_spellbuf".
5506 * This also removes the wordnr from the NUL byte entries to make
5507 * compression possible.
5508 */
5509 if (sug_maketable(spin) == FAIL)
5510 goto theend;
5511
5512 smsg(_("Number of words after soundfolding: %ld"),
5513 (long)spin->si_spellbuf->b_ml.ml_line_count);
5514
5515 /*
5516 * Compress the soundfold trie.
5517 */
5518 spell_message(spin, (char_u *)_(msg_compressing));
5519 wordtree_compress(spin, spin->si_foldroot, "case-folded");
5520
5521 /*
5522 * Write the .sug file.
5523 * Make the file name by changing ".spl" to ".sug".
5524 */
5525 fname = alloc(MAXPATHL);
5526 if (fname == NULL)
5527 goto theend;
5528 vim_strncpy(fname, wfname, MAXPATHL - 1);
5529 len = (int)STRLEN(fname);
5530 fname[len - 2] = 'u';
5531 fname[len - 1] = 'g';
5532 sug_write(spin, fname);
5533
5534 theend:
5535 vim_free(fname);
5536 if (free_slang)
5537 slang_free(slang);
5538 free_blocks(spin->si_blocks);
5539 close_spellbuf(spin->si_spellbuf);
5540 }
5541
5542 /*
5543 * Build the soundfold trie for language "slang".
5544 */
5545 static int
sug_filltree(spellinfo_T * spin,slang_T * slang)5546 sug_filltree(spellinfo_T *spin, slang_T *slang)
5547 {
5548 char_u *byts;
5549 idx_T *idxs;
5550 int depth;
5551 idx_T arridx[MAXWLEN];
5552 int curi[MAXWLEN];
5553 char_u tword[MAXWLEN];
5554 char_u tsalword[MAXWLEN];
5555 int c;
5556 idx_T n;
5557 unsigned words_done = 0;
5558 int wordcount[MAXWLEN];
5559
5560 // We use si_foldroot for the soundfolded trie.
5561 spin->si_foldroot = wordtree_alloc(spin);
5562 if (spin->si_foldroot == NULL)
5563 return FAIL;
5564
5565 // let tree_add_word() know we're adding to the soundfolded tree
5566 spin->si_sugtree = TRUE;
5567
5568 /*
5569 * Go through the whole case-folded tree, soundfold each word and put it
5570 * in the trie.
5571 */
5572 byts = slang->sl_fbyts;
5573 idxs = slang->sl_fidxs;
5574
5575 arridx[0] = 0;
5576 curi[0] = 1;
5577 wordcount[0] = 0;
5578
5579 depth = 0;
5580 while (depth >= 0 && !got_int)
5581 {
5582 if (curi[depth] > byts[arridx[depth]])
5583 {
5584 // Done all bytes at this node, go up one level.
5585 idxs[arridx[depth]] = wordcount[depth];
5586 if (depth > 0)
5587 wordcount[depth - 1] += wordcount[depth];
5588
5589 --depth;
5590 line_breakcheck();
5591 }
5592 else
5593 {
5594
5595 // Do one more byte at this node.
5596 n = arridx[depth] + curi[depth];
5597 ++curi[depth];
5598
5599 c = byts[n];
5600 if (c == 0)
5601 {
5602 // Sound-fold the word.
5603 tword[depth] = NUL;
5604 spell_soundfold(slang, tword, TRUE, tsalword);
5605
5606 // We use the "flags" field for the MSB of the wordnr,
5607 // "region" for the LSB of the wordnr.
5608 if (tree_add_word(spin, tsalword, spin->si_foldroot,
5609 words_done >> 16, words_done & 0xffff,
5610 0) == FAIL)
5611 return FAIL;
5612
5613 ++words_done;
5614 ++wordcount[depth];
5615
5616 // Reset the block count each time to avoid compression
5617 // kicking in.
5618 spin->si_blocks_cnt = 0;
5619
5620 // Skip over any other NUL bytes (same word with different
5621 // flags). But don't go over the end.
5622 while (n + 1 < slang->sl_fbyts_len && byts[n + 1] == 0)
5623 {
5624 ++n;
5625 ++curi[depth];
5626 }
5627 }
5628 else
5629 {
5630 // Normal char, go one level deeper.
5631 tword[depth++] = c;
5632 arridx[depth] = idxs[n];
5633 curi[depth] = 1;
5634 wordcount[depth] = 0;
5635 }
5636 }
5637 }
5638
5639 smsg(_("Total number of words: %d"), words_done);
5640
5641 return OK;
5642 }
5643
5644 /*
5645 * Make the table that links each word in the soundfold trie to the words it
5646 * can be produced from.
5647 * This is not unlike lines in a file, thus use a memfile to be able to access
5648 * the table efficiently.
5649 * Returns FAIL when out of memory.
5650 */
5651 static int
sug_maketable(spellinfo_T * spin)5652 sug_maketable(spellinfo_T *spin)
5653 {
5654 garray_T ga;
5655 int res = OK;
5656
5657 // Allocate a buffer, open a memline for it and create the swap file
5658 // (uses a temp file, not a .swp file).
5659 spin->si_spellbuf = open_spellbuf();
5660 if (spin->si_spellbuf == NULL)
5661 return FAIL;
5662
5663 // Use a buffer to store the line info, avoids allocating many small
5664 // pieces of memory.
5665 ga_init2(&ga, 1, 100);
5666
5667 // recursively go through the tree
5668 if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1)
5669 res = FAIL;
5670
5671 ga_clear(&ga);
5672 return res;
5673 }
5674
5675 /*
5676 * Fill the table for one node and its children.
5677 * Returns the wordnr at the start of the node.
5678 * Returns -1 when out of memory.
5679 */
5680 static int
sug_filltable(spellinfo_T * spin,wordnode_T * node,int startwordnr,garray_T * gap)5681 sug_filltable(
5682 spellinfo_T *spin,
5683 wordnode_T *node,
5684 int startwordnr,
5685 garray_T *gap) // place to store line of numbers
5686 {
5687 wordnode_T *p, *np;
5688 int wordnr = startwordnr;
5689 int nr;
5690 int prev_nr;
5691
5692 FOR_ALL_NODE_SIBLINGS(node, p)
5693 {
5694 if (p->wn_byte == NUL)
5695 {
5696 gap->ga_len = 0;
5697 prev_nr = 0;
5698 for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling)
5699 {
5700 if (ga_grow(gap, 10) == FAIL)
5701 return -1;
5702
5703 nr = (np->wn_flags << 16) + (np->wn_region & 0xffff);
5704 // Compute the offset from the previous nr and store the
5705 // offset in a way that it takes a minimum number of bytes.
5706 // It's a bit like utf-8, but without the need to mark
5707 // following bytes.
5708 nr -= prev_nr;
5709 prev_nr += nr;
5710 gap->ga_len += offset2bytes(nr,
5711 (char_u *)gap->ga_data + gap->ga_len);
5712 }
5713
5714 // add the NUL byte
5715 ((char_u *)gap->ga_data)[gap->ga_len++] = NUL;
5716
5717 if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr,
5718 gap->ga_data, gap->ga_len, TRUE) == FAIL)
5719 return -1;
5720 ++wordnr;
5721
5722 // Remove extra NUL entries, we no longer need them. We don't
5723 // bother freeing the nodes, the won't be reused anyway.
5724 while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL)
5725 p->wn_sibling = p->wn_sibling->wn_sibling;
5726
5727 // Clear the flags on the remaining NUL node, so that compression
5728 // works a lot better.
5729 p->wn_flags = 0;
5730 p->wn_region = 0;
5731 }
5732 else
5733 {
5734 wordnr = sug_filltable(spin, p->wn_child, wordnr, gap);
5735 if (wordnr == -1)
5736 return -1;
5737 }
5738 }
5739 return wordnr;
5740 }
5741
5742 /*
5743 * Convert an offset into a minimal number of bytes.
5744 * Similar to utf_char2byters, but use 8 bits in followup bytes and avoid NUL
5745 * bytes.
5746 */
5747 static int
offset2bytes(int nr,char_u * buf)5748 offset2bytes(int nr, char_u *buf)
5749 {
5750 int rem;
5751 int b1, b2, b3, b4;
5752
5753 // Split the number in parts of base 255. We need to avoid NUL bytes.
5754 b1 = nr % 255 + 1;
5755 rem = nr / 255;
5756 b2 = rem % 255 + 1;
5757 rem = rem / 255;
5758 b3 = rem % 255 + 1;
5759 b4 = rem / 255 + 1;
5760
5761 if (b4 > 1 || b3 > 0x1f) // 4 bytes
5762 {
5763 buf[0] = 0xe0 + b4;
5764 buf[1] = b3;
5765 buf[2] = b2;
5766 buf[3] = b1;
5767 return 4;
5768 }
5769 if (b3 > 1 || b2 > 0x3f ) // 3 bytes
5770 {
5771 buf[0] = 0xc0 + b3;
5772 buf[1] = b2;
5773 buf[2] = b1;
5774 return 3;
5775 }
5776 if (b2 > 1 || b1 > 0x7f ) // 2 bytes
5777 {
5778 buf[0] = 0x80 + b2;
5779 buf[1] = b1;
5780 return 2;
5781 }
5782 // 1 byte
5783 buf[0] = b1;
5784 return 1;
5785 }
5786
5787 /*
5788 * Write the .sug file in "fname".
5789 */
5790 static void
sug_write(spellinfo_T * spin,char_u * fname)5791 sug_write(spellinfo_T *spin, char_u *fname)
5792 {
5793 FILE *fd;
5794 wordnode_T *tree;
5795 int nodecount;
5796 int wcount;
5797 char_u *line;
5798 linenr_T lnum;
5799 int len;
5800
5801 // Create the file. Note that an existing file is silently overwritten!
5802 fd = mch_fopen((char *)fname, "w");
5803 if (fd == NULL)
5804 {
5805 semsg(_(e_notopen), fname);
5806 return;
5807 }
5808
5809 vim_snprintf((char *)IObuff, IOSIZE,
5810 _("Writing suggestion file %s..."), fname);
5811 spell_message(spin, IObuff);
5812
5813 /*
5814 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
5815 */
5816 if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) // <fileID>
5817 {
5818 emsg(_(e_write));
5819 goto theend;
5820 }
5821 putc(VIMSUGVERSION, fd); // <versionnr>
5822
5823 // Write si_sugtime to the file.
5824 put_time(fd, spin->si_sugtime); // <timestamp>
5825
5826 /*
5827 * <SUGWORDTREE>
5828 */
5829 spin->si_memtot = 0;
5830 tree = spin->si_foldroot->wn_sibling;
5831
5832 // Clear the index and wnode fields in the tree.
5833 clear_node(tree);
5834
5835 // Count the number of nodes. Needed to be able to allocate the
5836 // memory when reading the nodes. Also fills in index for shared
5837 // nodes.
5838 nodecount = put_node(NULL, tree, 0, 0, FALSE);
5839
5840 // number of nodes in 4 bytes
5841 put_bytes(fd, (long_u)nodecount, 4); // <nodecount>
5842 spin->si_memtot += nodecount + nodecount * sizeof(int);
5843
5844 // Write the nodes.
5845 (void)put_node(fd, tree, 0, 0, FALSE);
5846
5847 /*
5848 * <SUGTABLE>: <sugwcount> <sugline> ...
5849 */
5850 wcount = spin->si_spellbuf->b_ml.ml_line_count;
5851 put_bytes(fd, (long_u)wcount, 4); // <sugwcount>
5852
5853 for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum)
5854 {
5855 // <sugline>: <sugnr> ... NUL
5856 line = ml_get_buf(spin->si_spellbuf, lnum, FALSE);
5857 len = (int)STRLEN(line) + 1;
5858 if (fwrite(line, (size_t)len, (size_t)1, fd) == 0)
5859 {
5860 emsg(_(e_write));
5861 goto theend;
5862 }
5863 spin->si_memtot += len;
5864 }
5865
5866 // Write another byte to check for errors.
5867 if (putc(0, fd) == EOF)
5868 emsg(_(e_write));
5869
5870 vim_snprintf((char *)IObuff, IOSIZE,
5871 _("Estimated runtime memory use: %d bytes"), spin->si_memtot);
5872 spell_message(spin, IObuff);
5873
5874 theend:
5875 // close the file
5876 fclose(fd);
5877 }
5878
5879
5880 /*
5881 * Create a Vim spell file from one or more word lists.
5882 * "fnames[0]" is the output file name.
5883 * "fnames[fcount - 1]" is the last input file name.
5884 * Exception: when "fnames[0]" ends in ".add" it's used as the input file name
5885 * and ".spl" is appended to make the output file name.
5886 */
5887 void
mkspell(int fcount,char_u ** fnames,int ascii,int over_write,int added_word)5888 mkspell(
5889 int fcount,
5890 char_u **fnames,
5891 int ascii, // -ascii argument given
5892 int over_write, // overwrite existing output file
5893 int added_word) // invoked through "zg"
5894 {
5895 char_u *fname = NULL;
5896 char_u *wfname;
5897 char_u **innames;
5898 int incount;
5899 afffile_T *(afile[MAXREGIONS]);
5900 int i;
5901 int len;
5902 stat_T st;
5903 int error = FALSE;
5904 spellinfo_T spin;
5905
5906 CLEAR_FIELD(spin);
5907 spin.si_verbose = !added_word;
5908 spin.si_ascii = ascii;
5909 spin.si_followup = TRUE;
5910 spin.si_rem_accents = TRUE;
5911 ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20);
5912 ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20);
5913 ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20);
5914 ga_init2(&spin.si_map, (int)sizeof(char_u), 100);
5915 ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20);
5916 ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50);
5917 hash_init(&spin.si_commonwords);
5918 spin.si_newcompID = 127; // start compound ID at first maximum
5919
5920 // default: fnames[0] is output file, following are input files
5921 // When "fcount" is 1 there is only one file.
5922 innames = &fnames[fcount == 1 ? 0 : 1];
5923 incount = fcount - 1;
5924
5925 wfname = alloc(MAXPATHL);
5926 if (wfname == NULL)
5927 return;
5928
5929 if (fcount >= 1)
5930 {
5931 len = (int)STRLEN(fnames[0]);
5932 if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0)
5933 {
5934 // For ":mkspell path/en.latin1.add" output file is
5935 // "path/en.latin1.add.spl".
5936 incount = 1;
5937 vim_snprintf((char *)wfname, MAXPATHL, "%s.spl", fnames[0]);
5938 }
5939 else if (fcount == 1)
5940 {
5941 // For ":mkspell path/vim" output file is "path/vim.latin1.spl".
5942 incount = 1;
5943 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL,
5944 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc());
5945 }
5946 else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0)
5947 {
5948 // Name ends in ".spl", use as the file name.
5949 vim_strncpy(wfname, fnames[0], MAXPATHL - 1);
5950 }
5951 else
5952 // Name should be language, make the file name from it.
5953 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL,
5954 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc());
5955
5956 // Check for .ascii.spl.
5957 if (strstr((char *)gettail(wfname), SPL_FNAME_ASCII) != NULL)
5958 spin.si_ascii = TRUE;
5959
5960 // Check for .add.spl.
5961 if (strstr((char *)gettail(wfname), SPL_FNAME_ADD) != NULL)
5962 spin.si_add = TRUE;
5963 }
5964
5965 if (incount <= 0)
5966 emsg(_(e_invarg)); // need at least output and input names
5967 else if (vim_strchr(gettail(wfname), '_') != NULL)
5968 emsg(_("E751: Output file name must not have region name"));
5969 else if (incount > MAXREGIONS)
5970 semsg(_("E754: Only up to %d regions supported"), MAXREGIONS);
5971 else
5972 {
5973 // Check for overwriting before doing things that may take a lot of
5974 // time.
5975 if (!over_write && mch_stat((char *)wfname, &st) >= 0)
5976 {
5977 emsg(_(e_file_exists));
5978 goto theend;
5979 }
5980 if (mch_isdir(wfname))
5981 {
5982 semsg(_(e_src_is_directory), wfname);
5983 goto theend;
5984 }
5985
5986 fname = alloc(MAXPATHL);
5987 if (fname == NULL)
5988 goto theend;
5989
5990 /*
5991 * Init the aff and dic pointers.
5992 * Get the region names if there are more than 2 arguments.
5993 */
5994 for (i = 0; i < incount; ++i)
5995 {
5996 afile[i] = NULL;
5997
5998 if (incount > 1)
5999 {
6000 len = (int)STRLEN(innames[i]);
6001 if (STRLEN(gettail(innames[i])) < 5
6002 || innames[i][len - 3] != '_')
6003 {
6004 semsg(_("E755: Invalid region in %s"), innames[i]);
6005 goto theend;
6006 }
6007 spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]);
6008 spin.si_region_name[i * 2 + 1] =
6009 TOLOWER_ASC(innames[i][len - 1]);
6010 }
6011 }
6012 spin.si_region_count = incount;
6013
6014 spin.si_foldroot = wordtree_alloc(&spin);
6015 spin.si_keeproot = wordtree_alloc(&spin);
6016 spin.si_prefroot = wordtree_alloc(&spin);
6017 if (spin.si_foldroot == NULL
6018 || spin.si_keeproot == NULL
6019 || spin.si_prefroot == NULL)
6020 {
6021 free_blocks(spin.si_blocks);
6022 goto theend;
6023 }
6024
6025 // When not producing a .add.spl file clear the character table when
6026 // we encounter one in the .aff file. This means we dump the current
6027 // one in the .spl file if the .aff file doesn't define one. That's
6028 // better than guessing the contents, the table will match a
6029 // previously loaded spell file.
6030 if (!spin.si_add)
6031 spin.si_clear_chartab = TRUE;
6032
6033 /*
6034 * Read all the .aff and .dic files.
6035 * Text is converted to 'encoding'.
6036 * Words are stored in the case-folded and keep-case trees.
6037 */
6038 for (i = 0; i < incount && !error; ++i)
6039 {
6040 spin.si_conv.vc_type = CONV_NONE;
6041 spin.si_region = 1 << i;
6042
6043 vim_snprintf((char *)fname, MAXPATHL, "%s.aff", innames[i]);
6044 if (mch_stat((char *)fname, &st) >= 0)
6045 {
6046 // Read the .aff file. Will init "spin->si_conv" based on the
6047 // "SET" line.
6048 afile[i] = spell_read_aff(&spin, fname);
6049 if (afile[i] == NULL)
6050 error = TRUE;
6051 else
6052 {
6053 // Read the .dic file and store the words in the trees.
6054 vim_snprintf((char *)fname, MAXPATHL, "%s.dic",
6055 innames[i]);
6056 if (spell_read_dic(&spin, fname, afile[i]) == FAIL)
6057 error = TRUE;
6058 }
6059 }
6060 else
6061 {
6062 // No .aff file, try reading the file as a word list. Store
6063 // the words in the trees.
6064 if (spell_read_wordfile(&spin, innames[i]) == FAIL)
6065 error = TRUE;
6066 }
6067
6068 // Free any conversion stuff.
6069 convert_setup(&spin.si_conv, NULL, NULL);
6070 }
6071
6072 if (spin.si_compflags != NULL && spin.si_nobreak)
6073 msg(_("Warning: both compounding and NOBREAK specified"));
6074
6075 if (!error && !got_int)
6076 {
6077 /*
6078 * Combine tails in the tree.
6079 */
6080 spell_message(&spin, (char_u *)_(msg_compressing));
6081 wordtree_compress(&spin, spin.si_foldroot, "case-folded");
6082 wordtree_compress(&spin, spin.si_keeproot, "keep-case");
6083 wordtree_compress(&spin, spin.si_prefroot, "prefixes");
6084 }
6085
6086 if (!error && !got_int)
6087 {
6088 /*
6089 * Write the info in the spell file.
6090 */
6091 vim_snprintf((char *)IObuff, IOSIZE,
6092 _("Writing spell file %s..."), wfname);
6093 spell_message(&spin, IObuff);
6094
6095 error = write_vim_spell(&spin, wfname) == FAIL;
6096
6097 spell_message(&spin, (char_u *)_("Done!"));
6098 vim_snprintf((char *)IObuff, IOSIZE,
6099 _("Estimated runtime memory use: %d bytes"), spin.si_memtot);
6100 spell_message(&spin, IObuff);
6101
6102 /*
6103 * If the file is loaded need to reload it.
6104 */
6105 if (!error)
6106 spell_reload_one(wfname, added_word);
6107 }
6108
6109 // Free the allocated memory.
6110 ga_clear(&spin.si_rep);
6111 ga_clear(&spin.si_repsal);
6112 ga_clear(&spin.si_sal);
6113 ga_clear(&spin.si_map);
6114 ga_clear(&spin.si_comppat);
6115 ga_clear(&spin.si_prefcond);
6116 hash_clear_all(&spin.si_commonwords, 0);
6117
6118 // Free the .aff file structures.
6119 for (i = 0; i < incount; ++i)
6120 if (afile[i] != NULL)
6121 spell_free_aff(afile[i]);
6122
6123 // Free all the bits and pieces at once.
6124 free_blocks(spin.si_blocks);
6125
6126 /*
6127 * If there is soundfolding info and no NOSUGFILE item create the
6128 * .sug file with the soundfolded word trie.
6129 */
6130 if (spin.si_sugtime != 0 && !error && !got_int)
6131 spell_make_sugfile(&spin, wfname);
6132
6133 }
6134
6135 theend:
6136 vim_free(fname);
6137 vim_free(wfname);
6138 }
6139
6140 /*
6141 * Display a message for spell file processing when 'verbose' is set or using
6142 * ":mkspell". "str" can be IObuff.
6143 */
6144 static void
spell_message(spellinfo_T * spin,char_u * str)6145 spell_message(spellinfo_T *spin, char_u *str)
6146 {
6147 if (spin->si_verbose || p_verbose > 2)
6148 {
6149 if (!spin->si_verbose)
6150 verbose_enter();
6151 msg((char *)str);
6152 out_flush();
6153 if (!spin->si_verbose)
6154 verbose_leave();
6155 }
6156 }
6157
6158 /*
6159 * ":[count]spellgood {word}"
6160 * ":[count]spellwrong {word}"
6161 * ":[count]spellundo {word}"
6162 * ":[count]spellrare {word}"
6163 */
6164 void
ex_spell(exarg_T * eap)6165 ex_spell(exarg_T *eap)
6166 {
6167 spell_add_word(eap->arg, (int)STRLEN(eap->arg),
6168 eap->cmdidx == CMD_spellwrong ? SPELL_ADD_BAD :
6169 eap->cmdidx == CMD_spellrare ? SPELL_ADD_RARE : SPELL_ADD_GOOD,
6170 eap->forceit ? 0 : (int)eap->line2,
6171 eap->cmdidx == CMD_spellundo);
6172 }
6173
6174 /*
6175 * Add "word[len]" to 'spellfile' as a good, rare or bad word.
6176 */
6177 void
spell_add_word(char_u * word,int len,int what,int idx,int undo)6178 spell_add_word(
6179 char_u *word,
6180 int len,
6181 int what, // SPELL_ADD_ values
6182 int idx, // "zG" and "zW": zero, otherwise index in
6183 // 'spellfile'
6184 int undo) // TRUE for "zug", "zuG", "zuw" and "zuW"
6185 {
6186 FILE *fd = NULL;
6187 buf_T *buf = NULL;
6188 int new_spf = FALSE;
6189 char_u *fname;
6190 char_u *fnamebuf = NULL;
6191 char_u line[MAXWLEN * 2];
6192 long fpos, fpos_next = 0;
6193 int i;
6194 char_u *spf;
6195
6196 if (idx == 0) // use internal wordlist
6197 {
6198 if (int_wordlist == NULL)
6199 {
6200 int_wordlist = vim_tempname('s', FALSE);
6201 if (int_wordlist == NULL)
6202 return;
6203 }
6204 fname = int_wordlist;
6205 }
6206 else
6207 {
6208 // If 'spellfile' isn't set figure out a good default value.
6209 if (*curwin->w_s->b_p_spf == NUL)
6210 {
6211 init_spellfile();
6212 new_spf = TRUE;
6213 }
6214
6215 if (*curwin->w_s->b_p_spf == NUL)
6216 {
6217 semsg(_(e_notset), "spellfile");
6218 return;
6219 }
6220 fnamebuf = alloc(MAXPATHL);
6221 if (fnamebuf == NULL)
6222 return;
6223
6224 for (spf = curwin->w_s->b_p_spf, i = 1; *spf != NUL; ++i)
6225 {
6226 copy_option_part(&spf, fnamebuf, MAXPATHL, ",");
6227 if (i == idx)
6228 break;
6229 if (*spf == NUL)
6230 {
6231 semsg(_("E765: 'spellfile' does not have %d entries"), idx);
6232 vim_free(fnamebuf);
6233 return;
6234 }
6235 }
6236
6237 // Check that the user isn't editing the .add file somewhere.
6238 buf = buflist_findname_exp(fnamebuf);
6239 if (buf != NULL && buf->b_ml.ml_mfp == NULL)
6240 buf = NULL;
6241 if (buf != NULL && bufIsChanged(buf))
6242 {
6243 emsg(_(e_bufloaded));
6244 vim_free(fnamebuf);
6245 return;
6246 }
6247
6248 fname = fnamebuf;
6249 }
6250
6251 if (what == SPELL_ADD_BAD || undo)
6252 {
6253 // When the word appears as good word we need to remove that one,
6254 // since its flags sort before the one with WF_BANNED.
6255 fd = mch_fopen((char *)fname, "r");
6256 if (fd != NULL)
6257 {
6258 while (!vim_fgets(line, MAXWLEN * 2, fd))
6259 {
6260 fpos = fpos_next;
6261 fpos_next = ftell(fd);
6262 if (STRNCMP(word, line, len) == 0
6263 && (line[len] == '/' || line[len] < ' '))
6264 {
6265 // Found duplicate word. Remove it by writing a '#' at
6266 // the start of the line. Mixing reading and writing
6267 // doesn't work for all systems, close the file first.
6268 fclose(fd);
6269 fd = mch_fopen((char *)fname, "r+");
6270 if (fd == NULL)
6271 break;
6272 if (fseek(fd, fpos, SEEK_SET) == 0)
6273 {
6274 fputc('#', fd);
6275 if (undo)
6276 {
6277 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
6278 smsg(_("Word '%.*s' removed from %s"),
6279 len, word, NameBuff);
6280 }
6281 }
6282 if (fseek(fd, fpos_next, SEEK_SET) != 0)
6283 {
6284 PERROR(_("Seek error in spellfile"));
6285 break;
6286 }
6287 }
6288 }
6289 if (fd != NULL)
6290 fclose(fd);
6291 }
6292 }
6293
6294 if (!undo)
6295 {
6296 fd = mch_fopen((char *)fname, "a");
6297 if (fd == NULL && new_spf)
6298 {
6299 char_u *p;
6300
6301 // We just initialized the 'spellfile' option and can't open the
6302 // file. We may need to create the "spell" directory first. We
6303 // already checked the runtime directory is writable in
6304 // init_spellfile().
6305 if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname)
6306 {
6307 int c = *p;
6308
6309 // The directory doesn't exist. Try creating it and opening
6310 // the file again.
6311 *p = NUL;
6312 vim_mkdir(fname, 0755);
6313 *p = c;
6314 fd = mch_fopen((char *)fname, "a");
6315 }
6316 }
6317
6318 if (fd == NULL)
6319 semsg(_(e_notopen), fname);
6320 else
6321 {
6322 if (what == SPELL_ADD_BAD)
6323 fprintf(fd, "%.*s/!\n", len, word);
6324 else if (what == SPELL_ADD_RARE)
6325 fprintf(fd, "%.*s/?\n", len, word);
6326 else
6327 fprintf(fd, "%.*s\n", len, word);
6328 fclose(fd);
6329
6330 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
6331 smsg(_("Word '%.*s' added to %s"), len, word, NameBuff);
6332 }
6333 }
6334
6335 if (fd != NULL)
6336 {
6337 // Update the .add.spl file.
6338 mkspell(1, &fname, FALSE, TRUE, TRUE);
6339
6340 // If the .add file is edited somewhere, reload it.
6341 if (buf != NULL)
6342 buf_reload(buf, buf->b_orig_mode);
6343
6344 redraw_all_later(SOME_VALID);
6345 }
6346 vim_free(fnamebuf);
6347 }
6348
6349 /*
6350 * Initialize 'spellfile' for the current buffer.
6351 */
6352 static void
init_spellfile(void)6353 init_spellfile(void)
6354 {
6355 char_u *buf;
6356 int l;
6357 char_u *fname;
6358 char_u *rtp;
6359 char_u *lend;
6360 int aspath = FALSE;
6361 char_u *lstart = curbuf->b_s.b_p_spl;
6362
6363 if (*curwin->w_s->b_p_spl != NUL && curwin->w_s->b_langp.ga_len > 0)
6364 {
6365 buf = alloc(MAXPATHL);
6366 if (buf == NULL)
6367 return;
6368
6369 // Find the end of the language name. Exclude the region. If there
6370 // is a path separator remember the start of the tail.
6371 for (lend = curwin->w_s->b_p_spl; *lend != NUL
6372 && vim_strchr((char_u *)",._", *lend) == NULL; ++lend)
6373 if (vim_ispathsep(*lend))
6374 {
6375 aspath = TRUE;
6376 lstart = lend + 1;
6377 }
6378
6379 // Loop over all entries in 'runtimepath'. Use the first one where we
6380 // are allowed to write.
6381 rtp = p_rtp;
6382 while (*rtp != NUL)
6383 {
6384 if (aspath)
6385 // Use directory of an entry with path, e.g., for
6386 // "/dir/lg.utf-8.spl" use "/dir".
6387 vim_strncpy(buf, curbuf->b_s.b_p_spl,
6388 lstart - curbuf->b_s.b_p_spl - 1);
6389 else
6390 // Copy the path from 'runtimepath' to buf[].
6391 copy_option_part(&rtp, buf, MAXPATHL, ",");
6392 if (filewritable(buf) == 2)
6393 {
6394 // Use the first language name from 'spelllang' and the
6395 // encoding used in the first loaded .spl file.
6396 if (aspath)
6397 vim_strncpy(buf, curbuf->b_s.b_p_spl,
6398 lend - curbuf->b_s.b_p_spl);
6399 else
6400 {
6401 // Create the "spell" directory if it doesn't exist yet.
6402 l = (int)STRLEN(buf);
6403 vim_snprintf((char *)buf + l, MAXPATHL - l, "/spell");
6404 if (filewritable(buf) != 2)
6405 vim_mkdir(buf, 0755);
6406
6407 l = (int)STRLEN(buf);
6408 vim_snprintf((char *)buf + l, MAXPATHL - l,
6409 "/%.*s", (int)(lend - lstart), lstart);
6410 }
6411 l = (int)STRLEN(buf);
6412 fname = LANGP_ENTRY(curwin->w_s->b_langp, 0)
6413 ->lp_slang->sl_fname;
6414 vim_snprintf((char *)buf + l, MAXPATHL - l, ".%s.add",
6415 fname != NULL
6416 && strstr((char *)gettail(fname), ".ascii.") != NULL
6417 ? (char_u *)"ascii" : spell_enc());
6418 set_option_value((char_u *)"spellfile", 0L, buf, OPT_LOCAL);
6419 break;
6420 }
6421 aspath = FALSE;
6422 }
6423
6424 vim_free(buf);
6425 }
6426 }
6427
6428
6429
6430 /*
6431 * Set the spell character tables from strings in the affix file.
6432 */
6433 static int
set_spell_chartab(char_u * fol,char_u * low,char_u * upp)6434 set_spell_chartab(char_u *fol, char_u *low, char_u *upp)
6435 {
6436 // We build the new tables here first, so that we can compare with the
6437 // previous one.
6438 spelltab_T new_st;
6439 char_u *pf = fol, *pl = low, *pu = upp;
6440 int f, l, u;
6441
6442 clear_spell_chartab(&new_st);
6443
6444 while (*pf != NUL)
6445 {
6446 if (*pl == NUL || *pu == NUL)
6447 {
6448 emsg(_(e_affform));
6449 return FAIL;
6450 }
6451 f = mb_ptr2char_adv(&pf);
6452 l = mb_ptr2char_adv(&pl);
6453 u = mb_ptr2char_adv(&pu);
6454
6455 // Every character that appears is a word character.
6456 if (f < 256)
6457 new_st.st_isw[f] = TRUE;
6458 if (l < 256)
6459 new_st.st_isw[l] = TRUE;
6460 if (u < 256)
6461 new_st.st_isw[u] = TRUE;
6462
6463 // if "LOW" and "FOL" are not the same the "LOW" char needs
6464 // case-folding
6465 if (l < 256 && l != f)
6466 {
6467 if (f >= 256)
6468 {
6469 emsg(_(e_affrange));
6470 return FAIL;
6471 }
6472 new_st.st_fold[l] = f;
6473 }
6474
6475 // if "UPP" and "FOL" are not the same the "UPP" char needs
6476 // case-folding, it's upper case and the "UPP" is the upper case of
6477 // "FOL" .
6478 if (u < 256 && u != f)
6479 {
6480 if (f >= 256)
6481 {
6482 emsg(_(e_affrange));
6483 return FAIL;
6484 }
6485 new_st.st_fold[u] = f;
6486 new_st.st_isu[u] = TRUE;
6487 new_st.st_upper[f] = u;
6488 }
6489 }
6490
6491 if (*pl != NUL || *pu != NUL)
6492 {
6493 emsg(_(e_affform));
6494 return FAIL;
6495 }
6496
6497 return set_spell_finish(&new_st);
6498 }
6499
6500 /*
6501 * Set the spell character tables from strings in the .spl file.
6502 */
6503 static void
set_spell_charflags(char_u * flags,int cnt,char_u * fol)6504 set_spell_charflags(
6505 char_u *flags,
6506 int cnt, // length of "flags"
6507 char_u *fol)
6508 {
6509 // We build the new tables here first, so that we can compare with the
6510 // previous one.
6511 spelltab_T new_st;
6512 int i;
6513 char_u *p = fol;
6514 int c;
6515
6516 clear_spell_chartab(&new_st);
6517
6518 for (i = 0; i < 128; ++i)
6519 {
6520 if (i < cnt)
6521 {
6522 new_st.st_isw[i + 128] = (flags[i] & CF_WORD) != 0;
6523 new_st.st_isu[i + 128] = (flags[i] & CF_UPPER) != 0;
6524 }
6525
6526 if (*p != NUL)
6527 {
6528 c = mb_ptr2char_adv(&p);
6529 new_st.st_fold[i + 128] = c;
6530 if (i + 128 != c && new_st.st_isu[i + 128] && c < 256)
6531 new_st.st_upper[c] = i + 128;
6532 }
6533 }
6534
6535 (void)set_spell_finish(&new_st);
6536 }
6537
6538 static int
set_spell_finish(spelltab_T * new_st)6539 set_spell_finish(spelltab_T *new_st)
6540 {
6541 int i;
6542
6543 if (did_set_spelltab)
6544 {
6545 // check that it's the same table
6546 for (i = 0; i < 256; ++i)
6547 {
6548 if (spelltab.st_isw[i] != new_st->st_isw[i]
6549 || spelltab.st_isu[i] != new_st->st_isu[i]
6550 || spelltab.st_fold[i] != new_st->st_fold[i]
6551 || spelltab.st_upper[i] != new_st->st_upper[i])
6552 {
6553 emsg(_("E763: Word characters differ between spell files"));
6554 return FAIL;
6555 }
6556 }
6557 }
6558 else
6559 {
6560 // copy the new spelltab into the one being used
6561 spelltab = *new_st;
6562 did_set_spelltab = TRUE;
6563 }
6564
6565 return OK;
6566 }
6567
6568 /*
6569 * Write the table with prefix conditions to the .spl file.
6570 * When "fd" is NULL only count the length of what is written.
6571 */
6572 static int
write_spell_prefcond(FILE * fd,garray_T * gap)6573 write_spell_prefcond(FILE *fd, garray_T *gap)
6574 {
6575 int i;
6576 char_u *p;
6577 int len;
6578 int totlen;
6579 size_t x = 1; // collect return value of fwrite()
6580
6581 if (fd != NULL)
6582 put_bytes(fd, (long_u)gap->ga_len, 2); // <prefcondcnt>
6583
6584 totlen = 2 + gap->ga_len; // length of <prefcondcnt> and <condlen> bytes
6585
6586 for (i = 0; i < gap->ga_len; ++i)
6587 {
6588 // <prefcond> : <condlen> <condstr>
6589 p = ((char_u **)gap->ga_data)[i];
6590 if (p != NULL)
6591 {
6592 len = (int)STRLEN(p);
6593 if (fd != NULL)
6594 {
6595 fputc(len, fd);
6596 x &= fwrite(p, (size_t)len, (size_t)1, fd);
6597 }
6598 totlen += len;
6599 }
6600 else if (fd != NULL)
6601 fputc(0, fd);
6602 }
6603
6604 return totlen;
6605 }
6606
6607
6608 /*
6609 * Use map string "map" for languages "lp".
6610 */
6611 static void
set_map_str(slang_T * lp,char_u * map)6612 set_map_str(slang_T *lp, char_u *map)
6613 {
6614 char_u *p;
6615 int headc = 0;
6616 int c;
6617 int i;
6618
6619 if (*map == NUL)
6620 {
6621 lp->sl_has_map = FALSE;
6622 return;
6623 }
6624 lp->sl_has_map = TRUE;
6625
6626 // Init the array and hash tables empty.
6627 for (i = 0; i < 256; ++i)
6628 lp->sl_map_array[i] = 0;
6629 hash_init(&lp->sl_map_hash);
6630
6631 /*
6632 * The similar characters are stored separated with slashes:
6633 * "aaa/bbb/ccc/". Fill sl_map_array[c] with the character before c and
6634 * before the same slash. For characters above 255 sl_map_hash is used.
6635 */
6636 for (p = map; *p != NUL; )
6637 {
6638 c = mb_cptr2char_adv(&p);
6639 if (c == '/')
6640 headc = 0;
6641 else
6642 {
6643 if (headc == 0)
6644 headc = c;
6645
6646 // Characters above 255 don't fit in sl_map_array[], put them in
6647 // the hash table. Each entry is the char, a NUL the headchar and
6648 // a NUL.
6649 if (c >= 256)
6650 {
6651 int cl = mb_char2len(c);
6652 int headcl = mb_char2len(headc);
6653 char_u *b;
6654 hash_T hash;
6655 hashitem_T *hi;
6656
6657 b = alloc(cl + headcl + 2);
6658 if (b == NULL)
6659 return;
6660 mb_char2bytes(c, b);
6661 b[cl] = NUL;
6662 mb_char2bytes(headc, b + cl + 1);
6663 b[cl + 1 + headcl] = NUL;
6664 hash = hash_hash(b);
6665 hi = hash_lookup(&lp->sl_map_hash, b, hash);
6666 if (HASHITEM_EMPTY(hi))
6667 hash_add_item(&lp->sl_map_hash, hi, b, hash);
6668 else
6669 {
6670 // This should have been checked when generating the .spl
6671 // file.
6672 emsg(_("E783: duplicate char in MAP entry"));
6673 vim_free(b);
6674 }
6675 }
6676 else
6677 lp->sl_map_array[c] = headc;
6678 }
6679 }
6680 }
6681
6682 #endif // FEAT_SPELL
6683