1 /* xgettext YCP backend.
2 Copyright (C) 2001-2003, 2005-2009, 2011, 2018-2020 Free Software Foundation, Inc.
3
4 This file was written by Bruno Haible <haible@clisp.cons.org>, 2001.
5
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program. If not, see <https://www.gnu.org/licenses/>. */
18
19 #ifdef HAVE_CONFIG_H
20 # include "config.h"
21 #endif
22
23 /* Specification. */
24 #include "x-ycp.h"
25
26 #include <errno.h>
27 #include <limits.h>
28 #include <stdbool.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31
32 #include "message.h"
33 #include "rc-str-list.h"
34 #include "xgettext.h"
35 #include "xg-pos.h"
36 #include "xg-arglist-context.h"
37 #include "xg-message.h"
38 #include "error.h"
39 #include "xalloc.h"
40 #include "gettext.h"
41
42 #define _(s) gettext(s)
43
44 #define SIZEOF(a) (sizeof(a) / sizeof(a[0]))
45
46
47 /* The YCP syntax is defined in libycp/doc/syntax.html.
48 See also libycp/src/scanner.ll.
49 Both are part of the yast2-core package in SuSE Linux distributions. */
50
51
52 void
init_flag_table_ycp()53 init_flag_table_ycp ()
54 {
55 xgettext_record_flag ("sformat:1:ycp-format");
56 xgettext_record_flag ("y2debug:1:ycp-format");
57 xgettext_record_flag ("y2milestone:1:ycp-format");
58 xgettext_record_flag ("y2warning:1:ycp-format");
59 xgettext_record_flag ("y2error:1:ycp-format");
60 xgettext_record_flag ("y2security:1:ycp-format");
61 xgettext_record_flag ("y2internal:1:ycp-format");
62 }
63
64
65 /* ======================== Reading of characters. ======================== */
66
67 /* Position in the current line. */
68 static int char_in_line;
69
70 /* The input file stream. */
71 static FILE *fp;
72
73 /* These are for tracking whether comments count as immediately before
74 keyword. */
75 static int last_comment_line;
76 static int last_non_comment_line;
77
78
79 /* 1. line_number handling. */
80
81 static int
phase1_getc()82 phase1_getc ()
83 {
84 int c = getc (fp);
85
86 if (c == EOF)
87 {
88 if (ferror (fp))
89 error (EXIT_FAILURE, errno, _("error while reading \"%s\""),
90 real_file_name);
91 return EOF;
92 }
93
94 if (c == '\n')
95 {
96 line_number++;
97 char_in_line = 0;
98 }
99 else
100 char_in_line++;
101
102 return c;
103 }
104
105 /* Supports only one pushback character. */
106 static void
phase1_ungetc(int c)107 phase1_ungetc (int c)
108 {
109 if (c != EOF)
110 {
111 if (c == '\n')
112 {
113 --line_number;
114 char_in_line = INT_MAX;
115 }
116 else
117 --char_in_line;
118
119 ungetc (c, fp);
120 }
121 }
122
123
124 /* 2. Replace each comment that is not inside a character constant or
125 string literal with a space character. We need to remember the
126 comment for later, because it may be attached to a keyword string.
127 YCP comments can be in C comment syntax, C++ comment syntax or sh
128 comment syntax. */
129
130 static unsigned char phase2_pushback[1];
131 static int phase2_pushback_length;
132
133 static int
phase2_getc()134 phase2_getc ()
135 {
136 static char *buffer;
137 static size_t bufmax;
138 size_t buflen;
139 int lineno;
140 int c;
141 bool last_was_star;
142
143 if (phase2_pushback_length)
144 return phase2_pushback[--phase2_pushback_length];
145
146 if (char_in_line == 0)
147 {
148 /* Eat whitespace, to recognize ^[\t ]*# pattern. */
149 do
150 c = phase1_getc ();
151 while (c == '\t' || c == ' ');
152
153 if (c == '#')
154 {
155 /* sh comment. */
156 buflen = 0;
157 lineno = line_number;
158 for (;;)
159 {
160 c = phase1_getc ();
161 if (c == '\n' || c == EOF)
162 break;
163 /* We skip all leading white space, but not EOLs. */
164 if (!(buflen == 0 && (c == ' ' || c == '\t')))
165 {
166 if (buflen >= bufmax)
167 {
168 bufmax = 2 * bufmax + 10;
169 buffer = xrealloc (buffer, bufmax);
170 }
171 buffer[buflen++] = c;
172 }
173 }
174 if (buflen >= bufmax)
175 {
176 bufmax = 2 * bufmax + 10;
177 buffer = xrealloc (buffer, bufmax);
178 }
179 buffer[buflen] = '\0';
180 savable_comment_add (buffer);
181 last_comment_line = lineno;
182 return '\n';
183 }
184 }
185 else
186 c = phase1_getc ();
187
188 if (c == '/')
189 {
190 c = phase1_getc ();
191
192 switch (c)
193 {
194 default:
195 phase1_ungetc (c);
196 return '/';
197
198 case '*':
199 /* C comment. */
200 buflen = 0;
201 lineno = line_number;
202 last_was_star = false;
203 for (;;)
204 {
205 c = phase1_getc ();
206 if (c == EOF)
207 break;
208 /* We skip all leading white space, but not EOLs. */
209 if (buflen == 0 && (c == ' ' || c == '\t'))
210 continue;
211 if (buflen >= bufmax)
212 {
213 bufmax = 2 * bufmax + 10;
214 buffer = xrealloc (buffer, bufmax);
215 }
216 buffer[buflen++] = c;
217 switch (c)
218 {
219 case '\n':
220 --buflen;
221 while (buflen >= 1
222 && (buffer[buflen - 1] == ' '
223 || buffer[buflen - 1] == '\t'))
224 --buflen;
225 buffer[buflen] = '\0';
226 savable_comment_add (buffer);
227 buflen = 0;
228 lineno = line_number;
229 last_was_star = false;
230 continue;
231
232 case '*':
233 last_was_star = true;
234 continue;
235
236 case '/':
237 if (last_was_star)
238 {
239 buflen -= 2;
240 while (buflen >= 1
241 && (buffer[buflen - 1] == ' '
242 || buffer[buflen - 1] == '\t'))
243 --buflen;
244 buffer[buflen] = '\0';
245 savable_comment_add (buffer);
246 break;
247 }
248 /* FALLTHROUGH */
249
250 default:
251 last_was_star = false;
252 continue;
253 }
254 break;
255 }
256 last_comment_line = lineno;
257 return ' ';
258
259 case '/':
260 /* C++ comment. */
261 buflen = 0;
262 lineno = line_number;
263 for (;;)
264 {
265 c = phase1_getc ();
266 if (c == '\n' || c == EOF)
267 break;
268 /* We skip all leading white space, but not EOLs. */
269 if (!(buflen == 0 && (c == ' ' || c == '\t')))
270 {
271 if (buflen >= bufmax)
272 {
273 bufmax = 2 * bufmax + 10;
274 buffer = xrealloc (buffer, bufmax);
275 }
276 buffer[buflen++] = c;
277 }
278 }
279 if (buflen >= bufmax)
280 {
281 bufmax = 2 * bufmax + 10;
282 buffer = xrealloc (buffer, bufmax);
283 }
284 buffer[buflen] = '\0';
285 savable_comment_add (buffer);
286 last_comment_line = lineno;
287 return '\n';
288 }
289 }
290 else
291 return c;
292 }
293
294 /* Supports only one pushback character. */
295 static void
phase2_ungetc(int c)296 phase2_ungetc (int c)
297 {
298 if (c != EOF)
299 {
300 if (phase2_pushback_length == SIZEOF (phase2_pushback))
301 abort ();
302 phase2_pushback[phase2_pushback_length++] = c;
303 }
304 }
305
306
307 /* ========================== Reading of tokens. ========================== */
308
309
310 enum token_type_ty
311 {
312 token_type_eof,
313 token_type_lparen, /* ( */
314 token_type_rparen, /* ) */
315 token_type_comma, /* , */
316 token_type_i18n, /* _( */
317 token_type_string_literal, /* "abc" */
318 token_type_symbol, /* symbol, number */
319 token_type_other /* misc. operator */
320 };
321 typedef enum token_type_ty token_type_ty;
322
323 typedef struct token_ty token_ty;
324 struct token_ty
325 {
326 token_type_ty type;
327 char *string; /* for token_type_string_literal, token_type_symbol */
328 refcounted_string_list_ty *comment; /* for token_type_string_literal */
329 int line_number;
330 };
331
332
333 /* 7. Replace escape sequences within character strings with their
334 single character equivalents. */
335
336 #define P7_QUOTES (1000 + '"')
337
338 static int
phase7_getc()339 phase7_getc ()
340 {
341 int c;
342
343 for (;;)
344 {
345 /* Use phase 1, because phase 2 elides comments. */
346 c = phase1_getc ();
347
348 if (c == '"')
349 return P7_QUOTES;
350 if (c != '\\')
351 return c;
352 c = phase1_getc ();
353 if (c != '\n')
354 switch (c)
355 {
356 case 'b':
357 return '\b';
358 case 'f':
359 return '\f';
360 case 'n':
361 return '\n';
362 case 'r':
363 return '\r';
364 case 't':
365 return '\t';
366
367 /* FIXME: What is the octal escape syntax?
368 syntax.html says: [0] [0-7]+
369 scanner.ll says: [0-7] [0-7] [0-7]
370 */
371 #if 0
372 case '0': case '1': case '2': case '3':
373 case '4': case '5': case '6': case '7':
374 {
375 int n, j;
376
377 n = 0;
378 for (j = 0; j < 3; ++j)
379 {
380 n = n * 8 + c - '0';
381 c = phase1_getc ();
382 switch (c)
383 {
384 default:
385 break;
386
387 case '0': case '1': case '2': case '3':
388 case '4': case '5': case '6': case '7':
389 continue;
390 }
391 break;
392 }
393 phase1_ungetc (c);
394 return n;
395 }
396 #endif
397
398 default:
399 return c;
400 }
401 }
402 }
403
404
405 /* Free the memory pointed to by a 'struct token_ty'. */
406 static inline void
free_token(token_ty * tp)407 free_token (token_ty *tp)
408 {
409 if (tp->type == token_type_string_literal || tp->type == token_type_symbol)
410 free (tp->string);
411 if (tp->type == token_type_string_literal)
412 drop_reference (tp->comment);
413 }
414
415
416 /* Combine characters into tokens. Discard whitespace. */
417
418 static token_ty phase5_pushback[1];
419 static int phase5_pushback_length;
420
421 static void
phase5_get(token_ty * tp)422 phase5_get (token_ty *tp)
423 {
424 static char *buffer;
425 static int bufmax;
426 int bufpos;
427 int c;
428
429 if (phase5_pushback_length)
430 {
431 *tp = phase5_pushback[--phase5_pushback_length];
432 return;
433 }
434 for (;;)
435 {
436 tp->line_number = line_number;
437 c = phase2_getc ();
438
439 switch (c)
440 {
441 case EOF:
442 tp->type = token_type_eof;
443 return;
444
445 case '\n':
446 if (last_non_comment_line > last_comment_line)
447 savable_comment_reset ();
448 /* FALLTHROUGH */
449 case '\r':
450 case '\t':
451 case ' ':
452 /* Ignore whitespace and comments. */
453 continue;
454 }
455
456 last_non_comment_line = tp->line_number;
457
458 switch (c)
459 {
460 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
461 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
462 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
463 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
464 case 'Y': case 'Z':
465 case '_':
466 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
467 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
468 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
469 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
470 case 'y': case 'z':
471 case '0': case '1': case '2': case '3': case '4':
472 case '5': case '6': case '7': case '8': case '9':
473 /* Symbol, or part of a number. */
474 bufpos = 0;
475 for (;;)
476 {
477 if (bufpos >= bufmax)
478 {
479 bufmax = 2 * bufmax + 10;
480 buffer = xrealloc (buffer, bufmax);
481 }
482 buffer[bufpos++] = c;
483 c = phase2_getc ();
484 switch (c)
485 {
486 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
487 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
488 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
489 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
490 case 'Y': case 'Z':
491 case '_':
492 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
493 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
494 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
495 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
496 case 'y': case 'z':
497 case '0': case '1': case '2': case '3': case '4':
498 case '5': case '6': case '7': case '8': case '9':
499 continue;
500 default:
501 if (bufpos == 1 && buffer[0] == '_' && c == '(')
502 {
503 tp->type = token_type_i18n;
504 return;
505 }
506 phase2_ungetc (c);
507 break;
508 }
509 break;
510 }
511 if (bufpos >= bufmax)
512 {
513 bufmax = 2 * bufmax + 10;
514 buffer = xrealloc (buffer, bufmax);
515 }
516 buffer[bufpos] = '\0';
517 tp->string = xstrdup (buffer);
518 tp->type = token_type_symbol;
519 return;
520
521 case '"':
522 bufpos = 0;
523 for (;;)
524 {
525 c = phase7_getc ();
526 if (c == EOF || c == P7_QUOTES)
527 break;
528 if (bufpos >= bufmax)
529 {
530 bufmax = 2 * bufmax + 10;
531 buffer = xrealloc (buffer, bufmax);
532 }
533 buffer[bufpos++] = c;
534 }
535 if (bufpos >= bufmax)
536 {
537 bufmax = 2 * bufmax + 10;
538 buffer = xrealloc (buffer, bufmax);
539 }
540 buffer[bufpos] = '\0';
541 tp->string = xstrdup (buffer);
542 tp->type = token_type_string_literal;
543 tp->comment = add_reference (savable_comment);
544 return;
545
546 case '(':
547 tp->type = token_type_lparen;
548 return;
549
550 case ')':
551 tp->type = token_type_rparen;
552 return;
553
554 case ',':
555 tp->type = token_type_comma;
556 return;
557
558 default:
559 /* We could carefully recognize each of the 2 and 3 character
560 operators, but it is not necessary, as we only need to recognize
561 gettext invocations. Don't bother. */
562 tp->type = token_type_other;
563 return;
564 }
565 }
566 }
567
568 /* Supports only one pushback token. */
569 static void
phase5_unget(token_ty * tp)570 phase5_unget (token_ty *tp)
571 {
572 if (tp->type != token_type_eof)
573 {
574 if (phase5_pushback_length == SIZEOF (phase5_pushback))
575 abort ();
576 phase5_pushback[phase5_pushback_length++] = *tp;
577 }
578 }
579
580
581 /* Concatenate adjacent string literals to form single string literals.
582 (See libycp/src/parser.yy, rule 'string' vs. terminal 'STRING'.) */
583
584 static token_ty phase8_pushback[1];
585 static int phase8_pushback_length;
586
587 static void
phase8_get(token_ty * tp)588 phase8_get (token_ty *tp)
589 {
590 if (phase8_pushback_length)
591 {
592 *tp = phase8_pushback[--phase8_pushback_length];
593 return;
594 }
595 phase5_get (tp);
596 if (tp->type != token_type_string_literal)
597 return;
598 for (;;)
599 {
600 token_ty tmp;
601 size_t len;
602
603 phase5_get (&tmp);
604 if (tmp.type != token_type_string_literal)
605 {
606 phase5_unget (&tmp);
607 return;
608 }
609 len = strlen (tp->string);
610 tp->string = xrealloc (tp->string, len + strlen (tmp.string) + 1);
611 strcpy (tp->string + len, tmp.string);
612 free_token (&tmp);
613 }
614 }
615
616 /* Supports only one pushback token. */
617 static void
phase8_unget(token_ty * tp)618 phase8_unget (token_ty *tp)
619 {
620 if (tp->type != token_type_eof)
621 {
622 if (phase8_pushback_length == SIZEOF (phase8_pushback))
623 abort ();
624 phase8_pushback[phase8_pushback_length++] = *tp;
625 }
626 }
627
628
629 /* ========================= Extracting strings. ========================== */
630
631
632 /* Context lookup table. */
633 static flag_context_list_table_ty *flag_context_list_table;
634
635
636 /* The file is broken into tokens.
637
638 Normal handling: Look for
639 [A] _( [B] msgid ... )
640 Plural handling: Look for
641 [A] _( [B] msgid [C] , [D] msgid_plural ... )
642 At point [A]: state == 0.
643 At point [B]: state == 1, plural_mp == NULL.
644 At point [C]: state == 2, plural_mp != NULL.
645 At point [D]: state == 1, plural_mp != NULL.
646
647 We use recursion because we have to set the context according to the given
648 flags. */
649
650
651 /* Extract messages until the next balanced closing parenthesis.
652 Extracted messages are added to MLP.
653 Return true upon eof, false upon closing parenthesis. */
654 static bool
extract_parenthesized(message_list_ty * mlp,flag_context_ty outer_context,flag_context_list_iterator_ty context_iter,bool in_i18n)655 extract_parenthesized (message_list_ty *mlp,
656 flag_context_ty outer_context,
657 flag_context_list_iterator_ty context_iter,
658 bool in_i18n)
659 {
660 int state; /* 1 or 2 inside _( ... ), otherwise 0 */
661 int plural_state = 0; /* defined only when in states 1 and 2 */
662 message_ty *plural_mp = NULL; /* defined only when in states 1 and 2 */
663 /* Context iterator that will be used if the next token is a '('. */
664 flag_context_list_iterator_ty next_context_iter =
665 passthrough_context_list_iterator;
666 /* Current context. */
667 flag_context_ty inner_context =
668 inherited_context (outer_context,
669 flag_context_list_iterator_advance (&context_iter));
670
671 /* Start state is 0 or 1. */
672 state = (in_i18n ? 1 : 0);
673
674 for (;;)
675 {
676 token_ty token;
677
678 if (in_i18n)
679 phase8_get (&token);
680 else
681 phase5_get (&token);
682
683 switch (token.type)
684 {
685 case token_type_i18n:
686 if (extract_parenthesized (mlp, inner_context, next_context_iter,
687 true))
688 return true;
689 next_context_iter = null_context_list_iterator;
690 state = 0;
691 continue;
692
693 case token_type_string_literal:
694 if (state == 1)
695 {
696 lex_pos_ty pos;
697 pos.file_name = logical_file_name;
698 pos.line_number = token.line_number;
699
700 if (plural_state == 0)
701 {
702 /* Seen an msgid. */
703 token_ty token2;
704
705 if (in_i18n)
706 phase8_get (&token2);
707 else
708 phase5_get (&token2);
709
710 plural_mp =
711 remember_a_message (mlp, NULL, token.string, false,
712 token2.type == token_type_comma,
713 inner_context, &pos,
714 NULL, token.comment, false);
715
716 if (in_i18n)
717 phase8_unget (&token2);
718 else
719 phase5_unget (&token2);
720
721 plural_state = 1;
722 state = 2;
723 }
724 else
725 {
726 /* Seen an msgid_plural. */
727 if (plural_mp != NULL)
728 remember_a_message_plural (plural_mp, token.string, false,
729 inner_context, &pos,
730 token.comment, false);
731 state = 0;
732 }
733 drop_reference (token.comment);
734 }
735 else
736 {
737 free_token (&token);
738 state = 0;
739 }
740 next_context_iter = null_context_list_iterator;
741 continue;
742
743 case token_type_symbol:
744 next_context_iter =
745 flag_context_list_iterator (
746 flag_context_list_table_lookup (
747 flag_context_list_table,
748 token.string, strlen (token.string)));
749 free_token (&token);
750 state = 0;
751 continue;
752
753 case token_type_lparen:
754 if (extract_parenthesized (mlp, inner_context, next_context_iter,
755 false))
756 return true;
757 next_context_iter = null_context_list_iterator;
758 state = 0;
759 continue;
760
761 case token_type_rparen:
762 return false;
763
764 case token_type_comma:
765 if (state == 2)
766 state = 1;
767 else
768 state = 0;
769 inner_context =
770 inherited_context (outer_context,
771 flag_context_list_iterator_advance (
772 &context_iter));
773 next_context_iter = passthrough_context_list_iterator;
774 continue;
775
776 case token_type_other:
777 next_context_iter = null_context_list_iterator;
778 state = 0;
779 continue;
780
781 case token_type_eof:
782 return true;
783
784 default:
785 abort ();
786 }
787 }
788 }
789
790
791 void
extract_ycp(FILE * f,const char * real_filename,const char * logical_filename,flag_context_list_table_ty * flag_table,msgdomain_list_ty * mdlp)792 extract_ycp (FILE *f,
793 const char *real_filename, const char *logical_filename,
794 flag_context_list_table_ty *flag_table,
795 msgdomain_list_ty *mdlp)
796 {
797 message_list_ty *mlp = mdlp->item[0]->messages;
798
799 fp = f;
800 real_file_name = real_filename;
801 logical_file_name = xstrdup (logical_filename);
802 line_number = 1;
803 char_in_line = 0;
804
805 last_comment_line = -1;
806 last_non_comment_line = -1;
807
808 phase2_pushback_length = 0;
809 phase5_pushback_length = 0;
810 phase8_pushback_length = 0;
811
812 flag_context_list_table = flag_table;
813
814 /* Eat tokens until eof is seen. When extract_parenthesized returns
815 due to an unbalanced closing parenthesis, just restart it. */
816 while (!extract_parenthesized (mlp, null_context, null_context_list_iterator,
817 false))
818 ;
819
820 fp = NULL;
821 real_file_name = NULL;
822 logical_file_name = NULL;
823 line_number = 0;
824 char_in_line = 0;
825 }
826