1 /*
2  * This file is part of DGD, https://github.com/dworkin/dgd
3  * Copyright (C) 1993-2010 Dworkin B.V.
4  * Copyright (C) 2010-2013 DGD Authors (see the commit log for details)
5  *
6  * This program is free software: you can redistribute it and/or modify
7  * it under the terms of the GNU Affero General Public License as
8  * published by the Free Software Foundation, either version 3 of the
9  * License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU Affero General Public License for more details.
15  *
16  * You should have received a copy of the GNU Affero General Public License
17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 # include "lex.h"
21 # include "macro.h"
22 # include "special.h"
23 # include "ppstr.h"
24 # include "token.h"
25 # include "path.h"
26 # include "ppcontrol.h"
27 
28 /*
29  * Get a token from a file, handling preprocessor control directives.
30  */
31 
32 # define ICHUNKSZ	8
33 
34 typedef struct _ifstate_ {
35     bool active;		/* is this ifstate active? */
36     bool skipping;		/* skipping this part? */
37     bool expect_else;		/* expect #else or #endif? */
38     char level;			/* include level */
39     struct _ifstate_ *prev;	/* previous ifstate */
40 } ifstate;
41 
42 typedef struct _ichunk_ {
43     struct _ichunk_ *next;	/* next in list */
44     ifstate i[ICHUNKSZ];	/* chunk of ifstates */
45 } ichunk;
46 
47 static char **idirs;		/* include directory array */
48 static char pri[NR_TOKENS + 1];	/* operator priority table */
49 static bool init_pri;		/* has the priority table been initialized? */
50 static int include_level;	/* current #include level */
51 static ichunk *ilist;		/* list of ifstate chunks */
52 static int ichunksz;		/* ifstate chunk size */
53 static ifstate *flist;		/* free ifstate list */
54 static ifstate *ifs;		/* current conditional inclusion state */
55 
56 static ifstate top = {		/* initial ifstate */
57     TRUE, FALSE, FALSE, 0, (ifstate *) NULL
58 };
59 
60 # define UNARY	0x10
61 
62 /*
63  * NAME:	pp->init()
64  * DESCRIPTION:	initialize preprocessor. Return TRUE if the input file could
65  *		be opened.
66  */
pp_init(char * file,char ** id,string ** strs,int nstr,int level)67 bool pp_init(char *file, char **id, string **strs, int nstr, int level)
68 {
69     tk_init();
70     if (strs != (string **) NULL) {
71 	tk_include(file, strs, nstr);
72     } else if (!tk_include(file, (string **) NULL, 0)) {
73 	tk_clear();
74 	return FALSE;
75     }
76     mc_init();
77     special_define();
78     mc_define("__DGD__", "\x0091\x009", -1);	/* HT 1 HT */
79 #ifdef NETWORK_EXTENSIONS
80     mc_define("__NETWORK_EXTENSIONS__", (char *) NULL, -1);
81 #endif
82     pps_init();
83     include_level = level;
84     ifs = &top;
85     ilist = (ichunk *) NULL;
86     ichunksz = ICHUNKSZ;
87     flist = (ifstate *) NULL;
88 
89     if (!init_pri) {
90 	/* #if operator priority table */
91 	pri['~' + 1]    =
92 	pri['!' + 1]    = UNARY;
93 	pri['*' + 1]    =
94 	pri['/' + 1]    =
95 	pri['%' + 1]    = 11;
96 	pri['+' + 1]    =
97 	pri['-' + 1]    = UNARY | 10;
98 	pri[LSHIFT + 1] =
99 	pri[RSHIFT + 1] = 9;
100 	pri['<' + 1]    =
101 	pri['>' + 1]    =
102 	pri[LE + 1]     =
103 	pri[GE + 1]     = 8;
104 	pri[EQ + 1]     =
105 	pri[NE + 1]     = 7;
106 	pri['&' + 1]    = 6;
107 	pri['^' + 1]    = 5;
108 	pri['|' + 1]    = 4;
109 	pri[LAND + 1]   = 3;
110 	pri[LOR + 1]    = 2;
111 	pri['?' + 1]    = 1;
112 
113 	init_pri = TRUE;
114     }
115     idirs = id;
116 
117     return TRUE;
118 }
119 
120 /*
121  * NAME:	push()
122  * DESCRIPTION:	push a new ifstate on the stack
123  */
push()124 static void push()
125 {
126     ifstate *s;
127 
128     if (flist != (ifstate *) NULL) {
129 	/* from free list */
130 	s = flist;
131 	flist = s->prev;
132     } else {
133 	/* allocate new one */
134 	if (ichunksz == ICHUNKSZ) {
135 	    ichunk *l;
136 
137 	    l = ALLOC(ichunk, 1);
138 	    l->next = ilist;
139 	    ilist = l;
140 	    ichunksz = 0;
141 	}
142 	s = &ilist->i[ichunksz++];
143     }
144     s->active = !ifs->skipping;
145     s->skipping = TRUE;	/* ! */
146     s->expect_else = TRUE;
147     s->level = include_level + 1;
148     s->prev = ifs;
149     ifs = s;
150 }
151 
152 /*
153  * NAME:	pop()
154  * DESCRIPTION:	pop an ifstate from the stack
155  */
pop()156 static void pop()
157 {
158     ifstate *s;
159 
160     s = ifs;
161     ifs = ifs->prev;
162 
163     s->prev = flist;
164     flist = s;
165 }
166 
167 /*
168  * NAME:	pp->clear()
169  * DESCRIPTION:	terminate preprocessor
170  */
pp_clear()171 void pp_clear()
172 {
173     ichunk *l, *f;
174 
175     pps_clear();
176     while (ifs != &top) {
177 	pop();
178     }
179     for (l = ilist; l != (ichunk *) NULL; ) {
180 	f = l;
181 	l = l->next;
182 	FREE(f);
183     }
184     ilist = (ichunk *) NULL;
185     mc_clear();
186     tk_clear();
187 }
188 
189 /*
190  * NAME:	wsgettok()
191  * DESCRIPTION:	get an unpreprocessed token, skipping white space
192  */
wsgettok()193 static int wsgettok()
194 {
195     int token;
196 
197     do {
198 	token = tk_gettok();
199     } while (token == ' ');
200 
201     return token;
202 }
203 
204 /*
205  * NAME:	mcgtok()
206  * DESCRIPTION:	get a token, while expanding macros
207  */
mcgtok()208 static int mcgtok()
209 {
210     int token;
211     macro *mc;
212 
213     for (;;) {
214 	token = tk_gettok();
215 	if (token == IDENTIFIER) {
216 	    mc = mc_lookup(yytext);
217 	    if (mc != (macro *) NULL && tk_expand(mc) > 0) {
218 		continue;
219 	    }
220 	}
221 	return token;
222     }
223 }
224 
225 /*
226  * NAME:	wsmcgtok()
227  * DESCRIPTION:	get a preprocessed token, skipping white space
228  */
wsmcgtok()229 static int wsmcgtok()
230 {
231     int token;
232 
233     do {
234 	token = mcgtok();
235     } while (token == ' ' || token == HT);
236 
237     return token;
238 }
239 
240 static int expr_keep;	/* single character unget buffer for expr_get() */
241 
242 # define expr_unget(c)	expr_keep = c
243 # define expr_uncng()	(expr_keep != EOF)
244 # define expr_unclr()	expr_keep = EOF
245 
246 /*
247  * NAME:	expr_get()
248  * DESCRIPTION:	get a token from the input stream, handling defined(IDENT),
249  *		replacing undefined identifiers by 0
250  */
expr_get()251 static int expr_get()
252 {
253     char buf[MAX_LINE_SIZE];
254     int token;
255 
256     if (expr_uncng()) {
257 	/* the unget buffer is not empty */
258 	token = expr_keep;
259 	expr_unclr();	/* clear unget buffer */
260     } else {
261 	token = wsmcgtok();
262 	if (token == IDENTIFIER) {
263 	    if (strcmp(yytext, "defined") == 0) {
264 		bool paren;
265 
266 		token = wsgettok();
267 		if (token == '(') {
268 		    paren = TRUE;
269 		    token = wsgettok();
270 		} else {
271 		    paren = FALSE;
272 		}
273 		strcpy(buf, yytext);
274 		if (token != IDENTIFIER) {
275 		    error("missing identifier in defined");
276 		}
277 		if (paren && token != ')') {
278 		    token = wsgettok();
279 		    if (token != ')') {
280 			error("missing ) in defined");
281 			expr_unget(token);
282 		    }
283 		}
284 		yynumber = (mc_lookup(buf) != (macro *) NULL);
285 	    } else {
286 		/* this identifier isn't a macro */
287 		yynumber = 0;
288 	    }
289 	    token = INT_CONST;
290 	}
291     }
292     return token;
293 }
294 
295 /*
296  * NAME:	eval_expr()
297  * DESCRIPTION:	evaluate an expression following #if
298  */
eval_expr(int priority)299 static long eval_expr(int priority)
300 {
301     int token;
302     long expr, expr2;
303 
304     token = expr_get();
305     if (token == '(') {
306 	/* get an expression between parenthesis */
307 	expr = eval_expr(0);
308 	token = expr_get();
309 	if (token != ')') {
310 	    error("missing ) in conditional control");
311 	    expr_unget(token);
312 	}
313     } else if (pri[token + 1] & UNARY) {
314 	/* unary operator */
315 	expr = eval_expr(11);
316 	switch (token) {
317 	case '~': expr = ~expr; break;
318 	case '!': expr = !expr; break;
319 	case '-': expr = -expr; break;
320 	}
321     } else if (token == INT_CONST) {
322 	/* integer constant */
323 	expr = yynumber;
324     } else {
325 	/* bad token */
326 	if (token == LF) {
327 	    error("missing expression in conditional control");
328 	}
329 	expr_unget(token);
330 	return 0;
331     }
332 
333     for (;;) {
334 	/* get (binary) operator, ), : or \n */
335 	token = expr_get();
336 	expr2 = pri[token + 1] & ~UNARY;
337 	if (expr2 == 0 || priority >= expr2) {
338 	    expr_unget(token);
339 	    break;	/* return current expression */
340 	} else {
341 	    /* get second operand */
342 	    expr2 = eval_expr((int) expr2);
343 	    if (expr2 == 0) {
344 		if (token == '/') {
345 		    error("division by zero in conditional control");
346 		    continue;
347 		} else if (token == '%') {
348 		    error("modulus by zero in conditional control");
349 		    continue;
350 		}
351 	    }
352 	    switch (token) {
353 	    case '/':		expr /= expr2;		break;
354 	    case '%':		expr %= expr2;		break;
355 	    case '*':		expr *= expr2;		break;
356 	    case '+':		expr += expr2;		break;
357 	    case '-':		expr -= expr2;		break;
358 	    case LSHIFT:	expr <<= expr2;		break;
359 	    case RSHIFT:	expr >>= expr2;		break;
360 	    case '<':		expr = expr < expr2;	break;
361 	    case '>':		expr = expr > expr2;	break;
362 	    case LE:		expr = expr <= expr2;	break;
363 	    case GE:		expr = expr >= expr2;	break;
364 	    case EQ:		expr = expr == expr2;	break;
365 	    case NE:		expr = expr != expr2;	break;
366 	    case '&':		expr &= expr2;		break;
367 	    case '^':		expr ^= expr2;		break;
368 	    case '|':		expr |= expr2;		break;
369 	    case LAND:		expr = expr && expr2;	break;
370 	    case LOR:		expr = expr || expr2;	break;
371 	    case '?':
372 		token = expr_get();
373 		if (token != ':') {
374 		    error("? without : in conditional control");
375 		    expr_unget(token);
376 		} else if (expr == 0) {
377 		    expr = eval_expr(0);
378 		} else {
379 		    eval_expr(0);
380 		    expr = expr2;
381 		}
382 	    }
383 	}
384     }
385 
386     return expr;
387 }
388 
389 
390 # define PP_ELSE	1
391 # define PP_ERROR	2
392 # define PP_LINE	3
393 # define PP_ELIF	4
394 # define PP_ENDIF	5
395 # define PP_IF		6
396 # define PP_DEFINE	7
397 # define PP_INCLUDE	8
398 # define PP_IFDEF	9
399 # define PP_IFNDEF	10
400 # define PP_UNDEF	11
401 # define PP_PRAGMA	12
402 
403 /*
404  * NAME:	pptokenz()
405  * DESCRIPTION:	return a number in the range 1..12 specifying which preprocessor
406  *		directive the argument is, or 0 if it isn't.
407  */
pptokenz(char * key,unsigned int len)408 static int pptokenz(char *key, unsigned int len)
409 {
410     static char *keyword[] = {
411       "else", "error", "line", "elif", "endif", "if", "define",
412       "include", "ifdef", "ifndef", "undef", "pragma"
413     };
414     static char value[] = {
415       9, 0, 0, 4, 0, 3, 0, 0, 4, 0, 0, 2, 0,
416       0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0
417     };
418 
419     len += value[key[0] - 'a'] + value[key[len - 1] - 'a'] - 3;
420     if (len < 1 || len > 12 || strcmp(keyword[len - 1], key) != 0) {
421 	return 0;
422     }
423     return len;
424 }
425 
426 # define FIRST_KEYWORD	GOTO
427 
428 /*
429  * NAME:	tokenz()
430  * DESCRIPTION:	return a number in the range 1..30 specifying which keyword
431  *		the argument is, or 0 if it isn't. Note that the keywords must
432  *		be given in the same order here as in parser.y.
433  */
tokenz(char * key,unsigned int len)434 static int tokenz(char *key, unsigned int len)
435 {
436     static char *keyword[] = {
437       "goto", "mapping", "nomask", "break", "else", "case", "for", "float",
438       "static", "continue", "rlimits", "default", "do", "mixed", "object",
439       "return", "function", "operator", "if", "int", "private", "catch",
440       "switch", "inherit", "while", "atomic", "string", "void", "varargs",
441       "nil"
442     };
443     static char value[] = {
444       18,  9,  1,  3,  0,  1, 19, 15, 15,  0, 19, 19,  5,
445        7,  7, 13,  0,  2,  1,  1,  0, 20, 19,  0,  0,  0
446     };
447 
448     len = (len + value[key[0] - 'a'] + value[key[len - 1] - 'a']) % 30;
449     if (strcmp(keyword[len], key) == 0) {
450 # ifndef CLOSURES
451 	if (len == FUNCTION - FIRST_KEYWORD) {
452 	    return 0;
453 	}
454 # endif
455 	return len + 1;
456     }
457     return 0;
458 }
459 
460 /*
461  * NAME:	unexpected()
462  * DESCRIPTION:	an error has occured, print appropriate errormessage and skip
463  *		till \n found
464  */
unexpected(int token,char * wanted,char * directive)465 static void unexpected(int token, char *wanted, char *directive)
466 {
467     if (token == LF) {
468 	error("missing %s in #%s", wanted, directive);
469     } else {
470 	error("unexpected token in #%s", directive);
471 	tk_skiptonl(FALSE);
472     }
473 }
474 
475 /*
476  * NAME:	do_include()
477  * DESCRIPTION:	handle an #include preprocessing directive
478  */
do_include()479 static void do_include()
480 {
481     char file[MAX_LINE_SIZE], path[STRINGSZ + MAX_LINE_SIZE], buf[STRINGSZ];
482     int token;
483     char **idir;
484     char *include;
485     string **strs;
486     int nstr;
487 
488     if (include_level == 8) {
489 	error("#include nesting too deep");
490 	tk_skiptonl(FALSE);
491 	return;
492     }
493 
494     tk_header(TRUE);
495     token = wsmcgtok();
496     tk_header(FALSE);
497 
498     if (idirs == (char **) NULL) {
499 	error("illegal #include from config file");
500 	return;
501     }
502     if (token == STRING_CONST) {
503 	strcpy(file, yytext);
504 	tk_skiptonl(TRUE);
505 
506 	/* first try the path direct */
507 	include = path_include(buf, tk_filename(), file, &strs, &nstr);
508 	if (tk_include(include, strs, nstr)) {
509 	    include_level++;
510 	    return;
511 	}
512     } else if (token == INCL_CONST) {
513 	strcpy(file, yytext);
514 	tk_skiptonl(TRUE);
515     } else {
516 	unexpected(token, "filename", "include");
517 	return;
518     }
519 
520     /* search in standard directories */
521     for (idir = idirs; *idir != (char *) NULL; idir++) {
522 	strcpy(path, *idir);
523 	strcat(path, "/");
524 	strcat(path, file);
525 	include = path_include(buf, tk_filename(), path, &strs, &nstr);
526 	if (tk_include(include, strs, nstr)) {
527 	    include_level++;
528 	    return;
529 	}
530     }
531     error("cannot include \"%s\"", file);
532 }
533 
534 /*
535  * NAME:	argnum()
536  * DESCRIPTION:	return the index in the parameter list if the supplied token is
537  *		a parameter, -1 otherwise
538  */
argnum(char ** args,int narg,int token)539 static int argnum(char **args, int narg, int token)
540 {
541     if (token == IDENTIFIER) {
542 	while (narg > 0) {
543 	    if (strcmp(args[--narg], yytext) == 0) {
544 		return narg;
545 	    }
546 	}
547     }
548     return -1;
549 }
550 
551 /*
552  * NAME:	do_define()
553  * DESCRIPTION:	handle a #define preprocessor directive
554  */
do_define()555 static void do_define()
556 {
557     char name[MAX_LINE_SIZE], buf[MAX_REPL_SIZE], *args[MAX_NARG], *arg;
558     int token, i, narg, errcount;
559     str *s;
560     bool seen_space;
561 
562     token = wsgettok();
563     if (token != IDENTIFIER) {
564 	unexpected(token, "identifier", "define");
565 	return;
566     }
567     strcpy(name, yytext);
568 
569     /* scan parameter list (if any) */
570     errcount = 0;
571     tk_setpp(TRUE);
572     token = tk_gettok();
573     if (token == '(') {
574 	narg = 0;
575 	token = wsgettok();
576 	if (token != ')') {
577 	    for (;;) {
578 		if (token == LF || token == EOF) {
579 		    error("unterminated macro definition");
580 		    errcount++;
581 		    break;
582 		}
583 		if (token != IDENTIFIER) {
584 		    error("unexpected token in macro parameter list");
585 		    errcount++;
586 		    tk_skiptonl(FALSE);
587 		    break;
588 		}
589 		if (narg < MAX_NARG) {
590 		    arg = ALLOCA(char, strlen(yytext) + 1);
591 		    args[narg++] = strcpy(arg, yytext);
592 		} else {
593 		    error("too many parameters in macro definition");
594 		    errcount++;
595 		    tk_skiptonl(FALSE);
596 		    break;
597 		}
598 		token = wsgettok();
599 		if (token == ')') {
600 		    break;
601 		}
602 		if (token == LF || token == EOF) {
603 		    error("unterminated macro definition");
604 		    errcount++;
605 		    break;
606 		}
607 		if (token != ',') {
608 		    error("unexpected token in macro parameter list");
609 		    errcount++;
610 		    tk_skiptonl(FALSE);
611 		    break;
612 		}
613 		token = wsgettok();
614 	    }
615 	    if (errcount > 0) {
616 		tk_setpp(FALSE);
617 		while (narg > 0) {
618 		    --narg;
619 		    AFREE(args[narg]);
620 		}
621 		return;
622 	    }
623 	}
624 	token = wsgettok();
625     } else {
626 	/* no parameter list */
627 	narg = -1;
628 	if (token == ' ') {
629 	    token = wsgettok();
630 	}
631     }
632 
633     s = pps_new(buf, sizeof(buf));
634     pps_ccat(s, HT);
635     seen_space = FALSE;
636 
637     /* scan replacement list */
638     while (token != LF && token != EOF) {
639 	if (token == ' ') {
640 	    seen_space = TRUE;
641 	} else {
642 	    if (token == HASH) {
643 		/* parameter must be made a string */
644 		token = wsgettok();
645 		i = argnum(args, narg, token);
646 		if (i >= 0) {
647 		    pps_scat(s, "\011\012");	/* HT LF */
648 		    pps_ccat(s, (i | MA_TAG | MA_STRING));
649 		    pps_ccat(s, HT);
650 		} else {
651 		    error("# must be followed by parameter");
652 		    errcount++;
653 		    tk_skiptonl(FALSE);
654 		    break;
655 		}
656 	    } else if (token == HASH_HASH) {
657 		/* concatenate previous token with next */
658 		if (s->len == 1) {
659 		    error("## at start of macro replacement list");
660 		    errcount++;
661 		    tk_skiptonl(FALSE);
662 		    break;
663 		}
664 		token = wsgettok();
665 		if (token == LF || token == EOF) {
666 		    error("## at end of macro replacement list");
667 		    errcount++;
668 		    break;
669 		}
670 		if (s->len >= 3 && buf[s->len - 3] == LF) {
671 		    /* previous token was a parameter: mark it "noexpand"
672 		       (this has no effect if it is a string) */
673 		    s->len--;
674 		    buf[s->len - 1] |= MA_NOEXPAND;
675 		}
676 		i = argnum(args, narg, token);
677 		if (i >= 0) {
678 		    pps_ccat(s, LF);
679 		    pps_ccat(s, (i | MA_TAG | MA_NOEXPAND));
680 		    pps_ccat(s, HT);
681 		} else {
682 		    pps_scat(s, yytext);
683 		}
684 	    } else {
685 		i = argnum(args, narg, token);
686 		if (i >= 0) {
687 		    pps_scat(s, "\011\012");	/* HT LF */
688 		    pps_ccat(s, (i | MA_TAG));
689 		    pps_ccat(s, HT);
690 		} else {
691 		    if (seen_space) {
692 			pps_ccat(s, ' ');
693 		    }
694 		    pps_scat(s, yytext);
695 		}
696 	    }
697 	    seen_space = FALSE;
698 	}
699 	token = tk_gettok();
700     }
701     pps_ccat(s, HT);
702     tk_setpp(FALSE);
703 
704     for (i = narg; i > 0; ) {
705 	--i;
706 	AFREE(args[i]);
707     }
708     i = s->len;
709     pps_del(s);
710 
711     if (errcount == 0) {
712 	if (i < 0) {
713 	    error("macro replacement list too large");
714 	} else if (special_replace(name) != (char *) NULL) {
715 	    error("#define of predefined macro");
716 	} else {
717 	    mc_define(name, buf, narg);
718 	}
719     }
720 }
721 
722 /*
723  * NAME:	pp->gettok()
724  * DESCRIPTION:	get a preprocessed token from the input stream, handling
725  *		preprocessor directives.
726  */
pp_gettok()727 int pp_gettok()
728 {
729     int token;
730     macro *mc;
731 
732     for (;;) {
733 	if (ifs->skipping) {
734 	    tk_setpp(TRUE);
735 	    token = wsgettok();
736 	    tk_setpp(FALSE);
737 	    if (token != '#' && token != LF && token != EOF) {
738 		tk_skiptonl(FALSE);
739 		continue;
740 	    }
741 	} else {
742 	    token = tk_gettok();
743 	}
744 	switch (token) {
745 	case EOF:
746 	    while (ifs->level > include_level) {
747 		error("missing #endif");
748 		pop();
749 	    }
750 	    if (include_level > 0) {
751 		--include_level;
752 		tk_endinclude();
753 		continue;
754 	    }
755 	    return token;
756 
757 	case ' ':
758 	case HT:
759 	case LF:
760 	    break;
761 
762 	case '!': case '%': case '&': case '(': case ')': case '*':
763 	case '+': case ',': case '-': case '/': case ':': case ';':
764 	case '<': case '=': case '>': case '?': case '[': case ']':
765 	case '^': case '{': case '|': case '}': case '~':
766 	case LARROW: case RARROW: case PLUS_PLUS: case MIN_MIN: case LSHIFT:
767 	case RSHIFT: case LE: case GE: case EQ: case NE: case LAND: case LOR:
768 	case PLUS_EQ: case MIN_EQ: case MULT_EQ: case DIV_EQ: case MOD_EQ:
769 	case LSHIFT_EQ: case RSHIFT_EQ: case AND_EQ: case XOR_EQ: case OR_EQ:
770 	case COLON_COLON: case DOT_DOT: case ELLIPSIS: case STRING_CONST:
771 	    /* legal operators and constants */
772 	    return token;
773 
774 	case INT_CONST:
775 	    /* integer constant */
776 	    yylval.number = yynumber;
777 	    return token;
778 
779 	case FLOAT_CONST:
780 	    /* floating point constant */
781 	    yylval.real = yyfloat;
782 	    return token;
783 
784 	case IDENTIFIER:
785 	    mc = mc_lookup(yytext);
786 	    if (mc != (macro *) NULL && tk_expand(mc) > 0) {
787 		break;
788 	    }
789 	    token = tokenz(yytext, yyleng);
790 	    if (token > 0) {
791 		return token + FIRST_KEYWORD - 1;
792 	    }
793 	    return IDENTIFIER;
794 
795 	case HASH:
796 	case HASH_HASH:
797 	    token = '#';
798 	    /* fall through */
799 	default:
800 	    error((token >= 32 && token < 127) ?
801 		   "illegal character: '%c'" : "illegal character: 0x%02x",
802 		  token);
803 	    break;
804 
805 	case '#':
806 	    /*
807 	     * Tk_gettok() returns HASH if a '#' is encountered in a macro
808 	     * expansion. So currently, no macro is being expanded.
809 	     */
810 	    token = wsgettok();
811 	    if (token == IDENTIFIER) {
812 		switch (pptokenz(yytext, strlen(yytext))) {
813 		case PP_IF:
814 		    push();
815 		    if (!ifs->active) {
816 			/* #if within unactive or skipped #if */
817 			tk_skiptonl(FALSE);
818 			break;
819 		    }
820 
821 		    /* get #if expression */
822 		    expr_unclr();
823 		    ifs->skipping = (eval_expr(0) == 0);
824 		    if (expr_get() != LF) {
825 			tk_skiptonl(TRUE);
826 		    }
827 		    break;
828 
829 		case PP_ELIF:
830 		    if (ifs == &top) {
831 			error("#elif without #if");
832 			tk_skiptonl(FALSE);
833 		    } else if (!ifs->expect_else) {
834 			error("#elif after #else");
835 			tk_skiptonl(FALSE);
836 		    } else if (!ifs->active || !ifs->skipping) {
837 			/* #elif within unactive/after non-skipped #if */
838 			ifs->active = FALSE;
839 			ifs->skipping = TRUE;
840 			tk_skiptonl(FALSE);
841 		    } else {
842 			/* get #elif expression */
843 			expr_unclr();
844 			ifs->skipping = (eval_expr(0) == 0);
845 			if (expr_get() != LF) {
846 			    tk_skiptonl(TRUE);
847 			}
848 		    }
849 		    break;
850 
851 		case PP_IFDEF:
852 		    push();
853 		    if (!ifs->active) {
854 			tk_skiptonl(FALSE);
855 			break;
856 		    }
857 		    token = wsgettok();
858 		    if (token == IDENTIFIER) {
859 			ifs->skipping =
860 			  (mc_lookup(yytext) == (macro *) NULL);
861 			tk_skiptonl(TRUE);
862 		    } else {
863 			unexpected(token, "identifier", "ifdef");
864 		    }
865 		    break;
866 
867 		case PP_IFNDEF:
868 		    push();
869 		    if (!ifs->active) {
870 			tk_skiptonl(FALSE);
871 			break;
872 		    }
873 		    token = wsgettok();
874 		    if (token == IDENTIFIER) {
875 			ifs->skipping =
876 			  (mc_lookup(yytext) != (macro *) NULL);
877 			tk_skiptonl(TRUE);
878 		    } else {
879 			unexpected(token, "identifier", "ifndef");
880 		    }
881 		    break;
882 
883 		case PP_ELSE:
884 		    if (ifs == &top) {
885 			error("#else without #if");
886 			tk_skiptonl(FALSE);
887 		    } else if (!ifs->expect_else) {
888 			error("#else after #else");
889 			tk_skiptonl(FALSE);
890 		    } else {
891 			ifs->expect_else = FALSE;
892 			ifs->skipping ^= ifs->active;
893 			tk_skiptonl(TRUE);
894 		    }
895 		    break;
896 
897 		case PP_ENDIF:
898 		    if (ifs->level <= include_level) {
899 			error("#endif without #if");
900 			tk_skiptonl(FALSE);
901 		    } else {
902 			pop();
903 			tk_skiptonl(TRUE);
904 		    }
905 		    break;
906 
907 		case PP_ERROR:
908 		    if (!ifs->skipping) {
909 			char buf[MAX_LINE_SIZE];
910 			str *s;
911 
912 			s = pps_new(buf, sizeof(buf));
913 			tk_setpp(TRUE);
914 			for (;;) {
915 			    token = mcgtok();
916 			    if (token == LF || token == EOF) {
917 				break;
918 			    }
919 			    if (token != HT) {
920 				pps_scat(s, yytext);
921 			    }
922 			}
923 			tk_setpp(FALSE);
924 			if (s->len == 0) {
925 			    error("#error directive");
926 			} else {
927 			    error("#error:%s", buf);
928 			}
929 			pps_del(s);
930 		    } else {
931 			tk_skiptonl(FALSE);
932 		    }
933 		    break;
934 
935 		case PP_LINE:
936 		    if (ifs->skipping) {
937 			tk_skiptonl(FALSE);
938 			break;
939 		    }
940 		    token = wsmcgtok();
941 		    if (token != INT_CONST) {
942 			unexpected(token, "number", "line");
943 			break;
944 		    }
945 		    tk_setline((unsigned short) yynumber - 1);
946 		    tk_header(TRUE);
947 		    token = wsmcgtok();
948 		    tk_header(FALSE);
949 		    if (token == STRING_CONST) {
950 			tk_setfilename(yytext);
951 			token = wsmcgtok();
952 		    }
953 		    if (token != LF) {
954 			unexpected(token, "number", "line");
955 			/* the "number" is fake, it's never used */
956 		    }
957 		    break;
958 
959 		case PP_INCLUDE:
960 		    if (ifs->skipping) {
961 			tk_skiptonl(FALSE);
962 			break;
963 		    }
964 		    do_include();
965 		    break;
966 
967 		case PP_DEFINE:
968 		    if (ifs->skipping) {
969 			tk_skiptonl(FALSE);
970 			break;
971 		    }
972 		    do_define();
973 		    break;
974 
975 		case PP_UNDEF:
976 		    if (ifs->skipping) {
977 			tk_skiptonl(FALSE);
978 			break;
979 		    }
980 		    token = wsgettok();
981 		    if (token == IDENTIFIER) {
982 			if (special_replace(yytext) != (char *) NULL) {
983 			    error("#undef of predefined macro");
984 			} else {
985 			    mc_undef(yytext);
986 			}
987 			tk_skiptonl(TRUE);
988 		    } else {
989 			unexpected(token, "identifier", "undef");
990 		    }
991 		    break;
992 
993 		case PP_PRAGMA:
994 		    /* no pragmas */
995 		    tk_skiptonl(FALSE);
996 		    break;
997 
998 		default:
999 		    error("undefined control");
1000 		    tk_skiptonl(FALSE);
1001 		    break;
1002 		}
1003 	    } else if (token != LF) {
1004 		error("undefined control");
1005 		tk_skiptonl(FALSE);
1006 	    }
1007 	    break;
1008 	}
1009     }
1010 }
1011