xref: /freebsd/contrib/one-true-awk/lex.c (revision c1d255d3) 
1 /****************************************************************
2 Copyright (C) Lucent Technologies 1997
3 All Rights Reserved
4 
5 Permission to use, copy, modify, and distribute this software and
6 its documentation for any purpose and without fee is hereby
7 granted, provided that the above copyright notice appear in all
8 copies and that both that the copyright notice and this
9 permission notice and warranty disclaimer appear in supporting
10 documentation, and that the name Lucent Technologies or any of
11 its entities not be used in advertising or publicity pertaining
12 to distribution of the software without specific, written prior
13 permission.
14 
15 LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
16 INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
17 IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY
18 SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
20 IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
21 ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
22 THIS SOFTWARE.
23 ****************************************************************/
24 
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <ctype.h>
29 #include "awk.h"
30 #include "awkgram.tab.h"
31 
32 extern YYSTYPE	yylval;
33 extern bool	infunc;
34 
35 int	lineno	= 1;
36 int	bracecnt = 0;
37 int	brackcnt  = 0;
38 int	parencnt = 0;
39 
40 typedef struct Keyword {
41 	const char *word;
42 	int	sub;
43 	int	type;
44 } Keyword;
45 
46 const Keyword keywords[] = {	/* keep sorted: binary searched */
47 	{ "BEGIN",	XBEGIN,		XBEGIN },
48 	{ "END",	XEND,		XEND },
49 	{ "NF",		VARNF,		VARNF },
50 	{ "and",	FAND,		BLTIN },
51 	{ "atan2",	FATAN,		BLTIN },
52 	{ "break",	BREAK,		BREAK },
53 	{ "close",	CLOSE,		CLOSE },
54 	{ "compl",	FCOMPL,		BLTIN },
55 	{ "continue",	CONTINUE,	CONTINUE },
56 	{ "cos",	FCOS,		BLTIN },
57 	{ "delete",	DELETE,		DELETE },
58 	{ "do",		DO,		DO },
59 	{ "else",	ELSE,		ELSE },
60 	{ "exit",	EXIT,		EXIT },
61 	{ "exp",	FEXP,		BLTIN },
62 	{ "fflush",	FFLUSH,		BLTIN },
63 	{ "for",	FOR,		FOR },
64 	{ "func",	FUNC,		FUNC },
65 	{ "function",	FUNC,		FUNC },
66 	{ "gensub",	GENSUB,		GENSUB },
67 	{ "getline",	GETLINE,	GETLINE },
68 	{ "gsub",	GSUB,		GSUB },
69 	{ "if",		IF,		IF },
70 	{ "in",		IN,		IN },
71 	{ "index",	INDEX,		INDEX },
72 	{ "int",	FINT,		BLTIN },
73 	{ "length",	FLENGTH,	BLTIN },
74 	{ "log",	FLOG,		BLTIN },
75 	{ "lshift",	FLSHIFT,	BLTIN },
76 	{ "match",	MATCHFCN,	MATCHFCN },
77 	{ "next",	NEXT,		NEXT },
78 	{ "nextfile",	NEXTFILE,	NEXTFILE },
79 	{ "or",		FFOR,		BLTIN },
80 	{ "print",	PRINT,		PRINT },
81 	{ "printf",	PRINTF,		PRINTF },
82 	{ "rand",	FRAND,		BLTIN },
83 	{ "return",	RETURN,		RETURN },
84 	{ "rshift",	FRSHIFT,	BLTIN },
85 	{ "sin",	FSIN,		BLTIN },
86 	{ "split",	SPLIT,		SPLIT },
87 	{ "sprintf",	SPRINTF,	SPRINTF },
88 	{ "sqrt",	FSQRT,		BLTIN },
89 	{ "srand",	FSRAND,		BLTIN },
90 	{ "strftime",	FSTRFTIME,	BLTIN },
91 	{ "sub",	SUB,		SUB },
92 	{ "substr",	SUBSTR,		SUBSTR },
93 	{ "system",	FSYSTEM,	BLTIN },
94 	{ "systime",	FSYSTIME,	BLTIN },
95 	{ "tolower",	FTOLOWER,	BLTIN },
96 	{ "toupper",	FTOUPPER,	BLTIN },
97 	{ "while",	WHILE,		WHILE },
98 	{ "xor",	FXOR,		BLTIN },
99 };
100 
101 #define	RET(x)	{ if(dbg)printf("lex %s\n", tokname(x)); return(x); }
102 
103 static int peek(void)
104 {
105 	int c = input();
106 	unput(c);
107 	return c;
108 }
109 
110 static int gettok(char **pbuf, int *psz)	/* get next input token */
111 {
112 	int c, retc;
113 	char *buf = *pbuf;
114 	int sz = *psz;
115 	char *bp = buf;
116 
117 	c = input();
118 	if (c == 0)
119 		return 0;
120 	buf[0] = c;
121 	buf[1] = 0;
122 	if (!isalnum(c) && c != '.' && c != '_')
123 		return c;
124 
125 	*bp++ = c;
126 	if (isalpha(c) || c == '_') {	/* it's a varname */
127 		for ( ; (c = input()) != 0; ) {
128 			if (bp-buf >= sz)
129 				if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
130 					FATAL( "out of space for name %.10s...", buf );
131 			if (isalnum(c) || c == '_')
132 				*bp++ = c;
133 			else {
134 				*bp = 0;
135 				unput(c);
136 				break;
137 			}
138 		}
139 		*bp = 0;
140 		retc = 'a';	/* alphanumeric */
141 	} else {	/* maybe it's a number, but could be . */
142 		char *rem;
143 		/* read input until can't be a number */
144 		for ( ; (c = input()) != 0; ) {
145 			if (bp-buf >= sz)
146 				if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
147 					FATAL( "out of space for number %.10s...", buf );
148 			if (isdigit(c) || c == 'e' || c == 'E'
149 			  || c == '.' || c == '+' || c == '-')
150 				*bp++ = c;
151 			else {
152 				unput(c);
153 				break;
154 			}
155 		}
156 		*bp = 0;
157 		strtod(buf, &rem);	/* parse the number */
158 		if (rem == buf) {	/* it wasn't a valid number at all */
159 			buf[1] = 0;	/* return one character as token */
160 			retc = (uschar)buf[0];	/* character is its own type */
161 			unputstr(rem+1); /* put rest back for later */
162 		} else {	/* some prefix was a number */
163 			unputstr(rem);	/* put rest back for later */
164 			rem[0] = 0;	/* truncate buf after number part */
165 			retc = '0';	/* type is number */
166 		}
167 	}
168 	*pbuf = buf;
169 	*psz = sz;
170 	return retc;
171 }
172 
173 int	word(char *);
174 int	string(void);
175 int	regexpr(void);
176 bool	sc	= false;	/* true => return a } right now */
177 bool	reg	= false;	/* true => return a REGEXPR now */
178 
179 int yylex(void)
180 {
181 	int c;
182 	static char *buf = NULL;
183 	static int bufsize = 5; /* BUG: setting this small causes core dump! */
184 
185 	if (buf == NULL && (buf = (char *) malloc(bufsize)) == NULL)
186 		FATAL( "out of space in yylex" );
187 	if (sc) {
188 		sc = false;
189 		RET('}');
190 	}
191 	if (reg) {
192 		reg = false;
193 		return regexpr();
194 	}
195 	for (;;) {
196 		c = gettok(&buf, &bufsize);
197 		if (c == 0)
198 			return 0;
199 		if (isalpha(c) || c == '_')
200 			return word(buf);
201 		if (isdigit(c)) {
202 			char *cp = tostring(buf);
203 			double result;
204 
205 			if (is_number(cp, & result))
206 				yylval.cp = setsymtab(buf, cp, result, CON|NUM, symtab);
207 			else
208 				yylval.cp = setsymtab(buf, cp, 0.0, STR, symtab);
209 			free(cp);
210 			/* should this also have STR set? */
211 			RET(NUMBER);
212 		}
213 
214 		yylval.i = c;
215 		switch (c) {
216 		case '\n':	/* {EOL} */
217 			lineno++;
218 			RET(NL);
219 		case '\r':	/* assume \n is coming */
220 		case ' ':	/* {WS}+ */
221 		case '\t':
222 			break;
223 		case '#':	/* #.* strip comments */
224 			while ((c = input()) != '\n' && c != 0)
225 				;
226 			unput(c);
227 			/*
228 			 * Next line is a hack, itcompensates for
229 			 * unput's treatment of \n.
230 			 */
231 			lineno++;
232 			break;
233 		case ';':
234 			RET(';');
235 		case '\\':
236 			if (peek() == '\n') {
237 				input();
238 				lineno++;
239 			} else if (peek() == '\r') {
240 				input(); input();	/* \n */
241 				lineno++;
242 			} else {
243 				RET(c);
244 			}
245 			break;
246 		case '&':
247 			if (peek() == '&') {
248 				input(); RET(AND);
249 			} else
250 				RET('&');
251 		case '|':
252 			if (peek() == '|') {
253 				input(); RET(BOR);
254 			} else
255 				RET('|');
256 		case '!':
257 			if (peek() == '=') {
258 				input(); yylval.i = NE; RET(NE);
259 			} else if (peek() == '~') {
260 				input(); yylval.i = NOTMATCH; RET(MATCHOP);
261 			} else
262 				RET(NOT);
263 		case '~':
264 			yylval.i = MATCH;
265 			RET(MATCHOP);
266 		case '<':
267 			if (peek() == '=') {
268 				input(); yylval.i = LE; RET(LE);
269 			} else {
270 				yylval.i = LT; RET(LT);
271 			}
272 		case '=':
273 			if (peek() == '=') {
274 				input(); yylval.i = EQ; RET(EQ);
275 			} else {
276 				yylval.i = ASSIGN; RET(ASGNOP);
277 			}
278 		case '>':
279 			if (peek() == '=') {
280 				input(); yylval.i = GE; RET(GE);
281 			} else if (peek() == '>') {
282 				input(); yylval.i = APPEND; RET(APPEND);
283 			} else {
284 				yylval.i = GT; RET(GT);
285 			}
286 		case '+':
287 			if (peek() == '+') {
288 				input(); yylval.i = INCR; RET(INCR);
289 			} else if (peek() == '=') {
290 				input(); yylval.i = ADDEQ; RET(ASGNOP);
291 			} else
292 				RET('+');
293 		case '-':
294 			if (peek() == '-') {
295 				input(); yylval.i = DECR; RET(DECR);
296 			} else if (peek() == '=') {
297 				input(); yylval.i = SUBEQ; RET(ASGNOP);
298 			} else
299 				RET('-');
300 		case '*':
301 			if (peek() == '=') {	/* *= */
302 				input(); yylval.i = MULTEQ; RET(ASGNOP);
303 			} else if (peek() == '*') {	/* ** or **= */
304 				input();	/* eat 2nd * */
305 				if (peek() == '=') {
306 					input(); yylval.i = POWEQ; RET(ASGNOP);
307 				} else {
308 					RET(POWER);
309 				}
310 			} else
311 				RET('*');
312 		case '/':
313 			RET('/');
314 		case '%':
315 			if (peek() == '=') {
316 				input(); yylval.i = MODEQ; RET(ASGNOP);
317 			} else
318 				RET('%');
319 		case '^':
320 			if (peek() == '=') {
321 				input(); yylval.i = POWEQ; RET(ASGNOP);
322 			} else
323 				RET(POWER);
324 
325 		case '$':
326 			/* BUG: awkward, if not wrong */
327 			c = gettok(&buf, &bufsize);
328 			if (isalpha(c)) {
329 				if (strcmp(buf, "NF") == 0) {	/* very special */
330 					unputstr("(NF)");
331 					RET(INDIRECT);
332 				}
333 				c = peek();
334 				if (c == '(' || c == '[' || (infunc && isarg(buf) >= 0)) {
335 					unputstr(buf);
336 					RET(INDIRECT);
337 				}
338 				yylval.cp = setsymtab(buf, "", 0.0, STR|NUM, symtab);
339 				RET(IVAR);
340 			} else if (c == 0) {	/*  */
341 				SYNTAX( "unexpected end of input after $" );
342 				RET(';');
343 			} else {
344 				unputstr(buf);
345 				RET(INDIRECT);
346 			}
347 
348 		case '}':
349 			if (--bracecnt < 0)
350 				SYNTAX( "extra }" );
351 			sc = true;
352 			RET(';');
353 		case ']':
354 			if (--brackcnt < 0)
355 				SYNTAX( "extra ]" );
356 			RET(']');
357 		case ')':
358 			if (--parencnt < 0)
359 				SYNTAX( "extra )" );
360 			RET(')');
361 		case '{':
362 			bracecnt++;
363 			RET('{');
364 		case '[':
365 			brackcnt++;
366 			RET('[');
367 		case '(':
368 			parencnt++;
369 			RET('(');
370 
371 		case '"':
372 			return string();	/* BUG: should be like tran.c ? */
373 
374 		default:
375 			RET(c);
376 		}
377 	}
378 }
379 
380 int string(void)
381 {
382 	int c, n;
383 	char *s, *bp;
384 	static char *buf = NULL;
385 	static int bufsz = 500;
386 
387 	if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL)
388 		FATAL("out of space for strings");
389 	for (bp = buf; (c = input()) != '"'; ) {
390 		if (!adjbuf(&buf, &bufsz, bp-buf+2, 500, &bp, "string"))
391 			FATAL("out of space for string %.10s...", buf);
392 		switch (c) {
393 		case '\n':
394 		case '\r':
395 		case 0:
396 			*bp = '\0';
397 			SYNTAX( "non-terminated string %.10s...", buf );
398 			if (c == 0)	/* hopeless */
399 				FATAL( "giving up" );
400 			lineno++;
401 			break;
402 		case '\\':
403 			c = input();
404 			switch (c) {
405 			case '\n': break;
406 			case '"': *bp++ = '"'; break;
407 			case 'n': *bp++ = '\n'; break;
408 			case 't': *bp++ = '\t'; break;
409 			case 'f': *bp++ = '\f'; break;
410 			case 'r': *bp++ = '\r'; break;
411 			case 'b': *bp++ = '\b'; break;
412 			case 'v': *bp++ = '\v'; break;
413 			case 'a': *bp++ = '\a'; break;
414 			case '\\': *bp++ = '\\'; break;
415 
416 			case '0': case '1': case '2': /* octal: \d \dd \ddd */
417 			case '3': case '4': case '5': case '6': case '7':
418 				n = c - '0';
419 				if ((c = peek()) >= '0' && c < '8') {
420 					n = 8 * n + input() - '0';
421 					if ((c = peek()) >= '0' && c < '8')
422 						n = 8 * n + input() - '0';
423 				}
424 				*bp++ = n;
425 				break;
426 
427 			case 'x':	/* hex  \x0-9a-fA-F + */
428 			    {	char xbuf[100], *px;
429 				for (px = xbuf; (c = input()) != 0 && px-xbuf < 100-2; ) {
430 					if (isdigit(c)
431 					 || (c >= 'a' && c <= 'f')
432 					 || (c >= 'A' && c <= 'F'))
433 						*px++ = c;
434 					else
435 						break;
436 				}
437 				*px = 0;
438 				unput(c);
439 	  			sscanf(xbuf, "%x", (unsigned int *) &n);
440 				*bp++ = n;
441 				break;
442 			    }
443 
444 			default:
445 				*bp++ = c;
446 				break;
447 			}
448 			break;
449 		default:
450 			*bp++ = c;
451 			break;
452 		}
453 	}
454 	*bp = 0;
455 	s = tostring(buf);
456 	*bp++ = ' '; *bp++ = '\0';
457 	yylval.cp = setsymtab(buf, s, 0.0, CON|STR|DONTFREE, symtab);
458 	free(s);
459 	RET(STRING);
460 }
461 
462 
463 static int binsearch(char *w, const Keyword *kp, int n)
464 {
465 	int cond, low, mid, high;
466 
467 	low = 0;
468 	high = n - 1;
469 	while (low <= high) {
470 		mid = (low + high) / 2;
471 		if ((cond = strcmp(w, kp[mid].word)) < 0)
472 			high = mid - 1;
473 		else if (cond > 0)
474 			low = mid + 1;
475 		else
476 			return mid;
477 	}
478 	return -1;
479 }
480 
481 int word(char *w)
482 {
483 	const Keyword *kp;
484 	int c, n;
485 
486 	n = binsearch(w, keywords, sizeof(keywords)/sizeof(keywords[0]));
487 	if (n != -1) {	/* found in table */
488 		kp = keywords + n;
489 		yylval.i = kp->sub;
490 		switch (kp->type) {	/* special handling */
491 		case BLTIN:
492 			if (kp->sub == FSYSTEM && safe)
493 				SYNTAX( "system is unsafe" );
494 			RET(kp->type);
495 		case FUNC:
496 			if (infunc)
497 				SYNTAX( "illegal nested function" );
498 			RET(kp->type);
499 		case RETURN:
500 			if (!infunc)
501 				SYNTAX( "return not in function" );
502 			RET(kp->type);
503 		case VARNF:
504 			yylval.cp = setsymtab("NF", "", 0.0, NUM, symtab);
505 			RET(VARNF);
506 		default:
507 			RET(kp->type);
508 		}
509 	}
510 	c = peek();	/* look for '(' */
511 	if (c != '(' && infunc && (n=isarg(w)) >= 0) {
512 		yylval.i = n;
513 		RET(ARG);
514 	} else {
515 		yylval.cp = setsymtab(w, "", 0.0, STR|NUM|DONTFREE, symtab);
516 		if (c == '(') {
517 			RET(CALL);
518 		} else {
519 			RET(VAR);
520 		}
521 	}
522 }
523 
524 void startreg(void)	/* next call to yylex will return a regular expression */
525 {
526 	reg = true;
527 }
528 
529 int regexpr(void)
530 {
531 	int c;
532 	static char *buf = NULL;
533 	static int bufsz = 500;
534 	char *bp;
535 
536 	if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL)
537 		FATAL("out of space for rex expr");
538 	bp = buf;
539 	for ( ; (c = input()) != '/' && c != 0; ) {
540 		if (!adjbuf(&buf, &bufsz, bp-buf+3, 500, &bp, "regexpr"))
541 			FATAL("out of space for reg expr %.10s...", buf);
542 		if (c == '\n') {
543 			*bp = '\0';
544 			SYNTAX( "newline in regular expression %.10s...", buf );
545 			unput('\n');
546 			break;
547 		} else if (c == '\\') {
548 			*bp++ = '\\';
549 			*bp++ = input();
550 		} else {
551 			*bp++ = c;
552 		}
553 	}
554 	*bp = 0;
555 	if (c == 0)
556 		SYNTAX("non-terminated regular expression %.10s...", buf);
557 	yylval.s = tostring(buf);
558 	unput('/');
559 	RET(REGEXPR);
560 }
561 
562 /* low-level lexical stuff, sort of inherited from lex */
563 
564 char	ebuf[300];
565 char	*ep = ebuf;
566 char	yysbuf[100];	/* pushback buffer */
567 char	*yysptr = yysbuf;
568 FILE	*yyin = NULL;
569 
570 int input(void)	/* get next lexical input character */
571 {
572 	int c;
573 	extern char *lexprog;
574 
575 	if (yysptr > yysbuf)
576 		c = (uschar)*--yysptr;
577 	else if (lexprog != NULL) {	/* awk '...' */
578 		if ((c = (uschar)*lexprog) != 0)
579 			lexprog++;
580 	} else				/* awk -f ... */
581 		c = pgetc();
582 	if (c == EOF)
583 		c = 0;
584 	if (ep >= ebuf + sizeof ebuf)
585 		ep = ebuf;
586 	*ep = c;
587 	if (c != 0) {
588 		ep++;
589 	}
590 	return (c);
591 }
592 
593 void unput(int c)	/* put lexical character back on input */
594 {
595 	if (c == '\n')
596 		lineno--;
597 	if (yysptr >= yysbuf + sizeof(yysbuf))
598 		FATAL("pushed back too much: %.20s...", yysbuf);
599 	*yysptr++ = c;
600 	if (--ep < ebuf)
601 		ep = ebuf + sizeof(ebuf) - 1;
602 }
603 
604 void unputstr(const char *s)	/* put a string back on input */
605 {
606 	int i;
607 
608 	for (i = strlen(s)-1; i >= 0; i--)
609 		unput(s[i]);
610 }
611