xref: /dragonfly/contrib/awk/lex.c (revision 65030a6a) 
1 /****************************************************************
2 Copyright (C) Lucent Technologies 1997
3 All Rights Reserved
4 
5 Permission to use, copy, modify, and distribute this software and
6 its documentation for any purpose and without fee is hereby
7 granted, provided that the above copyright notice appear in all
8 copies and that both that the copyright notice and this
9 permission notice and warranty disclaimer appear in supporting
10 documentation, and that the name Lucent Technologies or any of
11 its entities not be used in advertising or publicity pertaining
12 to distribution of the software without specific, written prior
13 permission.
14 
15 LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
16 INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
17 IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY
18 SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
20 IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
21 ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
22 THIS SOFTWARE.
23 ****************************************************************/
24 
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <ctype.h>
29 #include "awk.h"
30 #include "ytab.h"
31 
32 extern YYSTYPE	yylval;
33 extern bool	infunc;
34 
35 int	lineno	= 1;
36 int	bracecnt = 0;
37 int	brackcnt  = 0;
38 int	parencnt = 0;
39 
40 typedef struct Keyword {
41 	const char *word;
42 	int	sub;
43 	int	type;
44 } Keyword;
45 
46 const Keyword keywords[] = {	/* keep sorted: binary searched */
47 	{ "BEGIN",	XBEGIN,		XBEGIN },
48 	{ "END",	XEND,		XEND },
49 	{ "NF",		VARNF,		VARNF },
50 	{ "atan2",	FATAN,		BLTIN },
51 	{ "break",	BREAK,		BREAK },
52 	{ "close",	CLOSE,		CLOSE },
53 	{ "continue",	CONTINUE,	CONTINUE },
54 	{ "cos",	FCOS,		BLTIN },
55 	{ "delete",	DELETE,		DELETE },
56 	{ "do",		DO,		DO },
57 	{ "else",	ELSE,		ELSE },
58 	{ "exit",	EXIT,		EXIT },
59 	{ "exp",	FEXP,		BLTIN },
60 	{ "fflush",	FFLUSH,		BLTIN },
61 	{ "for",	FOR,		FOR },
62 	{ "func",	FUNC,		FUNC },
63 	{ "function",	FUNC,		FUNC },
64 	{ "getline",	GETLINE,	GETLINE },
65 	{ "gsub",	GSUB,		GSUB },
66 	{ "if",		IF,		IF },
67 	{ "in",		IN,		IN },
68 	{ "index",	INDEX,		INDEX },
69 	{ "int",	FINT,		BLTIN },
70 	{ "length",	FLENGTH,	BLTIN },
71 	{ "log",	FLOG,		BLTIN },
72 	{ "match",	MATCHFCN,	MATCHFCN },
73 	{ "next",	NEXT,		NEXT },
74 	{ "nextfile",	NEXTFILE,	NEXTFILE },
75 	{ "print",	PRINT,		PRINT },
76 	{ "printf",	PRINTF,		PRINTF },
77 	{ "rand",	FRAND,		BLTIN },
78 	{ "return",	RETURN,		RETURN },
79 	{ "sin",	FSIN,		BLTIN },
80 	{ "split",	SPLIT,		SPLIT },
81 	{ "sprintf",	SPRINTF,	SPRINTF },
82 	{ "sqrt",	FSQRT,		BLTIN },
83 	{ "srand",	FSRAND,		BLTIN },
84 	{ "sub",	SUB,		SUB },
85 	{ "substr",	SUBSTR,		SUBSTR },
86 	{ "system",	FSYSTEM,	BLTIN },
87 	{ "tolower",	FTOLOWER,	BLTIN },
88 	{ "toupper",	FTOUPPER,	BLTIN },
89 	{ "while",	WHILE,		WHILE },
90 };
91 
92 #define	RET(x)	{ if(dbg)printf("lex %s\n", tokname(x)); return(x); }
93 
94 static int peek(void)
95 {
96 	int c = input();
97 	unput(c);
98 	return c;
99 }
100 
101 static int gettok(char **pbuf, int *psz)	/* get next input token */
102 {
103 	int c, retc;
104 	char *buf = *pbuf;
105 	int sz = *psz;
106 	char *bp = buf;
107 
108 	c = input();
109 	if (c == 0)
110 		return 0;
111 	buf[0] = c;
112 	buf[1] = 0;
113 	if (!isalnum(c) && c != '.' && c != '_')
114 		return c;
115 
116 	*bp++ = c;
117 	if (isalpha(c) || c == '_') {	/* it's a varname */
118 		for ( ; (c = input()) != 0; ) {
119 			if (bp-buf >= sz)
120 				if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
121 					FATAL( "out of space for name %.10s...", buf );
122 			if (isalnum(c) || c == '_')
123 				*bp++ = c;
124 			else {
125 				*bp = 0;
126 				unput(c);
127 				break;
128 			}
129 		}
130 		*bp = 0;
131 		retc = 'a';	/* alphanumeric */
132 	} else {	/* maybe it's a number, but could be . */
133 		char *rem;
134 		/* read input until can't be a number */
135 		for ( ; (c = input()) != 0; ) {
136 			if (bp-buf >= sz)
137 				if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
138 					FATAL( "out of space for number %.10s...", buf );
139 			if (isdigit(c) || c == 'e' || c == 'E'
140 			  || c == '.' || c == '+' || c == '-')
141 				*bp++ = c;
142 			else {
143 				unput(c);
144 				break;
145 			}
146 		}
147 		*bp = 0;
148 		strtod(buf, &rem);	/* parse the number */
149 		if (rem == buf) {	/* it wasn't a valid number at all */
150 			buf[1] = 0;	/* return one character as token */
151 			retc = buf[0];	/* character is its own type */
152 			unputstr(rem+1); /* put rest back for later */
153 		} else {	/* some prefix was a number */
154 			unputstr(rem);	/* put rest back for later */
155 			rem[0] = 0;	/* truncate buf after number part */
156 			retc = '0';	/* type is number */
157 		}
158 	}
159 	*pbuf = buf;
160 	*psz = sz;
161 	return retc;
162 }
163 
164 int	word(char *);
165 int	string(void);
166 int	regexpr(void);
167 bool	sc	= false;	/* true => return a } right now */
168 bool	reg	= false;	/* true => return a REGEXPR now */
169 
170 int yylex(void)
171 {
172 	int c;
173 	static char *buf = NULL;
174 	static int bufsize = 5; /* BUG: setting this small causes core dump! */
175 
176 	if (buf == NULL && (buf = malloc(bufsize)) == NULL)
177 		FATAL( "out of space in yylex" );
178 	if (sc) {
179 		sc = false;
180 		RET('}');
181 	}
182 	if (reg) {
183 		reg = false;
184 		return regexpr();
185 	}
186 	for (;;) {
187 		c = gettok(&buf, &bufsize);
188 		if (c == 0)
189 			return 0;
190 		if (isalpha(c) || c == '_')
191 			return word(buf);
192 		if (isdigit(c)) {
193 			char *cp = tostring(buf);
194 			yylval.cp = setsymtab(buf, cp, atof(buf), CON|NUM, symtab);
195 			free(cp);
196 			/* should this also have STR set? */
197 			RET(NUMBER);
198 		}
199 
200 		yylval.i = c;
201 		switch (c) {
202 		case '\n':	/* {EOL} */
203 			lineno++;
204 			RET(NL);
205 		case '\r':	/* assume \n is coming */
206 		case ' ':	/* {WS}+ */
207 		case '\t':
208 			break;
209 		case '#':	/* #.* strip comments */
210 			while ((c = input()) != '\n' && c != 0)
211 				;
212 			unput(c);
213 			/*
214 			 * Next line is a hack, itcompensates for
215 			 * unput's treatment of \n.
216 			 */
217 			lineno++;
218 			break;
219 		case ';':
220 			RET(';');
221 		case '\\':
222 			if (peek() == '\n') {
223 				input();
224 				lineno++;
225 			} else if (peek() == '\r') {
226 				input(); input();	/* \n */
227 				lineno++;
228 			} else {
229 				RET(c);
230 			}
231 			break;
232 		case '&':
233 			if (peek() == '&') {
234 				input(); RET(AND);
235 			} else
236 				RET('&');
237 		case '|':
238 			if (peek() == '|') {
239 				input(); RET(BOR);
240 			} else
241 				RET('|');
242 		case '!':
243 			if (peek() == '=') {
244 				input(); yylval.i = NE; RET(NE);
245 			} else if (peek() == '~') {
246 				input(); yylval.i = NOTMATCH; RET(MATCHOP);
247 			} else
248 				RET(NOT);
249 		case '~':
250 			yylval.i = MATCH;
251 			RET(MATCHOP);
252 		case '<':
253 			if (peek() == '=') {
254 				input(); yylval.i = LE; RET(LE);
255 			} else {
256 				yylval.i = LT; RET(LT);
257 			}
258 		case '=':
259 			if (peek() == '=') {
260 				input(); yylval.i = EQ; RET(EQ);
261 			} else {
262 				yylval.i = ASSIGN; RET(ASGNOP);
263 			}
264 		case '>':
265 			if (peek() == '=') {
266 				input(); yylval.i = GE; RET(GE);
267 			} else if (peek() == '>') {
268 				input(); yylval.i = APPEND; RET(APPEND);
269 			} else {
270 				yylval.i = GT; RET(GT);
271 			}
272 		case '+':
273 			if (peek() == '+') {
274 				input(); yylval.i = INCR; RET(INCR);
275 			} else if (peek() == '=') {
276 				input(); yylval.i = ADDEQ; RET(ASGNOP);
277 			} else
278 				RET('+');
279 		case '-':
280 			if (peek() == '-') {
281 				input(); yylval.i = DECR; RET(DECR);
282 			} else if (peek() == '=') {
283 				input(); yylval.i = SUBEQ; RET(ASGNOP);
284 			} else
285 				RET('-');
286 		case '*':
287 			if (peek() == '=') {	/* *= */
288 				input(); yylval.i = MULTEQ; RET(ASGNOP);
289 			} else if (peek() == '*') {	/* ** or **= */
290 				input();	/* eat 2nd * */
291 				if (peek() == '=') {
292 					input(); yylval.i = POWEQ; RET(ASGNOP);
293 				} else {
294 					RET(POWER);
295 				}
296 			} else
297 				RET('*');
298 		case '/':
299 			RET('/');
300 		case '%':
301 			if (peek() == '=') {
302 				input(); yylval.i = MODEQ; RET(ASGNOP);
303 			} else
304 				RET('%');
305 		case '^':
306 			if (peek() == '=') {
307 				input(); yylval.i = POWEQ; RET(ASGNOP);
308 			} else
309 				RET(POWER);
310 
311 		case '$':
312 			/* BUG: awkward, if not wrong */
313 			c = gettok(&buf, &bufsize);
314 			if (isalpha(c)) {
315 				if (strcmp(buf, "NF") == 0) {	/* very special */
316 					unputstr("(NF)");
317 					RET(INDIRECT);
318 				}
319 				c = peek();
320 				if (c == '(' || c == '[' || (infunc && isarg(buf) >= 0)) {
321 					unputstr(buf);
322 					RET(INDIRECT);
323 				}
324 				yylval.cp = setsymtab(buf, "", 0.0, STR|NUM, symtab);
325 				RET(IVAR);
326 			} else if (c == 0) {	/*  */
327 				SYNTAX( "unexpected end of input after $" );
328 				RET(';');
329 			} else {
330 				unputstr(buf);
331 				RET(INDIRECT);
332 			}
333 
334 		case '}':
335 			if (--bracecnt < 0)
336 				SYNTAX( "extra }" );
337 			sc = true;
338 			RET(';');
339 		case ']':
340 			if (--brackcnt < 0)
341 				SYNTAX( "extra ]" );
342 			RET(']');
343 		case ')':
344 			if (--parencnt < 0)
345 				SYNTAX( "extra )" );
346 			RET(')');
347 		case '{':
348 			bracecnt++;
349 			RET('{');
350 		case '[':
351 			brackcnt++;
352 			RET('[');
353 		case '(':
354 			parencnt++;
355 			RET('(');
356 
357 		case '"':
358 			return string();	/* BUG: should be like tran.c ? */
359 
360 		default:
361 			RET(c);
362 		}
363 	}
364 }
365 
366 int string(void)
367 {
368 	int c, n;
369 	char *s, *bp;
370 	static char *buf = NULL;
371 	static int bufsz = 500;
372 
373 	if (buf == NULL && (buf = malloc(bufsz)) == NULL)
374 		FATAL("out of space for strings");
375 	for (bp = buf; (c = input()) != '"'; ) {
376 		if (!adjbuf(&buf, &bufsz, bp-buf+2, 500, &bp, "string"))
377 			FATAL("out of space for string %.10s...", buf);
378 		switch (c) {
379 		case '\n':
380 		case '\r':
381 		case 0:
382 			*bp = '\0';
383 			SYNTAX( "non-terminated string %.10s...", buf );
384 			if (c == 0)	/* hopeless */
385 				FATAL( "giving up" );
386 			lineno++;
387 			break;
388 		case '\\':
389 			c = input();
390 			switch (c) {
391 			case '\n': break;
392 			case '"': *bp++ = '"'; break;
393 			case 'n': *bp++ = '\n'; break;
394 			case 't': *bp++ = '\t'; break;
395 			case 'f': *bp++ = '\f'; break;
396 			case 'r': *bp++ = '\r'; break;
397 			case 'b': *bp++ = '\b'; break;
398 			case 'v': *bp++ = '\v'; break;
399 			case 'a': *bp++ = '\a'; break;
400 			case '\\': *bp++ = '\\'; break;
401 
402 			case '0': case '1': case '2': /* octal: \d \dd \ddd */
403 			case '3': case '4': case '5': case '6': case '7':
404 				n = c - '0';
405 				if ((c = peek()) >= '0' && c < '8') {
406 					n = 8 * n + input() - '0';
407 					if ((c = peek()) >= '0' && c < '8')
408 						n = 8 * n + input() - '0';
409 				}
410 				*bp++ = n;
411 				break;
412 
413 			case 'x':	/* hex  \x0-9a-fA-F + */
414 			    {	char xbuf[100], *px;
415 				for (px = xbuf; (c = input()) != 0 && px-xbuf < 100-2; ) {
416 					if (isdigit(c)
417 					 || (c >= 'a' && c <= 'f')
418 					 || (c >= 'A' && c <= 'F'))
419 						*px++ = c;
420 					else
421 						break;
422 				}
423 				*px = 0;
424 				unput(c);
425 	  			sscanf(xbuf, "%x", (unsigned int *) &n);
426 				*bp++ = n;
427 				break;
428 			    }
429 
430 			default:
431 				*bp++ = c;
432 				break;
433 			}
434 			break;
435 		default:
436 			*bp++ = c;
437 			break;
438 		}
439 	}
440 	*bp = 0;
441 	s = tostring(buf);
442 	*bp++ = ' '; *bp++ = '\0';
443 	yylval.cp = setsymtab(buf, s, 0.0, CON|STR|DONTFREE, symtab);
444 	free(s);
445 	RET(STRING);
446 }
447 
448 
449 static int binsearch(char *w, const Keyword *kp, int n)
450 {
451 	int cond, low, mid, high;
452 
453 	low = 0;
454 	high = n - 1;
455 	while (low <= high) {
456 		mid = (low + high) / 2;
457 		if ((cond = strcmp(w, kp[mid].word)) < 0)
458 			high = mid - 1;
459 		else if (cond > 0)
460 			low = mid + 1;
461 		else
462 			return mid;
463 	}
464 	return -1;
465 }
466 
467 int word(char *w)
468 {
469 	const Keyword *kp;
470 	int c, n;
471 
472 	n = binsearch(w, keywords, sizeof(keywords)/sizeof(keywords[0]));
473 	if (n != -1) {	/* found in table */
474 		kp = keywords + n;
475 		yylval.i = kp->sub;
476 		switch (kp->type) {	/* special handling */
477 		case BLTIN:
478 			if (kp->sub == FSYSTEM && safe)
479 				SYNTAX( "system is unsafe" );
480 			RET(kp->type);
481 		case FUNC:
482 			if (infunc)
483 				SYNTAX( "illegal nested function" );
484 			RET(kp->type);
485 		case RETURN:
486 			if (!infunc)
487 				SYNTAX( "return not in function" );
488 			RET(kp->type);
489 		case VARNF:
490 			yylval.cp = setsymtab("NF", "", 0.0, NUM, symtab);
491 			RET(VARNF);
492 		default:
493 			RET(kp->type);
494 		}
495 	}
496 	c = peek();	/* look for '(' */
497 	if (c != '(' && infunc && (n=isarg(w)) >= 0) {
498 		yylval.i = n;
499 		RET(ARG);
500 	} else {
501 		yylval.cp = setsymtab(w, "", 0.0, STR|NUM|DONTFREE, symtab);
502 		if (c == '(') {
503 			RET(CALL);
504 		} else {
505 			RET(VAR);
506 		}
507 	}
508 }
509 
510 void startreg(void)	/* next call to yylex will return a regular expression */
511 {
512 	reg = true;
513 }
514 
515 int regexpr(void)
516 {
517 	int c;
518 	static char *buf = NULL;
519 	static int bufsz = 500;
520 	char *bp;
521 
522 	if (buf == NULL && (buf = malloc(bufsz)) == NULL)
523 		FATAL("out of space for rex expr");
524 	bp = buf;
525 	for ( ; (c = input()) != '/' && c != 0; ) {
526 		if (!adjbuf(&buf, &bufsz, bp-buf+3, 500, &bp, "regexpr"))
527 			FATAL("out of space for reg expr %.10s...", buf);
528 		if (c == '\n') {
529 			*bp = '\0';
530 			SYNTAX( "newline in regular expression %.10s...", buf );
531 			unput('\n');
532 			break;
533 		} else if (c == '\\') {
534 			*bp++ = '\\';
535 			*bp++ = input();
536 		} else {
537 			*bp++ = c;
538 		}
539 	}
540 	*bp = 0;
541 	if (c == 0)
542 		SYNTAX("non-terminated regular expression %.10s...", buf);
543 	yylval.s = tostring(buf);
544 	unput('/');
545 	RET(REGEXPR);
546 }
547 
548 /* low-level lexical stuff, sort of inherited from lex */
549 
550 char	ebuf[300];
551 char	*ep = ebuf;
552 char	yysbuf[100];	/* pushback buffer */
553 char	*yysptr = yysbuf;
554 FILE	*yyin = NULL;
555 
556 int input(void)	/* get next lexical input character */
557 {
558 	int c;
559 	extern char *lexprog;
560 
561 	if (yysptr > yysbuf)
562 		c = (uschar)*--yysptr;
563 	else if (lexprog != NULL) {	/* awk '...' */
564 		if ((c = (uschar)*lexprog) != 0)
565 			lexprog++;
566 	} else				/* awk -f ... */
567 		c = pgetc();
568 	if (c == EOF)
569 		c = 0;
570 	if (ep >= ebuf + sizeof ebuf)
571 		ep = ebuf;
572 	*ep = c;
573 	if (c != 0) {
574 		ep++;
575 	}
576 	return (c);
577 }
578 
579 void unput(int c)	/* put lexical character back on input */
580 {
581 	if (c == '\n')
582 		lineno--;
583 	if (yysptr >= yysbuf + sizeof(yysbuf))
584 		FATAL("pushed back too much: %.20s...", yysbuf);
585 	*yysptr++ = c;
586 	if (--ep < ebuf)
587 		ep = ebuf + sizeof(ebuf) - 1;
588 }
589 
590 void unputstr(const char *s)	/* put a string back on input */
591 {
592 	int i;
593 
594 	for (i = strlen(s)-1; i >= 0; i--)
595 		unput(s[i]);
596 }
597