1 /****************************************************************************\
2 Copyright (c) 2002, NVIDIA Corporation.
3 
4 NVIDIA Corporation("NVIDIA") supplies this software to you in
5 consideration of your agreement to the following terms, and your use,
6 installation, modification or redistribution of this NVIDIA software
7 constitutes acceptance of these terms.  If you do not agree with these
8 terms, please do not use, install, modify or redistribute this NVIDIA
9 software.
10 
11 In consideration of your agreement to abide by the following terms, and
12 subject to these terms, NVIDIA grants you a personal, non-exclusive
13 license, under NVIDIA's copyrights in this original NVIDIA software (the
14 "NVIDIA Software"), to use, reproduce, modify and redistribute the
15 NVIDIA Software, with or without modifications, in source and/or binary
16 forms; provided that if you redistribute the NVIDIA Software, you must
17 retain the copyright notice of NVIDIA, this notice and the following
18 text and disclaimers in all such redistributions of the NVIDIA Software.
19 Neither the name, trademarks, service marks nor logos of NVIDIA
20 Corporation may be used to endorse or promote products derived from the
21 NVIDIA Software without specific prior written permission from NVIDIA.
22 Except as expressly stated in this notice, no other rights or licenses
23 express or implied, are granted by NVIDIA herein, including but not
24 limited to any patent rights that may be infringed by your derivative
25 works or by other works in which the NVIDIA Software may be
26 incorporated. No hardware is licensed hereunder.
27 
28 THE NVIDIA SOFTWARE IS BEING PROVIDED ON AN "AS IS" BASIS, WITHOUT
29 WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED,
30 INCLUDING WITHOUT LIMITATION, WARRANTIES OR CONDITIONS OF TITLE,
31 NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
32 ITS USE AND OPERATION EITHER ALONE OR IN COMBINATION WITH OTHER
33 PRODUCTS.
34 
35 IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT,
36 INCIDENTAL, EXEMPLARY, CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
37 TO, LOST PROFITS; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
38 USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) OR ARISING IN ANY WAY
39 OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF THE
40 NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
41 TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
42 NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 \****************************************************************************/
44 //
45 // tokens.c
46 //
47 
48 #include <stdlib.h>
49 #include <stdio.h>
50 #include <string.h>
51 #include <ctype.h>
52 
53 #include "compiler/debug.h"
54 #include "compiler/preprocessor/slglobals.h"
55 #include "compiler/util.h"
56 
57 ///////////////////////////////////////////////////////////////////////////////////////////////
58 //////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
59 ///////////////////////////////////////////////////////////////////////////////////////////////
60 
61 /*
62  * idstr()
63  * Copy a string to a malloc'ed block and convert it into something suitable
64  * for an ID
65  *
66  */
67 
idstr(const char * fstr,MemoryPool * pool)68 static char *idstr(const char *fstr, MemoryPool *pool)
69 {
70     size_t len;
71     char *str, *t;
72     const char *f;
73 
74     len = strlen(fstr);
75     if (!pool)
76         str = (char *) malloc(len + 1);
77     else
78         str = (char *) mem_Alloc(pool, len + 1);
79 
80     for (f=fstr, t=str; *f; f++) {
81         if (isalnum(*f)) *t++ = *f;
82         else if (*f == '.' || *f == '/') *t++ = '_';
83     }
84     *t = 0;
85     return str;
86 } // idstr
87 
88 
89 /*
90  * lNewBlock()
91  *
92  */
93 
lNewBlock(TokenStream * fTok,MemoryPool * pool)94 static TokenBlock *lNewBlock(TokenStream *fTok, MemoryPool *pool)
95 {
96     TokenBlock *lBlock;
97 
98     if (!pool)
99         lBlock = (TokenBlock *) malloc(sizeof(TokenBlock) + 256);
100     else
101         lBlock = (TokenBlock *) mem_Alloc(pool, sizeof(TokenBlock) + 256);
102     lBlock->count = 0;
103     lBlock->current = 0;
104     lBlock->data = (unsigned char *) lBlock + sizeof(TokenBlock);
105     lBlock->max = 256;
106     lBlock->next = NULL;
107     if (fTok->head) {
108         fTok->current->next = lBlock;
109     } else {
110         fTok->head = lBlock;
111     }
112     fTok->current = lBlock;
113     return lBlock;
114 } // lNewBlock
115 
116 /*
117  * lAddByte()
118  *
119  */
120 
lAddByte(TokenStream * fTok,unsigned char fVal)121 static void lAddByte(TokenStream *fTok, unsigned char fVal)
122 {
123     TokenBlock *lBlock;
124     lBlock = fTok->current;
125     if (lBlock->count >= lBlock->max)
126         lBlock = lNewBlock(fTok, 0);
127     lBlock->data[lBlock->count++] = fVal;
128 } // lAddByte
129 
130 
131 
132 /*
133  * lReadByte() - Get the next byte from a stream.
134  *
135  */
136 
lReadByte(TokenStream * pTok)137 static int lReadByte(TokenStream *pTok)
138 {
139     TokenBlock *lBlock;
140     int lval = -1;
141 
142     lBlock = pTok->current;
143     if (lBlock) {
144         if (lBlock->current >= lBlock->count) {
145             lBlock = lBlock->next;
146             if (lBlock)
147                 lBlock->current = 0;
148             pTok->current = lBlock;
149         }
150         if (lBlock)
151             lval = lBlock->data[lBlock->current++];
152     }
153     return lval;
154 } // lReadByte
155 
156 /////////////////////////////////////// Global Functions://////////////////////////////////////
157 
158 /*
159  * NewTokenStream()
160  *
161  */
162 
NewTokenStream(const char * name,MemoryPool * pool)163 TokenStream *NewTokenStream(const char *name, MemoryPool *pool)
164 {
165     TokenStream *pTok;
166 
167     if (!pool)
168         pTok = (TokenStream *) malloc(sizeof(TokenStream));
169     else
170         pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
171     pTok->next = NULL;
172     pTok->name = idstr(name, pool);
173     pTok->head = NULL;
174     pTok->current = NULL;
175     lNewBlock(pTok, pool);
176     return pTok;
177 } // NewTokenStream
178 
179 /*
180  * DeleteTokenStream()
181  *
182  */
183 
DeleteTokenStream(TokenStream * pTok)184 void DeleteTokenStream(TokenStream *pTok)
185 {
186     TokenBlock *pBlock, *nBlock;
187 
188     if (pTok) {
189         pBlock = pTok->head;
190         while (pBlock) {
191             nBlock = pBlock->next;
192             free(pBlock);
193             pBlock = nBlock;
194         }
195         if (pTok->name)
196             free(pTok->name);
197         free(pTok);
198     }
199 } // DeleteTokenStream
200 
201 /*
202  * RecordToken() - Add a token to the end of a list for later playback or printout.
203  *
204  */
205 
RecordToken(TokenStream * pTok,int token,yystypepp * yylvalpp)206 void RecordToken(TokenStream *pTok, int token, yystypepp * yylvalpp)
207 {
208     const char *s;
209     char *str=NULL;
210 
211     if (token > 256)
212         lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
213     else
214         lAddByte(pTok, (unsigned char)(token & 0x7f));
215     switch (token) {
216     case CPP_IDENTIFIER:
217     case CPP_TYPEIDENTIFIER:
218     case CPP_STRCONSTANT:
219         s = GetAtomString(atable, yylvalpp->sc_ident);
220         while (*s)
221             lAddByte(pTok, (unsigned char) *s++);
222         lAddByte(pTok, 0);
223         break;
224     case CPP_FLOATCONSTANT:
225     case CPP_INTCONSTANT:
226          str=yylvalpp->symbol_name;
227          while (*str){
228             lAddByte(pTok, (unsigned char) *str++);
229          }
230          lAddByte(pTok, 0);
231          break;
232     case '(':
233         lAddByte(pTok, (unsigned char)(yylvalpp->sc_int ? 1 : 0));
234     default:
235         break;
236     }
237 } // RecordToken
238 
239 /*
240  * RewindTokenStream() - Reset a token stream in preperation for reading.
241  *
242  */
243 
RewindTokenStream(TokenStream * pTok)244 void RewindTokenStream(TokenStream *pTok)
245 {
246     if (pTok->head) {
247         pTok->current = pTok->head;
248         pTok->current->current = 0;
249     }
250 } // RewindTokenStream
251 
252 /*
253  * ReadToken() - Read the next token from a stream.
254  *
255  */
256 
ReadToken(TokenStream * pTok,yystypepp * yylvalpp)257 int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
258 {
259     char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
260     char string_val[MAX_STRING_LEN + 1];
261     int ltoken, len;
262     char ch;
263 
264     ltoken = lReadByte(pTok);
265     if (ltoken >= 0) {
266         if (ltoken > 127)
267             ltoken += 128;
268         switch (ltoken) {
269         case CPP_IDENTIFIER:
270         case CPP_TYPEIDENTIFIER:
271             len = 0;
272             ch = lReadByte(pTok);
273             while ((ch >= 'a' && ch <= 'z') ||
274                      (ch >= 'A' && ch <= 'Z') ||
275                      (ch >= '0' && ch <= '9') ||
276                      ch == '_')
277             {
278                 if (len < MAX_SYMBOL_NAME_LEN) {
279                     symbol_name[len++] = ch;
280                     ch = lReadByte(pTok);
281                 }
282             }
283             symbol_name[len] = '\0';
284             assert(ch == '\0');
285             yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
286             return CPP_IDENTIFIER;
287             break;
288         case CPP_STRCONSTANT:
289             len = 0;
290             while ((ch = lReadByte(pTok)) != 0)
291                 if (len < MAX_STRING_LEN)
292                     string_val[len++] = ch;
293             string_val[len] = '\0';
294             yylvalpp->sc_ident = LookUpAddString(atable, string_val);
295             break;
296         case CPP_FLOATCONSTANT:
297             len = 0;
298             ch = lReadByte(pTok);
299             while ((ch >= '0' && ch <= '9')||(ch=='e'||ch=='E'||ch=='.')||(ch=='+'||ch=='-'))
300             {
301                 if (len < MAX_SYMBOL_NAME_LEN) {
302                     symbol_name[len++] = ch;
303                     ch = lReadByte(pTok);
304                 }
305             }
306             symbol_name[len] = '\0';
307             assert(ch == '\0');
308             strcpy(yylvalpp->symbol_name,symbol_name);
309             yylvalpp->sc_fval=(float)atof_dot(yylvalpp->symbol_name);
310             break;
311         case CPP_INTCONSTANT:
312             len = 0;
313             ch = lReadByte(pTok);
314             while ((ch >= '0' && ch <= '9'))
315             {
316                 if (len < MAX_SYMBOL_NAME_LEN) {
317                     symbol_name[len++] = ch;
318                     ch = lReadByte(pTok);
319                 }
320             }
321             symbol_name[len] = '\0';
322             assert(ch == '\0');
323             strcpy(yylvalpp->symbol_name,symbol_name);
324             yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
325             break;
326         case '(':
327             yylvalpp->sc_int = lReadByte(pTok);
328             break;
329         }
330         return ltoken;
331     }
332     return EOF_SY;
333 } // ReadToken
334 
335 typedef struct TokenInputSrc {
336     InputSrc            base;
337     TokenStream         *tokens;
338     int                 (*final)(CPPStruct *);
339 } TokenInputSrc;
340 
scan_token(TokenInputSrc * in,yystypepp * yylvalpp)341 static int scan_token(TokenInputSrc *in, yystypepp * yylvalpp)
342 {
343     int token = ReadToken(in->tokens, yylvalpp);
344     int (*final)(CPPStruct *);
345     cpp->tokenLoc->file = cpp->currentInput->name;
346     cpp->tokenLoc->line = cpp->currentInput->line;
347     if (token == '\n') {
348         in->base.line++;
349         return token;
350     }
351     if (token > 0) return token;
352     cpp->currentInput = in->base.prev;
353     final = in->final;
354     free(in);
355     if (final && !final(cpp)) return -1;
356     return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
357 }
358 
ReadFromTokenStream(TokenStream * ts,int name,int (* final)(CPPStruct *))359 int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(CPPStruct *))
360 {
361     TokenInputSrc *in = malloc(sizeof(TokenInputSrc));
362     memset(in, 0, sizeof(TokenInputSrc));
363     in->base.name = name;
364     in->base.prev = cpp->currentInput;
365     in->base.scan = (int (*)(InputSrc *, yystypepp *))scan_token;
366     in->base.line = 1;
367     in->tokens = ts;
368     in->final = final;
369     RewindTokenStream(ts);
370     cpp->currentInput = &in->base;
371     return 1;
372 }
373 
374 typedef struct UngotToken {
375     InputSrc    base;
376     int         token;
377     yystypepp     lval;
378 } UngotToken;
379 
reget_token(UngotToken * t,yystypepp * yylvalpp)380 static int reget_token(UngotToken *t, yystypepp * yylvalpp)
381 {
382     int token = t->token;
383     *yylvalpp = t->lval;
384     cpp->currentInput = t->base.prev;
385     free(t);
386     return token;
387 }
388 
UngetToken(int token,yystypepp * yylvalpp)389 void UngetToken(int token, yystypepp * yylvalpp) {
390     UngotToken *t = malloc(sizeof(UngotToken));
391     memset(t, 0, sizeof(UngotToken));
392     t->token = token;
393     t->lval = *yylvalpp;
394     t->base.scan = (void *)reget_token;
395     t->base.prev = cpp->currentInput;
396     t->base.name = cpp->currentInput->name;
397     t->base.line = cpp->currentInput->line;
398     cpp->currentInput = &t->base;
399 }
400 
401 
DumpTokenStream(FILE * fp,TokenStream * s,yystypepp * yylvalpp)402 void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
403     int token;
404     char str[100];
405 
406     if (fp == 0) fp = stdout;
407     RewindTokenStream(s);
408     while ((token = ReadToken(s, yylvalpp)) > 0) {
409         switch (token) {
410         case CPP_IDENTIFIER:
411         case CPP_TYPEIDENTIFIER:
412             sprintf(str, "%s ", GetAtomString(atable, yylvalpp->sc_ident));
413             break;
414         case CPP_STRCONSTANT:
415             sprintf(str, "\"%s\"", GetAtomString(atable, yylvalpp->sc_ident));
416             break;
417         case CPP_FLOATCONSTANT:
418             //printf("%g9.6 ", yylvalpp->sc_fval);
419             break;
420         case CPP_INTCONSTANT:
421             //printf("%d ", yylvalpp->sc_int);
422             break;
423         default:
424             if (token >= 127)
425                 sprintf(str, "%s ", GetAtomString(atable, token));
426             else
427                 sprintf(str, "%c", token);
428             break;
429         }
430         CPPDebugLogMsg(str);
431     }
432 }
433 
434 ///////////////////////////////////////////////////////////////////////////////////////////////
435 /////////////////////////////////////// End of tokens.c ///////////////////////////////////////
436 ///////////////////////////////////////////////////////////////////////////////////////////////
437