1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Implement the Lexer for TableGen.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "TGLexer.h"
15 #include "llvm/ADT/StringSwitch.h"
16 #include "llvm/ADT/Twine.h"
17 #include "llvm/Config/config.h" // for strtoull()/strtoll() define
18 #include "llvm/Support/MemoryBuffer.h"
19 #include "llvm/Support/SourceMgr.h"
20 #include "llvm/TableGen/Error.h"
21 #include <cctype>
22 #include <cerrno>
23 #include <cstdio>
24 #include <cstdlib>
25 #include <cstring>
26 
27 using namespace llvm;
28 
TGLexer(SourceMgr & SM)29 TGLexer::TGLexer(SourceMgr &SM) : SrcMgr(SM) {
30   CurBuffer = SrcMgr.getMainFileID();
31   CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
32   CurPtr = CurBuf.begin();
33   TokStart = nullptr;
34 }
35 
getLoc() const36 SMLoc TGLexer::getLoc() const {
37   return SMLoc::getFromPointer(TokStart);
38 }
39 
40 /// ReturnError - Set the error to the specified string at the specified
41 /// location.  This is defined to always return tgtok::Error.
ReturnError(const char * Loc,const Twine & Msg)42 tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
43   PrintError(Loc, Msg);
44   return tgtok::Error;
45 }
46 
getNextChar()47 int TGLexer::getNextChar() {
48   char CurChar = *CurPtr++;
49   switch (CurChar) {
50   default:
51     return (unsigned char)CurChar;
52   case 0: {
53     // A nul character in the stream is either the end of the current buffer or
54     // a random nul in the file.  Disambiguate that here.
55     if (CurPtr-1 != CurBuf.end())
56       return 0;  // Just whitespace.
57 
58     // If this is the end of an included file, pop the parent file off the
59     // include stack.
60     SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
61     if (ParentIncludeLoc != SMLoc()) {
62       CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
63       CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
64       CurPtr = ParentIncludeLoc.getPointer();
65       return getNextChar();
66     }
67 
68     // Otherwise, return end of file.
69     --CurPtr;  // Another call to lex will return EOF again.
70     return EOF;
71   }
72   case '\n':
73   case '\r':
74     // Handle the newline character by ignoring it and incrementing the line
75     // count.  However, be careful about 'dos style' files with \n\r in them.
76     // Only treat a \n\r or \r\n as a single line.
77     if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
78         *CurPtr != CurChar)
79       ++CurPtr;  // Eat the two char newline sequence.
80     return '\n';
81   }
82 }
83 
peekNextChar(int Index)84 int TGLexer::peekNextChar(int Index) {
85   return *(CurPtr + Index);
86 }
87 
LexToken()88 tgtok::TokKind TGLexer::LexToken() {
89   TokStart = CurPtr;
90   // This always consumes at least one character.
91   int CurChar = getNextChar();
92 
93   switch (CurChar) {
94   default:
95     // Handle letters: [a-zA-Z_]
96     if (isalpha(CurChar) || CurChar == '_')
97       return LexIdentifier();
98 
99     // Unknown character, emit an error.
100     return ReturnError(TokStart, "Unexpected character");
101   case EOF: return tgtok::Eof;
102   case ':': return tgtok::colon;
103   case ';': return tgtok::semi;
104   case '.': return tgtok::period;
105   case ',': return tgtok::comma;
106   case '<': return tgtok::less;
107   case '>': return tgtok::greater;
108   case ']': return tgtok::r_square;
109   case '{': return tgtok::l_brace;
110   case '}': return tgtok::r_brace;
111   case '(': return tgtok::l_paren;
112   case ')': return tgtok::r_paren;
113   case '=': return tgtok::equal;
114   case '?': return tgtok::question;
115   case '#': return tgtok::paste;
116 
117   case 0:
118   case ' ':
119   case '\t':
120   case '\n':
121   case '\r':
122     // Ignore whitespace.
123     return LexToken();
124   case '/':
125     // If this is the start of a // comment, skip until the end of the line or
126     // the end of the buffer.
127     if (*CurPtr == '/')
128       SkipBCPLComment();
129     else if (*CurPtr == '*') {
130       if (SkipCComment())
131         return tgtok::Error;
132     } else // Otherwise, this is an error.
133       return ReturnError(TokStart, "Unexpected character");
134     return LexToken();
135   case '-': case '+':
136   case '0': case '1': case '2': case '3': case '4': case '5': case '6':
137   case '7': case '8': case '9': {
138     int NextChar = 0;
139     if (isdigit(CurChar)) {
140       // Allow identifiers to start with a number if it is followed by
141       // an identifier.  This can happen with paste operations like
142       // foo#8i.
143       int i = 0;
144       do {
145         NextChar = peekNextChar(i++);
146       } while (isdigit(NextChar));
147 
148       if (NextChar == 'x' || NextChar == 'b') {
149         // If this is [0-9]b[01] or [0-9]x[0-9A-fa-f] this is most
150         // likely a number.
151         int NextNextChar = peekNextChar(i);
152         switch (NextNextChar) {
153         default:
154           break;
155         case '0': case '1':
156           if (NextChar == 'b')
157             return LexNumber();
158           // Fallthrough
159         case '2': case '3': case '4': case '5':
160         case '6': case '7': case '8': case '9':
161         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
162         case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
163           if (NextChar == 'x')
164             return LexNumber();
165           break;
166         }
167       }
168     }
169 
170     if (isalpha(NextChar) || NextChar == '_')
171       return LexIdentifier();
172 
173     return LexNumber();
174   }
175   case '"': return LexString();
176   case '$': return LexVarName();
177   case '[': return LexBracket();
178   case '!': return LexExclaim();
179   }
180 }
181 
182 /// LexString - Lex "[^"]*"
LexString()183 tgtok::TokKind TGLexer::LexString() {
184   const char *StrStart = CurPtr;
185 
186   CurStrVal = "";
187 
188   while (*CurPtr != '"') {
189     // If we hit the end of the buffer, report an error.
190     if (*CurPtr == 0 && CurPtr == CurBuf.end())
191       return ReturnError(StrStart, "End of file in string literal");
192 
193     if (*CurPtr == '\n' || *CurPtr == '\r')
194       return ReturnError(StrStart, "End of line in string literal");
195 
196     if (*CurPtr != '\\') {
197       CurStrVal += *CurPtr++;
198       continue;
199     }
200 
201     ++CurPtr;
202 
203     switch (*CurPtr) {
204     case '\\': case '\'': case '"':
205       // These turn into their literal character.
206       CurStrVal += *CurPtr++;
207       break;
208     case 't':
209       CurStrVal += '\t';
210       ++CurPtr;
211       break;
212     case 'n':
213       CurStrVal += '\n';
214       ++CurPtr;
215       break;
216 
217     case '\n':
218     case '\r':
219       return ReturnError(CurPtr, "escaped newlines not supported in tblgen");
220 
221     // If we hit the end of the buffer, report an error.
222     case '\0':
223       if (CurPtr == CurBuf.end())
224         return ReturnError(StrStart, "End of file in string literal");
225       // FALL THROUGH
226     default:
227       return ReturnError(CurPtr, "invalid escape in string literal");
228     }
229   }
230 
231   ++CurPtr;
232   return tgtok::StrVal;
233 }
234 
LexVarName()235 tgtok::TokKind TGLexer::LexVarName() {
236   if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
237     return ReturnError(TokStart, "Invalid variable name");
238 
239   // Otherwise, we're ok, consume the rest of the characters.
240   const char *VarNameStart = CurPtr++;
241 
242   while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
243     ++CurPtr;
244 
245   CurStrVal.assign(VarNameStart, CurPtr);
246   return tgtok::VarName;
247 }
248 
249 
LexIdentifier()250 tgtok::TokKind TGLexer::LexIdentifier() {
251   // The first letter is [a-zA-Z_#].
252   const char *IdentStart = TokStart;
253 
254   // Match the rest of the identifier regex: [0-9a-zA-Z_#]*
255   while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
256     ++CurPtr;
257 
258   // Check to see if this identifier is a keyword.
259   StringRef Str(IdentStart, CurPtr-IdentStart);
260 
261   if (Str == "include") {
262     if (LexInclude()) return tgtok::Error;
263     return Lex();
264   }
265 
266   tgtok::TokKind Kind = StringSwitch<tgtok::TokKind>(Str)
267     .Case("int", tgtok::Int)
268     .Case("bit", tgtok::Bit)
269     .Case("bits", tgtok::Bits)
270     .Case("string", tgtok::String)
271     .Case("list", tgtok::List)
272     .Case("code", tgtok::Code)
273     .Case("dag", tgtok::Dag)
274     .Case("class", tgtok::Class)
275     .Case("def", tgtok::Def)
276     .Case("foreach", tgtok::Foreach)
277     .Case("defm", tgtok::Defm)
278     .Case("multiclass", tgtok::MultiClass)
279     .Case("field", tgtok::Field)
280     .Case("let", tgtok::Let)
281     .Case("in", tgtok::In)
282     .Default(tgtok::Id);
283 
284   if (Kind == tgtok::Id)
285     CurStrVal.assign(Str.begin(), Str.end());
286   return Kind;
287 }
288 
289 /// LexInclude - We just read the "include" token.  Get the string token that
290 /// comes next and enter the include.
LexInclude()291 bool TGLexer::LexInclude() {
292   // The token after the include must be a string.
293   tgtok::TokKind Tok = LexToken();
294   if (Tok == tgtok::Error) return true;
295   if (Tok != tgtok::StrVal) {
296     PrintError(getLoc(), "Expected filename after include");
297     return true;
298   }
299 
300   // Get the string.
301   std::string Filename = CurStrVal;
302   std::string IncludedFile;
303 
304 
305   CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
306                                     IncludedFile);
307   if (!CurBuffer) {
308     PrintError(getLoc(), "Could not find include file '" + Filename + "'");
309     return true;
310   }
311 
312   DependenciesMapTy::const_iterator Found = Dependencies.find(IncludedFile);
313   if (Found != Dependencies.end()) {
314     PrintError(getLoc(),
315                "File '" + IncludedFile + "' has already been included.");
316     SrcMgr.PrintMessage(Found->second, SourceMgr::DK_Note,
317                         "previously included here");
318     return true;
319   }
320   Dependencies.insert(std::make_pair(IncludedFile, getLoc()));
321   // Save the line number and lex buffer of the includer.
322   CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
323   CurPtr = CurBuf.begin();
324   return false;
325 }
326 
SkipBCPLComment()327 void TGLexer::SkipBCPLComment() {
328   ++CurPtr;  // skip the second slash.
329   while (1) {
330     switch (*CurPtr) {
331     case '\n':
332     case '\r':
333       return;  // Newline is end of comment.
334     case 0:
335       // If this is the end of the buffer, end the comment.
336       if (CurPtr == CurBuf.end())
337         return;
338       break;
339     }
340     // Otherwise, skip the character.
341     ++CurPtr;
342   }
343 }
344 
345 /// SkipCComment - This skips C-style /**/ comments.  The only difference from C
346 /// is that we allow nesting.
SkipCComment()347 bool TGLexer::SkipCComment() {
348   ++CurPtr;  // skip the star.
349   unsigned CommentDepth = 1;
350 
351   while (1) {
352     int CurChar = getNextChar();
353     switch (CurChar) {
354     case EOF:
355       PrintError(TokStart, "Unterminated comment!");
356       return true;
357     case '*':
358       // End of the comment?
359       if (CurPtr[0] != '/') break;
360 
361       ++CurPtr;   // End the */.
362       if (--CommentDepth == 0)
363         return false;
364       break;
365     case '/':
366       // Start of a nested comment?
367       if (CurPtr[0] != '*') break;
368       ++CurPtr;
369       ++CommentDepth;
370       break;
371     }
372   }
373 }
374 
375 /// LexNumber - Lex:
376 ///    [-+]?[0-9]+
377 ///    0x[0-9a-fA-F]+
378 ///    0b[01]+
LexNumber()379 tgtok::TokKind TGLexer::LexNumber() {
380   if (CurPtr[-1] == '0') {
381     if (CurPtr[0] == 'x') {
382       ++CurPtr;
383       const char *NumStart = CurPtr;
384       while (isxdigit(CurPtr[0]))
385         ++CurPtr;
386 
387       // Requires at least one hex digit.
388       if (CurPtr == NumStart)
389         return ReturnError(TokStart, "Invalid hexadecimal number");
390 
391       errno = 0;
392       CurIntVal = strtoll(NumStart, nullptr, 16);
393       if (errno == EINVAL)
394         return ReturnError(TokStart, "Invalid hexadecimal number");
395       if (errno == ERANGE) {
396         errno = 0;
397         CurIntVal = (int64_t)strtoull(NumStart, nullptr, 16);
398         if (errno == EINVAL)
399           return ReturnError(TokStart, "Invalid hexadecimal number");
400         if (errno == ERANGE)
401           return ReturnError(TokStart, "Hexadecimal number out of range");
402       }
403       return tgtok::IntVal;
404     } else if (CurPtr[0] == 'b') {
405       ++CurPtr;
406       const char *NumStart = CurPtr;
407       while (CurPtr[0] == '0' || CurPtr[0] == '1')
408         ++CurPtr;
409 
410       // Requires at least one binary digit.
411       if (CurPtr == NumStart)
412         return ReturnError(CurPtr-2, "Invalid binary number");
413       CurIntVal = strtoll(NumStart, nullptr, 2);
414       return tgtok::BinaryIntVal;
415     }
416   }
417 
418   // Check for a sign without a digit.
419   if (!isdigit(CurPtr[0])) {
420     if (CurPtr[-1] == '-')
421       return tgtok::minus;
422     else if (CurPtr[-1] == '+')
423       return tgtok::plus;
424   }
425 
426   while (isdigit(CurPtr[0]))
427     ++CurPtr;
428   CurIntVal = strtoll(TokStart, nullptr, 10);
429   return tgtok::IntVal;
430 }
431 
432 /// LexBracket - We just read '['.  If this is a code block, return it,
433 /// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
LexBracket()434 tgtok::TokKind TGLexer::LexBracket() {
435   if (CurPtr[0] != '{')
436     return tgtok::l_square;
437   ++CurPtr;
438   const char *CodeStart = CurPtr;
439   while (1) {
440     int Char = getNextChar();
441     if (Char == EOF) break;
442 
443     if (Char != '}') continue;
444 
445     Char = getNextChar();
446     if (Char == EOF) break;
447     if (Char == ']') {
448       CurStrVal.assign(CodeStart, CurPtr-2);
449       return tgtok::CodeFragment;
450     }
451   }
452 
453   return ReturnError(CodeStart-2, "Unterminated Code Block");
454 }
455 
456 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
LexExclaim()457 tgtok::TokKind TGLexer::LexExclaim() {
458   if (!isalpha(*CurPtr))
459     return ReturnError(CurPtr - 1, "Invalid \"!operator\"");
460 
461   const char *Start = CurPtr++;
462   while (isalpha(*CurPtr))
463     ++CurPtr;
464 
465   // Check to see which operator this is.
466   tgtok::TokKind Kind =
467     StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
468     .Case("eq", tgtok::XEq)
469     .Case("if", tgtok::XIf)
470     .Case("head", tgtok::XHead)
471     .Case("tail", tgtok::XTail)
472     .Case("con", tgtok::XConcat)
473     .Case("add", tgtok::XADD)
474     .Case("and", tgtok::XAND)
475     .Case("shl", tgtok::XSHL)
476     .Case("sra", tgtok::XSRA)
477     .Case("srl", tgtok::XSRL)
478     .Case("cast", tgtok::XCast)
479     .Case("empty", tgtok::XEmpty)
480     .Case("subst", tgtok::XSubst)
481     .Case("foreach", tgtok::XForEach)
482     .Case("listconcat", tgtok::XListConcat)
483     .Case("strconcat", tgtok::XStrConcat)
484     .Default(tgtok::Error);
485 
486   return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
487 }
488 
489