1 /* 2 // 3 // Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 // 7 8 This file contains the Lex specification for GLSL ES preprocessor. 9 Based on Microsoft Visual Studio 2010 Preprocessor Grammar: 10 http://msdn.microsoft.com/en-us/library/2scxys89.aspx 11 12 IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. 13 */ 14 15 %top{ 16 // 17 // Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved. 18 // Use of this source code is governed by a BSD-style license that can be 19 // found in the LICENSE file. 20 // 21 22 // This file is auto-generated by generate_parser.sh. DO NOT EDIT! 23 } 24 25 %{ 26 #if defined(_MSC_VER) 27 #pragma warning(disable: 4005) 28 #endif 29 30 #include "compiler/preprocessor/Tokenizer.h" 31 32 #include "compiler/preprocessor/DiagnosticsBase.h" 33 #include "compiler/preprocessor/Token.h" 34 35 #if defined(__GNUC__) 36 // Triggered by the auto-generated yy_fatal_error function. 37 #pragma GCC diagnostic ignored "-Wmissing-noreturn" 38 #elif defined(_MSC_VER) 39 #pragma warning(disable: 4244) 40 #endif 41 42 // Workaround for flex using the register keyword, deprecated in C++11. 43 #ifdef __cplusplus 44 #if __cplusplus > 199711L 45 #define register 46 #endif 47 #endif 48 49 typedef std::string YYSTYPE; 50 typedef pp::SourceLocation YYLTYPE; 51 52 // Use the unused yycolumn variable to track file (string) number. 53 #define yyfileno yycolumn 54 55 #define YY_USER_INIT \ 56 do { \ 57 yyfileno = 0; \ 58 yylineno = 1; \ 59 yyextra->leadingSpace = false; \ 60 yyextra->lineStart = true; \ 61 } while(0); 62 63 #define YY_USER_ACTION \ 64 do \ 65 { \ 66 pp::Input* input = &yyextra->input; \ 67 pp::Input::Location* scanLoc = &yyextra->scanLoc; \ 68 while ((scanLoc->sIndex < input->count()) && \ 69 (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ 70 { \ 71 scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ 72 ++yyfileno; yylineno = 1; \ 73 } \ 74 yylloc->file = yyfileno; \ 75 yylloc->line = yylineno; \ 76 scanLoc->cIndex += yyleng; \ 77 } while(0); 78 79 #define YY_INPUT(buf, result, maxSize) \ 80 result = yyextra->input.read(buf, maxSize, &yylineno); 81 82 %} 83 84 %option noyywrap nounput never-interactive 85 %option reentrant bison-bridge bison-locations 86 %option prefix="pp" 87 %option extra-type="pp::Tokenizer::Context*" 88 %x COMMENT 89 90 NEWLINE \n|\r|\r\n 91 IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* 92 PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] 93 94 DECIMAL_CONSTANT [1-9][0-9]*[uU]? 95 OCTAL_CONSTANT 0[0-7]*[uU]? 96 HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+[uU]? 97 98 DIGIT [0-9] 99 EXPONENT_PART [eE][+-]?{DIGIT}+ 100 FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") 101 102 %% 103 104 /* Line comment */ 105 "//"[^\r\n]* 106 107 /* Block comment */ 108 /* Line breaks are just counted - not returned. */ 109 /* The comment is replaced by a single space. */ 110 "/*" { BEGIN(COMMENT); } 111 <COMMENT>[^*\r\n]+ 112 <COMMENT>"*" 113 <COMMENT>{NEWLINE} { 114 if (yylineno == INT_MAX) 115 { 116 *yylval = "Integer overflow on line number"; 117 return pp::Token::GOT_ERROR; 118 } 119 ++yylineno; 120 } 121 <COMMENT>"*/" { 122 yyextra->leadingSpace = true; 123 BEGIN(INITIAL); 124 } 125 126 # { 127 // # is only valid at start of line for preprocessor directives. 128 yylval->assign(1, yytext[0]); 129 return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; 130 } 131 132 {IDENTIFIER} { 133 yylval->assign(yytext, yyleng); 134 return pp::Token::IDENTIFIER; 135 } 136 137 ({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) { 138 yylval->assign(yytext, yyleng); 139 return pp::Token::CONST_INT; 140 } 141 142 ({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) { 143 yylval->assign(yytext, yyleng); 144 return pp::Token::CONST_FLOAT; 145 } 146 147 /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ 148 /* Rule to catch all invalid integers and floats. */ 149 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { 150 yylval->assign(yytext, yyleng); 151 return pp::Token::PP_NUMBER; 152 } 153 154 "++" { 155 yylval->assign(yytext, yyleng); 156 return pp::Token::OP_INC; 157 } 158 "--" { 159 yylval->assign(yytext, yyleng); 160 return pp::Token::OP_DEC; 161 } 162 "<<" { 163 yylval->assign(yytext, yyleng); 164 return pp::Token::OP_LEFT; 165 } 166 ">>" { 167 yylval->assign(yytext, yyleng); 168 return pp::Token::OP_RIGHT; 169 } 170 "<=" { 171 yylval->assign(yytext, yyleng); 172 return pp::Token::OP_LE; 173 } 174 ">=" { 175 yylval->assign(yytext, yyleng); 176 return pp::Token::OP_GE; 177 } 178 "==" { 179 yylval->assign(yytext, yyleng); 180 return pp::Token::OP_EQ; 181 } 182 "!=" { 183 yylval->assign(yytext, yyleng); 184 return pp::Token::OP_NE; 185 } 186 "&&" { 187 yylval->assign(yytext, yyleng); 188 return pp::Token::OP_AND; 189 } 190 "^^" { 191 yylval->assign(yytext, yyleng); 192 return pp::Token::OP_XOR; 193 } 194 "||" { 195 yylval->assign(yytext, yyleng); 196 return pp::Token::OP_OR; 197 } 198 "+=" { 199 yylval->assign(yytext, yyleng); 200 return pp::Token::OP_ADD_ASSIGN; 201 } 202 "-=" { 203 yylval->assign(yytext, yyleng); 204 return pp::Token::OP_SUB_ASSIGN; 205 } 206 "*=" { 207 yylval->assign(yytext, yyleng); 208 return pp::Token::OP_MUL_ASSIGN; 209 } 210 "/=" { 211 yylval->assign(yytext, yyleng); 212 return pp::Token::OP_DIV_ASSIGN; 213 } 214 "%=" { 215 yylval->assign(yytext, yyleng); 216 return pp::Token::OP_MOD_ASSIGN; 217 } 218 "<<=" { 219 yylval->assign(yytext, yyleng); 220 return pp::Token::OP_LEFT_ASSIGN; 221 } 222 ">>=" { 223 yylval->assign(yytext, yyleng); 224 return pp::Token::OP_RIGHT_ASSIGN; 225 } 226 "&=" { 227 yylval->assign(yytext, yyleng); 228 return pp::Token::OP_AND_ASSIGN; 229 } 230 "^=" { 231 yylval->assign(yytext, yyleng); 232 return pp::Token::OP_XOR_ASSIGN; 233 } 234 "|=" { 235 yylval->assign(yytext, yyleng); 236 return pp::Token::OP_OR_ASSIGN; 237 } 238 239 {PUNCTUATOR} { 240 yylval->assign(1, yytext[0]); 241 return yytext[0]; 242 } 243 244 [ \t\v\f]+ { yyextra->leadingSpace = true; } 245 246 {NEWLINE} { 247 if (yylineno == INT_MAX) 248 { 249 *yylval = "Integer overflow on line number"; 250 return pp::Token::GOT_ERROR; 251 } 252 ++yylineno; 253 yylval->assign(1, '\n'); 254 return '\n'; 255 } 256 257 . { 258 yylval->assign(1, yytext[0]); 259 return pp::Token::PP_OTHER; 260 } 261 262 <*><<EOF>> { 263 // YY_USER_ACTION is not invoked for handling EOF. 264 // Set the location for EOF token manually. 265 pp::Input* input = &yyextra->input; 266 pp::Input::Location* scanLoc = &yyextra->scanLoc; 267 yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0; 268 if (scanLoc->sIndex != sIndexMax) 269 { 270 // We can only reach here if there are empty strings at the 271 // end of the input. 272 scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; 273 // FIXME: this is not 64-bit clean. 274 yyfileno = static_cast<int>(sIndexMax); yylineno = 1; 275 } 276 yylloc->file = yyfileno; 277 yylloc->line = yylineno; 278 yylval->clear(); 279 280 // Line number overflows fake EOFs to exit early, check for this case. 281 if (yylineno == INT_MAX) { 282 yyextra->diagnostics->report(pp::Diagnostics::PP_TOKENIZER_ERROR, 283 pp::SourceLocation(yyfileno, yylineno), 284 "Integer overflow on line number"); 285 } 286 else if (YY_START == COMMENT) 287 { 288 yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT, 289 pp::SourceLocation(yyfileno, yylineno), 290 "EOF while in a comment"); 291 } 292 yyterminate(); 293 } 294 295 %% 296 297 namespace pp { 298 299 Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(256) 300 { 301 mContext.diagnostics = diagnostics; 302 } 303 304 Tokenizer::~Tokenizer() 305 { 306 destroyScanner(); 307 } 308 309 bool Tokenizer::init(size_t count, const char * const string[], const int length[]) 310 { 311 if ((count > 0) && (string == 0)) 312 return false; 313 314 mContext.input = Input(count, string, length); 315 return initScanner(); 316 } 317 318 void Tokenizer::setFileNumber(int file) 319 { 320 // We use column number as file number. 321 // See macro yyfileno. 322 yyset_column(file, mHandle); 323 } 324 325 void Tokenizer::setLineNumber(int line) 326 { 327 yyset_lineno(line, mHandle); 328 } 329 330 void Tokenizer::setMaxTokenSize(size_t maxTokenSize) 331 { 332 mMaxTokenSize = maxTokenSize; 333 } 334 335 void Tokenizer::lex(Token *token) 336 { 337 int tokenType = yylex(&token->text, &token->location, mHandle); 338 339 if (tokenType == Token::GOT_ERROR) 340 { 341 mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text); 342 token->type = Token::LAST; 343 } 344 else 345 { 346 token->type = tokenType; 347 } 348 349 if (token->text.size() > mMaxTokenSize) 350 { 351 mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, 352 token->location, token->text); 353 token->text.erase(mMaxTokenSize); 354 } 355 356 token->flags = 0; 357 358 token->setAtStartOfLine(mContext.lineStart); 359 mContext.lineStart = token->type == '\n'; 360 361 token->setHasLeadingSpace(mContext.leadingSpace); 362 mContext.leadingSpace = false; 363 } 364 365 bool Tokenizer::initScanner() 366 { 367 if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle)) 368 return false; 369 370 yyrestart(0, mHandle); 371 return true; 372 } 373 374 void Tokenizer::destroyScanner() 375 { 376 if (mHandle == nullptr) 377 return; 378 379 yylex_destroy(mHandle); 380 mHandle = nullptr; 381 } 382 383 } // namespace pp 384 385