1 //#include <iostream> 2 #include "LexTokenStream.hpp" 3 #include "JavaTokenTypes.hpp" 4 #include "antlr/CommonToken.hpp" 5 6 extern "C" { 7 extern char* yytext; 8 extern char string_buf[]; // used for strings 9 extern int yylex(); 10 extern int yyrestart(std::FILE*); 11 } 12 13 static LexTokenStream* currentLexer=0; 14 15 extern "C" newLine()16 void newLine() 17 { 18 currentLexer->newLine(); 19 } 20 LexTokenStream(std::FILE * fp)21LexTokenStream::LexTokenStream(std::FILE* fp) 22 { 23 yyrestart(fp); 24 reachedEOF = false; 25 line = 1; 26 } 27 newLine()28void LexTokenStream::newLine() 29 { 30 ++line; 31 } 32 ANTLR_USE_NAMESPACE(antlr)33ANTLR_USE_NAMESPACE(antlr)RefToken LexTokenStream::nextToken() 34 { 35 ANTLR_USE_NAMESPACE(antlr)RefToken ret; 36 if (!reachedEOF) 37 { 38 int type; 39 40 currentLexer = this; 41 while ((type = yylex()) == -1) // Token::SKIP 42 ; 43 44 if (type != 0) 45 { 46 if( type != JavaTokenTypes::STRING_LITERAL ) 47 ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,yytext)); 48 else 49 ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,string_buf)); 50 51 // ANTLR_USE_NAMESPACE(std)cout << "Read token " << type << " [" << ret->getText() << "]" << ANTLR_USE_NAMESPACE(std)endl; 52 } 53 else 54 reachedEOF = true; 55 } 56 if (reachedEOF) 57 { 58 ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(1,"EOF")); 59 // cout << "EOF reached" << endl; 60 } 61 ret->setLine(line); 62 return ret; 63 } 64