1/* 2 * Copyright (C) 2009 Google Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are 6 * met: 7 * 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above 11 * copyright notice, this list of conditions and the following disclaimer 12 * in the documentation and/or other materials provided with the 13 * distribution. 14 * * Neither the name of Google Inc. nor the names of its 15 * contributors may be used to endorse or promote products derived from 16 * this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31// Generate js file as follows: 32// 33// re2c -isc WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js \ 34// | sed 's|^yy\([^:]*\)*\:|case \1:|' \ 35// | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \ 36// | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \ 37// | sed 's|[*]cursor|this._charAt(cursor)|' \ 38// | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \ 39// | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \ 40// | sed 's|unsigned\ int|var|' \ 41// | sed 's|var\ yych|case 1: var yych|' 42 43WebInspector.SourceJavaScriptTokenizer = function() 44{ 45 WebInspector.SourceTokenizer.call(this); 46 47 this._keywords = [ 48 "null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for", 49 "instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if", 50 "this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger", 51 "class", "enum", "export", "extends", "import", "super", "get", "set", "with" 52 ].keySet(); 53 54 this._lexConditions = { 55 DIV: 0, 56 NODIV: 1, 57 COMMENT: 2, 58 DSTRING: 3, 59 SSTRING: 4, 60 REGEX: 5 61 }; 62 63 this.case_DIV = 1000; 64 this.case_NODIV = 1001; 65 this.case_COMMENT = 1002; 66 this.case_DSTRING = 1003; 67 this.case_SSTRING = 1004; 68 this.case_REGEX = 1005; 69 70 this.initialCondition = { lexCondition: this._lexConditions.NODIV } 71 this.condition = this.initialCondition; 72} 73 74WebInspector.SourceJavaScriptTokenizer.prototype = { 75 nextToken: function(cursor) 76 { 77 var cursorOnEnter = cursor; 78 var gotoCase = 1; 79 while (1) { 80 switch (gotoCase) 81 // Following comment is replaced with generated state machine. 82 /*!re2c 83 re2c:define:YYCTYPE = "var"; 84 re2c:define:YYCURSOR = cursor; 85 re2c:define:YYGETCONDITION = "this.getLexCondition"; 86 re2c:define:YYSETCONDITION = "this.setLexCondition"; 87 re2c:condprefix = "case this.case_"; 88 re2c:condenumprefix = "this._lexConditions."; 89 re2c:yyfill:enable = 0; 90 re2c:labelprefix = "case "; 91 re2c:indent:top = 2; 92 re2c:indent:string = " "; 93 94 LineComment = "//" [^\r\n]*; 95 CommentContent = ([^*\r\n] | ("*"+[^/*]))*; 96 Comment = "/*" CommentContent "*"+ "/"; 97 CommentStart = "/*" CommentContent [\r\n]; 98 CommentEnd = CommentContent "*"+ "/"; 99 100 DecimalDigit = [0-9]; 101 NonZeroDigit = [1-9]; 102 OctalDigit = [0-7]; 103 HexDigit = [0-9a-fA-F]; 104 SignedInteger = ("+"|"-")? DecimalDigit+; 105 ExponentPart = ("e" | "E") SignedInteger; 106 DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*; 107 DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?; 108 HexIntegerLiteral = "0" ("x"|"X") HexDigit+; 109 OctalIntegerLiteral = "0" OctalDigit+; 110 NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral; 111 112 Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<=" | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||"; 113 Division = "/" | "/="; 114 RightParen = ")"; 115 116 Letter = [a-zA-Z\x80-\xFF]; 117 UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit; 118 119 IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence; 120 IdentifierPart = IdentifierStart | DecimalDigit; 121 Identifier = IdentifierStart IdentifierPart *; 122 123 DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*; 124 SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*; 125 StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'"; 126 DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n]; 127 DoubleStringEnd = DoubleStringContent "\""; 128 SingleStringStart = "'" SingleStringContent "\\" [\r\n]; 129 SingleStringEnd = SingleStringContent "'"; 130 131 BackslashSequence = "\\" [^\r\n]; 132 RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]"; 133 RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet; 134 RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet; 135 RegexContent = RegexChar*; 136 Regex = "/" RegexFirstChar RegexContent "/" [igm]*; 137 RegexStart = "/" RegexFirstChar RegexContent "\\"; 138 RegexEnd = RegexContent "/" [igm]*; 139 140 <DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; } 141 <DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; } 142 <DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; } 143 <COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; } 144 <COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; } 145 146 <DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; } 147 <DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; } 148 <DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; } 149 <DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; } 150 <DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; } 151 <SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; } 152 <SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; } 153 154 <NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; } 155 <NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; } 156 <REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; } 157 <REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; } 158 159 <DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; } 160 <DIV,NODIV> Identifier => DIV 161 { 162 var token = this._line.substring(cursorOnEnter, cursor); 163 if (token in this._keywords) 164 this.tokenType = "javascript-keyword"; 165 else 166 this.tokenType = "javascript-ident"; 167 return cursor; 168 } 169 <DIV,NODIV> RightParen => DIV { this.tokenType = null; return cursor; } 170 <DIV,NODIV> Punctuation => NODIV { this.tokenType = null; return cursor; } 171 <DIV> Division => NODIV { this.tokenType = null; return cursor; } 172 <*> [^] { this.tokenType = null; return cursor; } 173 */ 174 } 175 } 176} 177 178WebInspector.SourceJavaScriptTokenizer.prototype.__proto__ = WebInspector.SourceTokenizer.prototype; 179