1// Package js is an ECMAScript5.1 lexer following the specifications at http://www.ecma-international.org/ecma-262/5.1/.
2package js
3
4import (
5	"unicode"
6	"unicode/utf8"
7
8	"github.com/tdewolff/parse/v2"
9)
10
11var identifierStart = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Other_ID_Start}
12var identifierContinue = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Mn, unicode.Mc, unicode.Nd, unicode.Pc, unicode.Other_ID_Continue}
13
14// IsIdentifierStart returns true if the byte-slice start is the start of an identifier
15func IsIdentifierStart(b []byte) bool {
16	r, _ := utf8.DecodeRune(b)
17	return r == '$' || r == '\\' || r == '_' || unicode.IsOneOf(identifierStart, r)
18}
19
20// IsIdentifierContinue returns true if the byte-slice start is a continuation of an identifier
21func IsIdentifierContinue(b []byte) bool {
22	r, _ := utf8.DecodeRune(b)
23	return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
24}
25
26// IsIdentifierEnd returns true if the byte-slice end is a start or continuation of an identifier
27func IsIdentifierEnd(b []byte) bool {
28	r, _ := utf8.DecodeLastRune(b)
29	return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
30}
31
32////////////////////////////////////////////////////////////////
33
34// Lexer is the state for the lexer.
35type Lexer struct {
36	r                  *parse.Input
37	err                error
38	prevLineTerminator bool
39	prevNumericLiteral bool
40	level              int
41	templateLevels     []int
42}
43
44// NewLexer returns a new Lexer for a given io.Reader.
45func NewLexer(r *parse.Input) *Lexer {
46	return &Lexer{
47		r:                  r,
48		prevLineTerminator: true,
49		level:              0,
50		templateLevels:     []int{},
51	}
52}
53
54// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
55func (l *Lexer) Err() error {
56	if l.err != nil {
57		return l.err
58	}
59	return l.r.Err()
60}
61
62// RegExp reparses the input stream for a regular expression. It is assumed that we just received DivToken or DivEqToken with Next(). This function will go back and read that as a regular expression.
63func (l *Lexer) RegExp() (TokenType, []byte) {
64	if 0 < l.r.Offset() && l.r.Peek(-1) == '/' {
65		l.r.Move(-1)
66	} else if 1 < l.r.Offset() && l.r.Peek(-1) == '=' && l.r.Peek(-2) == '/' {
67		l.r.Move(-2)
68	} else {
69		l.err = parse.NewErrorLexer(l.r, "expected / or /=")
70		return ErrorToken, nil
71	}
72	l.r.Skip() // trick to set start = pos
73
74	if l.consumeRegExpToken() {
75		return RegExpToken, l.r.Shift()
76	}
77	l.err = parse.NewErrorLexer(l.r, "unexpected EOF or newline")
78	return ErrorToken, nil
79}
80
81// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
82func (l *Lexer) Next() (TokenType, []byte) {
83	prevLineTerminator := l.prevLineTerminator
84	l.prevLineTerminator = false
85
86	prevNumericLiteral := l.prevNumericLiteral
87	l.prevNumericLiteral = false
88
89	// study on 50x jQuery shows:
90	// spaces: 20k
91	// alpha: 16k
92	// newlines: 14.4k
93	// operators: 4k
94	// numbers and dot: 3.6k
95	// (): 3.4k
96	// {}: 1.8k
97	// []: 0.9k
98	// "': 1k
99	// semicolon: 2.4k
100	// colon: 0.8k
101	// comma: 2.4k
102	// slash: 1.4k
103	// `~: almost 0
104
105	c := l.r.Peek(0)
106	switch c {
107	case ' ', '\t', '\v', '\f':
108		l.r.Move(1)
109		for l.consumeWhitespace() {
110		}
111		l.prevLineTerminator = prevLineTerminator
112		return WhitespaceToken, l.r.Shift()
113	case '\n', '\r':
114		l.r.Move(1)
115		for l.consumeLineTerminator() {
116		}
117		l.prevLineTerminator = true
118		return LineTerminatorToken, l.r.Shift()
119	case '>', '=', '!', '+', '*', '%', '&', '|', '^', '~', '?':
120		if tt := l.consumeOperatorToken(); tt != ErrorToken {
121			return tt, l.r.Shift()
122		}
123	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
124		if tt := l.consumeNumericToken(); tt != ErrorToken || l.r.Pos() != 0 {
125			l.prevNumericLiteral = true
126			return tt, l.r.Shift()
127		} else if c == '.' {
128			l.r.Move(1)
129			if l.r.Peek(0) == '.' && l.r.Peek(1) == '.' {
130				l.r.Move(2)
131				return EllipsisToken, l.r.Shift()
132			}
133			return DotToken, l.r.Shift()
134		}
135	case ',':
136		l.r.Move(1)
137		return CommaToken, l.r.Shift()
138	case ';':
139		l.r.Move(1)
140		return SemicolonToken, l.r.Shift()
141	case '(':
142		l.level++
143		l.r.Move(1)
144		return OpenParenToken, l.r.Shift()
145	case ')':
146		l.level--
147		l.r.Move(1)
148		return CloseParenToken, l.r.Shift()
149	case '/':
150		if tt := l.consumeCommentToken(); tt != ErrorToken {
151			return tt, l.r.Shift()
152		} else if tt := l.consumeOperatorToken(); tt != ErrorToken {
153			return tt, l.r.Shift()
154		}
155	case '{':
156		l.level++
157		l.r.Move(1)
158		return OpenBraceToken, l.r.Shift()
159	case '}':
160		l.level--
161		if len(l.templateLevels) != 0 && l.level == l.templateLevels[len(l.templateLevels)-1] {
162			return l.consumeTemplateToken(), l.r.Shift()
163		}
164		l.r.Move(1)
165		return CloseBraceToken, l.r.Shift()
166	case ':':
167		l.r.Move(1)
168		return ColonToken, l.r.Shift()
169	case '\'', '"':
170		if l.consumeStringToken() {
171			return StringToken, l.r.Shift()
172		}
173	case ']':
174		l.r.Move(1)
175		return CloseBracketToken, l.r.Shift()
176	case '[':
177		l.r.Move(1)
178		return OpenBracketToken, l.r.Shift()
179	case '<', '-':
180		if l.consumeHTMLLikeCommentToken(prevLineTerminator) {
181			return CommentToken, l.r.Shift()
182		} else if tt := l.consumeOperatorToken(); tt != ErrorToken {
183			return tt, l.r.Shift()
184		}
185	case '`':
186		l.templateLevels = append(l.templateLevels, l.level)
187		return l.consumeTemplateToken(), l.r.Shift()
188	case '#':
189		l.r.Move(1)
190		if l.consumeIdentifierToken() {
191			return PrivateIdentifierToken, l.r.Shift()
192		}
193		return ErrorToken, nil
194	default:
195		if l.consumeIdentifierToken() {
196			if prevNumericLiteral {
197				l.err = parse.NewErrorLexer(l.r, "unexpected identifier after number")
198				return ErrorToken, nil
199			} else if keyword, ok := Keywords[string(l.r.Lexeme())]; ok {
200				return keyword, l.r.Shift()
201			}
202			return IdentifierToken, l.r.Shift()
203		}
204		if 0xC0 <= c {
205			if l.consumeWhitespace() {
206				for l.consumeWhitespace() {
207				}
208				l.prevLineTerminator = prevLineTerminator
209				return WhitespaceToken, l.r.Shift()
210			} else if l.consumeLineTerminator() {
211				for l.consumeLineTerminator() {
212				}
213				l.prevLineTerminator = true
214				return LineTerminatorToken, l.r.Shift()
215			}
216		} else if c == 0 && l.r.Err() != nil {
217			return ErrorToken, nil
218		}
219	}
220
221	r, _ := l.r.PeekRune(0)
222	l.err = parse.NewErrorLexer(l.r, "unexpected %s", parse.Printable(r))
223	return ErrorToken, l.r.Shift()
224}
225
226////////////////////////////////////////////////////////////////
227
228/*
229The following functions follow the specifications at http://www.ecma-international.org/ecma-262/5.1/
230*/
231
232func (l *Lexer) consumeWhitespace() bool {
233	c := l.r.Peek(0)
234	if c == ' ' || c == '\t' || c == '\v' || c == '\f' {
235		l.r.Move(1)
236		return true
237	} else if 0xC0 <= c {
238		if r, n := l.r.PeekRune(0); r == '\u00A0' || r == '\uFEFF' || unicode.Is(unicode.Zs, r) {
239			l.r.Move(n)
240			return true
241		}
242	}
243	return false
244}
245
246func (l *Lexer) isLineTerminator() bool {
247	c := l.r.Peek(0)
248	if c == '\n' || c == '\r' {
249		return true
250	} else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
251		return true
252	}
253	return false
254}
255
256func (l *Lexer) consumeLineTerminator() bool {
257	c := l.r.Peek(0)
258	if c == '\n' {
259		l.r.Move(1)
260		return true
261	} else if c == '\r' {
262		if l.r.Peek(1) == '\n' {
263			l.r.Move(2)
264		} else {
265			l.r.Move(1)
266		}
267		return true
268	} else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
269		l.r.Move(3)
270		return true
271	}
272	return false
273}
274
275func (l *Lexer) consumeDigit() bool {
276	if c := l.r.Peek(0); c >= '0' && c <= '9' {
277		l.r.Move(1)
278		return true
279	}
280	return false
281}
282
283func (l *Lexer) consumeHexDigit() bool {
284	if c := l.r.Peek(0); (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') {
285		l.r.Move(1)
286		return true
287	}
288	return false
289}
290
291func (l *Lexer) consumeBinaryDigit() bool {
292	if c := l.r.Peek(0); c == '0' || c == '1' {
293		l.r.Move(1)
294		return true
295	}
296	return false
297}
298
299func (l *Lexer) consumeOctalDigit() bool {
300	if c := l.r.Peek(0); c >= '0' && c <= '7' {
301		l.r.Move(1)
302		return true
303	}
304	return false
305}
306
307func (l *Lexer) consumeUnicodeEscape() bool {
308	if l.r.Peek(0) != '\\' || l.r.Peek(1) != 'u' {
309		return false
310	}
311	mark := l.r.Pos()
312	l.r.Move(2)
313	if c := l.r.Peek(0); c == '{' {
314		l.r.Move(1)
315		if l.consumeHexDigit() {
316			for l.consumeHexDigit() {
317			}
318			if c := l.r.Peek(0); c == '}' {
319				l.r.Move(1)
320				return true
321			}
322		}
323		l.r.Rewind(mark)
324		return false
325	} else if !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() {
326		l.r.Rewind(mark)
327		return false
328	}
329	return true
330}
331
332func (l *Lexer) consumeSingleLineComment() {
333	for {
334		c := l.r.Peek(0)
335		if c == '\r' || c == '\n' || c == 0 && l.r.Err() != nil {
336			break
337		} else if 0xC0 <= c {
338			if r, _ := l.r.PeekRune(0); r == '\u2028' || r == '\u2029' {
339				break
340			}
341		}
342		l.r.Move(1)
343	}
344}
345
346////////////////////////////////////////////////////////////////
347
348func (l *Lexer) consumeHTMLLikeCommentToken(prevLineTerminator bool) bool {
349	c := l.r.Peek(0)
350	if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
351		// opening HTML-style single line comment
352		l.r.Move(4)
353		l.consumeSingleLineComment()
354		return true
355	} else if prevLineTerminator && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
356		// closing HTML-style single line comment
357		// (only if current line didn't contain any meaningful tokens)
358		l.r.Move(3)
359		l.consumeSingleLineComment()
360		return true
361	}
362	return false
363}
364
365func (l *Lexer) consumeCommentToken() TokenType {
366	c := l.r.Peek(1)
367	if c == '/' {
368		// single line comment
369		l.r.Move(2)
370		l.consumeSingleLineComment()
371		return CommentToken
372	} else if c == '*' {
373		l.r.Move(2)
374		tt := CommentToken
375		for {
376			c := l.r.Peek(0)
377			if c == '*' && l.r.Peek(1) == '/' {
378				l.r.Move(2)
379				break
380			} else if c == 0 && l.r.Err() != nil {
381				break
382			} else if l.consumeLineTerminator() {
383				l.prevLineTerminator = true
384				tt = CommentLineTerminatorToken
385			} else {
386				l.r.Move(1)
387			}
388		}
389		return tt
390	}
391	return ErrorToken
392}
393
394var opTokens = map[byte]TokenType{
395	'=': EqToken,
396	'!': NotToken,
397	'<': LtToken,
398	'>': GtToken,
399	'+': AddToken,
400	'-': SubToken,
401	'*': MulToken,
402	'/': DivToken,
403	'%': ModToken,
404	'&': BitAndToken,
405	'|': BitOrToken,
406	'^': BitXorToken,
407	'~': BitNotToken,
408	'?': QuestionToken,
409}
410
411var opEqTokens = map[byte]TokenType{
412	'=': EqEqToken,
413	'!': NotEqToken,
414	'<': LtEqToken,
415	'>': GtEqToken,
416	'+': AddEqToken,
417	'-': SubEqToken,
418	'*': MulEqToken,
419	'/': DivEqToken,
420	'%': ModEqToken,
421	'&': BitAndEqToken,
422	'|': BitOrEqToken,
423	'^': BitXorEqToken,
424}
425
426var opOpTokens = map[byte]TokenType{
427	'<': LtLtToken,
428	'+': IncrToken,
429	'-': DecrToken,
430	'*': ExpToken,
431	'&': AndToken,
432	'|': OrToken,
433	'?': NullishToken,
434}
435
436var opOpEqTokens = map[byte]TokenType{
437	'<': LtLtEqToken,
438	'*': ExpEqToken,
439	'&': AndEqToken,
440	'|': OrEqToken,
441	'?': NullishEqToken,
442}
443
444func (l *Lexer) consumeOperatorToken() TokenType {
445	c := l.r.Peek(0)
446	l.r.Move(1)
447	if l.r.Peek(0) == '=' {
448		l.r.Move(1)
449		if l.r.Peek(0) == '=' && (c == '!' || c == '=') {
450			l.r.Move(1)
451			if c == '!' {
452				return NotEqEqToken
453			}
454			return EqEqEqToken
455		}
456		return opEqTokens[c]
457	} else if l.r.Peek(0) == c && (c == '+' || c == '-' || c == '*' || c == '&' || c == '|' || c == '?' || c == '<') {
458		l.r.Move(1)
459		if l.r.Peek(0) == '=' && c != '+' && c != '-' {
460			l.r.Move(1)
461			return opOpEqTokens[c]
462		}
463		return opOpTokens[c]
464	} else if c == '?' && l.r.Peek(0) == '.' && (l.r.Peek(1) < '0' || l.r.Peek(1) > '9') {
465		l.r.Move(1)
466		return OptChainToken
467	} else if c == '=' && l.r.Peek(0) == '>' {
468		l.r.Move(1)
469		return ArrowToken
470	} else if c == '>' && l.r.Peek(0) == '>' {
471		l.r.Move(1)
472		if l.r.Peek(0) == '>' {
473			l.r.Move(1)
474			if l.r.Peek(0) == '=' {
475				l.r.Move(1)
476				return GtGtGtEqToken
477			}
478			return GtGtGtToken
479		} else if l.r.Peek(0) == '=' {
480			l.r.Move(1)
481			return GtGtEqToken
482		}
483		return GtGtToken
484	}
485	return opTokens[c]
486}
487
488func (l *Lexer) consumeIdentifierToken() bool {
489	c := l.r.Peek(0)
490	if identifierStartTable[c] {
491		l.r.Move(1)
492	} else if 0xC0 <= c {
493		if r, n := l.r.PeekRune(0); unicode.IsOneOf(identifierStart, r) {
494			l.r.Move(n)
495		} else {
496			return false
497		}
498	} else if !l.consumeUnicodeEscape() {
499		return false
500	}
501	for {
502		c := l.r.Peek(0)
503		if identifierTable[c] {
504			l.r.Move(1)
505		} else if 0xC0 <= c {
506			if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
507				l.r.Move(n)
508			} else {
509				break
510			}
511		} else {
512			break
513		}
514	}
515	return true
516}
517
518func (l *Lexer) consumeNumericToken() TokenType {
519	// assume to be on 0 1 2 3 4 5 6 7 8 9 .
520	first := l.r.Peek(0)
521	if first == '0' {
522		l.r.Move(1)
523		if l.r.Peek(0) == 'x' || l.r.Peek(0) == 'X' {
524			l.r.Move(1)
525			if l.consumeHexDigit() {
526				for l.consumeHexDigit() {
527				}
528				return HexadecimalToken
529			}
530			l.err = parse.NewErrorLexer(l.r, "invalid hexadecimal number")
531			return ErrorToken
532		} else if l.r.Peek(0) == 'b' || l.r.Peek(0) == 'B' {
533			l.r.Move(1)
534			if l.consumeBinaryDigit() {
535				for l.consumeBinaryDigit() {
536				}
537				return BinaryToken
538			}
539			l.err = parse.NewErrorLexer(l.r, "invalid binary number")
540			return ErrorToken
541		} else if l.r.Peek(0) == 'o' || l.r.Peek(0) == 'O' {
542			l.r.Move(1)
543			if l.consumeOctalDigit() {
544				for l.consumeOctalDigit() {
545				}
546				return OctalToken
547			}
548			l.err = parse.NewErrorLexer(l.r, "invalid octal number")
549			return ErrorToken
550		} else if l.r.Peek(0) == 'n' {
551			l.r.Move(1)
552			return BigIntToken
553		} else if '0' <= l.r.Peek(0) && l.r.Peek(0) <= '9' {
554			l.err = parse.NewErrorLexer(l.r, "legacy octal numbers are not supported")
555			return ErrorToken
556		}
557	} else if first != '.' {
558		for l.consumeDigit() {
559		}
560	}
561	// we have parsed a 0 or an integer number
562	c := l.r.Peek(0)
563	if c == '.' {
564		l.r.Move(1)
565		if l.consumeDigit() {
566			for l.consumeDigit() {
567			}
568			c = l.r.Peek(0)
569		} else if first == '.' {
570			// number starts with a dot and must be followed by digits
571			l.r.Move(-1)
572			return ErrorToken // may be dot or ellipsis
573		} else {
574			c = l.r.Peek(0)
575		}
576	} else if c == 'n' {
577		l.r.Move(1)
578		return BigIntToken
579	}
580	if c == 'e' || c == 'E' {
581		l.r.Move(1)
582		c = l.r.Peek(0)
583		if c == '+' || c == '-' {
584			l.r.Move(1)
585		}
586		if !l.consumeDigit() {
587			l.err = parse.NewErrorLexer(l.r, "invalid number")
588			return ErrorToken
589		}
590		for l.consumeDigit() {
591		}
592	}
593	return DecimalToken
594}
595
596func (l *Lexer) consumeStringToken() bool {
597	// assume to be on ' or "
598	mark := l.r.Pos()
599	delim := l.r.Peek(0)
600	l.r.Move(1)
601	for {
602		c := l.r.Peek(0)
603		if c == delim {
604			l.r.Move(1)
605			break
606		} else if c == '\\' {
607			l.r.Move(1)
608			if !l.consumeLineTerminator() {
609				if c := l.r.Peek(0); c == delim || c == '\\' {
610					l.r.Move(1)
611				}
612			}
613			continue
614		} else if c == '\n' || c == '\r' || c == 0 && l.r.Err() != nil {
615			l.r.Rewind(mark)
616			return false
617		}
618		l.r.Move(1)
619	}
620	return true
621}
622
623func (l *Lexer) consumeRegExpToken() bool {
624	// assume to be on /
625	l.r.Move(1)
626	inClass := false
627	for {
628		c := l.r.Peek(0)
629		if !inClass && c == '/' {
630			l.r.Move(1)
631			break
632		} else if c == '[' {
633			inClass = true
634		} else if c == ']' {
635			inClass = false
636		} else if c == '\\' {
637			l.r.Move(1)
638			if l.isLineTerminator() || l.r.Peek(0) == 0 && l.r.Err() != nil {
639				return false
640			}
641		} else if l.isLineTerminator() || c == 0 && l.r.Err() != nil {
642			return false
643		}
644		l.r.Move(1)
645	}
646	// flags
647	for {
648		c := l.r.Peek(0)
649		if identifierTable[c] {
650			l.r.Move(1)
651		} else if 0xC0 <= c {
652			if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
653				l.r.Move(n)
654			} else {
655				break
656			}
657		} else {
658			break
659		}
660	}
661	return true
662}
663
664func (l *Lexer) consumeTemplateToken() TokenType {
665	// assume to be on ` or } when already within template
666	continuation := l.r.Peek(0) == '}'
667	l.r.Move(1)
668	for {
669		c := l.r.Peek(0)
670		if c == '`' {
671			l.templateLevels = l.templateLevels[:len(l.templateLevels)-1]
672			l.r.Move(1)
673			if continuation {
674				return TemplateEndToken
675			}
676			return TemplateToken
677		} else if c == '$' && l.r.Peek(1) == '{' {
678			l.level++
679			l.r.Move(2)
680			if continuation {
681				return TemplateMiddleToken
682			}
683			return TemplateStartToken
684		} else if c == '\\' {
685			l.r.Move(1)
686			if c := l.r.Peek(0); c != 0 {
687				l.r.Move(1)
688			}
689			continue
690		} else if c == 0 && l.r.Err() != nil {
691			if continuation {
692				return TemplateEndToken
693			}
694			return TemplateToken
695		}
696		l.r.Move(1)
697	}
698}
699
700var identifierStartTable = [256]bool{
701	// ASCII
702	false, false, false, false, false, false, false, false,
703	false, false, false, false, false, false, false, false,
704	false, false, false, false, false, false, false, false,
705	false, false, false, false, false, false, false, false,
706
707	false, false, false, false, true, false, false, false, // $
708	false, false, false, false, false, false, false, false,
709	false, false, false, false, false, false, false, false,
710	false, false, false, false, false, false, false, false,
711
712	false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
713	true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
714	true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
715	true, true, true, false, false, false, false, true, // X, Y, Z, _
716
717	false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
718	true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
719	true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
720	true, true, true, false, false, false, false, false, // x, y, z
721
722	// non-ASCII
723	false, false, false, false, false, false, false, false,
724	false, false, false, false, false, false, false, false,
725	false, false, false, false, false, false, false, false,
726	false, false, false, false, false, false, false, false,
727
728	false, false, false, false, false, false, false, false,
729	false, false, false, false, false, false, false, false,
730	false, false, false, false, false, false, false, false,
731	false, false, false, false, false, false, false, false,
732
733	false, false, false, false, false, false, false, false,
734	false, false, false, false, false, false, false, false,
735	false, false, false, false, false, false, false, false,
736	false, false, false, false, false, false, false, false,
737
738	false, false, false, false, false, false, false, false,
739	false, false, false, false, false, false, false, false,
740	false, false, false, false, false, false, false, false,
741	false, false, false, false, false, false, false, false,
742}
743
744var identifierTable = [256]bool{
745	// ASCII
746	false, false, false, false, false, false, false, false,
747	false, false, false, false, false, false, false, false,
748	false, false, false, false, false, false, false, false,
749	false, false, false, false, false, false, false, false,
750
751	false, false, false, false, true, false, false, false, // $
752	false, false, false, false, false, false, false, false,
753	true, true, true, true, true, true, true, true, // 0, 1, 2, 3, 4, 5, 6, 7
754	true, true, false, false, false, false, false, false, // 8, 9
755
756	false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
757	true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
758	true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
759	true, true, true, false, false, false, false, true, // X, Y, Z, _
760
761	false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
762	true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
763	true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
764	true, true, true, false, false, false, false, false, // x, y, z
765
766	// non-ASCII
767	false, false, false, false, false, false, false, false,
768	false, false, false, false, false, false, false, false,
769	false, false, false, false, false, false, false, false,
770	false, false, false, false, false, false, false, false,
771
772	false, false, false, false, false, false, false, false,
773	false, false, false, false, false, false, false, false,
774	false, false, false, false, false, false, false, false,
775	false, false, false, false, false, false, false, false,
776
777	false, false, false, false, false, false, false, false,
778	false, false, false, false, false, false, false, false,
779	false, false, false, false, false, false, false, false,
780	false, false, false, false, false, false, false, false,
781
782	false, false, false, false, false, false, false, false,
783	false, false, false, false, false, false, false, false,
784	false, false, false, false, false, false, false, false,
785	false, false, false, false, false, false, false, false,
786}
787