1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Package token defines constants representing the lexical tokens of the Go
6// programming language and basic operations on tokens (printing, predicates).
7//
8package token
9
10import (
11	"strconv"
12	"unicode"
13	"unicode/utf8"
14)
15
16// Token is the set of lexical tokens of the Go programming language.
17type Token int
18
19// The list of tokens.
20const (
21	// Special tokens
22	ILLEGAL Token = iota
23	EOF
24	COMMENT
25
26	literal_beg
27	// Identifiers and basic type literals
28	// (these tokens stand for classes of literals)
29	IDENT  // main
30	INT    // 12345
31	FLOAT  // 123.45
32	IMAG   // 123.45i
33	CHAR   // 'a'
34	STRING // "abc"
35	literal_end
36
37	operator_beg
38	// Operators and delimiters
39	ADD // +
40	SUB // -
41	MUL // *
42	QUO // /
43	REM // %
44
45	AND     // &
46	OR      // |
47	XOR     // ^
48	SHL     // <<
49	SHR     // >>
50	AND_NOT // &^
51
52	ADD_ASSIGN // +=
53	SUB_ASSIGN // -=
54	MUL_ASSIGN // *=
55	QUO_ASSIGN // /=
56	REM_ASSIGN // %=
57
58	AND_ASSIGN     // &=
59	OR_ASSIGN      // |=
60	XOR_ASSIGN     // ^=
61	SHL_ASSIGN     // <<=
62	SHR_ASSIGN     // >>=
63	AND_NOT_ASSIGN // &^=
64
65	LAND  // &&
66	LOR   // ||
67	ARROW // <-
68	INC   // ++
69	DEC   // --
70
71	EQL    // ==
72	LSS    // <
73	GTR    // >
74	ASSIGN // =
75	NOT    // !
76
77	NEQ      // !=
78	LEQ      // <=
79	GEQ      // >=
80	DEFINE   // :=
81	ELLIPSIS // ...
82
83	LPAREN // (
84	LBRACK // [
85	LBRACE // {
86	COMMA  // ,
87	PERIOD // .
88
89	RPAREN    // )
90	RBRACK    // ]
91	RBRACE    // }
92	SEMICOLON // ;
93	COLON     // :
94	operator_end
95
96	keyword_beg
97	// Keywords
98	BREAK
99	CASE
100	CHAN
101	CONST
102	CONTINUE
103
104	DEFAULT
105	DEFER
106	ELSE
107	FALLTHROUGH
108	FOR
109
110	FUNC
111	GO
112	GOTO
113	IF
114	IMPORT
115
116	INTERFACE
117	MAP
118	PACKAGE
119	RANGE
120	RETURN
121
122	SELECT
123	STRUCT
124	SWITCH
125	TYPE
126	VAR
127	keyword_end
128)
129
130var tokens = [...]string{
131	ILLEGAL: "ILLEGAL",
132
133	EOF:     "EOF",
134	COMMENT: "COMMENT",
135
136	IDENT:  "IDENT",
137	INT:    "INT",
138	FLOAT:  "FLOAT",
139	IMAG:   "IMAG",
140	CHAR:   "CHAR",
141	STRING: "STRING",
142
143	ADD: "+",
144	SUB: "-",
145	MUL: "*",
146	QUO: "/",
147	REM: "%",
148
149	AND:     "&",
150	OR:      "|",
151	XOR:     "^",
152	SHL:     "<<",
153	SHR:     ">>",
154	AND_NOT: "&^",
155
156	ADD_ASSIGN: "+=",
157	SUB_ASSIGN: "-=",
158	MUL_ASSIGN: "*=",
159	QUO_ASSIGN: "/=",
160	REM_ASSIGN: "%=",
161
162	AND_ASSIGN:     "&=",
163	OR_ASSIGN:      "|=",
164	XOR_ASSIGN:     "^=",
165	SHL_ASSIGN:     "<<=",
166	SHR_ASSIGN:     ">>=",
167	AND_NOT_ASSIGN: "&^=",
168
169	LAND:  "&&",
170	LOR:   "||",
171	ARROW: "<-",
172	INC:   "++",
173	DEC:   "--",
174
175	EQL:    "==",
176	LSS:    "<",
177	GTR:    ">",
178	ASSIGN: "=",
179	NOT:    "!",
180
181	NEQ:      "!=",
182	LEQ:      "<=",
183	GEQ:      ">=",
184	DEFINE:   ":=",
185	ELLIPSIS: "...",
186
187	LPAREN: "(",
188	LBRACK: "[",
189	LBRACE: "{",
190	COMMA:  ",",
191	PERIOD: ".",
192
193	RPAREN:    ")",
194	RBRACK:    "]",
195	RBRACE:    "}",
196	SEMICOLON: ";",
197	COLON:     ":",
198
199	BREAK:    "break",
200	CASE:     "case",
201	CHAN:     "chan",
202	CONST:    "const",
203	CONTINUE: "continue",
204
205	DEFAULT:     "default",
206	DEFER:       "defer",
207	ELSE:        "else",
208	FALLTHROUGH: "fallthrough",
209	FOR:         "for",
210
211	FUNC:   "func",
212	GO:     "go",
213	GOTO:   "goto",
214	IF:     "if",
215	IMPORT: "import",
216
217	INTERFACE: "interface",
218	MAP:       "map",
219	PACKAGE:   "package",
220	RANGE:     "range",
221	RETURN:    "return",
222
223	SELECT: "select",
224	STRUCT: "struct",
225	SWITCH: "switch",
226	TYPE:   "type",
227	VAR:    "var",
228}
229
230// String returns the string corresponding to the token tok.
231// For operators, delimiters, and keywords the string is the actual
232// token character sequence (e.g., for the token ADD, the string is
233// "+"). For all other tokens the string corresponds to the token
234// constant name (e.g. for the token IDENT, the string is "IDENT").
235//
236func (tok Token) String() string {
237	s := ""
238	if 0 <= tok && tok < Token(len(tokens)) {
239		s = tokens[tok]
240	}
241	if s == "" {
242		s = "token(" + strconv.Itoa(int(tok)) + ")"
243	}
244	return s
245}
246
247// A set of constants for precedence-based expression parsing.
248// Non-operators have lowest precedence, followed by operators
249// starting with precedence 1 up to unary operators. The highest
250// precedence serves as "catch-all" precedence for selector,
251// indexing, and other operator and delimiter tokens.
252//
253const (
254	LowestPrec  = 0 // non-operators
255	UnaryPrec   = 6
256	HighestPrec = 7
257)
258
259// Precedence returns the operator precedence of the binary
260// operator op. If op is not a binary operator, the result
261// is LowestPrecedence.
262//
263func (op Token) Precedence() int {
264	switch op {
265	case LOR:
266		return 1
267	case LAND:
268		return 2
269	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
270		return 3
271	case ADD, SUB, OR, XOR:
272		return 4
273	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
274		return 5
275	}
276	return LowestPrec
277}
278
279var keywords map[string]Token
280
281func init() {
282	keywords = make(map[string]Token)
283	for i := keyword_beg + 1; i < keyword_end; i++ {
284		keywords[tokens[i]] = i
285	}
286}
287
288// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
289//
290func Lookup(ident string) Token {
291	if tok, is_keyword := keywords[ident]; is_keyword {
292		return tok
293	}
294	return IDENT
295}
296
297// Predicates
298
299// IsLiteral returns true for tokens corresponding to identifiers
300// and basic type literals; it returns false otherwise.
301//
302func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
303
304// IsOperator returns true for tokens corresponding to operators and
305// delimiters; it returns false otherwise.
306//
307func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
308
309// IsKeyword returns true for tokens corresponding to keywords;
310// it returns false otherwise.
311//
312func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
313
314// IsExported reports whether name starts with an upper-case letter.
315//
316func IsExported(name string) bool {
317	ch, _ := utf8.DecodeRuneInString(name)
318	return unicode.IsUpper(ch)
319}
320
321// IsKeyword reports whether name is a Go keyword, such as "func" or "return".
322//
323func IsKeyword(name string) bool {
324	// TODO: opt: use a perfect hash function instead of a global map.
325	_, ok := keywords[name]
326	return ok
327}
328
329// IsIdentifier reports whether name is a Go identifier, that is, a non-empty
330// string made up of letters, digits, and underscores, where the first character
331// is not a digit. Keywords are not identifiers.
332//
333func IsIdentifier(name string) bool {
334	for i, c := range name {
335		if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) {
336			return false
337		}
338	}
339	return name != "" && !IsKeyword(name)
340}
341