1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package token defines constants representing the lexical tokens of the Go 6// programming language and basic operations on tokens (printing, predicates). 7// 8package token 9 10import ( 11 "strconv" 12 "unicode" 13 "unicode/utf8" 14) 15 16// Token is the set of lexical tokens of the Go programming language. 17type Token int 18 19// The list of tokens. 20const ( 21 // Special tokens 22 ILLEGAL Token = iota 23 EOF 24 COMMENT 25 26 literal_beg 27 // Identifiers and basic type literals 28 // (these tokens stand for classes of literals) 29 IDENT // main 30 INT // 12345 31 FLOAT // 123.45 32 IMAG // 123.45i 33 CHAR // 'a' 34 STRING // "abc" 35 literal_end 36 37 operator_beg 38 // Operators and delimiters 39 ADD // + 40 SUB // - 41 MUL // * 42 QUO // / 43 REM // % 44 45 AND // & 46 OR // | 47 XOR // ^ 48 SHL // << 49 SHR // >> 50 AND_NOT // &^ 51 52 ADD_ASSIGN // += 53 SUB_ASSIGN // -= 54 MUL_ASSIGN // *= 55 QUO_ASSIGN // /= 56 REM_ASSIGN // %= 57 58 AND_ASSIGN // &= 59 OR_ASSIGN // |= 60 XOR_ASSIGN // ^= 61 SHL_ASSIGN // <<= 62 SHR_ASSIGN // >>= 63 AND_NOT_ASSIGN // &^= 64 65 LAND // && 66 LOR // || 67 ARROW // <- 68 INC // ++ 69 DEC // -- 70 71 EQL // == 72 LSS // < 73 GTR // > 74 ASSIGN // = 75 NOT // ! 76 77 NEQ // != 78 LEQ // <= 79 GEQ // >= 80 DEFINE // := 81 ELLIPSIS // ... 82 83 LPAREN // ( 84 LBRACK // [ 85 LBRACE // { 86 COMMA // , 87 PERIOD // . 88 89 RPAREN // ) 90 RBRACK // ] 91 RBRACE // } 92 SEMICOLON // ; 93 COLON // : 94 operator_end 95 96 keyword_beg 97 // Keywords 98 BREAK 99 CASE 100 CHAN 101 CONST 102 CONTINUE 103 104 DEFAULT 105 DEFER 106 ELSE 107 FALLTHROUGH 108 FOR 109 110 FUNC 111 GO 112 GOTO 113 IF 114 IMPORT 115 116 INTERFACE 117 MAP 118 PACKAGE 119 RANGE 120 RETURN 121 122 SELECT 123 STRUCT 124 SWITCH 125 TYPE 126 VAR 127 keyword_end 128) 129 130var tokens = [...]string{ 131 ILLEGAL: "ILLEGAL", 132 133 EOF: "EOF", 134 COMMENT: "COMMENT", 135 136 IDENT: "IDENT", 137 INT: "INT", 138 FLOAT: "FLOAT", 139 IMAG: "IMAG", 140 CHAR: "CHAR", 141 STRING: "STRING", 142 143 ADD: "+", 144 SUB: "-", 145 MUL: "*", 146 QUO: "/", 147 REM: "%", 148 149 AND: "&", 150 OR: "|", 151 XOR: "^", 152 SHL: "<<", 153 SHR: ">>", 154 AND_NOT: "&^", 155 156 ADD_ASSIGN: "+=", 157 SUB_ASSIGN: "-=", 158 MUL_ASSIGN: "*=", 159 QUO_ASSIGN: "/=", 160 REM_ASSIGN: "%=", 161 162 AND_ASSIGN: "&=", 163 OR_ASSIGN: "|=", 164 XOR_ASSIGN: "^=", 165 SHL_ASSIGN: "<<=", 166 SHR_ASSIGN: ">>=", 167 AND_NOT_ASSIGN: "&^=", 168 169 LAND: "&&", 170 LOR: "||", 171 ARROW: "<-", 172 INC: "++", 173 DEC: "--", 174 175 EQL: "==", 176 LSS: "<", 177 GTR: ">", 178 ASSIGN: "=", 179 NOT: "!", 180 181 NEQ: "!=", 182 LEQ: "<=", 183 GEQ: ">=", 184 DEFINE: ":=", 185 ELLIPSIS: "...", 186 187 LPAREN: "(", 188 LBRACK: "[", 189 LBRACE: "{", 190 COMMA: ",", 191 PERIOD: ".", 192 193 RPAREN: ")", 194 RBRACK: "]", 195 RBRACE: "}", 196 SEMICOLON: ";", 197 COLON: ":", 198 199 BREAK: "break", 200 CASE: "case", 201 CHAN: "chan", 202 CONST: "const", 203 CONTINUE: "continue", 204 205 DEFAULT: "default", 206 DEFER: "defer", 207 ELSE: "else", 208 FALLTHROUGH: "fallthrough", 209 FOR: "for", 210 211 FUNC: "func", 212 GO: "go", 213 GOTO: "goto", 214 IF: "if", 215 IMPORT: "import", 216 217 INTERFACE: "interface", 218 MAP: "map", 219 PACKAGE: "package", 220 RANGE: "range", 221 RETURN: "return", 222 223 SELECT: "select", 224 STRUCT: "struct", 225 SWITCH: "switch", 226 TYPE: "type", 227 VAR: "var", 228} 229 230// String returns the string corresponding to the token tok. 231// For operators, delimiters, and keywords the string is the actual 232// token character sequence (e.g., for the token ADD, the string is 233// "+"). For all other tokens the string corresponds to the token 234// constant name (e.g. for the token IDENT, the string is "IDENT"). 235// 236func (tok Token) String() string { 237 s := "" 238 if 0 <= tok && tok < Token(len(tokens)) { 239 s = tokens[tok] 240 } 241 if s == "" { 242 s = "token(" + strconv.Itoa(int(tok)) + ")" 243 } 244 return s 245} 246 247// A set of constants for precedence-based expression parsing. 248// Non-operators have lowest precedence, followed by operators 249// starting with precedence 1 up to unary operators. The highest 250// precedence serves as "catch-all" precedence for selector, 251// indexing, and other operator and delimiter tokens. 252// 253const ( 254 LowestPrec = 0 // non-operators 255 UnaryPrec = 6 256 HighestPrec = 7 257) 258 259// Precedence returns the operator precedence of the binary 260// operator op. If op is not a binary operator, the result 261// is LowestPrecedence. 262// 263func (op Token) Precedence() int { 264 switch op { 265 case LOR: 266 return 1 267 case LAND: 268 return 2 269 case EQL, NEQ, LSS, LEQ, GTR, GEQ: 270 return 3 271 case ADD, SUB, OR, XOR: 272 return 4 273 case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: 274 return 5 275 } 276 return LowestPrec 277} 278 279var keywords map[string]Token 280 281func init() { 282 keywords = make(map[string]Token) 283 for i := keyword_beg + 1; i < keyword_end; i++ { 284 keywords[tokens[i]] = i 285 } 286} 287 288// Lookup maps an identifier to its keyword token or IDENT (if not a keyword). 289// 290func Lookup(ident string) Token { 291 if tok, is_keyword := keywords[ident]; is_keyword { 292 return tok 293 } 294 return IDENT 295} 296 297// Predicates 298 299// IsLiteral returns true for tokens corresponding to identifiers 300// and basic type literals; it returns false otherwise. 301// 302func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } 303 304// IsOperator returns true for tokens corresponding to operators and 305// delimiters; it returns false otherwise. 306// 307func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end } 308 309// IsKeyword returns true for tokens corresponding to keywords; 310// it returns false otherwise. 311// 312func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } 313 314// IsExported reports whether name starts with an upper-case letter. 315// 316func IsExported(name string) bool { 317 ch, _ := utf8.DecodeRuneInString(name) 318 return unicode.IsUpper(ch) 319} 320 321// IsKeyword reports whether name is a Go keyword, such as "func" or "return". 322// 323func IsKeyword(name string) bool { 324 // TODO: opt: use a perfect hash function instead of a global map. 325 _, ok := keywords[name] 326 return ok 327} 328 329// IsIdentifier reports whether name is a Go identifier, that is, a non-empty 330// string made up of letters, digits, and underscores, where the first character 331// is not a digit. Keywords are not identifiers. 332// 333func IsIdentifier(name string) bool { 334 for i, c := range name { 335 if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) { 336 return false 337 } 338 } 339 return name != "" && !IsKeyword(name) 340} 341