1// Copyright (c) 2019-2020 Alexander Medvednikov. All rights reserved. 2// Use of this source code is governed by an MIT license 3// that can be found in the LICENSE file. 4module token 5 6pub struct Token { 7pub: 8 kind Kind // the token number/enum; for quick comparisons 9 lit string // literal representation of the token 10 line_nr int // the line number in the source where the token occured 11 // name_idx int // name table index for O(1) lookup 12 pos int // the position of the token in scanner text 13 len int // length of the literal 14 tidx int // the index of the token 15} 16 17pub enum Kind { 18 unknown 19 eof 20 name // user 21 number // 123 22 string // 'foo' 23 str_inter // 'name=$user.name' 24 chartoken // `A` 25 plus 26 minus 27 mul 28 div 29 mod 30 xor // ^ 31 pipe // | 32 inc // ++ 33 dec // -- 34 and // && 35 logical_or 36 not 37 bit_not 38 question 39 comma 40 semicolon 41 colon 42 arrow // => 43 amp 44 hash 45 dollar 46 str_dollar 47 left_shift 48 right_shift 49 not_in // !in 50 not_is // !is 51 // at // @ 52 assign // = 53 decl_assign // := 54 plus_assign // += 55 minus_assign // -= 56 div_assign 57 mult_assign 58 xor_assign 59 mod_assign 60 or_assign 61 and_assign 62 right_shift_assign 63 left_shift_assign 64 // {} () [] 65 lcbr 66 rcbr 67 lpar 68 rpar 69 lsbr 70 rsbr 71 // == != <= < >= > 72 eq 73 ne 74 gt 75 lt 76 ge 77 le 78 comment 79 nl 80 dot 81 dotdot 82 ellipsis 83 // keywords 84 keyword_beg 85 key_as 86 key_asm 87 key_assert 88 key_atomic 89 key_break 90 key_const 91 key_continue 92 key_defer 93 key_else 94 key_embed 95 key_enum 96 key_false 97 key_for 98 key_fn 99 key_global 100 key_go 101 key_goto 102 key_if 103 key_import 104 key_in 105 key_interface 106 key_is 107 // key_it 108 key_match 109 key_module 110 key_mut 111 key_shared 112 key_lock 113 key_rlock 114 key_none 115 key_return 116 key_select 117 key_sizeof 118 key_likely 119 key_unlikely 120 key_offsetof 121 key_struct 122 key_true 123 key_type 124 key_typeof 125 key_orelse 126 key_union 127 key_pub 128 key_static 129 key_unsafe 130 keyword_end 131 _end_ 132} 133 134const ( 135 assign_tokens = [Kind.assign, .plus_assign, .minus_assign, .mult_assign, 136 .div_assign, .xor_assign, .mod_assign, .or_assign, .and_assign, 137 .right_shift_assign, .left_shift_assign] 138 nr_tokens = int(Kind._end_) 139) 140// build_keys genereates a map with keywords' string values: 141// Keywords['return'] == .key_return 142fn build_keys() map[string]Kind { 143 mut res := map[string]Kind 144 for t in int(Kind.keyword_beg) + 1 .. int(Kind.keyword_end) { 145 key := token_str[t] 146 res[key] = t 147 } 148 return res 149} 150 151// TODO remove once we have `enum Kind { name('name') if('if') ... }` 152fn build_token_str() []string { 153 mut s := []string{len:(nr_tokens)} 154 s[Kind.unknown] = 'unknown' 155 s[Kind.eof] = 'eof' 156 s[Kind.name] = 'name' 157 s[Kind.number] = 'number' 158 s[Kind.string] = 'string' 159 s[Kind.chartoken] = 'char' 160 s[Kind.plus] = '+' 161 s[Kind.minus] = '-' 162 s[Kind.mul] = '*' 163 s[Kind.div] = '/' 164 s[Kind.mod] = '%' 165 s[Kind.xor] = '^' 166 s[Kind.bit_not] = '~' 167 s[Kind.pipe] = '|' 168 s[Kind.hash] = '#' 169 s[Kind.amp] = '&' 170 s[Kind.inc] = '++' 171 s[Kind.dec] = '--' 172 s[Kind.and] = '&&' 173 s[Kind.logical_or] = '||' 174 s[Kind.not] = '!' 175 s[Kind.dot] = '.' 176 s[Kind.dotdot] = '..' 177 s[Kind.ellipsis] = '...' 178 s[Kind.comma] = ',' 179 s[Kind.not_in] = '!in' 180 s[Kind.not_is] = '!is' 181 // s[Kind.at] = '@' 182 s[Kind.semicolon] = ';' 183 s[Kind.colon] = ':' 184 s[Kind.arrow] = '=>' 185 s[Kind.assign] = '=' 186 s[Kind.decl_assign] = ':=' 187 s[Kind.plus_assign] = '+=' 188 s[Kind.minus_assign] = '-=' 189 s[Kind.mult_assign] = '*=' 190 s[Kind.div_assign] = '/=' 191 s[Kind.xor_assign] = '^=' 192 s[Kind.mod_assign] = '%=' 193 s[Kind.or_assign] = '|=' 194 s[Kind.and_assign] = '&=' 195 s[Kind.right_shift_assign] = '>>=' 196 s[Kind.left_shift_assign] = '<<=' 197 s[Kind.lcbr] = '{' 198 s[Kind.rcbr] = '}' 199 s[Kind.lpar] = '(' 200 s[Kind.rpar] = ')' 201 s[Kind.lsbr] = '[' 202 s[Kind.rsbr] = ']' 203 s[Kind.eq] = '==' 204 s[Kind.ne] = '!=' 205 s[Kind.gt] = '>' 206 s[Kind.lt] = '<' 207 s[Kind.ge] = '>=' 208 s[Kind.le] = '<=' 209 s[Kind.question] = '?' 210 s[Kind.left_shift] = '<<' 211 s[Kind.right_shift] = '>>' 212 s[Kind.comment] = '// comment' 213 s[Kind.nl] = 'NLL' 214 s[Kind.dollar] = '$' 215 s[Kind.str_dollar] = '$2' 216 s[Kind.key_assert] = 'assert' 217 s[Kind.key_struct] = 'struct' 218 s[Kind.key_if] = 'if' 219 // s[Kind.key_it] = 'it' 220 s[Kind.key_else] = 'else' 221 s[Kind.key_asm] = 'asm' 222 s[Kind.key_return] = 'return' 223 s[Kind.key_module] = 'module' 224 s[Kind.key_sizeof] = 'sizeof' 225 s[Kind.key_likely] = '_likely_' 226 s[Kind.key_unlikely] = '_unlikely_' 227 s[Kind.key_go] = 'go' 228 s[Kind.key_goto] = 'goto' 229 s[Kind.key_const] = 'const' 230 s[Kind.key_mut] = 'mut' 231 s[Kind.key_shared] = 'shared' 232 s[Kind.key_lock] = 'lock' 233 s[Kind.key_rlock] = 'rlock' 234 s[Kind.key_type] = 'type' 235 s[Kind.key_for] = 'for' 236 s[Kind.key_fn] = 'fn' 237 s[Kind.key_true] = 'true' 238 s[Kind.key_false] = 'false' 239 s[Kind.key_continue] = 'continue' 240 s[Kind.key_break] = 'break' 241 s[Kind.key_import] = 'import' 242 s[Kind.key_embed] = 'embed' 243 s[Kind.key_unsafe] = 'unsafe' 244 s[Kind.key_typeof] = 'typeof' 245 s[Kind.key_enum] = 'enum' 246 s[Kind.key_interface] = 'interface' 247 s[Kind.key_pub] = 'pub' 248 s[Kind.key_in] = 'in' 249 s[Kind.key_atomic] = 'atomic' 250 s[Kind.key_orelse] = 'or' 251 s[Kind.key_global] = '__global' 252 s[Kind.key_union] = 'union' 253 s[Kind.key_static] = 'static' 254 s[Kind.key_as] = 'as' 255 s[Kind.key_defer] = 'defer' 256 s[Kind.key_match] = 'match' 257 s[Kind.key_select] = 'select' 258 s[Kind.key_none] = 'none' 259 s[Kind.key_offsetof] = '__offsetof' 260 s[Kind.key_is] = 'is' 261 return s 262} 263 264const ( 265 token_str = build_token_str() 266 keywords = build_keys() 267) 268 269pub fn key_to_token(key string) Kind { 270 a := Kind(keywords[key]) 271 return a 272} 273 274pub fn is_key(key string) bool { 275 return int(key_to_token(key)) > 0 276} 277 278pub fn is_decl(t Kind) bool { 279 return t in [.key_enum, .key_interface, .key_fn, .key_struct, .key_type, .key_const, .key_pub, .eof] 280} 281 282pub fn (t Kind) is_assign() bool { 283 return t in assign_tokens 284} 285 286fn (t []Kind) contains(val Kind) bool { 287 for tt in t { 288 if tt == val { 289 return true 290 } 291 } 292 return false 293} 294 295pub fn (t Kind) str() string { 296 return token_str[int(t)] 297} 298 299pub fn (t Token) str() string { 300 return '$t.kind.str() "$t.lit"' 301} 302 303// Representation of highest and lowest precedence 304/* 305pub const ( 306 lowest_prec = 0 307 highest_prec = 8 308) 309*/ 310 311pub enum Precedence { 312 lowest 313 cond // OR or AND 314 in_as 315 assign // = 316 eq // == or != 317 // less_greater // > or < 318 sum // + - | ^ 319 product // * / << >> & 320 // mod // % 321 prefix // -X or !X 322 postfix // ++ or -- 323 call // func(X) or foo.method(X) 324 index // array[index], map[key] 325} 326 327pub fn build_precedences() []Precedence { 328 mut p := []Precedence{len:int(Kind._end_), cap:int(Kind._end_)} 329 330 p[Kind.lsbr] = .index 331 p[Kind.dot] = .call 332 // `++` | `--` 333 p[Kind.inc] = .postfix 334 p[Kind.dec] = .postfix 335 // `*` | `/` | `%` | `<<` | `>>` | `&` 336 p[Kind.mul] = .product 337 p[Kind.div] = .product 338 p[Kind.mod] = .product 339 p[Kind.left_shift] = .product 340 p[Kind.right_shift] = .product 341 p[Kind.amp] = .product 342 // `+` | `-` | `|` | `^` 343 p[Kind.plus] = .sum 344 p[Kind.minus] = .sum 345 p[Kind.pipe] = .sum 346 p[Kind.xor] = .sum 347 // `==` | `!=` | `<` | `<=` | `>` | `>=` 348 p[Kind.eq] = .eq 349 p[Kind.ne] = .eq 350 p[Kind.lt] = .eq 351 p[Kind.le] = .eq 352 p[Kind.gt] = .eq 353 p[Kind.ge] = .eq 354 // `=` | `+=` | ... 355 p[Kind.assign] = .assign 356 p[Kind.plus_assign] = .assign 357 p[Kind.minus_assign] = .assign 358 p[Kind.div_assign] = .assign 359 p[Kind.mod_assign] = .assign 360 p[Kind.or_assign] = .assign 361 p[Kind.and_assign] = .assign 362 // <<= | *= | ... 363 p[Kind.left_shift_assign] = .assign 364 p[Kind.right_shift_assign] = .assign 365 p[Kind.mult_assign] = .assign 366 p[Kind.xor_assign] = .assign 367 368 p[Kind.key_in] = .in_as 369 p[Kind.not_in] = .in_as 370 p[Kind.key_as] = .in_as 371 p[Kind.key_is] = .in_as 372 p[Kind.not_is] = .in_as 373 p[Kind.logical_or] = .cond 374 p[Kind.and] = .cond 375 376 return p 377} 378 379const ( 380 precedences = build_precedences() 381) 382 383// precedence returns a tokens precedence if defined, otherwise lowest_prec 384pub fn (tok Token) precedence() int { 385 return int(precedences[tok.kind]) 386} 387 388// is_scalar returns true if the token is a scalar 389pub fn (tok Token) is_scalar() bool { 390 return tok.kind in [.number, .string] 391} 392 393// is_unary returns true if the token can be in a unary expression 394pub fn (tok Token) is_unary() bool { 395 return tok.kind in [ 396 // `+` | `-` | `!` | `~` | `*` | `&` 397 .plus, .minus, .not, .bit_not, .mul, .amp] 398} 399 400pub fn (tok Kind) is_relational() bool { 401 return tok in [ 402 // `<` | `<=` | `>` | `>=` 403 .lt, .le, .gt, .ge, .eq, .ne] 404} 405 406pub fn (k Kind) is_start_of_type() bool { 407 return k in [.name, .lpar, .amp, .lsbr, .question] 408} 409 410pub fn (kind Kind) is_infix() bool { 411 return kind in [.plus, .minus, .mod, .mul, .div, .eq, .ne, .gt, .lt, .key_in, 412 // 413 .key_as, .ge, .le, .logical_or, .xor, .not_in, .key_is, .not_is, 414 // 415 .and, .dot, .pipe, .amp, .left_shift, .right_shift] 416} 417