1// Copyright (c) 2019-2020 Alexander Medvednikov. All rights reserved.
2// Use of this source code is governed by an MIT license
3// that can be found in the LICENSE file.
4module token
5
6pub struct Token {
7pub:
8	kind    Kind // the token number/enum; for quick comparisons
9	lit     string // literal representation of the token
10	line_nr int // the line number in the source where the token occured
11	// name_idx int // name table index for O(1) lookup
12	pos     int // the position of the token in scanner text
13	len     int // length of the literal
14	tidx    int // the index of the token
15}
16
17pub enum Kind {
18	unknown
19	eof
20	name // user
21	number // 123
22	string // 'foo'
23	str_inter // 'name=$user.name'
24	chartoken // `A`
25	plus
26	minus
27	mul
28	div
29	mod
30	xor // ^
31	pipe // |
32	inc // ++
33	dec // --
34	and // &&
35	logical_or
36	not
37	bit_not
38	question
39	comma
40	semicolon
41	colon
42	arrow // =>
43	amp
44	hash
45	dollar
46	str_dollar
47	left_shift
48	right_shift
49	not_in // !in
50	not_is // !is
51	// at // @
52	assign // =
53	decl_assign // :=
54	plus_assign // +=
55	minus_assign // -=
56	div_assign
57	mult_assign
58	xor_assign
59	mod_assign
60	or_assign
61	and_assign
62	right_shift_assign
63	left_shift_assign
64	// {}  () []
65	lcbr
66	rcbr
67	lpar
68	rpar
69	lsbr
70	rsbr
71	// == != <= < >= >
72	eq
73	ne
74	gt
75	lt
76	ge
77	le
78	comment
79	nl
80	dot
81	dotdot
82	ellipsis
83	// keywords
84	keyword_beg
85	key_as
86	key_asm
87	key_assert
88	key_atomic
89	key_break
90	key_const
91	key_continue
92	key_defer
93	key_else
94	key_embed
95	key_enum
96	key_false
97	key_for
98	key_fn
99	key_global
100	key_go
101	key_goto
102	key_if
103	key_import
104	key_in
105	key_interface
106	key_is
107	// key_it
108	key_match
109	key_module
110	key_mut
111	key_shared
112	key_lock
113	key_rlock
114	key_none
115	key_return
116	key_select
117	key_sizeof
118	key_likely
119	key_unlikely
120	key_offsetof
121	key_struct
122	key_true
123	key_type
124	key_typeof
125	key_orelse
126	key_union
127	key_pub
128	key_static
129	key_unsafe
130	keyword_end
131	_end_
132}
133
134const (
135	assign_tokens = [Kind.assign, .plus_assign, .minus_assign, .mult_assign,
136	.div_assign, .xor_assign, .mod_assign, .or_assign, .and_assign,
137	.right_shift_assign, .left_shift_assign]
138	nr_tokens = int(Kind._end_)
139)
140// build_keys genereates a map with keywords' string values:
141// Keywords['return'] == .key_return
142fn build_keys() map[string]Kind {
143	mut res := map[string]Kind
144	for t in int(Kind.keyword_beg) + 1 .. int(Kind.keyword_end) {
145		key := token_str[t]
146		res[key] = t
147	}
148	return res
149}
150
151// TODO remove once we have `enum Kind { name('name') if('if') ... }`
152fn build_token_str() []string {
153	mut s := []string{len:(nr_tokens)}
154	s[Kind.unknown] = 'unknown'
155	s[Kind.eof] = 'eof'
156	s[Kind.name] = 'name'
157	s[Kind.number] = 'number'
158	s[Kind.string] = 'string'
159	s[Kind.chartoken] = 'char'
160	s[Kind.plus] = '+'
161	s[Kind.minus] = '-'
162	s[Kind.mul] = '*'
163	s[Kind.div] = '/'
164	s[Kind.mod] = '%'
165	s[Kind.xor] = '^'
166	s[Kind.bit_not] = '~'
167	s[Kind.pipe] = '|'
168	s[Kind.hash] = '#'
169	s[Kind.amp] = '&'
170	s[Kind.inc] = '++'
171	s[Kind.dec] = '--'
172	s[Kind.and] = '&&'
173	s[Kind.logical_or] = '||'
174	s[Kind.not] = '!'
175	s[Kind.dot] = '.'
176	s[Kind.dotdot] = '..'
177	s[Kind.ellipsis] = '...'
178	s[Kind.comma] = ','
179	s[Kind.not_in] = '!in'
180	s[Kind.not_is] = '!is'
181	// s[Kind.at] = '@'
182	s[Kind.semicolon] = ';'
183	s[Kind.colon] = ':'
184	s[Kind.arrow] = '=>'
185	s[Kind.assign] = '='
186	s[Kind.decl_assign] = ':='
187	s[Kind.plus_assign] = '+='
188	s[Kind.minus_assign] = '-='
189	s[Kind.mult_assign] = '*='
190	s[Kind.div_assign] = '/='
191	s[Kind.xor_assign] = '^='
192	s[Kind.mod_assign] = '%='
193	s[Kind.or_assign] = '|='
194	s[Kind.and_assign] = '&='
195	s[Kind.right_shift_assign] = '>>='
196	s[Kind.left_shift_assign] = '<<='
197	s[Kind.lcbr] = '{'
198	s[Kind.rcbr] = '}'
199	s[Kind.lpar] = '('
200	s[Kind.rpar] = ')'
201	s[Kind.lsbr] = '['
202	s[Kind.rsbr] = ']'
203	s[Kind.eq] = '=='
204	s[Kind.ne] = '!='
205	s[Kind.gt] = '>'
206	s[Kind.lt] = '<'
207	s[Kind.ge] = '>='
208	s[Kind.le] = '<='
209	s[Kind.question] = '?'
210	s[Kind.left_shift] = '<<'
211	s[Kind.right_shift] = '>>'
212	s[Kind.comment] = '// comment'
213	s[Kind.nl] = 'NLL'
214	s[Kind.dollar] = '$'
215	s[Kind.str_dollar] = '$2'
216	s[Kind.key_assert] = 'assert'
217	s[Kind.key_struct] = 'struct'
218	s[Kind.key_if] = 'if'
219	// s[Kind.key_it] = 'it'
220	s[Kind.key_else] = 'else'
221	s[Kind.key_asm] = 'asm'
222	s[Kind.key_return] = 'return'
223	s[Kind.key_module] = 'module'
224	s[Kind.key_sizeof] = 'sizeof'
225	s[Kind.key_likely] = '_likely_'
226	s[Kind.key_unlikely] = '_unlikely_'
227	s[Kind.key_go] = 'go'
228	s[Kind.key_goto] = 'goto'
229	s[Kind.key_const] = 'const'
230	s[Kind.key_mut] = 'mut'
231	s[Kind.key_shared] = 'shared'
232	s[Kind.key_lock] = 'lock'
233	s[Kind.key_rlock] = 'rlock'
234	s[Kind.key_type] = 'type'
235	s[Kind.key_for] = 'for'
236	s[Kind.key_fn] = 'fn'
237	s[Kind.key_true] = 'true'
238	s[Kind.key_false] = 'false'
239	s[Kind.key_continue] = 'continue'
240	s[Kind.key_break] = 'break'
241	s[Kind.key_import] = 'import'
242	s[Kind.key_embed] = 'embed'
243	s[Kind.key_unsafe] = 'unsafe'
244	s[Kind.key_typeof] = 'typeof'
245	s[Kind.key_enum] = 'enum'
246	s[Kind.key_interface] = 'interface'
247	s[Kind.key_pub] = 'pub'
248	s[Kind.key_in] = 'in'
249	s[Kind.key_atomic] = 'atomic'
250	s[Kind.key_orelse] = 'or'
251	s[Kind.key_global] = '__global'
252	s[Kind.key_union] = 'union'
253	s[Kind.key_static] = 'static'
254	s[Kind.key_as] = 'as'
255	s[Kind.key_defer] = 'defer'
256	s[Kind.key_match] = 'match'
257	s[Kind.key_select] = 'select'
258	s[Kind.key_none] = 'none'
259	s[Kind.key_offsetof] = '__offsetof'
260	s[Kind.key_is] = 'is'
261	return s
262}
263
264const (
265	token_str = build_token_str()
266	keywords = build_keys()
267)
268
269pub fn key_to_token(key string) Kind {
270	a := Kind(keywords[key])
271	return a
272}
273
274pub fn is_key(key string) bool {
275	return int(key_to_token(key)) > 0
276}
277
278pub fn is_decl(t Kind) bool {
279	return t in [.key_enum, .key_interface, .key_fn, .key_struct, .key_type, .key_const, .key_pub, .eof]
280}
281
282pub fn (t Kind) is_assign() bool {
283	return t in assign_tokens
284}
285
286fn (t []Kind) contains(val Kind) bool {
287	for tt in t {
288		if tt == val {
289			return true
290		}
291	}
292	return false
293}
294
295pub fn (t Kind) str() string {
296	return token_str[int(t)]
297}
298
299pub fn (t Token) str() string {
300	return '$t.kind.str() "$t.lit"'
301}
302
303// Representation of highest and lowest precedence
304/*
305pub const (
306	lowest_prec = 0
307	highest_prec = 8
308)
309*/
310
311pub enum Precedence {
312	lowest
313	cond // OR or AND
314	in_as
315	assign // =
316	eq // == or !=
317	// less_greater // > or <
318	sum // + - | ^
319	product // * / << >> &
320	// mod // %
321	prefix // -X or !X
322	postfix // ++ or --
323	call // func(X) or foo.method(X)
324	index // array[index], map[key]
325}
326
327pub fn build_precedences() []Precedence {
328	mut p := []Precedence{len:int(Kind._end_), cap:int(Kind._end_)}
329
330	p[Kind.lsbr] = .index
331	p[Kind.dot] = .call
332	// `++` | `--`
333	p[Kind.inc] = .postfix
334	p[Kind.dec] = .postfix
335	// `*` |  `/` | `%` | `<<` | `>>` | `&`
336	p[Kind.mul] = .product
337	p[Kind.div] = .product
338	p[Kind.mod] = .product
339	p[Kind.left_shift] = .product
340	p[Kind.right_shift] = .product
341	p[Kind.amp] = .product
342	// `+` |  `-` |  `|` | `^`
343	p[Kind.plus] = .sum
344	p[Kind.minus] = .sum
345	p[Kind.pipe] = .sum
346	p[Kind.xor] = .sum
347	// `==` | `!=` | `<` | `<=` | `>` | `>=`
348	p[Kind.eq] = .eq
349	p[Kind.ne] = .eq
350	p[Kind.lt] = .eq
351	p[Kind.le] = .eq
352	p[Kind.gt] = .eq
353	p[Kind.ge] = .eq
354	// `=` | `+=` | ...
355	p[Kind.assign] = .assign
356	p[Kind.plus_assign] = .assign
357	p[Kind.minus_assign] = .assign
358	p[Kind.div_assign] = .assign
359	p[Kind.mod_assign] = .assign
360	p[Kind.or_assign] = .assign
361	p[Kind.and_assign] = .assign
362	// <<= | *= | ...
363	p[Kind.left_shift_assign] = .assign
364	p[Kind.right_shift_assign] = .assign
365	p[Kind.mult_assign] = .assign
366	p[Kind.xor_assign] = .assign
367
368	p[Kind.key_in] = .in_as
369	p[Kind.not_in] = .in_as
370	p[Kind.key_as] = .in_as
371	p[Kind.key_is] = .in_as
372	p[Kind.not_is] = .in_as
373	p[Kind.logical_or] = .cond
374	p[Kind.and] = .cond
375
376	return p
377}
378
379const (
380	precedences = build_precedences()
381)
382
383// precedence returns a tokens precedence if defined, otherwise lowest_prec
384pub fn (tok Token) precedence() int {
385	return int(precedences[tok.kind])
386}
387
388// is_scalar returns true if the token is a scalar
389pub fn (tok Token) is_scalar() bool {
390	return tok.kind in [.number, .string]
391}
392
393// is_unary returns true if the token can be in a unary expression
394pub fn (tok Token) is_unary() bool {
395	return tok.kind in [
396	// `+` | `-` | `!` | `~` | `*` | `&`
397	.plus, .minus, .not, .bit_not, .mul, .amp]
398}
399
400pub fn (tok Kind) is_relational() bool {
401	return tok in [
402	// `<` | `<=` | `>` | `>=`
403	.lt, .le, .gt, .ge, .eq, .ne]
404}
405
406pub fn (k Kind) is_start_of_type() bool {
407	return k in [.name, .lpar, .amp, .lsbr, .question]
408}
409
410pub fn (kind Kind) is_infix() bool {
411	return kind in [.plus, .minus, .mod, .mul, .div, .eq, .ne, .gt, .lt, .key_in,
412	//
413	.key_as, .ge, .le, .logical_or, .xor, .not_in, .key_is, .not_is,
414	//
415	.and, .dot, .pipe, .amp, .left_shift, .right_shift]
416}
417