1package toml
2
3import (
4	"strings"
5	"testing"
6)
7
8func testFlow(t *testing.T, input string, expectedFlow []token) {
9	ch := lexToml(strings.NewReader(input))
10	for _, expected := range expectedFlow {
11		token := <-ch
12		if token != expected {
13			t.Log("While testing: ", input)
14			t.Log("compared (got)", token, "to (expected)", expected)
15			t.Log("\tvalue:", token.val, "<->", expected.val)
16			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
17			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
18			t.Log("\tline:", token.Line, "<->", expected.Line)
19			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
20			t.Log("compared", token, "to", expected)
21			t.FailNow()
22		}
23	}
24
25	tok, ok := <-ch
26	if ok {
27		t.Log("channel is not closed!")
28		t.Log(len(ch)+1, "tokens remaining:")
29
30		t.Log("token ->", tok)
31		for token := range ch {
32			t.Log("token ->", token)
33		}
34		t.FailNow()
35	}
36}
37
38func TestValidKeyGroup(t *testing.T) {
39	testFlow(t, "[hello world]", []token{
40		{Position{1, 1}, tokenLeftBracket, "["},
41		{Position{1, 2}, tokenKeyGroup, "hello world"},
42		{Position{1, 13}, tokenRightBracket, "]"},
43		{Position{1, 14}, tokenEOF, ""},
44	})
45}
46
47func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
48	testFlow(t, `[ j . "ʞ" . l ]`, []token{
49		{Position{1, 1}, tokenLeftBracket, "["},
50		{Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
51		{Position{1, 15}, tokenRightBracket, "]"},
52		{Position{1, 16}, tokenEOF, ""},
53	})
54}
55
56func TestUnclosedKeyGroup(t *testing.T) {
57	testFlow(t, "[hello world", []token{
58		{Position{1, 1}, tokenLeftBracket, "["},
59		{Position{1, 2}, tokenError, "unclosed table key"},
60	})
61}
62
63func TestComment(t *testing.T) {
64	testFlow(t, "# blahblah", []token{
65		{Position{1, 11}, tokenEOF, ""},
66	})
67}
68
69func TestKeyGroupComment(t *testing.T) {
70	testFlow(t, "[hello world] # blahblah", []token{
71		{Position{1, 1}, tokenLeftBracket, "["},
72		{Position{1, 2}, tokenKeyGroup, "hello world"},
73		{Position{1, 13}, tokenRightBracket, "]"},
74		{Position{1, 25}, tokenEOF, ""},
75	})
76}
77
78func TestMultipleKeyGroupsComment(t *testing.T) {
79	testFlow(t, "[hello world] # blahblah\n[test]", []token{
80		{Position{1, 1}, tokenLeftBracket, "["},
81		{Position{1, 2}, tokenKeyGroup, "hello world"},
82		{Position{1, 13}, tokenRightBracket, "]"},
83		{Position{2, 1}, tokenLeftBracket, "["},
84		{Position{2, 2}, tokenKeyGroup, "test"},
85		{Position{2, 6}, tokenRightBracket, "]"},
86		{Position{2, 7}, tokenEOF, ""},
87	})
88}
89
90func TestSimpleWindowsCRLF(t *testing.T) {
91	testFlow(t, "a=4\r\nb=2", []token{
92		{Position{1, 1}, tokenKey, "a"},
93		{Position{1, 2}, tokenEqual, "="},
94		{Position{1, 3}, tokenInteger, "4"},
95		{Position{2, 1}, tokenKey, "b"},
96		{Position{2, 2}, tokenEqual, "="},
97		{Position{2, 3}, tokenInteger, "2"},
98		{Position{2, 4}, tokenEOF, ""},
99	})
100}
101
102func TestBasicKey(t *testing.T) {
103	testFlow(t, "hello", []token{
104		{Position{1, 1}, tokenKey, "hello"},
105		{Position{1, 6}, tokenEOF, ""},
106	})
107}
108
109func TestBasicKeyWithUnderscore(t *testing.T) {
110	testFlow(t, "hello_hello", []token{
111		{Position{1, 1}, tokenKey, "hello_hello"},
112		{Position{1, 12}, tokenEOF, ""},
113	})
114}
115
116func TestBasicKeyWithDash(t *testing.T) {
117	testFlow(t, "hello-world", []token{
118		{Position{1, 1}, tokenKey, "hello-world"},
119		{Position{1, 12}, tokenEOF, ""},
120	})
121}
122
123func TestBasicKeyWithUppercaseMix(t *testing.T) {
124	testFlow(t, "helloHELLOHello", []token{
125		{Position{1, 1}, tokenKey, "helloHELLOHello"},
126		{Position{1, 16}, tokenEOF, ""},
127	})
128}
129
130func TestBasicKeyWithInternationalCharacters(t *testing.T) {
131	testFlow(t, "héllÖ", []token{
132		{Position{1, 1}, tokenKey, "héllÖ"},
133		{Position{1, 6}, tokenEOF, ""},
134	})
135}
136
137func TestBasicKeyAndEqual(t *testing.T) {
138	testFlow(t, "hello =", []token{
139		{Position{1, 1}, tokenKey, "hello"},
140		{Position{1, 7}, tokenEqual, "="},
141		{Position{1, 8}, tokenEOF, ""},
142	})
143}
144
145func TestKeyWithSharpAndEqual(t *testing.T) {
146	testFlow(t, "key#name = 5", []token{
147		{Position{1, 1}, tokenError, "keys cannot contain # character"},
148	})
149}
150
151func TestKeyWithSymbolsAndEqual(t *testing.T) {
152	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
153		{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
154	})
155}
156
157func TestKeyEqualStringEscape(t *testing.T) {
158	testFlow(t, `foo = "hello\""`, []token{
159		{Position{1, 1}, tokenKey, "foo"},
160		{Position{1, 5}, tokenEqual, "="},
161		{Position{1, 8}, tokenString, "hello\""},
162		{Position{1, 16}, tokenEOF, ""},
163	})
164}
165
166func TestKeyEqualStringUnfinished(t *testing.T) {
167	testFlow(t, `foo = "bar`, []token{
168		{Position{1, 1}, tokenKey, "foo"},
169		{Position{1, 5}, tokenEqual, "="},
170		{Position{1, 8}, tokenError, "unclosed string"},
171	})
172}
173
174func TestKeyEqualString(t *testing.T) {
175	testFlow(t, `foo = "bar"`, []token{
176		{Position{1, 1}, tokenKey, "foo"},
177		{Position{1, 5}, tokenEqual, "="},
178		{Position{1, 8}, tokenString, "bar"},
179		{Position{1, 12}, tokenEOF, ""},
180	})
181}
182
183func TestKeyEqualTrue(t *testing.T) {
184	testFlow(t, "foo = true", []token{
185		{Position{1, 1}, tokenKey, "foo"},
186		{Position{1, 5}, tokenEqual, "="},
187		{Position{1, 7}, tokenTrue, "true"},
188		{Position{1, 11}, tokenEOF, ""},
189	})
190}
191
192func TestKeyEqualFalse(t *testing.T) {
193	testFlow(t, "foo = false", []token{
194		{Position{1, 1}, tokenKey, "foo"},
195		{Position{1, 5}, tokenEqual, "="},
196		{Position{1, 7}, tokenFalse, "false"},
197		{Position{1, 12}, tokenEOF, ""},
198	})
199}
200
201func TestArrayNestedString(t *testing.T) {
202	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
203		{Position{1, 1}, tokenKey, "a"},
204		{Position{1, 3}, tokenEqual, "="},
205		{Position{1, 5}, tokenLeftBracket, "["},
206		{Position{1, 7}, tokenLeftBracket, "["},
207		{Position{1, 9}, tokenString, "hello"},
208		{Position{1, 15}, tokenComma, ","},
209		{Position{1, 18}, tokenString, "world"},
210		{Position{1, 24}, tokenRightBracket, "]"},
211		{Position{1, 26}, tokenRightBracket, "]"},
212		{Position{1, 27}, tokenEOF, ""},
213	})
214}
215
216func TestArrayNestedInts(t *testing.T) {
217	testFlow(t, "a = [ [42, 21], [10] ]", []token{
218		{Position{1, 1}, tokenKey, "a"},
219		{Position{1, 3}, tokenEqual, "="},
220		{Position{1, 5}, tokenLeftBracket, "["},
221		{Position{1, 7}, tokenLeftBracket, "["},
222		{Position{1, 8}, tokenInteger, "42"},
223		{Position{1, 10}, tokenComma, ","},
224		{Position{1, 12}, tokenInteger, "21"},
225		{Position{1, 14}, tokenRightBracket, "]"},
226		{Position{1, 15}, tokenComma, ","},
227		{Position{1, 17}, tokenLeftBracket, "["},
228		{Position{1, 18}, tokenInteger, "10"},
229		{Position{1, 20}, tokenRightBracket, "]"},
230		{Position{1, 22}, tokenRightBracket, "]"},
231		{Position{1, 23}, tokenEOF, ""},
232	})
233}
234
235func TestArrayInts(t *testing.T) {
236	testFlow(t, "a = [ 42, 21, 10, ]", []token{
237		{Position{1, 1}, tokenKey, "a"},
238		{Position{1, 3}, tokenEqual, "="},
239		{Position{1, 5}, tokenLeftBracket, "["},
240		{Position{1, 7}, tokenInteger, "42"},
241		{Position{1, 9}, tokenComma, ","},
242		{Position{1, 11}, tokenInteger, "21"},
243		{Position{1, 13}, tokenComma, ","},
244		{Position{1, 15}, tokenInteger, "10"},
245		{Position{1, 17}, tokenComma, ","},
246		{Position{1, 19}, tokenRightBracket, "]"},
247		{Position{1, 20}, tokenEOF, ""},
248	})
249}
250
251func TestMultilineArrayComments(t *testing.T) {
252	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
253		{Position{1, 1}, tokenKey, "a"},
254		{Position{1, 3}, tokenEqual, "="},
255		{Position{1, 5}, tokenLeftBracket, "["},
256		{Position{1, 6}, tokenInteger, "1"},
257		{Position{1, 7}, tokenComma, ","},
258		{Position{2, 1}, tokenInteger, "2"},
259		{Position{2, 2}, tokenComma, ","},
260		{Position{3, 1}, tokenInteger, "3"},
261		{Position{3, 2}, tokenComma, ","},
262		{Position{4, 1}, tokenRightBracket, "]"},
263		{Position{4, 2}, tokenEOF, ""},
264	})
265}
266
267func TestNestedArraysComment(t *testing.T) {
268	toml := `
269someArray = [
270# does not work
271["entry1"]
272]`
273	testFlow(t, toml, []token{
274		{Position{2, 1}, tokenKey, "someArray"},
275		{Position{2, 11}, tokenEqual, "="},
276		{Position{2, 13}, tokenLeftBracket, "["},
277		{Position{4, 1}, tokenLeftBracket, "["},
278		{Position{4, 3}, tokenString, "entry1"},
279		{Position{4, 10}, tokenRightBracket, "]"},
280		{Position{5, 1}, tokenRightBracket, "]"},
281		{Position{5, 2}, tokenEOF, ""},
282	})
283}
284
285func TestKeyEqualArrayBools(t *testing.T) {
286	testFlow(t, "foo = [true, false, true]", []token{
287		{Position{1, 1}, tokenKey, "foo"},
288		{Position{1, 5}, tokenEqual, "="},
289		{Position{1, 7}, tokenLeftBracket, "["},
290		{Position{1, 8}, tokenTrue, "true"},
291		{Position{1, 12}, tokenComma, ","},
292		{Position{1, 14}, tokenFalse, "false"},
293		{Position{1, 19}, tokenComma, ","},
294		{Position{1, 21}, tokenTrue, "true"},
295		{Position{1, 25}, tokenRightBracket, "]"},
296		{Position{1, 26}, tokenEOF, ""},
297	})
298}
299
300func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
301	testFlow(t, "foo = [true, false, true] # YEAH", []token{
302		{Position{1, 1}, tokenKey, "foo"},
303		{Position{1, 5}, tokenEqual, "="},
304		{Position{1, 7}, tokenLeftBracket, "["},
305		{Position{1, 8}, tokenTrue, "true"},
306		{Position{1, 12}, tokenComma, ","},
307		{Position{1, 14}, tokenFalse, "false"},
308		{Position{1, 19}, tokenComma, ","},
309		{Position{1, 21}, tokenTrue, "true"},
310		{Position{1, 25}, tokenRightBracket, "]"},
311		{Position{1, 33}, tokenEOF, ""},
312	})
313}
314
315func TestDateRegexp(t *testing.T) {
316	if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
317		t.Error("basic lexing")
318	}
319	if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
320		t.Error("offset lexing")
321	}
322	if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
323		t.Error("nano precision lexing")
324	}
325}
326
327func TestKeyEqualDate(t *testing.T) {
328	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
329		{Position{1, 1}, tokenKey, "foo"},
330		{Position{1, 5}, tokenEqual, "="},
331		{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
332		{Position{1, 27}, tokenEOF, ""},
333	})
334	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
335		{Position{1, 1}, tokenKey, "foo"},
336		{Position{1, 5}, tokenEqual, "="},
337		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
338		{Position{1, 32}, tokenEOF, ""},
339	})
340	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
341		{Position{1, 1}, tokenKey, "foo"},
342		{Position{1, 5}, tokenEqual, "="},
343		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
344		{Position{1, 39}, tokenEOF, ""},
345	})
346}
347
348func TestFloatEndingWithDot(t *testing.T) {
349	testFlow(t, "foo = 42.", []token{
350		{Position{1, 1}, tokenKey, "foo"},
351		{Position{1, 5}, tokenEqual, "="},
352		{Position{1, 7}, tokenError, "float cannot end with a dot"},
353	})
354}
355
356func TestFloatWithTwoDots(t *testing.T) {
357	testFlow(t, "foo = 4.2.", []token{
358		{Position{1, 1}, tokenKey, "foo"},
359		{Position{1, 5}, tokenEqual, "="},
360		{Position{1, 7}, tokenError, "cannot have two dots in one float"},
361	})
362}
363
364func TestFloatWithExponent1(t *testing.T) {
365	testFlow(t, "a = 5e+22", []token{
366		{Position{1, 1}, tokenKey, "a"},
367		{Position{1, 3}, tokenEqual, "="},
368		{Position{1, 5}, tokenFloat, "5e+22"},
369		{Position{1, 10}, tokenEOF, ""},
370	})
371}
372
373func TestFloatWithExponent2(t *testing.T) {
374	testFlow(t, "a = 5E+22", []token{
375		{Position{1, 1}, tokenKey, "a"},
376		{Position{1, 3}, tokenEqual, "="},
377		{Position{1, 5}, tokenFloat, "5E+22"},
378		{Position{1, 10}, tokenEOF, ""},
379	})
380}
381
382func TestFloatWithExponent3(t *testing.T) {
383	testFlow(t, "a = -5e+22", []token{
384		{Position{1, 1}, tokenKey, "a"},
385		{Position{1, 3}, tokenEqual, "="},
386		{Position{1, 5}, tokenFloat, "-5e+22"},
387		{Position{1, 11}, tokenEOF, ""},
388	})
389}
390
391func TestFloatWithExponent4(t *testing.T) {
392	testFlow(t, "a = -5e-22", []token{
393		{Position{1, 1}, tokenKey, "a"},
394		{Position{1, 3}, tokenEqual, "="},
395		{Position{1, 5}, tokenFloat, "-5e-22"},
396		{Position{1, 11}, tokenEOF, ""},
397	})
398}
399
400func TestFloatWithExponent5(t *testing.T) {
401	testFlow(t, "a = 6.626e-34", []token{
402		{Position{1, 1}, tokenKey, "a"},
403		{Position{1, 3}, tokenEqual, "="},
404		{Position{1, 5}, tokenFloat, "6.626e-34"},
405		{Position{1, 14}, tokenEOF, ""},
406	})
407}
408
409func TestInvalidEsquapeSequence(t *testing.T) {
410	testFlow(t, `foo = "\x"`, []token{
411		{Position{1, 1}, tokenKey, "foo"},
412		{Position{1, 5}, tokenEqual, "="},
413		{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
414	})
415}
416
417func TestNestedArrays(t *testing.T) {
418	testFlow(t, "foo = [[[]]]", []token{
419		{Position{1, 1}, tokenKey, "foo"},
420		{Position{1, 5}, tokenEqual, "="},
421		{Position{1, 7}, tokenLeftBracket, "["},
422		{Position{1, 8}, tokenLeftBracket, "["},
423		{Position{1, 9}, tokenLeftBracket, "["},
424		{Position{1, 10}, tokenRightBracket, "]"},
425		{Position{1, 11}, tokenRightBracket, "]"},
426		{Position{1, 12}, tokenRightBracket, "]"},
427		{Position{1, 13}, tokenEOF, ""},
428	})
429}
430
431func TestKeyEqualNumber(t *testing.T) {
432	testFlow(t, "foo = 42", []token{
433		{Position{1, 1}, tokenKey, "foo"},
434		{Position{1, 5}, tokenEqual, "="},
435		{Position{1, 7}, tokenInteger, "42"},
436		{Position{1, 9}, tokenEOF, ""},
437	})
438
439	testFlow(t, "foo = +42", []token{
440		{Position{1, 1}, tokenKey, "foo"},
441		{Position{1, 5}, tokenEqual, "="},
442		{Position{1, 7}, tokenInteger, "+42"},
443		{Position{1, 10}, tokenEOF, ""},
444	})
445
446	testFlow(t, "foo = -42", []token{
447		{Position{1, 1}, tokenKey, "foo"},
448		{Position{1, 5}, tokenEqual, "="},
449		{Position{1, 7}, tokenInteger, "-42"},
450		{Position{1, 10}, tokenEOF, ""},
451	})
452
453	testFlow(t, "foo = 4.2", []token{
454		{Position{1, 1}, tokenKey, "foo"},
455		{Position{1, 5}, tokenEqual, "="},
456		{Position{1, 7}, tokenFloat, "4.2"},
457		{Position{1, 10}, tokenEOF, ""},
458	})
459
460	testFlow(t, "foo = +4.2", []token{
461		{Position{1, 1}, tokenKey, "foo"},
462		{Position{1, 5}, tokenEqual, "="},
463		{Position{1, 7}, tokenFloat, "+4.2"},
464		{Position{1, 11}, tokenEOF, ""},
465	})
466
467	testFlow(t, "foo = -4.2", []token{
468		{Position{1, 1}, tokenKey, "foo"},
469		{Position{1, 5}, tokenEqual, "="},
470		{Position{1, 7}, tokenFloat, "-4.2"},
471		{Position{1, 11}, tokenEOF, ""},
472	})
473
474	testFlow(t, "foo = 1_000", []token{
475		{Position{1, 1}, tokenKey, "foo"},
476		{Position{1, 5}, tokenEqual, "="},
477		{Position{1, 7}, tokenInteger, "1_000"},
478		{Position{1, 12}, tokenEOF, ""},
479	})
480
481	testFlow(t, "foo = 5_349_221", []token{
482		{Position{1, 1}, tokenKey, "foo"},
483		{Position{1, 5}, tokenEqual, "="},
484		{Position{1, 7}, tokenInteger, "5_349_221"},
485		{Position{1, 16}, tokenEOF, ""},
486	})
487
488	testFlow(t, "foo = 1_2_3_4_5", []token{
489		{Position{1, 1}, tokenKey, "foo"},
490		{Position{1, 5}, tokenEqual, "="},
491		{Position{1, 7}, tokenInteger, "1_2_3_4_5"},
492		{Position{1, 16}, tokenEOF, ""},
493	})
494
495	testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
496		{Position{1, 1}, tokenKey, "flt8"},
497		{Position{1, 6}, tokenEqual, "="},
498		{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
499		{Position{1, 33}, tokenEOF, ""},
500	})
501
502	testFlow(t, "foo = +", []token{
503		{Position{1, 1}, tokenKey, "foo"},
504		{Position{1, 5}, tokenEqual, "="},
505		{Position{1, 7}, tokenError, "no digit in that number"},
506	})
507}
508
509func TestMultiline(t *testing.T) {
510	testFlow(t, "foo = 42\nbar=21", []token{
511		{Position{1, 1}, tokenKey, "foo"},
512		{Position{1, 5}, tokenEqual, "="},
513		{Position{1, 7}, tokenInteger, "42"},
514		{Position{2, 1}, tokenKey, "bar"},
515		{Position{2, 4}, tokenEqual, "="},
516		{Position{2, 5}, tokenInteger, "21"},
517		{Position{2, 7}, tokenEOF, ""},
518	})
519}
520
521func TestKeyEqualStringUnicodeEscape(t *testing.T) {
522	testFlow(t, `foo = "hello \u2665"`, []token{
523		{Position{1, 1}, tokenKey, "foo"},
524		{Position{1, 5}, tokenEqual, "="},
525		{Position{1, 8}, tokenString, "hello ♥"},
526		{Position{1, 21}, tokenEOF, ""},
527	})
528	testFlow(t, `foo = "hello \U000003B4"`, []token{
529		{Position{1, 1}, tokenKey, "foo"},
530		{Position{1, 5}, tokenEqual, "="},
531		{Position{1, 8}, tokenString, "hello δ"},
532		{Position{1, 25}, tokenEOF, ""},
533	})
534	testFlow(t, `foo = "\uabcd"`, []token{
535		{Position{1, 1}, tokenKey, "foo"},
536		{Position{1, 5}, tokenEqual, "="},
537		{Position{1, 8}, tokenString, "\uabcd"},
538		{Position{1, 15}, tokenEOF, ""},
539	})
540	testFlow(t, `foo = "\uABCD"`, []token{
541		{Position{1, 1}, tokenKey, "foo"},
542		{Position{1, 5}, tokenEqual, "="},
543		{Position{1, 8}, tokenString, "\uABCD"},
544		{Position{1, 15}, tokenEOF, ""},
545	})
546	testFlow(t, `foo = "\U000bcdef"`, []token{
547		{Position{1, 1}, tokenKey, "foo"},
548		{Position{1, 5}, tokenEqual, "="},
549		{Position{1, 8}, tokenString, "\U000bcdef"},
550		{Position{1, 19}, tokenEOF, ""},
551	})
552	testFlow(t, `foo = "\U000BCDEF"`, []token{
553		{Position{1, 1}, tokenKey, "foo"},
554		{Position{1, 5}, tokenEqual, "="},
555		{Position{1, 8}, tokenString, "\U000BCDEF"},
556		{Position{1, 19}, tokenEOF, ""},
557	})
558	testFlow(t, `foo = "\u2"`, []token{
559		{Position{1, 1}, tokenKey, "foo"},
560		{Position{1, 5}, tokenEqual, "="},
561		{Position{1, 8}, tokenError, "unfinished unicode escape"},
562	})
563	testFlow(t, `foo = "\U2"`, []token{
564		{Position{1, 1}, tokenKey, "foo"},
565		{Position{1, 5}, tokenEqual, "="},
566		{Position{1, 8}, tokenError, "unfinished unicode escape"},
567	})
568}
569
570func TestKeyEqualStringNoEscape(t *testing.T) {
571	testFlow(t, "foo = \"hello \u0002\"", []token{
572		{Position{1, 1}, tokenKey, "foo"},
573		{Position{1, 5}, tokenEqual, "="},
574		{Position{1, 8}, tokenError, "unescaped control character U+0002"},
575	})
576	testFlow(t, "foo = \"hello \u001F\"", []token{
577		{Position{1, 1}, tokenKey, "foo"},
578		{Position{1, 5}, tokenEqual, "="},
579		{Position{1, 8}, tokenError, "unescaped control character U+001F"},
580	})
581}
582
583func TestLiteralString(t *testing.T) {
584	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
585		{Position{1, 1}, tokenKey, "foo"},
586		{Position{1, 5}, tokenEqual, "="},
587		{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
588		{Position{1, 34}, tokenEOF, ""},
589	})
590	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
591		{Position{1, 1}, tokenKey, "foo"},
592		{Position{1, 5}, tokenEqual, "="},
593		{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
594		{Position{1, 35}, tokenEOF, ""},
595	})
596	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
597		{Position{1, 1}, tokenKey, "foo"},
598		{Position{1, 5}, tokenEqual, "="},
599		{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
600		{Position{1, 34}, tokenEOF, ""},
601	})
602	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
603		{Position{1, 1}, tokenKey, "foo"},
604		{Position{1, 5}, tokenEqual, "="},
605		{Position{1, 8}, tokenString, `<\i\c*\s*>`},
606		{Position{1, 19}, tokenEOF, ""},
607	})
608	testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
609		{Position{1, 1}, tokenKey, "foo"},
610		{Position{1, 5}, tokenEqual, "="},
611		{Position{1, 8}, tokenError, "unclosed string"},
612	})
613}
614
615func TestMultilineLiteralString(t *testing.T) {
616	testFlow(t, `foo = '''hello 'literal' world'''`, []token{
617		{Position{1, 1}, tokenKey, "foo"},
618		{Position{1, 5}, tokenEqual, "="},
619		{Position{1, 10}, tokenString, `hello 'literal' world`},
620		{Position{1, 34}, tokenEOF, ""},
621	})
622
623	testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
624		{Position{1, 1}, tokenKey, "foo"},
625		{Position{1, 5}, tokenEqual, "="},
626		{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
627		{Position{4, 9}, tokenEOF, ""},
628	})
629	testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
630		{Position{1, 1}, tokenKey, "foo"},
631		{Position{1, 5}, tokenEqual, "="},
632		{Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
633		{Position{4, 9}, tokenEOF, ""},
634	})
635}
636
637func TestMultilineString(t *testing.T) {
638	testFlow(t, `foo = """hello "literal" world"""`, []token{
639		{Position{1, 1}, tokenKey, "foo"},
640		{Position{1, 5}, tokenEqual, "="},
641		{Position{1, 10}, tokenString, `hello "literal" world`},
642		{Position{1, 34}, tokenEOF, ""},
643	})
644
645	testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
646		{Position{1, 1}, tokenKey, "foo"},
647		{Position{1, 5}, tokenEqual, "="},
648		{Position{2, 1}, tokenString, "hello\"literal\"world"},
649		{Position{4, 9}, tokenEOF, ""},
650	})
651
652	testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
653		{Position{1, 1}, tokenKey, "foo"},
654		{Position{1, 5}, tokenEqual, "="},
655		{Position{1, 10}, tokenString, "hellomultilineworld"},
656		{Position{6, 9}, tokenEOF, ""},
657	})
658
659	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
660		{Position{1, 1}, tokenKey, "key2"},
661		{Position{1, 6}, tokenEqual, "="},
662		{Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
663		{Position{6, 21}, tokenEOF, ""},
664	})
665
666	testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
667		{Position{1, 1}, tokenKey, "key2"},
668		{Position{1, 6}, tokenEqual, "="},
669		{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
670		{Position{5, 11}, tokenEOF, ""},
671	})
672
673	testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
674		{Position{1, 1}, tokenKey, "key2"},
675		{Position{1, 6}, tokenEqual, "="},
676		{Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
677		{Position{1, 41}, tokenEOF, ""},
678	})
679
680	testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
681		{Position{1, 1}, tokenKey, "key2"},
682		{Position{1, 6}, tokenEqual, "="},
683		{Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
684		{Position{3, 20}, tokenEOF, ""},
685	})
686}
687
688func TestUnicodeString(t *testing.T) {
689	testFlow(t, `foo = "hello ♥ world"`, []token{
690		{Position{1, 1}, tokenKey, "foo"},
691		{Position{1, 5}, tokenEqual, "="},
692		{Position{1, 8}, tokenString, "hello ♥ world"},
693		{Position{1, 22}, tokenEOF, ""},
694	})
695}
696func TestEscapeInString(t *testing.T) {
697	testFlow(t, `foo = "\b\f\/"`, []token{
698		{Position{1, 1}, tokenKey, "foo"},
699		{Position{1, 5}, tokenEqual, "="},
700		{Position{1, 8}, tokenString, "\b\f/"},
701		{Position{1, 15}, tokenEOF, ""},
702	})
703}
704
705func TestKeyGroupArray(t *testing.T) {
706	testFlow(t, "[[foo]]", []token{
707		{Position{1, 1}, tokenDoubleLeftBracket, "[["},
708		{Position{1, 3}, tokenKeyGroupArray, "foo"},
709		{Position{1, 6}, tokenDoubleRightBracket, "]]"},
710		{Position{1, 8}, tokenEOF, ""},
711	})
712}
713
714func TestQuotedKey(t *testing.T) {
715	testFlow(t, "\"a b\" = 42", []token{
716		{Position{1, 1}, tokenKey, "\"a b\""},
717		{Position{1, 7}, tokenEqual, "="},
718		{Position{1, 9}, tokenInteger, "42"},
719		{Position{1, 11}, tokenEOF, ""},
720	})
721}
722
723func TestKeyNewline(t *testing.T) {
724	testFlow(t, "a\n= 4", []token{
725		{Position{1, 1}, tokenError, "keys cannot contain new lines"},
726	})
727}
728
729func TestInvalidFloat(t *testing.T) {
730	testFlow(t, "a=7e1_", []token{
731		{Position{1, 1}, tokenKey, "a"},
732		{Position{1, 2}, tokenEqual, "="},
733		{Position{1, 3}, tokenFloat, "7e1_"},
734		{Position{1, 7}, tokenEOF, ""},
735	})
736}
737
738func TestLexUnknownRvalue(t *testing.T) {
739	testFlow(t, `a = !b`, []token{
740		{Position{1, 1}, tokenKey, "a"},
741		{Position{1, 3}, tokenEqual, "="},
742		{Position{1, 5}, tokenError, "no value can start with !"},
743	})
744
745	testFlow(t, `a = \b`, []token{
746		{Position{1, 1}, tokenKey, "a"},
747		{Position{1, 3}, tokenEqual, "="},
748		{Position{1, 5}, tokenError, `no value can start with \`},
749	})
750}
751