1package toml
2
3import (
4	"os"
5	"strings"
6	"testing"
7)
8
9func testFlow(t *testing.T, input string, expectedFlow []token) {
10	ch := lexToml(strings.NewReader(input))
11	for _, expected := range expectedFlow {
12		token := <-ch
13		if token != expected {
14			t.Log("While testing: ", input)
15			t.Log("compared (got)", token, "to (expected)", expected)
16			t.Log("\tvalue:", token.val, "<->", expected.val)
17			t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val))
18			t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String())
19			t.Log("\tline:", token.Line, "<->", expected.Line)
20			t.Log("\tcolumn:", token.Col, "<->", expected.Col)
21			t.Log("compared", token, "to", expected)
22			t.FailNow()
23		}
24	}
25
26	tok, ok := <-ch
27	if ok {
28		t.Log("channel is not closed!")
29		t.Log(len(ch)+1, "tokens remaining:")
30
31		t.Log("token ->", tok)
32		for token := range ch {
33			t.Log("token ->", token)
34		}
35		t.FailNow()
36	}
37}
38
39func TestValidKeyGroup(t *testing.T) {
40	testFlow(t, "[hello world]", []token{
41		{Position{1, 1}, tokenLeftBracket, "["},
42		{Position{1, 2}, tokenKeyGroup, "hello world"},
43		{Position{1, 13}, tokenRightBracket, "]"},
44		{Position{1, 14}, tokenEOF, ""},
45	})
46}
47
48func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
49	testFlow(t, `[ j . "ʞ" . l ]`, []token{
50		{Position{1, 1}, tokenLeftBracket, "["},
51		{Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
52		{Position{1, 15}, tokenRightBracket, "]"},
53		{Position{1, 16}, tokenEOF, ""},
54	})
55}
56
57func TestUnclosedKeyGroup(t *testing.T) {
58	testFlow(t, "[hello world", []token{
59		{Position{1, 1}, tokenLeftBracket, "["},
60		{Position{1, 2}, tokenError, "unclosed table key"},
61	})
62}
63
64func TestComment(t *testing.T) {
65	testFlow(t, "# blahblah", []token{
66		{Position{1, 11}, tokenEOF, ""},
67	})
68}
69
70func TestKeyGroupComment(t *testing.T) {
71	testFlow(t, "[hello world] # blahblah", []token{
72		{Position{1, 1}, tokenLeftBracket, "["},
73		{Position{1, 2}, tokenKeyGroup, "hello world"},
74		{Position{1, 13}, tokenRightBracket, "]"},
75		{Position{1, 25}, tokenEOF, ""},
76	})
77}
78
79func TestMultipleKeyGroupsComment(t *testing.T) {
80	testFlow(t, "[hello world] # blahblah\n[test]", []token{
81		{Position{1, 1}, tokenLeftBracket, "["},
82		{Position{1, 2}, tokenKeyGroup, "hello world"},
83		{Position{1, 13}, tokenRightBracket, "]"},
84		{Position{2, 1}, tokenLeftBracket, "["},
85		{Position{2, 2}, tokenKeyGroup, "test"},
86		{Position{2, 6}, tokenRightBracket, "]"},
87		{Position{2, 7}, tokenEOF, ""},
88	})
89}
90
91func TestSimpleWindowsCRLF(t *testing.T) {
92	testFlow(t, "a=4\r\nb=2", []token{
93		{Position{1, 1}, tokenKey, "a"},
94		{Position{1, 2}, tokenEqual, "="},
95		{Position{1, 3}, tokenInteger, "4"},
96		{Position{2, 1}, tokenKey, "b"},
97		{Position{2, 2}, tokenEqual, "="},
98		{Position{2, 3}, tokenInteger, "2"},
99		{Position{2, 4}, tokenEOF, ""},
100	})
101}
102
103func TestBasicKey(t *testing.T) {
104	testFlow(t, "hello", []token{
105		{Position{1, 1}, tokenKey, "hello"},
106		{Position{1, 6}, tokenEOF, ""},
107	})
108}
109
110func TestBasicKeyWithUnderscore(t *testing.T) {
111	testFlow(t, "hello_hello", []token{
112		{Position{1, 1}, tokenKey, "hello_hello"},
113		{Position{1, 12}, tokenEOF, ""},
114	})
115}
116
117func TestBasicKeyWithDash(t *testing.T) {
118	testFlow(t, "hello-world", []token{
119		{Position{1, 1}, tokenKey, "hello-world"},
120		{Position{1, 12}, tokenEOF, ""},
121	})
122}
123
124func TestBasicKeyWithUppercaseMix(t *testing.T) {
125	testFlow(t, "helloHELLOHello", []token{
126		{Position{1, 1}, tokenKey, "helloHELLOHello"},
127		{Position{1, 16}, tokenEOF, ""},
128	})
129}
130
131func TestBasicKeyWithInternationalCharacters(t *testing.T) {
132	testFlow(t, "héllÖ", []token{
133		{Position{1, 1}, tokenKey, "héllÖ"},
134		{Position{1, 6}, tokenEOF, ""},
135	})
136}
137
138func TestBasicKeyAndEqual(t *testing.T) {
139	testFlow(t, "hello =", []token{
140		{Position{1, 1}, tokenKey, "hello"},
141		{Position{1, 7}, tokenEqual, "="},
142		{Position{1, 8}, tokenEOF, ""},
143	})
144}
145
146func TestKeyWithSharpAndEqual(t *testing.T) {
147	testFlow(t, "key#name = 5", []token{
148		{Position{1, 1}, tokenError, "keys cannot contain # character"},
149	})
150}
151
152func TestKeyWithSymbolsAndEqual(t *testing.T) {
153	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
154		{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
155	})
156}
157
158func TestKeyEqualStringEscape(t *testing.T) {
159	testFlow(t, `foo = "hello\""`, []token{
160		{Position{1, 1}, tokenKey, "foo"},
161		{Position{1, 5}, tokenEqual, "="},
162		{Position{1, 8}, tokenString, "hello\""},
163		{Position{1, 16}, tokenEOF, ""},
164	})
165}
166
167func TestKeyEqualStringUnfinished(t *testing.T) {
168	testFlow(t, `foo = "bar`, []token{
169		{Position{1, 1}, tokenKey, "foo"},
170		{Position{1, 5}, tokenEqual, "="},
171		{Position{1, 8}, tokenError, "unclosed string"},
172	})
173}
174
175func TestKeyEqualString(t *testing.T) {
176	testFlow(t, `foo = "bar"`, []token{
177		{Position{1, 1}, tokenKey, "foo"},
178		{Position{1, 5}, tokenEqual, "="},
179		{Position{1, 8}, tokenString, "bar"},
180		{Position{1, 12}, tokenEOF, ""},
181	})
182}
183
184func TestKeyEqualTrue(t *testing.T) {
185	testFlow(t, "foo = true", []token{
186		{Position{1, 1}, tokenKey, "foo"},
187		{Position{1, 5}, tokenEqual, "="},
188		{Position{1, 7}, tokenTrue, "true"},
189		{Position{1, 11}, tokenEOF, ""},
190	})
191}
192
193func TestKeyEqualFalse(t *testing.T) {
194	testFlow(t, "foo = false", []token{
195		{Position{1, 1}, tokenKey, "foo"},
196		{Position{1, 5}, tokenEqual, "="},
197		{Position{1, 7}, tokenFalse, "false"},
198		{Position{1, 12}, tokenEOF, ""},
199	})
200}
201
202func TestArrayNestedString(t *testing.T) {
203	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
204		{Position{1, 1}, tokenKey, "a"},
205		{Position{1, 3}, tokenEqual, "="},
206		{Position{1, 5}, tokenLeftBracket, "["},
207		{Position{1, 7}, tokenLeftBracket, "["},
208		{Position{1, 9}, tokenString, "hello"},
209		{Position{1, 15}, tokenComma, ","},
210		{Position{1, 18}, tokenString, "world"},
211		{Position{1, 24}, tokenRightBracket, "]"},
212		{Position{1, 26}, tokenRightBracket, "]"},
213		{Position{1, 27}, tokenEOF, ""},
214	})
215}
216
217func TestArrayNestedInts(t *testing.T) {
218	testFlow(t, "a = [ [42, 21], [10] ]", []token{
219		{Position{1, 1}, tokenKey, "a"},
220		{Position{1, 3}, tokenEqual, "="},
221		{Position{1, 5}, tokenLeftBracket, "["},
222		{Position{1, 7}, tokenLeftBracket, "["},
223		{Position{1, 8}, tokenInteger, "42"},
224		{Position{1, 10}, tokenComma, ","},
225		{Position{1, 12}, tokenInteger, "21"},
226		{Position{1, 14}, tokenRightBracket, "]"},
227		{Position{1, 15}, tokenComma, ","},
228		{Position{1, 17}, tokenLeftBracket, "["},
229		{Position{1, 18}, tokenInteger, "10"},
230		{Position{1, 20}, tokenRightBracket, "]"},
231		{Position{1, 22}, tokenRightBracket, "]"},
232		{Position{1, 23}, tokenEOF, ""},
233	})
234}
235
236func TestArrayInts(t *testing.T) {
237	testFlow(t, "a = [ 42, 21, 10, ]", []token{
238		{Position{1, 1}, tokenKey, "a"},
239		{Position{1, 3}, tokenEqual, "="},
240		{Position{1, 5}, tokenLeftBracket, "["},
241		{Position{1, 7}, tokenInteger, "42"},
242		{Position{1, 9}, tokenComma, ","},
243		{Position{1, 11}, tokenInteger, "21"},
244		{Position{1, 13}, tokenComma, ","},
245		{Position{1, 15}, tokenInteger, "10"},
246		{Position{1, 17}, tokenComma, ","},
247		{Position{1, 19}, tokenRightBracket, "]"},
248		{Position{1, 20}, tokenEOF, ""},
249	})
250}
251
252func TestMultilineArrayComments(t *testing.T) {
253	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
254		{Position{1, 1}, tokenKey, "a"},
255		{Position{1, 3}, tokenEqual, "="},
256		{Position{1, 5}, tokenLeftBracket, "["},
257		{Position{1, 6}, tokenInteger, "1"},
258		{Position{1, 7}, tokenComma, ","},
259		{Position{2, 1}, tokenInteger, "2"},
260		{Position{2, 2}, tokenComma, ","},
261		{Position{3, 1}, tokenInteger, "3"},
262		{Position{3, 2}, tokenComma, ","},
263		{Position{4, 1}, tokenRightBracket, "]"},
264		{Position{4, 2}, tokenEOF, ""},
265	})
266}
267
268func TestNestedArraysComment(t *testing.T) {
269	toml := `
270someArray = [
271# does not work
272["entry1"]
273]`
274	testFlow(t, toml, []token{
275		{Position{2, 1}, tokenKey, "someArray"},
276		{Position{2, 11}, tokenEqual, "="},
277		{Position{2, 13}, tokenLeftBracket, "["},
278		{Position{4, 1}, tokenLeftBracket, "["},
279		{Position{4, 3}, tokenString, "entry1"},
280		{Position{4, 10}, tokenRightBracket, "]"},
281		{Position{5, 1}, tokenRightBracket, "]"},
282		{Position{5, 2}, tokenEOF, ""},
283	})
284}
285
286func TestKeyEqualArrayBools(t *testing.T) {
287	testFlow(t, "foo = [true, false, true]", []token{
288		{Position{1, 1}, tokenKey, "foo"},
289		{Position{1, 5}, tokenEqual, "="},
290		{Position{1, 7}, tokenLeftBracket, "["},
291		{Position{1, 8}, tokenTrue, "true"},
292		{Position{1, 12}, tokenComma, ","},
293		{Position{1, 14}, tokenFalse, "false"},
294		{Position{1, 19}, tokenComma, ","},
295		{Position{1, 21}, tokenTrue, "true"},
296		{Position{1, 25}, tokenRightBracket, "]"},
297		{Position{1, 26}, tokenEOF, ""},
298	})
299}
300
301func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
302	testFlow(t, "foo = [true, false, true] # YEAH", []token{
303		{Position{1, 1}, tokenKey, "foo"},
304		{Position{1, 5}, tokenEqual, "="},
305		{Position{1, 7}, tokenLeftBracket, "["},
306		{Position{1, 8}, tokenTrue, "true"},
307		{Position{1, 12}, tokenComma, ","},
308		{Position{1, 14}, tokenFalse, "false"},
309		{Position{1, 19}, tokenComma, ","},
310		{Position{1, 21}, tokenTrue, "true"},
311		{Position{1, 25}, tokenRightBracket, "]"},
312		{Position{1, 33}, tokenEOF, ""},
313	})
314}
315
316func TestDateRegexp(t *testing.T) {
317	if dateRegexp.FindString("1979-05-27T07:32:00Z") == "" {
318		t.Error("basic lexing")
319	}
320	if dateRegexp.FindString("1979-05-27T00:32:00-07:00") == "" {
321		t.Error("offset lexing")
322	}
323	if dateRegexp.FindString("1979-05-27T00:32:00.999999-07:00") == "" {
324		t.Error("nano precision lexing")
325	}
326}
327
328func TestKeyEqualDate(t *testing.T) {
329	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
330		{Position{1, 1}, tokenKey, "foo"},
331		{Position{1, 5}, tokenEqual, "="},
332		{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
333		{Position{1, 27}, tokenEOF, ""},
334	})
335	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
336		{Position{1, 1}, tokenKey, "foo"},
337		{Position{1, 5}, tokenEqual, "="},
338		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
339		{Position{1, 32}, tokenEOF, ""},
340	})
341	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
342		{Position{1, 1}, tokenKey, "foo"},
343		{Position{1, 5}, tokenEqual, "="},
344		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
345		{Position{1, 39}, tokenEOF, ""},
346	})
347}
348
349func TestFloatEndingWithDot(t *testing.T) {
350	testFlow(t, "foo = 42.", []token{
351		{Position{1, 1}, tokenKey, "foo"},
352		{Position{1, 5}, tokenEqual, "="},
353		{Position{1, 7}, tokenError, "float cannot end with a dot"},
354	})
355}
356
357func TestFloatWithTwoDots(t *testing.T) {
358	testFlow(t, "foo = 4.2.", []token{
359		{Position{1, 1}, tokenKey, "foo"},
360		{Position{1, 5}, tokenEqual, "="},
361		{Position{1, 7}, tokenError, "cannot have two dots in one float"},
362	})
363}
364
365func TestFloatWithExponent1(t *testing.T) {
366	testFlow(t, "a = 5e+22", []token{
367		{Position{1, 1}, tokenKey, "a"},
368		{Position{1, 3}, tokenEqual, "="},
369		{Position{1, 5}, tokenFloat, "5e+22"},
370		{Position{1, 10}, tokenEOF, ""},
371	})
372}
373
374func TestFloatWithExponent2(t *testing.T) {
375	testFlow(t, "a = 5E+22", []token{
376		{Position{1, 1}, tokenKey, "a"},
377		{Position{1, 3}, tokenEqual, "="},
378		{Position{1, 5}, tokenFloat, "5E+22"},
379		{Position{1, 10}, tokenEOF, ""},
380	})
381}
382
383func TestFloatWithExponent3(t *testing.T) {
384	testFlow(t, "a = -5e+22", []token{
385		{Position{1, 1}, tokenKey, "a"},
386		{Position{1, 3}, tokenEqual, "="},
387		{Position{1, 5}, tokenFloat, "-5e+22"},
388		{Position{1, 11}, tokenEOF, ""},
389	})
390}
391
392func TestFloatWithExponent4(t *testing.T) {
393	testFlow(t, "a = -5e-22", []token{
394		{Position{1, 1}, tokenKey, "a"},
395		{Position{1, 3}, tokenEqual, "="},
396		{Position{1, 5}, tokenFloat, "-5e-22"},
397		{Position{1, 11}, tokenEOF, ""},
398	})
399}
400
401func TestFloatWithExponent5(t *testing.T) {
402	testFlow(t, "a = 6.626e-34", []token{
403		{Position{1, 1}, tokenKey, "a"},
404		{Position{1, 3}, tokenEqual, "="},
405		{Position{1, 5}, tokenFloat, "6.626e-34"},
406		{Position{1, 14}, tokenEOF, ""},
407	})
408}
409
410func TestInvalidEsquapeSequence(t *testing.T) {
411	testFlow(t, `foo = "\x"`, []token{
412		{Position{1, 1}, tokenKey, "foo"},
413		{Position{1, 5}, tokenEqual, "="},
414		{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
415	})
416}
417
418func TestNestedArrays(t *testing.T) {
419	testFlow(t, "foo = [[[]]]", []token{
420		{Position{1, 1}, tokenKey, "foo"},
421		{Position{1, 5}, tokenEqual, "="},
422		{Position{1, 7}, tokenLeftBracket, "["},
423		{Position{1, 8}, tokenLeftBracket, "["},
424		{Position{1, 9}, tokenLeftBracket, "["},
425		{Position{1, 10}, tokenRightBracket, "]"},
426		{Position{1, 11}, tokenRightBracket, "]"},
427		{Position{1, 12}, tokenRightBracket, "]"},
428		{Position{1, 13}, tokenEOF, ""},
429	})
430}
431
432func TestKeyEqualNumber(t *testing.T) {
433	testFlow(t, "foo = 42", []token{
434		{Position{1, 1}, tokenKey, "foo"},
435		{Position{1, 5}, tokenEqual, "="},
436		{Position{1, 7}, tokenInteger, "42"},
437		{Position{1, 9}, tokenEOF, ""},
438	})
439
440	testFlow(t, "foo = +42", []token{
441		{Position{1, 1}, tokenKey, "foo"},
442		{Position{1, 5}, tokenEqual, "="},
443		{Position{1, 7}, tokenInteger, "+42"},
444		{Position{1, 10}, tokenEOF, ""},
445	})
446
447	testFlow(t, "foo = -42", []token{
448		{Position{1, 1}, tokenKey, "foo"},
449		{Position{1, 5}, tokenEqual, "="},
450		{Position{1, 7}, tokenInteger, "-42"},
451		{Position{1, 10}, tokenEOF, ""},
452	})
453
454	testFlow(t, "foo = 4.2", []token{
455		{Position{1, 1}, tokenKey, "foo"},
456		{Position{1, 5}, tokenEqual, "="},
457		{Position{1, 7}, tokenFloat, "4.2"},
458		{Position{1, 10}, tokenEOF, ""},
459	})
460
461	testFlow(t, "foo = +4.2", []token{
462		{Position{1, 1}, tokenKey, "foo"},
463		{Position{1, 5}, tokenEqual, "="},
464		{Position{1, 7}, tokenFloat, "+4.2"},
465		{Position{1, 11}, tokenEOF, ""},
466	})
467
468	testFlow(t, "foo = -4.2", []token{
469		{Position{1, 1}, tokenKey, "foo"},
470		{Position{1, 5}, tokenEqual, "="},
471		{Position{1, 7}, tokenFloat, "-4.2"},
472		{Position{1, 11}, tokenEOF, ""},
473	})
474
475	testFlow(t, "foo = 1_000", []token{
476		{Position{1, 1}, tokenKey, "foo"},
477		{Position{1, 5}, tokenEqual, "="},
478		{Position{1, 7}, tokenInteger, "1_000"},
479		{Position{1, 12}, tokenEOF, ""},
480	})
481
482	testFlow(t, "foo = 5_349_221", []token{
483		{Position{1, 1}, tokenKey, "foo"},
484		{Position{1, 5}, tokenEqual, "="},
485		{Position{1, 7}, tokenInteger, "5_349_221"},
486		{Position{1, 16}, tokenEOF, ""},
487	})
488
489	testFlow(t, "foo = 1_2_3_4_5", []token{
490		{Position{1, 1}, tokenKey, "foo"},
491		{Position{1, 5}, tokenEqual, "="},
492		{Position{1, 7}, tokenInteger, "1_2_3_4_5"},
493		{Position{1, 16}, tokenEOF, ""},
494	})
495
496	testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
497		{Position{1, 1}, tokenKey, "flt8"},
498		{Position{1, 6}, tokenEqual, "="},
499		{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
500		{Position{1, 33}, tokenEOF, ""},
501	})
502
503	testFlow(t, "foo = +", []token{
504		{Position{1, 1}, tokenKey, "foo"},
505		{Position{1, 5}, tokenEqual, "="},
506		{Position{1, 7}, tokenError, "no digit in that number"},
507	})
508}
509
510func TestMultiline(t *testing.T) {
511	testFlow(t, "foo = 42\nbar=21", []token{
512		{Position{1, 1}, tokenKey, "foo"},
513		{Position{1, 5}, tokenEqual, "="},
514		{Position{1, 7}, tokenInteger, "42"},
515		{Position{2, 1}, tokenKey, "bar"},
516		{Position{2, 4}, tokenEqual, "="},
517		{Position{2, 5}, tokenInteger, "21"},
518		{Position{2, 7}, tokenEOF, ""},
519	})
520}
521
522func TestKeyEqualStringUnicodeEscape(t *testing.T) {
523	testFlow(t, `foo = "hello \u2665"`, []token{
524		{Position{1, 1}, tokenKey, "foo"},
525		{Position{1, 5}, tokenEqual, "="},
526		{Position{1, 8}, tokenString, "hello ♥"},
527		{Position{1, 21}, tokenEOF, ""},
528	})
529	testFlow(t, `foo = "hello \U000003B4"`, []token{
530		{Position{1, 1}, tokenKey, "foo"},
531		{Position{1, 5}, tokenEqual, "="},
532		{Position{1, 8}, tokenString, "hello δ"},
533		{Position{1, 25}, tokenEOF, ""},
534	})
535	testFlow(t, `foo = "\uabcd"`, []token{
536		{Position{1, 1}, tokenKey, "foo"},
537		{Position{1, 5}, tokenEqual, "="},
538		{Position{1, 8}, tokenString, "\uabcd"},
539		{Position{1, 15}, tokenEOF, ""},
540	})
541	testFlow(t, `foo = "\uABCD"`, []token{
542		{Position{1, 1}, tokenKey, "foo"},
543		{Position{1, 5}, tokenEqual, "="},
544		{Position{1, 8}, tokenString, "\uABCD"},
545		{Position{1, 15}, tokenEOF, ""},
546	})
547	testFlow(t, `foo = "\U000bcdef"`, []token{
548		{Position{1, 1}, tokenKey, "foo"},
549		{Position{1, 5}, tokenEqual, "="},
550		{Position{1, 8}, tokenString, "\U000bcdef"},
551		{Position{1, 19}, tokenEOF, ""},
552	})
553	testFlow(t, `foo = "\U000BCDEF"`, []token{
554		{Position{1, 1}, tokenKey, "foo"},
555		{Position{1, 5}, tokenEqual, "="},
556		{Position{1, 8}, tokenString, "\U000BCDEF"},
557		{Position{1, 19}, tokenEOF, ""},
558	})
559	testFlow(t, `foo = "\u2"`, []token{
560		{Position{1, 1}, tokenKey, "foo"},
561		{Position{1, 5}, tokenEqual, "="},
562		{Position{1, 8}, tokenError, "unfinished unicode escape"},
563	})
564	testFlow(t, `foo = "\U2"`, []token{
565		{Position{1, 1}, tokenKey, "foo"},
566		{Position{1, 5}, tokenEqual, "="},
567		{Position{1, 8}, tokenError, "unfinished unicode escape"},
568	})
569}
570
571func TestKeyEqualStringNoEscape(t *testing.T) {
572	testFlow(t, "foo = \"hello \u0002\"", []token{
573		{Position{1, 1}, tokenKey, "foo"},
574		{Position{1, 5}, tokenEqual, "="},
575		{Position{1, 8}, tokenError, "unescaped control character U+0002"},
576	})
577	testFlow(t, "foo = \"hello \u001F\"", []token{
578		{Position{1, 1}, tokenKey, "foo"},
579		{Position{1, 5}, tokenEqual, "="},
580		{Position{1, 8}, tokenError, "unescaped control character U+001F"},
581	})
582}
583
584func TestLiteralString(t *testing.T) {
585	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
586		{Position{1, 1}, tokenKey, "foo"},
587		{Position{1, 5}, tokenEqual, "="},
588		{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
589		{Position{1, 34}, tokenEOF, ""},
590	})
591	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
592		{Position{1, 1}, tokenKey, "foo"},
593		{Position{1, 5}, tokenEqual, "="},
594		{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
595		{Position{1, 35}, tokenEOF, ""},
596	})
597	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
598		{Position{1, 1}, tokenKey, "foo"},
599		{Position{1, 5}, tokenEqual, "="},
600		{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
601		{Position{1, 34}, tokenEOF, ""},
602	})
603	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
604		{Position{1, 1}, tokenKey, "foo"},
605		{Position{1, 5}, tokenEqual, "="},
606		{Position{1, 8}, tokenString, `<\i\c*\s*>`},
607		{Position{1, 19}, tokenEOF, ""},
608	})
609	testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
610		{Position{1, 1}, tokenKey, "foo"},
611		{Position{1, 5}, tokenEqual, "="},
612		{Position{1, 8}, tokenError, "unclosed string"},
613	})
614}
615
616func TestMultilineLiteralString(t *testing.T) {
617	testFlow(t, `foo = '''hello 'literal' world'''`, []token{
618		{Position{1, 1}, tokenKey, "foo"},
619		{Position{1, 5}, tokenEqual, "="},
620		{Position{1, 10}, tokenString, `hello 'literal' world`},
621		{Position{1, 34}, tokenEOF, ""},
622	})
623
624	testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
625		{Position{1, 1}, tokenKey, "foo"},
626		{Position{1, 5}, tokenEqual, "="},
627		{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
628		{Position{4, 9}, tokenEOF, ""},
629	})
630	testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
631		{Position{1, 1}, tokenKey, "foo"},
632		{Position{1, 5}, tokenEqual, "="},
633		{Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
634		{Position{4, 9}, tokenEOF, ""},
635	})
636}
637
638func TestMultilineString(t *testing.T) {
639	testFlow(t, `foo = """hello "literal" world"""`, []token{
640		{Position{1, 1}, tokenKey, "foo"},
641		{Position{1, 5}, tokenEqual, "="},
642		{Position{1, 10}, tokenString, `hello "literal" world`},
643		{Position{1, 34}, tokenEOF, ""},
644	})
645
646	testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
647		{Position{1, 1}, tokenKey, "foo"},
648		{Position{1, 5}, tokenEqual, "="},
649		{Position{2, 1}, tokenString, "hello\"literal\"world"},
650		{Position{4, 9}, tokenEOF, ""},
651	})
652
653	testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
654		{Position{1, 1}, tokenKey, "foo"},
655		{Position{1, 5}, tokenEqual, "="},
656		{Position{1, 10}, tokenString, "hellomultilineworld"},
657		{Position{6, 9}, tokenEOF, ""},
658	})
659
660	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
661		{Position{1, 1}, tokenKey, "key2"},
662		{Position{1, 6}, tokenEqual, "="},
663		{Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
664		{Position{6, 21}, tokenEOF, ""},
665	})
666
667	testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
668		{Position{1, 1}, tokenKey, "key2"},
669		{Position{1, 6}, tokenEqual, "="},
670		{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
671		{Position{5, 11}, tokenEOF, ""},
672	})
673
674	testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
675		{Position{1, 1}, tokenKey, "key2"},
676		{Position{1, 6}, tokenEqual, "="},
677		{Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
678		{Position{1, 41}, tokenEOF, ""},
679	})
680
681	testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
682		{Position{1, 1}, tokenKey, "key2"},
683		{Position{1, 6}, tokenEqual, "="},
684		{Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
685		{Position{3, 20}, tokenEOF, ""},
686	})
687}
688
689func TestUnicodeString(t *testing.T) {
690	testFlow(t, `foo = "hello ♥ world"`, []token{
691		{Position{1, 1}, tokenKey, "foo"},
692		{Position{1, 5}, tokenEqual, "="},
693		{Position{1, 8}, tokenString, "hello ♥ world"},
694		{Position{1, 22}, tokenEOF, ""},
695	})
696}
697func TestEscapeInString(t *testing.T) {
698	testFlow(t, `foo = "\b\f\/"`, []token{
699		{Position{1, 1}, tokenKey, "foo"},
700		{Position{1, 5}, tokenEqual, "="},
701		{Position{1, 8}, tokenString, "\b\f/"},
702		{Position{1, 15}, tokenEOF, ""},
703	})
704}
705
706func TestKeyGroupArray(t *testing.T) {
707	testFlow(t, "[[foo]]", []token{
708		{Position{1, 1}, tokenDoubleLeftBracket, "[["},
709		{Position{1, 3}, tokenKeyGroupArray, "foo"},
710		{Position{1, 6}, tokenDoubleRightBracket, "]]"},
711		{Position{1, 8}, tokenEOF, ""},
712	})
713}
714
715func TestQuotedKey(t *testing.T) {
716	testFlow(t, "\"a b\" = 42", []token{
717		{Position{1, 1}, tokenKey, "\"a b\""},
718		{Position{1, 7}, tokenEqual, "="},
719		{Position{1, 9}, tokenInteger, "42"},
720		{Position{1, 11}, tokenEOF, ""},
721	})
722}
723
724func TestKeyNewline(t *testing.T) {
725	testFlow(t, "a\n= 4", []token{
726		{Position{1, 1}, tokenError, "keys cannot contain new lines"},
727	})
728}
729
730func TestInvalidFloat(t *testing.T) {
731	testFlow(t, "a=7e1_", []token{
732		{Position{1, 1}, tokenKey, "a"},
733		{Position{1, 2}, tokenEqual, "="},
734		{Position{1, 3}, tokenFloat, "7e1_"},
735		{Position{1, 7}, tokenEOF, ""},
736	})
737}
738
739func TestLexUnknownRvalue(t *testing.T) {
740	testFlow(t, `a = !b`, []token{
741		{Position{1, 1}, tokenKey, "a"},
742		{Position{1, 3}, tokenEqual, "="},
743		{Position{1, 5}, tokenError, "no value can start with !"},
744	})
745
746	testFlow(t, `a = \b`, []token{
747		{Position{1, 1}, tokenKey, "a"},
748		{Position{1, 3}, tokenEqual, "="},
749		{Position{1, 5}, tokenError, `no value can start with \`},
750	})
751}
752
753func BenchmarkLexer(b *testing.B) {
754	sample := `title = "Hugo: A Fast and Flexible Website Generator"
755baseurl = "http://gohugo.io/"
756MetaDataFormat = "yaml"
757pluralizeListTitles = false
758
759[params]
760  description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go"
761  author = "Steve Francia (spf13) and friends"
762  release = "0.22-DEV"
763
764[[menu.main]]
765	name = "Download Hugo"
766	pre = "<i class='fa fa-download'></i>"
767	url = "https://github.com/spf13/hugo/releases"
768	weight = -200
769`
770	rd := strings.NewReader(sample)
771
772	b.ResetTimer()
773	for i := 0; i < b.N; i++ {
774		rd.Seek(0, os.SEEK_SET)
775		ch := lexToml(rd)
776		for _ = range ch {
777		}
778	}
779}
780