1package toml
2
3import (
4	"reflect"
5	"testing"
6)
7
8func testFlow(t *testing.T, input string, expectedFlow []token) {
9	tokens := lexToml([]byte(input))
10	if !reflect.DeepEqual(tokens, expectedFlow) {
11		t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens)
12	}
13}
14
15func TestValidKeyGroup(t *testing.T) {
16	testFlow(t, "[hello world]", []token{
17		{Position{1, 1}, tokenLeftBracket, "["},
18		{Position{1, 2}, tokenKeyGroup, "hello world"},
19		{Position{1, 13}, tokenRightBracket, "]"},
20		{Position{1, 14}, tokenEOF, ""},
21	})
22}
23
24func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
25	testFlow(t, `[ j . "ʞ" . l ]`, []token{
26		{Position{1, 1}, tokenLeftBracket, "["},
27		{Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l `},
28		{Position{1, 15}, tokenRightBracket, "]"},
29		{Position{1, 16}, tokenEOF, ""},
30	})
31}
32
33func TestUnclosedKeyGroup(t *testing.T) {
34	testFlow(t, "[hello world", []token{
35		{Position{1, 1}, tokenLeftBracket, "["},
36		{Position{1, 2}, tokenError, "unclosed table key"},
37	})
38}
39
40func TestComment(t *testing.T) {
41	testFlow(t, "# blahblah", []token{
42		{Position{1, 11}, tokenEOF, ""},
43	})
44}
45
46func TestKeyGroupComment(t *testing.T) {
47	testFlow(t, "[hello world] # blahblah", []token{
48		{Position{1, 1}, tokenLeftBracket, "["},
49		{Position{1, 2}, tokenKeyGroup, "hello world"},
50		{Position{1, 13}, tokenRightBracket, "]"},
51		{Position{1, 25}, tokenEOF, ""},
52	})
53}
54
55func TestMultipleKeyGroupsComment(t *testing.T) {
56	testFlow(t, "[hello world] # blahblah\n[test]", []token{
57		{Position{1, 1}, tokenLeftBracket, "["},
58		{Position{1, 2}, tokenKeyGroup, "hello world"},
59		{Position{1, 13}, tokenRightBracket, "]"},
60		{Position{2, 1}, tokenLeftBracket, "["},
61		{Position{2, 2}, tokenKeyGroup, "test"},
62		{Position{2, 6}, tokenRightBracket, "]"},
63		{Position{2, 7}, tokenEOF, ""},
64	})
65}
66
67func TestSimpleWindowsCRLF(t *testing.T) {
68	testFlow(t, "a=4\r\nb=2", []token{
69		{Position{1, 1}, tokenKey, "a"},
70		{Position{1, 2}, tokenEqual, "="},
71		{Position{1, 3}, tokenInteger, "4"},
72		{Position{2, 1}, tokenKey, "b"},
73		{Position{2, 2}, tokenEqual, "="},
74		{Position{2, 3}, tokenInteger, "2"},
75		{Position{2, 4}, tokenEOF, ""},
76	})
77}
78
79func TestBasicKey(t *testing.T) {
80	testFlow(t, "hello", []token{
81		{Position{1, 1}, tokenKey, "hello"},
82		{Position{1, 6}, tokenEOF, ""},
83	})
84}
85
86func TestBasicKeyWithUnderscore(t *testing.T) {
87	testFlow(t, "hello_hello", []token{
88		{Position{1, 1}, tokenKey, "hello_hello"},
89		{Position{1, 12}, tokenEOF, ""},
90	})
91}
92
93func TestBasicKeyWithDash(t *testing.T) {
94	testFlow(t, "hello-world", []token{
95		{Position{1, 1}, tokenKey, "hello-world"},
96		{Position{1, 12}, tokenEOF, ""},
97	})
98}
99
100func TestBasicKeyWithUppercaseMix(t *testing.T) {
101	testFlow(t, "helloHELLOHello", []token{
102		{Position{1, 1}, tokenKey, "helloHELLOHello"},
103		{Position{1, 16}, tokenEOF, ""},
104	})
105}
106
107func TestBasicKeyWithInternationalCharacters(t *testing.T) {
108	testFlow(t, "héllÖ", []token{
109		{Position{1, 1}, tokenKey, "héllÖ"},
110		{Position{1, 6}, tokenEOF, ""},
111	})
112}
113
114func TestBasicKeyAndEqual(t *testing.T) {
115	testFlow(t, "hello =", []token{
116		{Position{1, 1}, tokenKey, "hello"},
117		{Position{1, 7}, tokenEqual, "="},
118		{Position{1, 8}, tokenEOF, ""},
119	})
120}
121
122func TestKeyWithSharpAndEqual(t *testing.T) {
123	testFlow(t, "key#name = 5", []token{
124		{Position{1, 1}, tokenError, "keys cannot contain # character"},
125	})
126}
127
128func TestKeyWithSymbolsAndEqual(t *testing.T) {
129	testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
130		{Position{1, 1}, tokenError, "keys cannot contain ~ character"},
131	})
132}
133
134func TestKeyEqualStringEscape(t *testing.T) {
135	testFlow(t, `foo = "hello\""`, []token{
136		{Position{1, 1}, tokenKey, "foo"},
137		{Position{1, 5}, tokenEqual, "="},
138		{Position{1, 8}, tokenString, "hello\""},
139		{Position{1, 16}, tokenEOF, ""},
140	})
141}
142
143func TestKeyEqualStringUnfinished(t *testing.T) {
144	testFlow(t, `foo = "bar`, []token{
145		{Position{1, 1}, tokenKey, "foo"},
146		{Position{1, 5}, tokenEqual, "="},
147		{Position{1, 8}, tokenError, "unclosed string"},
148	})
149}
150
151func TestKeyEqualString(t *testing.T) {
152	testFlow(t, `foo = "bar"`, []token{
153		{Position{1, 1}, tokenKey, "foo"},
154		{Position{1, 5}, tokenEqual, "="},
155		{Position{1, 8}, tokenString, "bar"},
156		{Position{1, 12}, tokenEOF, ""},
157	})
158}
159
160func TestKeyEqualTrue(t *testing.T) {
161	testFlow(t, "foo = true", []token{
162		{Position{1, 1}, tokenKey, "foo"},
163		{Position{1, 5}, tokenEqual, "="},
164		{Position{1, 7}, tokenTrue, "true"},
165		{Position{1, 11}, tokenEOF, ""},
166	})
167}
168
169func TestKeyEqualFalse(t *testing.T) {
170	testFlow(t, "foo = false", []token{
171		{Position{1, 1}, tokenKey, "foo"},
172		{Position{1, 5}, tokenEqual, "="},
173		{Position{1, 7}, tokenFalse, "false"},
174		{Position{1, 12}, tokenEOF, ""},
175	})
176}
177
178func TestArrayNestedString(t *testing.T) {
179	testFlow(t, `a = [ ["hello", "world"] ]`, []token{
180		{Position{1, 1}, tokenKey, "a"},
181		{Position{1, 3}, tokenEqual, "="},
182		{Position{1, 5}, tokenLeftBracket, "["},
183		{Position{1, 7}, tokenLeftBracket, "["},
184		{Position{1, 9}, tokenString, "hello"},
185		{Position{1, 15}, tokenComma, ","},
186		{Position{1, 18}, tokenString, "world"},
187		{Position{1, 24}, tokenRightBracket, "]"},
188		{Position{1, 26}, tokenRightBracket, "]"},
189		{Position{1, 27}, tokenEOF, ""},
190	})
191}
192
193func TestArrayNestedInts(t *testing.T) {
194	testFlow(t, "a = [ [42, 21], [10] ]", []token{
195		{Position{1, 1}, tokenKey, "a"},
196		{Position{1, 3}, tokenEqual, "="},
197		{Position{1, 5}, tokenLeftBracket, "["},
198		{Position{1, 7}, tokenLeftBracket, "["},
199		{Position{1, 8}, tokenInteger, "42"},
200		{Position{1, 10}, tokenComma, ","},
201		{Position{1, 12}, tokenInteger, "21"},
202		{Position{1, 14}, tokenRightBracket, "]"},
203		{Position{1, 15}, tokenComma, ","},
204		{Position{1, 17}, tokenLeftBracket, "["},
205		{Position{1, 18}, tokenInteger, "10"},
206		{Position{1, 20}, tokenRightBracket, "]"},
207		{Position{1, 22}, tokenRightBracket, "]"},
208		{Position{1, 23}, tokenEOF, ""},
209	})
210}
211
212func TestArrayInts(t *testing.T) {
213	testFlow(t, "a = [ 42, 21, 10, ]", []token{
214		{Position{1, 1}, tokenKey, "a"},
215		{Position{1, 3}, tokenEqual, "="},
216		{Position{1, 5}, tokenLeftBracket, "["},
217		{Position{1, 7}, tokenInteger, "42"},
218		{Position{1, 9}, tokenComma, ","},
219		{Position{1, 11}, tokenInteger, "21"},
220		{Position{1, 13}, tokenComma, ","},
221		{Position{1, 15}, tokenInteger, "10"},
222		{Position{1, 17}, tokenComma, ","},
223		{Position{1, 19}, tokenRightBracket, "]"},
224		{Position{1, 20}, tokenEOF, ""},
225	})
226}
227
228func TestMultilineArrayComments(t *testing.T) {
229	testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
230		{Position{1, 1}, tokenKey, "a"},
231		{Position{1, 3}, tokenEqual, "="},
232		{Position{1, 5}, tokenLeftBracket, "["},
233		{Position{1, 6}, tokenInteger, "1"},
234		{Position{1, 7}, tokenComma, ","},
235		{Position{2, 1}, tokenInteger, "2"},
236		{Position{2, 2}, tokenComma, ","},
237		{Position{3, 1}, tokenInteger, "3"},
238		{Position{3, 2}, tokenComma, ","},
239		{Position{4, 1}, tokenRightBracket, "]"},
240		{Position{4, 2}, tokenEOF, ""},
241	})
242}
243
244func TestNestedArraysComment(t *testing.T) {
245	toml := `
246someArray = [
247# does not work
248["entry1"]
249]`
250	testFlow(t, toml, []token{
251		{Position{2, 1}, tokenKey, "someArray"},
252		{Position{2, 11}, tokenEqual, "="},
253		{Position{2, 13}, tokenLeftBracket, "["},
254		{Position{4, 1}, tokenLeftBracket, "["},
255		{Position{4, 3}, tokenString, "entry1"},
256		{Position{4, 10}, tokenRightBracket, "]"},
257		{Position{5, 1}, tokenRightBracket, "]"},
258		{Position{5, 2}, tokenEOF, ""},
259	})
260}
261
262func TestKeyEqualArrayBools(t *testing.T) {
263	testFlow(t, "foo = [true, false, true]", []token{
264		{Position{1, 1}, tokenKey, "foo"},
265		{Position{1, 5}, tokenEqual, "="},
266		{Position{1, 7}, tokenLeftBracket, "["},
267		{Position{1, 8}, tokenTrue, "true"},
268		{Position{1, 12}, tokenComma, ","},
269		{Position{1, 14}, tokenFalse, "false"},
270		{Position{1, 19}, tokenComma, ","},
271		{Position{1, 21}, tokenTrue, "true"},
272		{Position{1, 25}, tokenRightBracket, "]"},
273		{Position{1, 26}, tokenEOF, ""},
274	})
275}
276
277func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
278	testFlow(t, "foo = [true, false, true] # YEAH", []token{
279		{Position{1, 1}, tokenKey, "foo"},
280		{Position{1, 5}, tokenEqual, "="},
281		{Position{1, 7}, tokenLeftBracket, "["},
282		{Position{1, 8}, tokenTrue, "true"},
283		{Position{1, 12}, tokenComma, ","},
284		{Position{1, 14}, tokenFalse, "false"},
285		{Position{1, 19}, tokenComma, ","},
286		{Position{1, 21}, tokenTrue, "true"},
287		{Position{1, 25}, tokenRightBracket, "]"},
288		{Position{1, 33}, tokenEOF, ""},
289	})
290}
291
292func TestDateRegexp(t *testing.T) {
293	cases := map[string]string{
294		"basic":               "1979-05-27T07:32:00Z",
295		"offset":              "1979-05-27T00:32:00-07:00",
296		"nano precision":      "1979-05-27T00:32:00.999999-07:00",
297		"basic-no-T":          "1979-05-27 07:32:00Z",
298		"offset-no-T":         "1979-05-27 00:32:00-07:00",
299		"nano precision-no-T": "1979-05-27 00:32:00.999999-07:00",
300		"no-tz":               "1979-05-27T07:32:00",
301		"no-tz-nano":          "1979-05-27T00:32:00.999999",
302		"no-tz-no-t":          "1979-05-27 07:32:00",
303		"no-tz-no-t-nano":     "1979-05-27 00:32:00.999999",
304		"date-no-tz":          "1979-05-27",
305		"time-no-tz":          "07:32:00",
306		"time-no-tz-nano":     "00:32:00.999999",
307	}
308
309	for name, value := range cases {
310		if dateRegexp.FindString(value) == "" {
311			t.Error("failed date regexp test", name)
312		}
313	}
314	if dateRegexp.FindString("1979-05-27 07:32:00Z") == "" {
315		t.Error("space delimiter lexing")
316	}
317}
318
319func TestKeyEqualDate(t *testing.T) {
320	testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
321		{Position{1, 1}, tokenKey, "foo"},
322		{Position{1, 5}, tokenEqual, "="},
323		{Position{1, 7}, tokenDate, "1979-05-27T07:32:00Z"},
324		{Position{1, 27}, tokenEOF, ""},
325	})
326	testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
327		{Position{1, 1}, tokenKey, "foo"},
328		{Position{1, 5}, tokenEqual, "="},
329		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00-07:00"},
330		{Position{1, 32}, tokenEOF, ""},
331	})
332	testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
333		{Position{1, 1}, tokenKey, "foo"},
334		{Position{1, 5}, tokenEqual, "="},
335		{Position{1, 7}, tokenDate, "1979-05-27T00:32:00.999999-07:00"},
336		{Position{1, 39}, tokenEOF, ""},
337	})
338	testFlow(t, "foo = 1979-05-27 07:32:00Z", []token{
339		{Position{1, 1}, tokenKey, "foo"},
340		{Position{1, 5}, tokenEqual, "="},
341		{Position{1, 7}, tokenDate, "1979-05-27 07:32:00Z"},
342		{Position{1, 27}, tokenEOF, ""},
343	})
344}
345
346func TestFloatEndingWithDot(t *testing.T) {
347	testFlow(t, "foo = 42.", []token{
348		{Position{1, 1}, tokenKey, "foo"},
349		{Position{1, 5}, tokenEqual, "="},
350		{Position{1, 7}, tokenError, "float cannot end with a dot"},
351	})
352}
353
354func TestFloatWithTwoDots(t *testing.T) {
355	testFlow(t, "foo = 4.2.", []token{
356		{Position{1, 1}, tokenKey, "foo"},
357		{Position{1, 5}, tokenEqual, "="},
358		{Position{1, 7}, tokenError, "cannot have two dots in one float"},
359	})
360}
361
362func TestFloatWithExponent1(t *testing.T) {
363	testFlow(t, "a = 5e+22", []token{
364		{Position{1, 1}, tokenKey, "a"},
365		{Position{1, 3}, tokenEqual, "="},
366		{Position{1, 5}, tokenFloat, "5e+22"},
367		{Position{1, 10}, tokenEOF, ""},
368	})
369}
370
371func TestFloatWithExponent2(t *testing.T) {
372	testFlow(t, "a = 5E+22", []token{
373		{Position{1, 1}, tokenKey, "a"},
374		{Position{1, 3}, tokenEqual, "="},
375		{Position{1, 5}, tokenFloat, "5E+22"},
376		{Position{1, 10}, tokenEOF, ""},
377	})
378}
379
380func TestFloatWithExponent3(t *testing.T) {
381	testFlow(t, "a = -5e+22", []token{
382		{Position{1, 1}, tokenKey, "a"},
383		{Position{1, 3}, tokenEqual, "="},
384		{Position{1, 5}, tokenFloat, "-5e+22"},
385		{Position{1, 11}, tokenEOF, ""},
386	})
387}
388
389func TestFloatWithExponent4(t *testing.T) {
390	testFlow(t, "a = -5e-22", []token{
391		{Position{1, 1}, tokenKey, "a"},
392		{Position{1, 3}, tokenEqual, "="},
393		{Position{1, 5}, tokenFloat, "-5e-22"},
394		{Position{1, 11}, tokenEOF, ""},
395	})
396}
397
398func TestFloatWithExponent5(t *testing.T) {
399	testFlow(t, "a = 6.626e-34", []token{
400		{Position{1, 1}, tokenKey, "a"},
401		{Position{1, 3}, tokenEqual, "="},
402		{Position{1, 5}, tokenFloat, "6.626e-34"},
403		{Position{1, 14}, tokenEOF, ""},
404	})
405}
406
407func TestInvalidEsquapeSequence(t *testing.T) {
408	testFlow(t, `foo = "\x"`, []token{
409		{Position{1, 1}, tokenKey, "foo"},
410		{Position{1, 5}, tokenEqual, "="},
411		{Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
412	})
413}
414
415func TestNestedArrays(t *testing.T) {
416	testFlow(t, "foo = [[[]]]", []token{
417		{Position{1, 1}, tokenKey, "foo"},
418		{Position{1, 5}, tokenEqual, "="},
419		{Position{1, 7}, tokenLeftBracket, "["},
420		{Position{1, 8}, tokenLeftBracket, "["},
421		{Position{1, 9}, tokenLeftBracket, "["},
422		{Position{1, 10}, tokenRightBracket, "]"},
423		{Position{1, 11}, tokenRightBracket, "]"},
424		{Position{1, 12}, tokenRightBracket, "]"},
425		{Position{1, 13}, tokenEOF, ""},
426	})
427}
428
429func TestKeyEqualNumber(t *testing.T) {
430	testFlow(t, "foo = 42", []token{
431		{Position{1, 1}, tokenKey, "foo"},
432		{Position{1, 5}, tokenEqual, "="},
433		{Position{1, 7}, tokenInteger, "42"},
434		{Position{1, 9}, tokenEOF, ""},
435	})
436
437	testFlow(t, "foo = +42", []token{
438		{Position{1, 1}, tokenKey, "foo"},
439		{Position{1, 5}, tokenEqual, "="},
440		{Position{1, 7}, tokenInteger, "+42"},
441		{Position{1, 10}, tokenEOF, ""},
442	})
443
444	testFlow(t, "foo = -42", []token{
445		{Position{1, 1}, tokenKey, "foo"},
446		{Position{1, 5}, tokenEqual, "="},
447		{Position{1, 7}, tokenInteger, "-42"},
448		{Position{1, 10}, tokenEOF, ""},
449	})
450
451	testFlow(t, "foo = 4.2", []token{
452		{Position{1, 1}, tokenKey, "foo"},
453		{Position{1, 5}, tokenEqual, "="},
454		{Position{1, 7}, tokenFloat, "4.2"},
455		{Position{1, 10}, tokenEOF, ""},
456	})
457
458	testFlow(t, "foo = +4.2", []token{
459		{Position{1, 1}, tokenKey, "foo"},
460		{Position{1, 5}, tokenEqual, "="},
461		{Position{1, 7}, tokenFloat, "+4.2"},
462		{Position{1, 11}, tokenEOF, ""},
463	})
464
465	testFlow(t, "foo = -4.2", []token{
466		{Position{1, 1}, tokenKey, "foo"},
467		{Position{1, 5}, tokenEqual, "="},
468		{Position{1, 7}, tokenFloat, "-4.2"},
469		{Position{1, 11}, tokenEOF, ""},
470	})
471
472	testFlow(t, "foo = 1_000", []token{
473		{Position{1, 1}, tokenKey, "foo"},
474		{Position{1, 5}, tokenEqual, "="},
475		{Position{1, 7}, tokenInteger, "1_000"},
476		{Position{1, 12}, tokenEOF, ""},
477	})
478
479	testFlow(t, "foo = 5_349_221", []token{
480		{Position{1, 1}, tokenKey, "foo"},
481		{Position{1, 5}, tokenEqual, "="},
482		{Position{1, 7}, tokenInteger, "5_349_221"},
483		{Position{1, 16}, tokenEOF, ""},
484	})
485
486	testFlow(t, "foo = 1_2_3_4_5", []token{
487		{Position{1, 1}, tokenKey, "foo"},
488		{Position{1, 5}, tokenEqual, "="},
489		{Position{1, 7}, tokenInteger, "1_2_3_4_5"},
490		{Position{1, 16}, tokenEOF, ""},
491	})
492
493	testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
494		{Position{1, 1}, tokenKey, "flt8"},
495		{Position{1, 6}, tokenEqual, "="},
496		{Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
497		{Position{1, 33}, tokenEOF, ""},
498	})
499
500	testFlow(t, "foo = +", []token{
501		{Position{1, 1}, tokenKey, "foo"},
502		{Position{1, 5}, tokenEqual, "="},
503		{Position{1, 7}, tokenError, "no digit in that number"},
504	})
505}
506
507func TestMultiline(t *testing.T) {
508	testFlow(t, "foo = 42\nbar=21", []token{
509		{Position{1, 1}, tokenKey, "foo"},
510		{Position{1, 5}, tokenEqual, "="},
511		{Position{1, 7}, tokenInteger, "42"},
512		{Position{2, 1}, tokenKey, "bar"},
513		{Position{2, 4}, tokenEqual, "="},
514		{Position{2, 5}, tokenInteger, "21"},
515		{Position{2, 7}, tokenEOF, ""},
516	})
517}
518
519func TestKeyEqualStringUnicodeEscape(t *testing.T) {
520	testFlow(t, `foo = "hello \u2665"`, []token{
521		{Position{1, 1}, tokenKey, "foo"},
522		{Position{1, 5}, tokenEqual, "="},
523		{Position{1, 8}, tokenString, "hello ♥"},
524		{Position{1, 21}, tokenEOF, ""},
525	})
526	testFlow(t, `foo = "hello \U000003B4"`, []token{
527		{Position{1, 1}, tokenKey, "foo"},
528		{Position{1, 5}, tokenEqual, "="},
529		{Position{1, 8}, tokenString, "hello δ"},
530		{Position{1, 25}, tokenEOF, ""},
531	})
532	testFlow(t, `foo = "\uabcd"`, []token{
533		{Position{1, 1}, tokenKey, "foo"},
534		{Position{1, 5}, tokenEqual, "="},
535		{Position{1, 8}, tokenString, "\uabcd"},
536		{Position{1, 15}, tokenEOF, ""},
537	})
538	testFlow(t, `foo = "\uABCD"`, []token{
539		{Position{1, 1}, tokenKey, "foo"},
540		{Position{1, 5}, tokenEqual, "="},
541		{Position{1, 8}, tokenString, "\uABCD"},
542		{Position{1, 15}, tokenEOF, ""},
543	})
544	testFlow(t, `foo = "\U000bcdef"`, []token{
545		{Position{1, 1}, tokenKey, "foo"},
546		{Position{1, 5}, tokenEqual, "="},
547		{Position{1, 8}, tokenString, "\U000bcdef"},
548		{Position{1, 19}, tokenEOF, ""},
549	})
550	testFlow(t, `foo = "\U000BCDEF"`, []token{
551		{Position{1, 1}, tokenKey, "foo"},
552		{Position{1, 5}, tokenEqual, "="},
553		{Position{1, 8}, tokenString, "\U000BCDEF"},
554		{Position{1, 19}, tokenEOF, ""},
555	})
556	testFlow(t, `foo = "\u2"`, []token{
557		{Position{1, 1}, tokenKey, "foo"},
558		{Position{1, 5}, tokenEqual, "="},
559		{Position{1, 8}, tokenError, "unfinished unicode escape"},
560	})
561	testFlow(t, `foo = "\U2"`, []token{
562		{Position{1, 1}, tokenKey, "foo"},
563		{Position{1, 5}, tokenEqual, "="},
564		{Position{1, 8}, tokenError, "unfinished unicode escape"},
565	})
566}
567
568func TestKeyEqualStringNoEscape(t *testing.T) {
569	testFlow(t, "foo = \"hello \u0002\"", []token{
570		{Position{1, 1}, tokenKey, "foo"},
571		{Position{1, 5}, tokenEqual, "="},
572		{Position{1, 8}, tokenError, "unescaped control character U+0002"},
573	})
574	testFlow(t, "foo = \"hello \u001F\"", []token{
575		{Position{1, 1}, tokenKey, "foo"},
576		{Position{1, 5}, tokenEqual, "="},
577		{Position{1, 8}, tokenError, "unescaped control character U+001F"},
578	})
579}
580
581func TestLiteralString(t *testing.T) {
582	testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
583		{Position{1, 1}, tokenKey, "foo"},
584		{Position{1, 5}, tokenEqual, "="},
585		{Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
586		{Position{1, 34}, tokenEOF, ""},
587	})
588	testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
589		{Position{1, 1}, tokenKey, "foo"},
590		{Position{1, 5}, tokenEqual, "="},
591		{Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
592		{Position{1, 35}, tokenEOF, ""},
593	})
594	testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
595		{Position{1, 1}, tokenKey, "foo"},
596		{Position{1, 5}, tokenEqual, "="},
597		{Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
598		{Position{1, 34}, tokenEOF, ""},
599	})
600	testFlow(t, `foo = '<\i\c*\s*>'`, []token{
601		{Position{1, 1}, tokenKey, "foo"},
602		{Position{1, 5}, tokenEqual, "="},
603		{Position{1, 8}, tokenString, `<\i\c*\s*>`},
604		{Position{1, 19}, tokenEOF, ""},
605	})
606	testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
607		{Position{1, 1}, tokenKey, "foo"},
608		{Position{1, 5}, tokenEqual, "="},
609		{Position{1, 8}, tokenError, "unclosed string"},
610	})
611}
612
613func TestMultilineLiteralString(t *testing.T) {
614	testFlow(t, `foo = '''hello 'literal' world'''`, []token{
615		{Position{1, 1}, tokenKey, "foo"},
616		{Position{1, 5}, tokenEqual, "="},
617		{Position{1, 10}, tokenString, `hello 'literal' world`},
618		{Position{1, 34}, tokenEOF, ""},
619	})
620
621	testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
622		{Position{1, 1}, tokenKey, "foo"},
623		{Position{1, 5}, tokenEqual, "="},
624		{Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
625		{Position{4, 9}, tokenEOF, ""},
626	})
627	testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
628		{Position{1, 1}, tokenKey, "foo"},
629		{Position{1, 5}, tokenEqual, "="},
630		{Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
631		{Position{4, 9}, tokenEOF, ""},
632	})
633}
634
635func TestMultilineString(t *testing.T) {
636	testFlow(t, `foo = """hello "literal" world"""`, []token{
637		{Position{1, 1}, tokenKey, "foo"},
638		{Position{1, 5}, tokenEqual, "="},
639		{Position{1, 10}, tokenString, `hello "literal" world`},
640		{Position{1, 34}, tokenEOF, ""},
641	})
642
643	testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
644		{Position{1, 1}, tokenKey, "foo"},
645		{Position{1, 5}, tokenEqual, "="},
646		{Position{2, 1}, tokenString, "hello\"literal\"world"},
647		{Position{4, 9}, tokenEOF, ""},
648	})
649
650	testFlow(t, "foo = \"\"\"\\\n    \\\n    \\\n    hello\\\nmultiline\\\nworld\"\"\"", []token{
651		{Position{1, 1}, tokenKey, "foo"},
652		{Position{1, 5}, tokenEqual, "="},
653		{Position{1, 10}, tokenString, "hellomultilineworld"},
654		{Position{6, 9}, tokenEOF, ""},
655	})
656
657	testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n  fox jumps over \\\n    the lazy dog.\"\"\"", []token{
658		{Position{1, 1}, tokenKey, "key2"},
659		{Position{1, 6}, tokenEqual, "="},
660		{Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
661		{Position{6, 21}, tokenEOF, ""},
662	})
663
664	testFlow(t, "key2 = \"\"\"\\\n       The quick brown \\\n       fox jumps over \\\n       the lazy dog.\\\n       \"\"\"", []token{
665		{Position{1, 1}, tokenKey, "key2"},
666		{Position{1, 6}, tokenEqual, "="},
667		{Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
668		{Position{5, 11}, tokenEOF, ""},
669	})
670
671	testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
672		{Position{1, 1}, tokenKey, "key2"},
673		{Position{1, 6}, tokenEqual, "="},
674		{Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
675		{Position{1, 41}, tokenEOF, ""},
676	})
677
678	testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
679		{Position{1, 1}, tokenKey, "key2"},
680		{Position{1, 6}, tokenEqual, "="},
681		{Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
682		{Position{3, 20}, tokenEOF, ""},
683	})
684}
685
686func TestUnicodeString(t *testing.T) {
687	testFlow(t, `foo = "hello ♥ world"`, []token{
688		{Position{1, 1}, tokenKey, "foo"},
689		{Position{1, 5}, tokenEqual, "="},
690		{Position{1, 8}, tokenString, "hello ♥ world"},
691		{Position{1, 22}, tokenEOF, ""},
692	})
693}
694func TestEscapeInString(t *testing.T) {
695	testFlow(t, `foo = "\b\f\/"`, []token{
696		{Position{1, 1}, tokenKey, "foo"},
697		{Position{1, 5}, tokenEqual, "="},
698		{Position{1, 8}, tokenString, "\b\f/"},
699		{Position{1, 15}, tokenEOF, ""},
700	})
701}
702
703func TestKeyGroupArray(t *testing.T) {
704	testFlow(t, "[[foo]]", []token{
705		{Position{1, 1}, tokenDoubleLeftBracket, "[["},
706		{Position{1, 3}, tokenKeyGroupArray, "foo"},
707		{Position{1, 6}, tokenDoubleRightBracket, "]]"},
708		{Position{1, 8}, tokenEOF, ""},
709	})
710}
711
712func TestQuotedKey(t *testing.T) {
713	testFlow(t, "\"a b\" = 42", []token{
714		{Position{1, 1}, tokenKey, "\"a b\""},
715		{Position{1, 7}, tokenEqual, "="},
716		{Position{1, 9}, tokenInteger, "42"},
717		{Position{1, 11}, tokenEOF, ""},
718	})
719}
720
721func TestKeyNewline(t *testing.T) {
722	testFlow(t, "a\n= 4", []token{
723		{Position{1, 1}, tokenError, "keys cannot contain new lines"},
724	})
725}
726
727func TestInvalidFloat(t *testing.T) {
728	testFlow(t, "a=7e1_", []token{
729		{Position{1, 1}, tokenKey, "a"},
730		{Position{1, 2}, tokenEqual, "="},
731		{Position{1, 3}, tokenFloat, "7e1_"},
732		{Position{1, 7}, tokenEOF, ""},
733	})
734}
735
736func TestLexUnknownRvalue(t *testing.T) {
737	testFlow(t, `a = !b`, []token{
738		{Position{1, 1}, tokenKey, "a"},
739		{Position{1, 3}, tokenEqual, "="},
740		{Position{1, 5}, tokenError, "no value can start with !"},
741	})
742
743	testFlow(t, `a = \b`, []token{
744		{Position{1, 1}, tokenKey, "a"},
745		{Position{1, 3}, tokenEqual, "="},
746		{Position{1, 5}, tokenError, `no value can start with \`},
747	})
748}
749
750func TestLexInlineTableBareKey(t *testing.T) {
751	testFlow(t, `foo = { bar = "baz" }`, []token{
752		{Position{1, 1}, tokenKey, "foo"},
753		{Position{1, 5}, tokenEqual, "="},
754		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
755		{Position{1, 9}, tokenKey, "bar"},
756		{Position{1, 13}, tokenEqual, "="},
757		{Position{1, 16}, tokenString, "baz"},
758		{Position{1, 21}, tokenRightCurlyBrace, "}"},
759		{Position{1, 22}, tokenEOF, ""},
760	})
761}
762
763func TestLexInlineTableBareKeyDash(t *testing.T) {
764	testFlow(t, `foo = { -bar = "baz" }`, []token{
765		{Position{1, 1}, tokenKey, "foo"},
766		{Position{1, 5}, tokenEqual, "="},
767		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
768		{Position{1, 9}, tokenKey, "-bar"},
769		{Position{1, 14}, tokenEqual, "="},
770		{Position{1, 17}, tokenString, "baz"},
771		{Position{1, 22}, tokenRightCurlyBrace, "}"},
772		{Position{1, 23}, tokenEOF, ""},
773	})
774}
775
776func TestLexInlineTableBareKeyUnderscore(t *testing.T) {
777	testFlow(t, `foo = { _bar = "baz" }`, []token{
778		{Position{1, 1}, tokenKey, "foo"},
779		{Position{1, 5}, tokenEqual, "="},
780		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
781		{Position{1, 9}, tokenKey, "_bar"},
782		{Position{1, 14}, tokenEqual, "="},
783		{Position{1, 17}, tokenString, "baz"},
784		{Position{1, 22}, tokenRightCurlyBrace, "}"},
785		{Position{1, 23}, tokenEOF, ""},
786	})
787}
788
789func TestLexInlineTableQuotedKey(t *testing.T) {
790	testFlow(t, `foo = { "bar" = "baz" }`, []token{
791		{Position{1, 1}, tokenKey, "foo"},
792		{Position{1, 5}, tokenEqual, "="},
793		{Position{1, 7}, tokenLeftCurlyBrace, "{"},
794		{Position{1, 9}, tokenKey, "\"bar\""},
795		{Position{1, 15}, tokenEqual, "="},
796		{Position{1, 18}, tokenString, "baz"},
797		{Position{1, 23}, tokenRightCurlyBrace, "}"},
798		{Position{1, 24}, tokenEOF, ""},
799	})
800}
801
802func BenchmarkLexer(b *testing.B) {
803	sample := `title = "Hugo: A Fast and Flexible Website Generator"
804baseurl = "http://gohugo.io/"
805MetaDataFormat = "yaml"
806pluralizeListTitles = false
807
808[params]
809  description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go"
810  author = "Steve Francia (spf13) and friends"
811  release = "0.22-DEV"
812
813[[menu.main]]
814	name = "Download Hugo"
815	pre = "<i class='fa fa-download'></i>"
816	url = "https://github.com/spf13/hugo/releases"
817	weight = -200
818`
819	b.ResetTimer()
820	for i := 0; i < b.N; i++ {
821		lexToml([]byte(sample))
822	}
823}
824