1// Copyright 2011 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package parse
6
7import (
8	"fmt"
9	"testing"
10)
11
12// Make the types prettyprint.
13var itemName = map[itemType]string{
14	itemError:        "error",
15	itemBool:         "bool",
16	itemChar:         "char",
17	itemCharConstant: "charconst",
18	itemComplex:      "complex",
19	itemColonEquals:  ":=",
20	itemEOF:          "EOF",
21	itemField:        "field",
22	itemIdentifier:   "identifier",
23	itemLeftDelim:    "left delim",
24	itemLeftParen:    "(",
25	itemNumber:       "number",
26	itemPipe:         "pipe",
27	itemRawString:    "raw string",
28	itemRightDelim:   "right delim",
29	itemElideNewline: "elide newline",
30	itemRightParen:   ")",
31	itemSpace:        "space",
32	itemString:       "string",
33	itemVariable:     "variable",
34
35	// keywords
36	itemDot:      ".",
37	itemDefine:   "define",
38	itemElse:     "else",
39	itemIf:       "if",
40	itemEnd:      "end",
41	itemNil:      "nil",
42	itemRange:    "range",
43	itemTemplate: "template",
44	itemWith:     "with",
45}
46
47func (i itemType) String() string {
48	s := itemName[i]
49	if s == "" {
50		return fmt.Sprintf("item%d", int(i))
51	}
52	return s
53}
54
55type lexTest struct {
56	name  string
57	input string
58	items []item
59}
60
61var (
62	tEOF          = item{itemEOF, 0, ""}
63	tFor          = item{itemIdentifier, 0, "for"}
64	tLeft         = item{itemLeftDelim, 0, "{{"}
65	tLpar         = item{itemLeftParen, 0, "("}
66	tPipe         = item{itemPipe, 0, "|"}
67	tQuote        = item{itemString, 0, `"abc \n\t\" "`}
68	tRange        = item{itemRange, 0, "range"}
69	tRight        = item{itemRightDelim, 0, "}}"}
70	tElideNewline = item{itemElideNewline, 0, "\\"}
71	tRpar         = item{itemRightParen, 0, ")"}
72	tSpace        = item{itemSpace, 0, " "}
73	raw           = "`" + `abc\n\t\" ` + "`"
74	tRawQuote     = item{itemRawString, 0, raw}
75)
76
77var lexTests = []lexTest{
78	{"empty", "", []item{tEOF}},
79	{"spaces", " \t\n", []item{{itemText, 0, " \t\n"}, tEOF}},
80	{"text", `now is the time`, []item{{itemText, 0, "now is the time"}, tEOF}},
81	{"elide newline", "{{}}\\", []item{tLeft, tRight, tElideNewline, tEOF}},
82	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
83		{itemText, 0, "hello-"},
84		{itemText, 0, "-world"},
85		tEOF,
86	}},
87	{"punctuation", "{{,@% }}", []item{
88		tLeft,
89		{itemChar, 0, ","},
90		{itemChar, 0, "@"},
91		{itemChar, 0, "%"},
92		tSpace,
93		tRight,
94		tEOF,
95	}},
96	{"parens", "{{((3))}}", []item{
97		tLeft,
98		tLpar,
99		tLpar,
100		{itemNumber, 0, "3"},
101		tRpar,
102		tRpar,
103		tRight,
104		tEOF,
105	}},
106	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
107	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
108	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
109	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
110	{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
111		tLeft,
112		{itemNumber, 0, "1"},
113		tSpace,
114		{itemNumber, 0, "02"},
115		tSpace,
116		{itemNumber, 0, "0x14"},
117		tSpace,
118		{itemNumber, 0, "-7.2i"},
119		tSpace,
120		{itemNumber, 0, "1e3"},
121		tSpace,
122		{itemNumber, 0, "+1.2e-4"},
123		tSpace,
124		{itemNumber, 0, "4.2i"},
125		tSpace,
126		{itemComplex, 0, "1+2i"},
127		tRight,
128		tEOF,
129	}},
130	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
131		tLeft,
132		{itemCharConstant, 0, `'a'`},
133		tSpace,
134		{itemCharConstant, 0, `'\n'`},
135		tSpace,
136		{itemCharConstant, 0, `'\''`},
137		tSpace,
138		{itemCharConstant, 0, `'\\'`},
139		tSpace,
140		{itemCharConstant, 0, `'\u00FF'`},
141		tSpace,
142		{itemCharConstant, 0, `'\xFF'`},
143		tSpace,
144		{itemCharConstant, 0, `'本'`},
145		tRight,
146		tEOF,
147	}},
148	{"bools", "{{true false}}", []item{
149		tLeft,
150		{itemBool, 0, "true"},
151		tSpace,
152		{itemBool, 0, "false"},
153		tRight,
154		tEOF,
155	}},
156	{"dot", "{{.}}", []item{
157		tLeft,
158		{itemDot, 0, "."},
159		tRight,
160		tEOF,
161	}},
162	{"nil", "{{nil}}", []item{
163		tLeft,
164		{itemNil, 0, "nil"},
165		tRight,
166		tEOF,
167	}},
168	{"dots", "{{.x . .2 .x.y.z}}", []item{
169		tLeft,
170		{itemField, 0, ".x"},
171		tSpace,
172		{itemDot, 0, "."},
173		tSpace,
174		{itemNumber, 0, ".2"},
175		tSpace,
176		{itemField, 0, ".x"},
177		{itemField, 0, ".y"},
178		{itemField, 0, ".z"},
179		tRight,
180		tEOF,
181	}},
182	{"keywords", "{{range if else end with}}", []item{
183		tLeft,
184		{itemRange, 0, "range"},
185		tSpace,
186		{itemIf, 0, "if"},
187		tSpace,
188		{itemElse, 0, "else"},
189		tSpace,
190		{itemEnd, 0, "end"},
191		tSpace,
192		{itemWith, 0, "with"},
193		tRight,
194		tEOF,
195	}},
196	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
197		tLeft,
198		{itemVariable, 0, "$c"},
199		tSpace,
200		{itemColonEquals, 0, ":="},
201		tSpace,
202		{itemIdentifier, 0, "printf"},
203		tSpace,
204		{itemVariable, 0, "$"},
205		tSpace,
206		{itemVariable, 0, "$hello"},
207		tSpace,
208		{itemVariable, 0, "$23"},
209		tSpace,
210		{itemVariable, 0, "$"},
211		tSpace,
212		{itemVariable, 0, "$var"},
213		{itemField, 0, ".Field"},
214		tSpace,
215		{itemField, 0, ".Method"},
216		tRight,
217		tEOF,
218	}},
219	{"variable invocation", "{{$x 23}}", []item{
220		tLeft,
221		{itemVariable, 0, "$x"},
222		tSpace,
223		{itemNumber, 0, "23"},
224		tRight,
225		tEOF,
226	}},
227	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
228		{itemText, 0, "intro "},
229		tLeft,
230		{itemIdentifier, 0, "echo"},
231		tSpace,
232		{itemIdentifier, 0, "hi"},
233		tSpace,
234		{itemNumber, 0, "1.2"},
235		tSpace,
236		tPipe,
237		{itemIdentifier, 0, "noargs"},
238		tPipe,
239		{itemIdentifier, 0, "args"},
240		tSpace,
241		{itemNumber, 0, "1"},
242		tSpace,
243		{itemString, 0, `"hi"`},
244		tRight,
245		{itemText, 0, " outro"},
246		tEOF,
247	}},
248	{"declaration", "{{$v := 3}}", []item{
249		tLeft,
250		{itemVariable, 0, "$v"},
251		tSpace,
252		{itemColonEquals, 0, ":="},
253		tSpace,
254		{itemNumber, 0, "3"},
255		tRight,
256		tEOF,
257	}},
258	{"2 declarations", "{{$v , $w := 3}}", []item{
259		tLeft,
260		{itemVariable, 0, "$v"},
261		tSpace,
262		{itemChar, 0, ","},
263		tSpace,
264		{itemVariable, 0, "$w"},
265		tSpace,
266		{itemColonEquals, 0, ":="},
267		tSpace,
268		{itemNumber, 0, "3"},
269		tRight,
270		tEOF,
271	}},
272	{"field of parenthesized expression", "{{(.X).Y}}", []item{
273		tLeft,
274		tLpar,
275		{itemField, 0, ".X"},
276		tRpar,
277		{itemField, 0, ".Y"},
278		tRight,
279		tEOF,
280	}},
281	// errors
282	{"badchar", "#{{\x01}}", []item{
283		{itemText, 0, "#"},
284		tLeft,
285		{itemError, 0, "unrecognized character in action: U+0001"},
286	}},
287	{"unclosed action", "{{\n}}", []item{
288		tLeft,
289		{itemError, 0, "unclosed action"},
290	}},
291	{"EOF in action", "{{range", []item{
292		tLeft,
293		tRange,
294		{itemError, 0, "unclosed action"},
295	}},
296	{"unclosed quote", "{{\"\n\"}}", []item{
297		tLeft,
298		{itemError, 0, "unterminated quoted string"},
299	}},
300	{"unclosed raw quote", "{{`xx\n`}}", []item{
301		tLeft,
302		{itemError, 0, "unterminated raw quoted string"},
303	}},
304	{"unclosed char constant", "{{'\n}}", []item{
305		tLeft,
306		{itemError, 0, "unterminated character constant"},
307	}},
308	{"bad number", "{{3k}}", []item{
309		tLeft,
310		{itemError, 0, `bad number syntax: "3k"`},
311	}},
312	{"unclosed paren", "{{(3}}", []item{
313		tLeft,
314		tLpar,
315		{itemNumber, 0, "3"},
316		{itemError, 0, `unclosed left paren`},
317	}},
318	{"extra right paren", "{{3)}}", []item{
319		tLeft,
320		{itemNumber, 0, "3"},
321		tRpar,
322		{itemError, 0, `unexpected right paren U+0029 ')'`},
323	}},
324
325	// Fixed bugs
326	// Many elements in an action blew the lookahead until
327	// we made lexInsideAction not loop.
328	{"long pipeline deadlock", "{{|||||}}", []item{
329		tLeft,
330		tPipe,
331		tPipe,
332		tPipe,
333		tPipe,
334		tPipe,
335		tRight,
336		tEOF,
337	}},
338	{"text with bad comment", "hello-{{/*/}}-world", []item{
339		{itemText, 0, "hello-"},
340		{itemError, 0, `unclosed comment`},
341	}},
342	{"text with comment close separted from delim", "hello-{{/* */ }}-world", []item{
343		{itemText, 0, "hello-"},
344		{itemError, 0, `comment ends before closing delimiter`},
345	}},
346	// This one is an error that we can't catch because it breaks templates with
347	// minimized JavaScript. Should have fixed it before Go 1.1.
348	{"unmatched right delimiter", "hello-{.}}-world", []item{
349		{itemText, 0, "hello-{.}}-world"},
350		tEOF,
351	}},
352}
353
354// collect gathers the emitted items into a slice.
355func collect(t *lexTest, left, right string) (items []item) {
356	l := lex(t.name, t.input, left, right)
357	for {
358		item := l.nextItem()
359		items = append(items, item)
360		if item.typ == itemEOF || item.typ == itemError {
361			break
362		}
363	}
364	return
365}
366
367func equal(i1, i2 []item, checkPos bool) bool {
368	if len(i1) != len(i2) {
369		return false
370	}
371	for k := range i1 {
372		if i1[k].typ != i2[k].typ {
373			return false
374		}
375		if i1[k].val != i2[k].val {
376			return false
377		}
378		if checkPos && i1[k].pos != i2[k].pos {
379			return false
380		}
381	}
382	return true
383}
384
385func TestLex(t *testing.T) {
386	for _, test := range lexTests {
387		items := collect(&test, "", "")
388		if !equal(items, test.items, false) {
389			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
390		}
391	}
392}
393
394// Some easy cases from above, but with delimiters $$ and @@
395var lexDelimTests = []lexTest{
396	{"punctuation", "$$,@%{{}}@@", []item{
397		tLeftDelim,
398		{itemChar, 0, ","},
399		{itemChar, 0, "@"},
400		{itemChar, 0, "%"},
401		{itemChar, 0, "{"},
402		{itemChar, 0, "{"},
403		{itemChar, 0, "}"},
404		{itemChar, 0, "}"},
405		tRightDelim,
406		tEOF,
407	}},
408	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
409	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
410	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
411	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
412}
413
414var (
415	tLeftDelim  = item{itemLeftDelim, 0, "$$"}
416	tRightDelim = item{itemRightDelim, 0, "@@"}
417)
418
419func TestDelims(t *testing.T) {
420	for _, test := range lexDelimTests {
421		items := collect(&test, "$$", "@@")
422		if !equal(items, test.items, false) {
423			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
424		}
425	}
426}
427
428var lexPosTests = []lexTest{
429	{"empty", "", []item{tEOF}},
430	{"punctuation", "{{,@%#}}", []item{
431		{itemLeftDelim, 0, "{{"},
432		{itemChar, 2, ","},
433		{itemChar, 3, "@"},
434		{itemChar, 4, "%"},
435		{itemChar, 5, "#"},
436		{itemRightDelim, 6, "}}"},
437		{itemEOF, 8, ""},
438	}},
439	{"sample", "0123{{hello}}xyz", []item{
440		{itemText, 0, "0123"},
441		{itemLeftDelim, 4, "{{"},
442		{itemIdentifier, 6, "hello"},
443		{itemRightDelim, 11, "}}"},
444		{itemText, 13, "xyz"},
445		{itemEOF, 16, ""},
446	}},
447}
448
449// The other tests don't check position, to make the test cases easier to construct.
450// This one does.
451func TestPos(t *testing.T) {
452	for _, test := range lexPosTests {
453		items := collect(&test, "", "")
454		if !equal(items, test.items, true) {
455			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
456			if len(items) == len(test.items) {
457				// Detailed print; avoid item.String() to expose the position value.
458				for i := range items {
459					if !equal(items[i:i+1], test.items[i:i+1], true) {
460						i1 := items[i]
461						i2 := test.items[i]
462						t.Errorf("\t#%d: got {%v %d %q} expected  {%v %d %q}", i, i1.typ, i1.pos, i1.val, i2.typ, i2.pos, i2.val)
463					}
464				}
465			}
466		}
467	}
468}
469