1// Copyright 2011 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package parse
6
7import (
8	"fmt"
9	"testing"
10)
11
12// Make the types prettyprint.
13var itemName = map[itemType]string{
14	itemError:        "error",
15	itemBool:         "bool",
16	itemChar:         "char",
17	itemCharConstant: "charconst",
18	itemComment:      "comment",
19	itemComplex:      "complex",
20	itemDeclare:      ":=",
21	itemEOF:          "EOF",
22	itemField:        "field",
23	itemIdentifier:   "identifier",
24	itemLeftDelim:    "left delim",
25	itemLeftParen:    "(",
26	itemNumber:       "number",
27	itemPipe:         "pipe",
28	itemRawString:    "raw string",
29	itemRightDelim:   "right delim",
30	itemRightParen:   ")",
31	itemSpace:        "space",
32	itemString:       "string",
33	itemVariable:     "variable",
34
35	// keywords
36	itemDot:      ".",
37	itemBlock:    "block",
38	itemBreak:    "break",
39	itemContinue: "continue",
40	itemDefine:   "define",
41	itemElse:     "else",
42	itemIf:       "if",
43	itemEnd:      "end",
44	itemNil:      "nil",
45	itemRange:    "range",
46	itemTemplate: "template",
47	itemWith:     "with",
48}
49
50func (i itemType) String() string {
51	s := itemName[i]
52	if s == "" {
53		return fmt.Sprintf("item%d", int(i))
54	}
55	return s
56}
57
58type lexTest struct {
59	name  string
60	input string
61	items []item
62}
63
64func mkItem(typ itemType, text string) item {
65	return item{
66		typ: typ,
67		val: text,
68	}
69}
70
71var (
72	tDot        = mkItem(itemDot, ".")
73	tBlock      = mkItem(itemBlock, "block")
74	tEOF        = mkItem(itemEOF, "")
75	tFor        = mkItem(itemIdentifier, "for")
76	tLeft       = mkItem(itemLeftDelim, "{{")
77	tLpar       = mkItem(itemLeftParen, "(")
78	tPipe       = mkItem(itemPipe, "|")
79	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
80	tRange      = mkItem(itemRange, "range")
81	tRight      = mkItem(itemRightDelim, "}}")
82	tRpar       = mkItem(itemRightParen, ")")
83	tSpace      = mkItem(itemSpace, " ")
84	raw         = "`" + `abc\n\t\" ` + "`"
85	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
86	tRawQuote   = mkItem(itemRawString, raw)
87	tRawQuoteNL = mkItem(itemRawString, rawNL)
88)
89
90var lexTests = []lexTest{
91	{"empty", "", []item{tEOF}},
92	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
93	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
94	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
95		mkItem(itemText, "hello-"),
96		mkItem(itemComment, "/* this is a comment */"),
97		mkItem(itemText, "-world"),
98		tEOF,
99	}},
100	{"punctuation", "{{,@% }}", []item{
101		tLeft,
102		mkItem(itemChar, ","),
103		mkItem(itemChar, "@"),
104		mkItem(itemChar, "%"),
105		tSpace,
106		tRight,
107		tEOF,
108	}},
109	{"parens", "{{((3))}}", []item{
110		tLeft,
111		tLpar,
112		tLpar,
113		mkItem(itemNumber, "3"),
114		tRpar,
115		tRpar,
116		tRight,
117		tEOF,
118	}},
119	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
120	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
121	{"block", `{{block "foo" .}}`, []item{
122		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
123	}},
124	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
125	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
126	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
127	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
128		tLeft,
129		mkItem(itemNumber, "1"),
130		tSpace,
131		mkItem(itemNumber, "02"),
132		tSpace,
133		mkItem(itemNumber, "0x14"),
134		tSpace,
135		mkItem(itemNumber, "0X14"),
136		tSpace,
137		mkItem(itemNumber, "-7.2i"),
138		tSpace,
139		mkItem(itemNumber, "1e3"),
140		tSpace,
141		mkItem(itemNumber, "1E3"),
142		tSpace,
143		mkItem(itemNumber, "+1.2e-4"),
144		tSpace,
145		mkItem(itemNumber, "4.2i"),
146		tSpace,
147		mkItem(itemComplex, "1+2i"),
148		tSpace,
149		mkItem(itemNumber, "1_2"),
150		tSpace,
151		mkItem(itemNumber, "0x1.e_fp4"),
152		tSpace,
153		mkItem(itemNumber, "0X1.E_FP4"),
154		tRight,
155		tEOF,
156	}},
157	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
158		tLeft,
159		mkItem(itemCharConstant, `'a'`),
160		tSpace,
161		mkItem(itemCharConstant, `'\n'`),
162		tSpace,
163		mkItem(itemCharConstant, `'\''`),
164		tSpace,
165		mkItem(itemCharConstant, `'\\'`),
166		tSpace,
167		mkItem(itemCharConstant, `'\u00FF'`),
168		tSpace,
169		mkItem(itemCharConstant, `'\xFF'`),
170		tSpace,
171		mkItem(itemCharConstant, `'本'`),
172		tRight,
173		tEOF,
174	}},
175	{"bools", "{{true false}}", []item{
176		tLeft,
177		mkItem(itemBool, "true"),
178		tSpace,
179		mkItem(itemBool, "false"),
180		tRight,
181		tEOF,
182	}},
183	{"dot", "{{.}}", []item{
184		tLeft,
185		tDot,
186		tRight,
187		tEOF,
188	}},
189	{"nil", "{{nil}}", []item{
190		tLeft,
191		mkItem(itemNil, "nil"),
192		tRight,
193		tEOF,
194	}},
195	{"dots", "{{.x . .2 .x.y.z}}", []item{
196		tLeft,
197		mkItem(itemField, ".x"),
198		tSpace,
199		tDot,
200		tSpace,
201		mkItem(itemNumber, ".2"),
202		tSpace,
203		mkItem(itemField, ".x"),
204		mkItem(itemField, ".y"),
205		mkItem(itemField, ".z"),
206		tRight,
207		tEOF,
208	}},
209	{"keywords", "{{range if else end with}}", []item{
210		tLeft,
211		mkItem(itemRange, "range"),
212		tSpace,
213		mkItem(itemIf, "if"),
214		tSpace,
215		mkItem(itemElse, "else"),
216		tSpace,
217		mkItem(itemEnd, "end"),
218		tSpace,
219		mkItem(itemWith, "with"),
220		tRight,
221		tEOF,
222	}},
223	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
224		tLeft,
225		mkItem(itemVariable, "$c"),
226		tSpace,
227		mkItem(itemDeclare, ":="),
228		tSpace,
229		mkItem(itemIdentifier, "printf"),
230		tSpace,
231		mkItem(itemVariable, "$"),
232		tSpace,
233		mkItem(itemVariable, "$hello"),
234		tSpace,
235		mkItem(itemVariable, "$23"),
236		tSpace,
237		mkItem(itemVariable, "$"),
238		tSpace,
239		mkItem(itemVariable, "$var"),
240		mkItem(itemField, ".Field"),
241		tSpace,
242		mkItem(itemField, ".Method"),
243		tRight,
244		tEOF,
245	}},
246	{"variable invocation", "{{$x 23}}", []item{
247		tLeft,
248		mkItem(itemVariable, "$x"),
249		tSpace,
250		mkItem(itemNumber, "23"),
251		tRight,
252		tEOF,
253	}},
254	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
255		mkItem(itemText, "intro "),
256		tLeft,
257		mkItem(itemIdentifier, "echo"),
258		tSpace,
259		mkItem(itemIdentifier, "hi"),
260		tSpace,
261		mkItem(itemNumber, "1.2"),
262		tSpace,
263		tPipe,
264		mkItem(itemIdentifier, "noargs"),
265		tPipe,
266		mkItem(itemIdentifier, "args"),
267		tSpace,
268		mkItem(itemNumber, "1"),
269		tSpace,
270		mkItem(itemString, `"hi"`),
271		tRight,
272		mkItem(itemText, " outro"),
273		tEOF,
274	}},
275	{"declaration", "{{$v := 3}}", []item{
276		tLeft,
277		mkItem(itemVariable, "$v"),
278		tSpace,
279		mkItem(itemDeclare, ":="),
280		tSpace,
281		mkItem(itemNumber, "3"),
282		tRight,
283		tEOF,
284	}},
285	{"2 declarations", "{{$v , $w := 3}}", []item{
286		tLeft,
287		mkItem(itemVariable, "$v"),
288		tSpace,
289		mkItem(itemChar, ","),
290		tSpace,
291		mkItem(itemVariable, "$w"),
292		tSpace,
293		mkItem(itemDeclare, ":="),
294		tSpace,
295		mkItem(itemNumber, "3"),
296		tRight,
297		tEOF,
298	}},
299	{"field of parenthesized expression", "{{(.X).Y}}", []item{
300		tLeft,
301		tLpar,
302		mkItem(itemField, ".X"),
303		tRpar,
304		mkItem(itemField, ".Y"),
305		tRight,
306		tEOF,
307	}},
308	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
309		mkItem(itemText, "hello-"),
310		tLeft,
311		mkItem(itemNumber, "3"),
312		tRight,
313		mkItem(itemText, "-world"),
314		tEOF,
315	}},
316	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
317		mkItem(itemText, "hello-"),
318		mkItem(itemComment, "/* hello */"),
319		mkItem(itemText, "-world"),
320		tEOF,
321	}},
322	// errors
323	{"badchar", "#{{\x01}}", []item{
324		mkItem(itemText, "#"),
325		tLeft,
326		mkItem(itemError, "unrecognized character in action: U+0001"),
327	}},
328	{"unclosed action", "{{", []item{
329		tLeft,
330		mkItem(itemError, "unclosed action"),
331	}},
332	{"EOF in action", "{{range", []item{
333		tLeft,
334		tRange,
335		mkItem(itemError, "unclosed action"),
336	}},
337	{"unclosed quote", "{{\"\n\"}}", []item{
338		tLeft,
339		mkItem(itemError, "unterminated quoted string"),
340	}},
341	{"unclosed raw quote", "{{`xx}}", []item{
342		tLeft,
343		mkItem(itemError, "unterminated raw quoted string"),
344	}},
345	{"unclosed char constant", "{{'\n}}", []item{
346		tLeft,
347		mkItem(itemError, "unterminated character constant"),
348	}},
349	{"bad number", "{{3k}}", []item{
350		tLeft,
351		mkItem(itemError, `bad number syntax: "3k"`),
352	}},
353	{"unclosed paren", "{{(3}}", []item{
354		tLeft,
355		tLpar,
356		mkItem(itemNumber, "3"),
357		mkItem(itemError, `unclosed left paren`),
358	}},
359	{"extra right paren", "{{3)}}", []item{
360		tLeft,
361		mkItem(itemNumber, "3"),
362		mkItem(itemError, "unexpected right paren"),
363	}},
364
365	// Fixed bugs
366	// Many elements in an action blew the lookahead until
367	// we made lexInsideAction not loop.
368	{"long pipeline deadlock", "{{|||||}}", []item{
369		tLeft,
370		tPipe,
371		tPipe,
372		tPipe,
373		tPipe,
374		tPipe,
375		tRight,
376		tEOF,
377	}},
378	{"text with bad comment", "hello-{{/*/}}-world", []item{
379		mkItem(itemText, "hello-"),
380		mkItem(itemError, `unclosed comment`),
381	}},
382	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
383		mkItem(itemText, "hello-"),
384		mkItem(itemError, `comment ends before closing delimiter`),
385	}},
386	// This one is an error that we can't catch because it breaks templates with
387	// minimized JavaScript. Should have fixed it before Go 1.1.
388	{"unmatched right delimiter", "hello-{.}}-world", []item{
389		mkItem(itemText, "hello-{.}}-world"),
390		tEOF,
391	}},
392}
393
394// collect gathers the emitted items into a slice.
395func collect(t *lexTest, left, right string) (items []item) {
396	l := lex(t.name, t.input, left, right)
397	l.options = lexOptions{
398		emitComment: true,
399		breakOK:     true,
400		continueOK:  true,
401	}
402	for {
403		item := l.nextItem()
404		items = append(items, item)
405		if item.typ == itemEOF || item.typ == itemError {
406			break
407		}
408	}
409	return
410}
411
412func equal(i1, i2 []item, checkPos bool) bool {
413	if len(i1) != len(i2) {
414		return false
415	}
416	for k := range i1 {
417		if i1[k].typ != i2[k].typ {
418			return false
419		}
420		if i1[k].val != i2[k].val {
421			return false
422		}
423		if checkPos && i1[k].pos != i2[k].pos {
424			return false
425		}
426		if checkPos && i1[k].line != i2[k].line {
427			return false
428		}
429	}
430	return true
431}
432
433func TestLex(t *testing.T) {
434	for _, test := range lexTests {
435		items := collect(&test, "", "")
436		if !equal(items, test.items, false) {
437			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
438			return // TODO
439		}
440		t.Log(test.name, "OK")
441	}
442}
443
444// Some easy cases from above, but with delimiters $$ and @@
445var lexDelimTests = []lexTest{
446	{"punctuation", "$$,@%{{}}@@", []item{
447		tLeftDelim,
448		mkItem(itemChar, ","),
449		mkItem(itemChar, "@"),
450		mkItem(itemChar, "%"),
451		mkItem(itemChar, "{"),
452		mkItem(itemChar, "{"),
453		mkItem(itemChar, "}"),
454		mkItem(itemChar, "}"),
455		tRightDelim,
456		tEOF,
457	}},
458	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
459	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
460	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
461	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
462}
463
464var (
465	tLeftDelim  = mkItem(itemLeftDelim, "$$")
466	tRightDelim = mkItem(itemRightDelim, "@@")
467)
468
469func TestDelims(t *testing.T) {
470	for _, test := range lexDelimTests {
471		items := collect(&test, "$$", "@@")
472		if !equal(items, test.items, false) {
473			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
474		}
475	}
476}
477
478func TestDelimsAlphaNumeric(t *testing.T) {
479	test := lexTest{"right delimiter with alphanumeric start", "{{hub .host hub}}", []item{
480		mkItem(itemLeftDelim, "{{hub"),
481		mkItem(itemSpace, " "),
482		mkItem(itemField, ".host"),
483		mkItem(itemSpace, " "),
484		mkItem(itemRightDelim, "hub}}"),
485		tEOF,
486	}}
487	items := collect(&test, "{{hub", "hub}}")
488
489	if !equal(items, test.items, false) {
490		t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
491	}
492}
493
494func TestDelimsAndMarkers(t *testing.T) {
495	test := lexTest{"delims that look like markers", "{{- .x -}} {{- - .x - -}}", []item{
496		mkItem(itemLeftDelim, "{{- "),
497		mkItem(itemField, ".x"),
498		mkItem(itemRightDelim, " -}}"),
499		mkItem(itemLeftDelim, "{{- "),
500		mkItem(itemField, ".x"),
501		mkItem(itemRightDelim, " -}}"),
502		tEOF,
503	}}
504	items := collect(&test, "{{- ", " -}}")
505
506	if !equal(items, test.items, false) {
507		t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
508	}
509}
510
511var lexPosTests = []lexTest{
512	{"empty", "", []item{{itemEOF, 0, "", 1}}},
513	{"punctuation", "{{,@%#}}", []item{
514		{itemLeftDelim, 0, "{{", 1},
515		{itemChar, 2, ",", 1},
516		{itemChar, 3, "@", 1},
517		{itemChar, 4, "%", 1},
518		{itemChar, 5, "#", 1},
519		{itemRightDelim, 6, "}}", 1},
520		{itemEOF, 8, "", 1},
521	}},
522	{"sample", "0123{{hello}}xyz", []item{
523		{itemText, 0, "0123", 1},
524		{itemLeftDelim, 4, "{{", 1},
525		{itemIdentifier, 6, "hello", 1},
526		{itemRightDelim, 11, "}}", 1},
527		{itemText, 13, "xyz", 1},
528		{itemEOF, 16, "", 1},
529	}},
530	{"trimafter", "{{x -}}\n{{y}}", []item{
531		{itemLeftDelim, 0, "{{", 1},
532		{itemIdentifier, 2, "x", 1},
533		{itemRightDelim, 5, "}}", 1},
534		{itemLeftDelim, 8, "{{", 2},
535		{itemIdentifier, 10, "y", 2},
536		{itemRightDelim, 11, "}}", 2},
537		{itemEOF, 13, "", 2},
538	}},
539	{"trimbefore", "{{x}}\n{{- y}}", []item{
540		{itemLeftDelim, 0, "{{", 1},
541		{itemIdentifier, 2, "x", 1},
542		{itemRightDelim, 3, "}}", 1},
543		{itemLeftDelim, 6, "{{", 2},
544		{itemIdentifier, 10, "y", 2},
545		{itemRightDelim, 11, "}}", 2},
546		{itemEOF, 13, "", 2},
547	}},
548}
549
550// The other tests don't check position, to make the test cases easier to construct.
551// This one does.
552func TestPos(t *testing.T) {
553	for _, test := range lexPosTests {
554		items := collect(&test, "", "")
555		if !equal(items, test.items, true) {
556			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
557			if len(items) == len(test.items) {
558				// Detailed print; avoid item.String() to expose the position value.
559				for i := range items {
560					if !equal(items[i:i+1], test.items[i:i+1], true) {
561						i1 := items[i]
562						i2 := test.items[i]
563						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
564							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
565					}
566				}
567			}
568		}
569	}
570}
571
572// parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
573// We expect an error, so the tree set and funcs list are explicitly nil.
574func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
575	defer t.recover(&err)
576	t.ParseName = t.Name
577	t.startParse(nil, lex, map[string]*Tree{})
578	t.parse()
579	t.add()
580	t.stopParse()
581	return t, nil
582}
583