1// Copyright 2010 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package textproto
6
7import (
8	"bufio"
9	"bytes"
10	"io"
11	"net"
12	"reflect"
13	"runtime"
14	"strings"
15	"sync"
16	"testing"
17)
18
19func reader(s string) *Reader {
20	return NewReader(bufio.NewReader(strings.NewReader(s)))
21}
22
23func TestReadLine(t *testing.T) {
24	r := reader("line1\nline2\n")
25	s, err := r.ReadLine()
26	if s != "line1" || err != nil {
27		t.Fatalf("Line 1: %s, %v", s, err)
28	}
29	s, err = r.ReadLine()
30	if s != "line2" || err != nil {
31		t.Fatalf("Line 2: %s, %v", s, err)
32	}
33	s, err = r.ReadLine()
34	if s != "" || err != io.EOF {
35		t.Fatalf("EOF: %s, %v", s, err)
36	}
37}
38
39func TestReadLineLongLine(t *testing.T) {
40	line := strings.Repeat("12345", 10000)
41	r := reader(line + "\r\n")
42	s, err := r.ReadLine()
43	if err != nil {
44		t.Fatalf("Line 1: %v", err)
45	}
46	if s != line {
47		t.Fatalf("%v-byte line does not match expected %v-byte line", len(s), len(line))
48	}
49}
50
51func TestReadContinuedLine(t *testing.T) {
52	r := reader("line1\nline\n 2\nline3\n")
53	s, err := r.ReadContinuedLine()
54	if s != "line1" || err != nil {
55		t.Fatalf("Line 1: %s, %v", s, err)
56	}
57	s, err = r.ReadContinuedLine()
58	if s != "line 2" || err != nil {
59		t.Fatalf("Line 2: %s, %v", s, err)
60	}
61	s, err = r.ReadContinuedLine()
62	if s != "line3" || err != nil {
63		t.Fatalf("Line 3: %s, %v", s, err)
64	}
65	s, err = r.ReadContinuedLine()
66	if s != "" || err != io.EOF {
67		t.Fatalf("EOF: %s, %v", s, err)
68	}
69}
70
71func TestReadCodeLine(t *testing.T) {
72	r := reader("123 hi\n234 bye\n345 no way\n")
73	code, msg, err := r.ReadCodeLine(0)
74	if code != 123 || msg != "hi" || err != nil {
75		t.Fatalf("Line 1: %d, %s, %v", code, msg, err)
76	}
77	code, msg, err = r.ReadCodeLine(23)
78	if code != 234 || msg != "bye" || err != nil {
79		t.Fatalf("Line 2: %d, %s, %v", code, msg, err)
80	}
81	code, msg, err = r.ReadCodeLine(346)
82	if code != 345 || msg != "no way" || err == nil {
83		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
84	}
85	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
86		t.Fatalf("Line 3: wrong error %v\n", err)
87	}
88	code, msg, err = r.ReadCodeLine(1)
89	if code != 0 || msg != "" || err != io.EOF {
90		t.Fatalf("EOF: %d, %s, %v", code, msg, err)
91	}
92}
93
94func TestReadDotLines(t *testing.T) {
95	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n")
96	s, err := r.ReadDotLines()
97	want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""}
98	if !reflect.DeepEqual(s, want) || err != nil {
99		t.Fatalf("ReadDotLines: %v, %v", s, err)
100	}
101
102	s, err = r.ReadDotLines()
103	want = []string{"another"}
104	if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF {
105		t.Fatalf("ReadDotLines2: %v, %v", s, err)
106	}
107}
108
109func TestReadDotBytes(t *testing.T) {
110	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n")
111	b, err := r.ReadDotBytes()
112	want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n")
113	if !reflect.DeepEqual(b, want) || err != nil {
114		t.Fatalf("ReadDotBytes: %q, %v", b, err)
115	}
116
117	b, err = r.ReadDotBytes()
118	want = []byte("anot.her\n")
119	if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF {
120		t.Fatalf("ReadDotBytes2: %q, %v", b, err)
121	}
122}
123
124func TestReadMIMEHeader(t *testing.T) {
125	r := reader("my-key: Value 1  \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
126	m, err := r.ReadMIMEHeader()
127	want := MIMEHeader{
128		"My-Key":   {"Value 1", "Value 2"},
129		"Long-Key": {"Even Longer Value"},
130	}
131	if !reflect.DeepEqual(m, want) || err != nil {
132		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
133	}
134}
135
136func TestReadMIMEHeaderSingle(t *testing.T) {
137	r := reader("Foo: bar\n\n")
138	m, err := r.ReadMIMEHeader()
139	want := MIMEHeader{"Foo": {"bar"}}
140	if !reflect.DeepEqual(m, want) || err != nil {
141		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
142	}
143}
144
145// TestReaderUpcomingHeaderKeys is testing an internal function, but it's very
146// difficult to test well via the external API.
147func TestReaderUpcomingHeaderKeys(t *testing.T) {
148	for _, test := range []struct {
149		input string
150		want  int
151	}{{
152		input: "",
153		want:  0,
154	}, {
155		input: "A: v",
156		want:  1,
157	}, {
158		input: "A: v\r\nB: v\r\n",
159		want:  2,
160	}, {
161		input: "A: v\nB: v\n",
162		want:  2,
163	}, {
164		input: "A: v\r\n  continued\r\n  still continued\r\nB: v\r\n\r\n",
165		want:  2,
166	}, {
167		input: "A: v\r\n\r\nB: v\r\nC: v\r\n",
168		want:  1,
169	}, {
170		input: "A: v" + strings.Repeat("\n", 1000),
171		want:  1,
172	}} {
173		r := reader(test.input)
174		got := r.upcomingHeaderKeys()
175		if test.want != got {
176			t.Fatalf("upcomingHeaderKeys(%q): %v; want %v", test.input, got, test.want)
177		}
178	}
179}
180
181func TestReadMIMEHeaderNoKey(t *testing.T) {
182	r := reader(": bar\ntest-1: 1\n\n")
183	m, err := r.ReadMIMEHeader()
184	want := MIMEHeader{}
185	if !reflect.DeepEqual(m, want) || err == nil {
186		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
187	}
188}
189
190func TestLargeReadMIMEHeader(t *testing.T) {
191	data := make([]byte, 16*1024)
192	for i := 0; i < len(data); i++ {
193		data[i] = 'x'
194	}
195	sdata := string(data)
196	r := reader("Cookie: " + sdata + "\r\n\n")
197	m, err := r.ReadMIMEHeader()
198	if err != nil {
199		t.Fatalf("ReadMIMEHeader: %v", err)
200	}
201	cookie := m.Get("Cookie")
202	if cookie != sdata {
203		t.Fatalf("ReadMIMEHeader: %v bytes, want %v bytes", len(cookie), len(sdata))
204	}
205}
206
207// TestReadMIMEHeaderNonCompliant checks that we don't normalize headers
208// with spaces before colons, and accept spaces in keys.
209func TestReadMIMEHeaderNonCompliant(t *testing.T) {
210	// These invalid headers will be rejected by net/http according to RFC 7230.
211	r := reader("Foo: bar\r\n" +
212		"Content-Language: en\r\n" +
213		"SID : 0\r\n" +
214		"Audio Mode : None\r\n" +
215		"Privilege : 127\r\n\r\n")
216	m, err := r.ReadMIMEHeader()
217	want := MIMEHeader{
218		"Foo":              {"bar"},
219		"Content-Language": {"en"},
220		"SID ":             {"0"},
221		"Audio Mode ":      {"None"},
222		"Privilege ":       {"127"},
223	}
224	if !reflect.DeepEqual(m, want) || err != nil {
225		t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want)
226	}
227}
228
229func TestReadMIMEHeaderMalformed(t *testing.T) {
230	inputs := []string{
231		"No colon first line\r\nFoo: foo\r\n\r\n",
232		" No colon first line with leading space\r\nFoo: foo\r\n\r\n",
233		"\tNo colon first line with leading tab\r\nFoo: foo\r\n\r\n",
234		" First: line with leading space\r\nFoo: foo\r\n\r\n",
235		"\tFirst: line with leading tab\r\nFoo: foo\r\n\r\n",
236		"Foo: foo\r\nNo colon second line\r\n\r\n",
237		"Foo-\n\tBar: foo\r\n\r\n",
238		"Foo-\r\n\tBar: foo\r\n\r\n",
239		"Foo\r\n\t: foo\r\n\r\n",
240		"Foo-\n\tBar",
241		"Foo \tBar: foo\r\n\r\n",
242		": empty key\r\n\r\n",
243	}
244	for _, input := range inputs {
245		r := reader(input)
246		if m, err := r.ReadMIMEHeader(); err == nil || err == io.EOF {
247			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want nil, err", input, m, err)
248		}
249	}
250}
251
252func TestReadMIMEHeaderBytes(t *testing.T) {
253	for i := 0; i <= 0xff; i++ {
254		s := "Foo" + string(rune(i)) + "Bar: foo\r\n\r\n"
255		r := reader(s)
256		wantErr := true
257		switch {
258		case i >= '0' && i <= '9':
259			wantErr = false
260		case i >= 'a' && i <= 'z':
261			wantErr = false
262		case i >= 'A' && i <= 'Z':
263			wantErr = false
264		case i == '!' || i == '#' || i == '$' || i == '%' || i == '&' || i == '\'' || i == '*' || i == '+' || i == '-' || i == '.' || i == '^' || i == '_' || i == '`' || i == '|' || i == '~':
265			wantErr = false
266		case i == ':':
267			// Special case: "Foo:Bar: foo" is the header "Foo".
268			wantErr = false
269		case i == ' ':
270			wantErr = false
271		}
272		m, err := r.ReadMIMEHeader()
273		if err != nil != wantErr {
274			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want error=%v", s, m, err, wantErr)
275		}
276	}
277	for i := 0; i <= 0xff; i++ {
278		s := "Foo: foo" + string(rune(i)) + "bar\r\n\r\n"
279		r := reader(s)
280		wantErr := true
281		switch {
282		case i >= 0x21 && i <= 0x7e:
283			wantErr = false
284		case i == ' ':
285			wantErr = false
286		case i == '\t':
287			wantErr = false
288		case i >= 0x80 && i <= 0xff:
289			wantErr = false
290		}
291		m, err := r.ReadMIMEHeader()
292		if (err != nil) != wantErr {
293			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want error=%v", s, m, err, wantErr)
294		}
295	}
296}
297
298// Test that continued lines are properly trimmed. Issue 11204.
299func TestReadMIMEHeaderTrimContinued(t *testing.T) {
300	// In this header, \n and \r\n terminated lines are mixed on purpose.
301	// We expect each line to be trimmed (prefix and suffix) before being concatenated.
302	// Keep the spaces as they are.
303	r := reader("" + // for code formatting purpose.
304		"a:\n" +
305		" 0 \r\n" +
306		"b:1 \t\r\n" +
307		"c: 2\r\n" +
308		" 3\t\n" +
309		"  \t 4  \r\n\n")
310	m, err := r.ReadMIMEHeader()
311	if err != nil {
312		t.Fatal(err)
313	}
314	want := MIMEHeader{
315		"A": {"0"},
316		"B": {"1"},
317		"C": {"2 3 4"},
318	}
319	if !reflect.DeepEqual(m, want) {
320		t.Fatalf("ReadMIMEHeader mismatch.\n got: %q\nwant: %q", m, want)
321	}
322}
323
324// Test that reading a header doesn't overallocate. Issue 58975.
325func TestReadMIMEHeaderAllocations(t *testing.T) {
326	var totalAlloc uint64
327	const count = 200
328	for i := 0; i < count; i++ {
329		r := reader("A: b\r\n\r\n" + strings.Repeat("\n", 4096))
330		var m1, m2 runtime.MemStats
331		runtime.ReadMemStats(&m1)
332		_, err := r.ReadMIMEHeader()
333		if err != nil {
334			t.Fatalf("ReadMIMEHeader: %v", err)
335		}
336		runtime.ReadMemStats(&m2)
337		totalAlloc += m2.TotalAlloc - m1.TotalAlloc
338	}
339	// 32k is large and we actually allocate substantially less,
340	// but prior to the fix for #58975 we allocated ~400k in this case.
341	if got, want := totalAlloc/count, uint64(32768); got > want {
342		t.Fatalf("ReadMIMEHeader allocated %v bytes, want < %v", got, want)
343	}
344}
345
346type readResponseTest struct {
347	in       string
348	inCode   int
349	wantCode int
350	wantMsg  string
351}
352
353var readResponseTests = []readResponseTest{
354	{"230-Anonymous access granted, restrictions apply\n" +
355		"Read the file README.txt,\n" +
356		"230  please",
357		23,
358		230,
359		"Anonymous access granted, restrictions apply\nRead the file README.txt,\n please",
360	},
361
362	{"230 Anonymous access granted, restrictions apply\n",
363		23,
364		230,
365		"Anonymous access granted, restrictions apply",
366	},
367
368	{"400-A\n400-B\n400 C",
369		4,
370		400,
371		"A\nB\nC",
372	},
373
374	{"400-A\r\n400-B\r\n400 C\r\n",
375		4,
376		400,
377		"A\nB\nC",
378	},
379}
380
381// See https://www.ietf.org/rfc/rfc959.txt page 36.
382func TestRFC959Lines(t *testing.T) {
383	for i, tt := range readResponseTests {
384		r := reader(tt.in + "\nFOLLOWING DATA")
385		code, msg, err := r.ReadResponse(tt.inCode)
386		if err != nil {
387			t.Errorf("#%d: ReadResponse: %v", i, err)
388			continue
389		}
390		if code != tt.wantCode {
391			t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode)
392		}
393		if msg != tt.wantMsg {
394			t.Errorf("#%d: msg=%q, want %q", i, msg, tt.wantMsg)
395		}
396	}
397}
398
399// Test that multi-line errors are appropriately and fully read. Issue 10230.
400func TestReadMultiLineError(t *testing.T) {
401	r := reader("550-5.1.1 The email account that you tried to reach does not exist. Please try\n" +
402		"550-5.1.1 double-checking the recipient's email address for typos or\n" +
403		"550-5.1.1 unnecessary spaces. Learn more at\n" +
404		"Unexpected but legal text!\n" +
405		"550 5.1.1 https://support.google.com/mail/answer/6596 h20si25154304pfd.166 - gsmtp\n")
406
407	wantMsg := "5.1.1 The email account that you tried to reach does not exist. Please try\n" +
408		"5.1.1 double-checking the recipient's email address for typos or\n" +
409		"5.1.1 unnecessary spaces. Learn more at\n" +
410		"Unexpected but legal text!\n" +
411		"5.1.1 https://support.google.com/mail/answer/6596 h20si25154304pfd.166 - gsmtp"
412
413	code, msg, err := r.ReadResponse(250)
414	if err == nil {
415		t.Errorf("ReadResponse: no error, want error")
416	}
417	if code != 550 {
418		t.Errorf("ReadResponse: code=%d, want %d", code, 550)
419	}
420	if msg != wantMsg {
421		t.Errorf("ReadResponse: msg=%q, want %q", msg, wantMsg)
422	}
423	if err != nil && err.Error() != "550 "+wantMsg {
424		t.Errorf("ReadResponse: error=%q, want %q", err.Error(), "550 "+wantMsg)
425	}
426}
427
428func TestCommonHeaders(t *testing.T) {
429	commonHeaderOnce.Do(initCommonHeader)
430	for h := range commonHeader {
431		if h != CanonicalMIMEHeaderKey(h) {
432			t.Errorf("Non-canonical header %q in commonHeader", h)
433		}
434	}
435	b := []byte("content-Length")
436	want := "Content-Length"
437	n := testing.AllocsPerRun(200, func() {
438		if x, _ := canonicalMIMEHeaderKey(b); x != want {
439			t.Fatalf("canonicalMIMEHeaderKey(%q) = %q; want %q", b, x, want)
440		}
441	})
442	if n > 0 {
443		t.Errorf("canonicalMIMEHeaderKey allocs = %v; want 0", n)
444	}
445}
446
447func TestIssue46363(t *testing.T) {
448	// Regression test for data race reported in issue 46363:
449	// ReadMIMEHeader reads commonHeader before commonHeader has been initialized.
450	// Run this test with the race detector enabled to catch the reported data race.
451
452	// Reset commonHeaderOnce, so that commonHeader will have to be initialized
453	commonHeaderOnce = sync.Once{}
454	commonHeader = nil
455
456	// Test for data race by calling ReadMIMEHeader and CanonicalMIMEHeaderKey concurrently
457
458	// Send MIME header over net.Conn
459	r, w := net.Pipe()
460	go func() {
461		// ReadMIMEHeader calls canonicalMIMEHeaderKey, which reads from commonHeader
462		NewConn(r).ReadMIMEHeader()
463	}()
464	w.Write([]byte("A: 1\r\nB: 2\r\nC: 3\r\n\r\n"))
465
466	// CanonicalMIMEHeaderKey calls commonHeaderOnce.Do(initCommonHeader) which initializes commonHeader
467	CanonicalMIMEHeaderKey("a")
468
469	if commonHeader == nil {
470		t.Fatal("CanonicalMIMEHeaderKey should initialize commonHeader")
471	}
472}
473
474var clientHeaders = strings.Replace(`Host: golang.org
475Connection: keep-alive
476Cache-Control: max-age=0
477Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
478User-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3
479Accept-Encoding: gzip,deflate,sdch
480Accept-Language: en-US,en;q=0.8,fr-CH;q=0.6
481Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
482COOKIE: __utma=000000000.0000000000.0000000000.0000000000.0000000000.00; __utmb=000000000.0.00.0000000000; __utmc=000000000; __utmz=000000000.0000000000.00.0.utmcsr=code.google.com|utmccn=(referral)|utmcmd=referral|utmcct=/p/go/issues/detail
483Non-Interned: test
484
485`, "\n", "\r\n", -1)
486
487var serverHeaders = strings.Replace(`Content-Type: text/html; charset=utf-8
488Content-Encoding: gzip
489Date: Thu, 27 Sep 2012 09:03:33 GMT
490Server: Google Frontend
491Cache-Control: private
492Content-Length: 2298
493VIA: 1.1 proxy.example.com:80 (XXX/n.n.n-nnn)
494Connection: Close
495Non-Interned: test
496
497`, "\n", "\r\n", -1)
498
499func BenchmarkReadMIMEHeader(b *testing.B) {
500	b.ReportAllocs()
501	for _, set := range []struct {
502		name    string
503		headers string
504	}{
505		{"client_headers", clientHeaders},
506		{"server_headers", serverHeaders},
507	} {
508		b.Run(set.name, func(b *testing.B) {
509			var buf bytes.Buffer
510			br := bufio.NewReader(&buf)
511			r := NewReader(br)
512
513			for i := 0; i < b.N; i++ {
514				buf.WriteString(set.headers)
515				if _, err := r.ReadMIMEHeader(); err != nil {
516					b.Fatal(err)
517				}
518			}
519		})
520	}
521}
522
523func BenchmarkUncommon(b *testing.B) {
524	b.ReportAllocs()
525	var buf bytes.Buffer
526	br := bufio.NewReader(&buf)
527	r := NewReader(br)
528	for i := 0; i < b.N; i++ {
529		buf.WriteString("uncommon-header-for-benchmark: foo\r\n\r\n")
530		h, err := r.ReadMIMEHeader()
531		if err != nil {
532			b.Fatal(err)
533		}
534		if _, ok := h["Uncommon-Header-For-Benchmark"]; !ok {
535			b.Fatal("Missing result header.")
536		}
537	}
538}
539