1// Copyright 2013 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package charset
6
7import (
8	"bytes"
9	"encoding/xml"
10	"io/ioutil"
11	"runtime"
12	"strings"
13	"testing"
14
15	"golang.org/x/text/transform"
16)
17
18func transformString(t transform.Transformer, s string) (string, error) {
19	r := transform.NewReader(strings.NewReader(s), t)
20	b, err := ioutil.ReadAll(r)
21	return string(b), err
22}
23
24type testCase struct {
25	utf8, other, otherEncoding string
26}
27
28// testCases for encoding and decoding.
29var testCases = []testCase{
30	{"Résumé", "Résumé", "utf8"},
31	{"Résumé", "R\xe9sum\xe9", "latin1"},
32	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
33	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
34	{"Hello, world", "Hello, world", "ASCII"},
35	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
36	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
37	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
38	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
39	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
40	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
41	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
42	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
43	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
44	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
45	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
46	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
47	{"Résumé", "R\x8esum\x8e", "macintosh"},
48	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
49	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
50	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
51	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
52	{"Kağan", "Ka\xf0an", "windows-1254"},
53	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
54	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
55	{"latviešu", "latvie\xf0u", "windows-1257"},
56	{"Việt", "Vi\xea\xf2t", "windows-1258"},
57	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
58	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
59	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
60	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
61	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
62	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
63	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
64	{"㧯", "\x82\x31\x89\x38", "gb18030"},
65	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
66	{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
67	{"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
68	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
69	{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
70	{"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
71}
72
73func TestDecode(t *testing.T) {
74	testCases := append(testCases, []testCase{
75		// Replace multi-byte maximum subpart of ill-formed subsequence with
76		// single replacement character (WhatWG requirement).
77		{"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
78	}...)
79	for _, tc := range testCases {
80		e, _ := Lookup(tc.otherEncoding)
81		if e == nil {
82			t.Errorf("%s: not found", tc.otherEncoding)
83			continue
84		}
85		s, err := transformString(e.NewDecoder(), tc.other)
86		if err != nil {
87			t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
88			continue
89		}
90		if s != tc.utf8 {
91			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
92		}
93	}
94}
95
96func TestEncode(t *testing.T) {
97	testCases := append(testCases, []testCase{
98		// Use Go-style replacement.
99		{"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
100		// U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
101		{"Gdańsk", "Gdańsk", "ISO-8859-11"},
102		{"\ufffd", "�", "ISO-8859-11"},
103		{"a\xe1\x80b", "a��b", "ISO-8859-11"},
104	}...)
105	for _, tc := range testCases {
106		e, _ := Lookup(tc.otherEncoding)
107		if e == nil {
108			t.Errorf("%s: not found", tc.otherEncoding)
109			continue
110		}
111		s, err := transformString(e.NewEncoder(), tc.utf8)
112		if err != nil {
113			t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
114			continue
115		}
116		if s != tc.other {
117			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
118		}
119	}
120}
121
122var sniffTestCases = []struct {
123	filename, declared, want string
124}{
125	{"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
126	{"UTF-16LE-BOM.html", "", "utf-16le"},
127	{"UTF-16BE-BOM.html", "", "utf-16be"},
128	{"meta-content-attribute.html", "text/html", "iso-8859-15"},
129	{"meta-charset-attribute.html", "text/html", "iso-8859-15"},
130	{"No-encoding-declaration.html", "text/html", "utf-8"},
131	{"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
132	{"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
133	{"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
134	{"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
135	{"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
136}
137
138func TestSniff(t *testing.T) {
139	switch runtime.GOOS {
140	case "nacl": // platforms that don't permit direct file system access
141		t.Skipf("not supported on %q", runtime.GOOS)
142	}
143
144	for _, tc := range sniffTestCases {
145		content, err := ioutil.ReadFile("testdata/" + tc.filename)
146		if err != nil {
147			t.Errorf("%s: error reading file: %v", tc.filename, err)
148			continue
149		}
150
151		_, name, _ := DetermineEncoding(content, tc.declared)
152		if name != tc.want {
153			t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
154			continue
155		}
156	}
157}
158
159func TestReader(t *testing.T) {
160	switch runtime.GOOS {
161	case "nacl": // platforms that don't permit direct file system access
162		t.Skipf("not supported on %q", runtime.GOOS)
163	}
164
165	for _, tc := range sniffTestCases {
166		content, err := ioutil.ReadFile("testdata/" + tc.filename)
167		if err != nil {
168			t.Errorf("%s: error reading file: %v", tc.filename, err)
169			continue
170		}
171
172		r, err := NewReader(bytes.NewReader(content), tc.declared)
173		if err != nil {
174			t.Errorf("%s: error creating reader: %v", tc.filename, err)
175			continue
176		}
177
178		got, err := ioutil.ReadAll(r)
179		if err != nil {
180			t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
181			continue
182		}
183
184		e, _ := Lookup(tc.want)
185		want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
186		if err != nil {
187			t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
188			continue
189		}
190
191		if !bytes.Equal(got, want) {
192			t.Errorf("%s: got %q, want %q", tc.filename, got, want)
193			continue
194		}
195	}
196}
197
198var metaTestCases = []struct {
199	meta, want string
200}{
201	{"", ""},
202	{"text/html", ""},
203	{"text/html; charset utf-8", ""},
204	{"text/html; charset=latin-2", "latin-2"},
205	{"text/html; charset; charset = utf-8", "utf-8"},
206	{`charset="big5"`, "big5"},
207	{"charset='shift_jis'", "shift_jis"},
208}
209
210func TestFromMeta(t *testing.T) {
211	for _, tc := range metaTestCases {
212		got := fromMetaElement(tc.meta)
213		if got != tc.want {
214			t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
215		}
216	}
217}
218
219func TestXML(t *testing.T) {
220	const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><a><Word>r\xe9sum\xe9</Word></a>"
221
222	d := xml.NewDecoder(strings.NewReader(s))
223	d.CharsetReader = NewReaderLabel
224
225	var a struct {
226		Word string
227	}
228	if err := d.Decode(&a); err != nil {
229		t.Fatalf("Decode: %v", err)
230	}
231
232	want := "résumé"
233	if a.Word != want {
234		t.Errorf("got %q, want %q", a.Word, want)
235	}
236}
237