1// Copyright 2015 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package simplifiedchinese 6 7import ( 8 "strings" 9 "testing" 10 11 "golang.org/x/text/encoding" 12 "golang.org/x/text/encoding/internal" 13 "golang.org/x/text/encoding/internal/enctest" 14 "golang.org/x/text/transform" 15) 16 17func dec(e encoding.Encoding) (dir string, t transform.Transformer, err error) { 18 return "Decode", e.NewDecoder(), nil 19} 20func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) { 21 return "Encode", e.NewEncoder(), internal.ErrASCIIReplacement 22} 23 24func TestNonRepertoire(t *testing.T) { 25 // Pick n large enough to overflow the destination buffer of transform.String. 26 const n = 10000 27 testCases := []struct { 28 init func(e encoding.Encoding) (string, transform.Transformer, error) 29 e encoding.Encoding 30 src, want string 31 }{ 32 {dec, GBK, "a\xfe\xfeb", "a\ufffdb"}, 33 {dec, HZGB2312, "~{z~", "\ufffd"}, 34 35 {enc, GBK, "갂", ""}, 36 {enc, GBK, "a갂", "a"}, 37 {enc, GBK, "\u4e02갂", "\x81@"}, 38 39 {enc, HZGB2312, "갂", ""}, 40 {enc, HZGB2312, "a갂", "a"}, 41 {enc, HZGB2312, "\u6cf5갂", "~{1C~}"}, 42 43 {dec, GB18030, "\x80", "€"}, 44 {dec, GB18030, "\x81", "\ufffd"}, 45 {dec, GB18030, "\x81\x20", "\ufffd "}, 46 {dec, GB18030, "\xfe\xfe", "\ufffd"}, 47 {dec, GB18030, "\xfe\xff", "\ufffd\ufffd"}, 48 {dec, GB18030, "\xfe\x30", "\ufffd0"}, 49 {dec, GB18030, "\xfe\x30\x30 ", "\ufffd00 "}, 50 {dec, GB18030, "\xfe\x30\xff ", "\ufffd0\ufffd "}, 51 {dec, GB18030, "\xfe\x30\x81\x21", "\ufffd0\ufffd!"}, 52 53 {dec, GB18030, strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd0", n)}, 54 55 {dec, HZGB2312, "~/", "\ufffd"}, 56 {dec, HZGB2312, "~", "\ufffd"}, 57 {dec, HZGB2312, "~~~", "~\ufffd"}, 58 {dec, HZGB2312, "~{a\x80", "\ufffd"}, 59 {dec, HZGB2312, "~{a\x80", "\ufffd"}, 60 {dec, HZGB2312, "~{" + strings.Repeat("z~", n), strings.Repeat("\ufffd", n)}, 61 {dec, HZGB2312, "~{" + strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd", n*2)}, 62 } 63 for _, tc := range testCases { 64 dir, tr, wantErr := tc.init(tc.e) 65 66 dst, _, err := transform.String(tr, tc.src) 67 if err != wantErr { 68 t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr) 69 } 70 if got := string(dst); got != tc.want { 71 t.Errorf("%s %v(%q):\ngot %q\nwant %q", dir, tc.e, tc.src, got, tc.want) 72 } 73 } 74} 75 76func TestBasics(t *testing.T) { 77 // The encoded forms can be verified by the iconv program: 78 // $ echo 月日は百代 | iconv -f UTF-8 -t SHIFT-JIS | xxd 79 testCases := []struct { 80 e encoding.Encoding 81 encPrefix string 82 encoded string 83 utf8 string 84 }{{ 85 // "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000" is a 86 // nonsense string that contains GB18030 encodable codepoints of which 87 // only U+00E0 and U+00E1 are GBK encodable. 88 // 89 // "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€" is a nonsense 90 // string that contains ASCII and GBK encodable codepoints from Levels 91 // 1-5 as well as the Euro sign. 92 // 93 // "A\u43f0\u4c32\U00027267\u3000\U0002910d\u79d4Z€" is a nonsense string 94 // that contains ASCII and Big5 encodable codepoints from the Basic 95 // Multilingual Plane and the Supplementary Ideographic Plane as well as 96 // the Euro sign. 97 // 98 // "花间一壶酒,独酌无相亲。" (simplified) and 99 // "花間一壺酒,獨酌無相親。" (traditional) 100 // are from the 8th century poem "Yuè Xià Dú Zhuó". 101 e: GB18030, 102 encoded: "\x81\x30\x81\x31\x81\x30\x89\x37\x81\x30\x89\x38\xa8\xa4\xa8\xa2" + 103 "\x81\x30\x89\x39\x81\x30\x8a\x30\x84\x31\xa4\x39\x90\x30\x81\x30", 104 utf8: "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000", 105 }, { 106 e: GB18030, 107 encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" + 108 "\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3", 109 utf8: "花间一壶酒,独酌无相亲。", 110 }, { 111 e: GBK, 112 encoded: "A\xa1\xa1\xb0\xa1\x81\x40\x81\x80\xaa\x40\xaa\x80\xa8\x40\xa8\x80Z\x80", 113 utf8: "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€", 114 }, { 115 e: GBK, 116 encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" + 117 "\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3", 118 utf8: "花间一壶酒,独酌无相亲。", 119 }, { 120 e: HZGB2312, 121 encoded: "A~{\x21\x21~~\x30\x21~}Z~~", 122 utf8: "A\u3000~\u554aZ~", 123 }, { 124 e: HZGB2312, 125 encPrefix: "~{", 126 encoded: ";(<dR;:x>F#,6@WCN^O`GW!#", 127 utf8: "花间一壶酒,独酌无相亲。", 128 }} 129 130 for _, tc := range testCases { 131 enctest.TestEncoding(t, tc.e, tc.encoded, tc.utf8, tc.encPrefix, "") 132 } 133} 134 135func TestFiles(t *testing.T) { 136 enctest.TestFile(t, GB18030) 137 enctest.TestFile(t, GBK) 138 enctest.TestFile(t, HZGB2312) 139} 140 141func BenchmarkEncoding(b *testing.B) { 142 enctest.Benchmark(b, GB18030) 143 enctest.Benchmark(b, GBK) 144 enctest.Benchmark(b, HZGB2312) 145} 146