1// Copyright 2013 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package language 6 7import ( 8 "bytes" 9 "strings" 10 "testing" 11 12 "golang.org/x/text/internal/tag" 13) 14 15type scanTest struct { 16 ok bool // true if scanning does not result in an error 17 in string 18 tok []string // the expected tokens 19} 20 21var tests = []scanTest{ 22 {true, "", []string{}}, 23 {true, "1", []string{"1"}}, 24 {true, "en", []string{"en"}}, 25 {true, "root", []string{"root"}}, 26 {true, "maxchars", []string{"maxchars"}}, 27 {false, "bad/", []string{}}, 28 {false, "morethan8", []string{}}, 29 {false, "-", []string{}}, 30 {false, "----", []string{}}, 31 {false, "_", []string{}}, 32 {true, "en-US", []string{"en", "US"}}, 33 {true, "en_US", []string{"en", "US"}}, 34 {false, "en-US-", []string{"en", "US"}}, 35 {false, "en-US--", []string{"en", "US"}}, 36 {false, "en-US---", []string{"en", "US"}}, 37 {false, "en--US", []string{"en", "US"}}, 38 {false, "-en-US", []string{"en", "US"}}, 39 {false, "-en--US-", []string{"en", "US"}}, 40 {false, "-en--US-", []string{"en", "US"}}, 41 {false, "en-.-US", []string{"en", "US"}}, 42 {false, ".-en--US-.", []string{"en", "US"}}, 43 {false, "en-u.-US", []string{"en", "US"}}, 44 {true, "en-u1-US", []string{"en", "u1", "US"}}, 45 {true, "maxchar1_maxchar2-maxchar3", []string{"maxchar1", "maxchar2", "maxchar3"}}, 46 {false, "moreThan8-moreThan8-e", []string{"e"}}, 47} 48 49func TestScan(t *testing.T) { 50 for i, tt := range tests { 51 scan := makeScannerString(tt.in) 52 for j := 0; !scan.done; j++ { 53 if j >= len(tt.tok) { 54 t.Errorf("%d: extra token %q", i, scan.token) 55 } else if tag.Compare(tt.tok[j], scan.token) != 0 { 56 t.Errorf("%d: token %d: found %q; want %q", i, j, scan.token, tt.tok[j]) 57 break 58 } 59 scan.scan() 60 } 61 if s := strings.Join(tt.tok, "-"); tag.Compare(s, bytes.Replace(scan.b, b("_"), b("-"), -1)) != 0 { 62 t.Errorf("%d: input: found %q; want %q", i, scan.b, s) 63 } 64 if (scan.err == nil) != tt.ok { 65 t.Errorf("%d: ok: found %v; want %v", i, scan.err == nil, tt.ok) 66 } 67 } 68} 69 70func TestAcceptMinSize(t *testing.T) { 71 for i, tt := range tests { 72 // count number of successive tokens with a minimum size. 73 for sz := 1; sz <= 8; sz++ { 74 scan := makeScannerString(tt.in) 75 scan.end, scan.next = 0, 0 76 end := scan.acceptMinSize(sz) 77 n := 0 78 for i := 0; i < len(tt.tok) && len(tt.tok[i]) >= sz; i++ { 79 n += len(tt.tok[i]) 80 if i > 0 { 81 n++ 82 } 83 } 84 if end != n { 85 t.Errorf("%d:%d: found len %d; want %d", i, sz, end, n) 86 } 87 } 88 } 89} 90 91type parseTest struct { 92 i int // the index of this test 93 in string 94 lang, script, region string 95 variants, ext string 96 extList []string // only used when more than one extension is present 97 invalid bool 98 rewrite bool // special rewrite not handled by parseTag 99 changed bool // string needed to be reformatted 100} 101 102func parseTests() []parseTest { 103 tests := []parseTest{ 104 {in: "root", lang: "und"}, 105 {in: "und", lang: "und"}, 106 {in: "en", lang: "en"}, 107 {in: "xy", lang: "und", invalid: true}, 108 {in: "en-ZY", lang: "en", invalid: true}, 109 {in: "gsw", lang: "gsw"}, 110 {in: "sr_Latn", lang: "sr", script: "Latn"}, 111 {in: "af-Arab", lang: "af", script: "Arab"}, 112 {in: "nl-BE", lang: "nl", region: "BE"}, 113 {in: "es-419", lang: "es", region: "419"}, 114 {in: "und-001", lang: "und", region: "001"}, 115 {in: "de-latn-be", lang: "de", script: "Latn", region: "BE"}, 116 // Variants 117 {in: "de-1901", lang: "de", variants: "1901"}, 118 // Accept with unsuppressed script. 119 {in: "de-Latn-1901", lang: "de", script: "Latn", variants: "1901"}, 120 // Specialized. 121 {in: "sl-rozaj", lang: "sl", variants: "rozaj"}, 122 {in: "sl-rozaj-lipaw", lang: "sl", variants: "rozaj-lipaw"}, 123 {in: "sl-rozaj-biske", lang: "sl", variants: "rozaj-biske"}, 124 {in: "sl-rozaj-biske-1994", lang: "sl", variants: "rozaj-biske-1994"}, 125 {in: "sl-rozaj-1994", lang: "sl", variants: "rozaj-1994"}, 126 // Maximum number of variants while adhering to prefix rules. 127 {in: "sl-rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp"}, 128 129 // Sorting. 130 {in: "sl-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true}, 131 {in: "sl-rozaj-biske-1994-alalc97-fonupa-fonipa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", changed: true}, 132 {in: "nl-fonxsamp-alalc97-fonipa-fonupa", lang: "nl", variants: "alalc97-fonipa-fonupa-fonxsamp", changed: true}, 133 134 // Duplicates variants are removed, but not an error. 135 {in: "nl-fonupa-fonupa", lang: "nl", variants: "fonupa"}, 136 137 // Variants that do not have correct prefixes. We still accept these. 138 {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"}, 139 {in: "sl-rozaj-lipaw-1994", lang: "sl", variants: "rozaj-lipaw-1994"}, 140 {in: "sl-1994-biske-rozaj-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true}, 141 {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"}, 142 143 // Invalid variant. 144 {in: "de-1902", lang: "de", variants: "", invalid: true}, 145 146 {in: "EN_CYRL", lang: "en", script: "Cyrl"}, 147 // private use and extensions 148 {in: "x-a-b-c-d", ext: "x-a-b-c-d"}, 149 {in: "x_A.-B-C_D", ext: "x-b-c-d", invalid: true, changed: true}, 150 {in: "x-aa-bbbb-cccccccc-d", ext: "x-aa-bbbb-cccccccc-d"}, 151 {in: "en-c_cc-b-bbb-a-aaa", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc"}}, 152 {in: "en-x_cc-b-bbb-a-aaa", lang: "en", ext: "x-cc-b-bbb-a-aaa", changed: true}, 153 {in: "en-c_cc-b-bbb-a-aaa-x-x", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc", "x-x"}}, 154 {in: "en-v-c", lang: "en", ext: "", invalid: true}, 155 {in: "en-v-abcdefghi", lang: "en", ext: "", invalid: true}, 156 {in: "en-v-abc-x", lang: "en", ext: "v-abc", invalid: true}, 157 {in: "en-v-abc-x-", lang: "en", ext: "v-abc", invalid: true}, 158 {in: "en-v-abc-w-x-xx", lang: "en", extList: []string{"v-abc", "x-xx"}, invalid: true, changed: true}, 159 {in: "en-v-abc-w-y-yx", lang: "en", extList: []string{"v-abc", "y-yx"}, invalid: true, changed: true}, 160 {in: "en-v-c-abc", lang: "en", ext: "c-abc", invalid: true, changed: true}, 161 {in: "en-v-w-abc", lang: "en", ext: "w-abc", invalid: true, changed: true}, 162 {in: "en-v-x-abc", lang: "en", ext: "x-abc", invalid: true, changed: true}, 163 {in: "en-v-x-a", lang: "en", ext: "x-a", invalid: true, changed: true}, 164 {in: "en-9-aa-0-aa-z-bb-x-a", lang: "en", extList: []string{"0-aa", "9-aa", "z-bb", "x-a"}, changed: true}, 165 {in: "en-u-c", lang: "en", ext: "", invalid: true}, 166 {in: "en-u-co-phonebk", lang: "en", ext: "u-co-phonebk"}, 167 {in: "en-u-co-phonebk-ca", lang: "en", ext: "u-co-phonebk", invalid: true}, 168 {in: "en-u-nu-arabic-co-phonebk-ca", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, 169 {in: "en-u-nu-arabic-co-phonebk-ca-x", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, 170 {in: "en-u-nu-arabic-co-phonebk-ca-s", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, 171 {in: "en-u-nu-arabic-co-phonebk-ca-a12345678", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true}, 172 {in: "en-u-co-phonebook", lang: "en", ext: "", invalid: true}, 173 {in: "en-u-co-phonebook-cu-xau", lang: "en", ext: "u-cu-xau", invalid: true, changed: true}, 174 {in: "en-Cyrl-u-co-phonebk", lang: "en", script: "Cyrl", ext: "u-co-phonebk"}, 175 {in: "en-US-u-co-phonebk", lang: "en", region: "US", ext: "u-co-phonebk"}, 176 {in: "en-US-u-co-phonebk-cu-xau", lang: "en", region: "US", ext: "u-co-phonebk-cu-xau"}, 177 {in: "en-scotland-u-co-phonebk", lang: "en", variants: "scotland", ext: "u-co-phonebk"}, 178 {in: "en-u-cu-xua-co-phonebk", lang: "en", ext: "u-co-phonebk-cu-xua", changed: true}, 179 {in: "en-u-def-abc-cu-xua-co-phonebk", lang: "en", ext: "u-abc-def-co-phonebk-cu-xua", changed: true}, 180 {in: "en-u-def-abc", lang: "en", ext: "u-abc-def", changed: true}, 181 {in: "en-u-cu-xua-co-phonebk-a-cd", lang: "en", extList: []string{"a-cd", "u-co-phonebk-cu-xua"}, changed: true}, 182 // Invalid "u" extension. Drop invalid parts. 183 {in: "en-u-cu-co-phonebk", lang: "en", extList: []string{"u-co-phonebk"}, invalid: true, changed: true}, 184 {in: "en-u-cu-xau-co", lang: "en", extList: []string{"u-cu-xau"}, invalid: true}, 185 // LDML spec is not specific about it, but remove duplicates and return an error if the values differ. 186 {in: "en-u-cu-xau-co-phonebk-cu-xau", lang: "en", ext: "u-co-phonebk-cu-xau", changed: true}, 187 // No change as the result is a substring of the original! 188 {in: "en-US-u-cu-xau-cu-eur", lang: "en", region: "US", ext: "u-cu-xau", invalid: true, changed: false}, 189 {in: "en-t-en-Cyrl-NL-fonipa", lang: "en", ext: "t-en-cyrl-nl-fonipa", changed: true}, 190 {in: "en-t-en-Cyrl-NL-fonipa-t0-abc-def", lang: "en", ext: "t-en-cyrl-nl-fonipa-t0-abc-def", changed: true}, 191 {in: "en-t-t0-abcd", lang: "en", ext: "t-t0-abcd"}, 192 // Not necessary to have changed here. 193 {in: "en-t-nl-abcd", lang: "en", ext: "t-nl", invalid: true}, 194 {in: "en-t-nl-latn", lang: "en", ext: "t-nl-latn"}, 195 {in: "en-t-t0-abcd-x-a", lang: "en", extList: []string{"t-t0-abcd", "x-a"}}, 196 // invalid 197 {in: "", lang: "und", invalid: true}, 198 {in: "-", lang: "und", invalid: true}, 199 {in: "x", lang: "und", invalid: true}, 200 {in: "x-", lang: "und", invalid: true}, 201 {in: "x--", lang: "und", invalid: true}, 202 {in: "a-a-b-c-d", lang: "und", invalid: true}, 203 {in: "en-", lang: "en", invalid: true}, 204 {in: "enne-", lang: "und", invalid: true}, 205 {in: "en.", lang: "und", invalid: true}, 206 {in: "en.-latn", lang: "und", invalid: true}, 207 {in: "en.-en", lang: "en", invalid: true}, 208 {in: "x-a-tooManyChars-c-d", ext: "x-a-c-d", invalid: true, changed: true}, 209 {in: "a-tooManyChars-c-d", lang: "und", invalid: true}, 210 // TODO: check key-value validity 211 // { in: "en-u-cu-xd", lang: "en", ext: "u-cu-xd", invalid: true }, 212 {in: "en-t-abcd", lang: "en", invalid: true}, 213 {in: "en-Latn-US-en", lang: "en", script: "Latn", region: "US", invalid: true}, 214 // rewrites (more tests in TestGrandfathered) 215 {in: "zh-min-nan", lang: "nan"}, 216 {in: "zh-yue", lang: "yue"}, 217 {in: "zh-xiang", lang: "hsn", rewrite: true}, 218 {in: "zh-guoyu", lang: "cmn", rewrite: true}, 219 {in: "iw", lang: "iw"}, 220 {in: "sgn-BE-FR", lang: "sfb", rewrite: true}, 221 {in: "i-klingon", lang: "tlh", rewrite: true}, 222 } 223 for i, tt := range tests { 224 tests[i].i = i 225 if tt.extList != nil { 226 tests[i].ext = strings.Join(tt.extList, "-") 227 } 228 if tt.ext != "" && tt.extList == nil { 229 tests[i].extList = []string{tt.ext} 230 } 231 } 232 return tests 233} 234 235func TestParseExtensions(t *testing.T) { 236 for i, tt := range parseTests() { 237 if tt.ext == "" || tt.rewrite { 238 continue 239 } 240 scan := makeScannerString(tt.in) 241 if len(scan.b) > 1 && scan.b[1] != '-' { 242 scan.end = nextExtension(string(scan.b), 0) 243 scan.next = scan.end + 1 244 scan.scan() 245 } 246 start := scan.start 247 scan.toLower(start, len(scan.b)) 248 parseExtensions(&scan) 249 ext := string(scan.b[start:]) 250 if ext != tt.ext { 251 t.Errorf("%d(%s): ext was %v; want %v", i, tt.in, ext, tt.ext) 252 } 253 if changed := !strings.HasPrefix(tt.in[start:], ext); changed != tt.changed { 254 t.Errorf("%d(%s): changed was %v; want %v", i, tt.in, changed, tt.changed) 255 } 256 } 257} 258 259// partChecks runs checks for each part by calling the function returned by f. 260func partChecks(t *testing.T, f func(*testing.T, *parseTest) (Tag, bool)) { 261 for i, tt := range parseTests() { 262 t.Run(tt.in, func(t *testing.T) { 263 tag, skip := f(t, &tt) 264 if skip { 265 return 266 } 267 if l, _ := getLangID(b(tt.lang)); l != tag.LangID { 268 t.Errorf("%d: lang was %q; want %q", i, tag.LangID, l) 269 } 270 if sc, _ := getScriptID(script, b(tt.script)); sc != tag.ScriptID { 271 t.Errorf("%d: script was %q; want %q", i, tag.ScriptID, sc) 272 } 273 if r, _ := getRegionID(b(tt.region)); r != tag.RegionID { 274 t.Errorf("%d: region was %q; want %q", i, tag.RegionID, r) 275 } 276 if tag.str == "" { 277 return 278 } 279 p := int(tag.pVariant) 280 if p < int(tag.pExt) { 281 p++ 282 } 283 if s, g := tag.str[p:tag.pExt], tt.variants; s != g { 284 t.Errorf("%d: variants was %q; want %q", i, s, g) 285 } 286 p = int(tag.pExt) 287 if p > 0 && p < len(tag.str) { 288 p++ 289 } 290 if s, g := (tag.str)[p:], tt.ext; s != g { 291 t.Errorf("%d: extensions were %q; want %q", i, s, g) 292 } 293 }) 294 } 295} 296 297func TestParseTag(t *testing.T) { 298 partChecks(t, func(t *testing.T, tt *parseTest) (id Tag, skip bool) { 299 if strings.HasPrefix(tt.in, "x-") || tt.rewrite { 300 return Tag{}, true 301 } 302 scan := makeScannerString(tt.in) 303 id, end := parseTag(&scan) 304 id.str = string(scan.b[:end]) 305 tt.ext = "" 306 tt.extList = []string{} 307 return id, false 308 }) 309} 310 311func TestParse(t *testing.T) { 312 partChecks(t, func(t *testing.T, tt *parseTest) (id Tag, skip bool) { 313 id, err := Parse(tt.in) 314 ext := "" 315 if id.str != "" { 316 if strings.HasPrefix(id.str, "x-") { 317 ext = id.str 318 } else if int(id.pExt) < len(id.str) && id.pExt > 0 { 319 ext = id.str[id.pExt+1:] 320 } 321 } 322 if tag, _ := Parse(id.String()); tag.String() != id.String() { 323 t.Errorf("%d:%s: reparse was %q; want %q", tt.i, tt.in, id.String(), tag.String()) 324 } 325 if ext != tt.ext { 326 t.Errorf("%d:%s: ext was %q; want %q", tt.i, tt.in, ext, tt.ext) 327 } 328 changed := id.str != "" && !strings.HasPrefix(tt.in, id.str) 329 if changed != tt.changed { 330 t.Errorf("%d:%s: changed was %v; want %v", tt.i, tt.in, changed, tt.changed) 331 } 332 if (err != nil) != tt.invalid { 333 t.Errorf("%d:%s: invalid was %v; want %v. Error: %v", tt.i, tt.in, err != nil, tt.invalid, err) 334 } 335 return id, false 336 }) 337} 338 339func TestErrors(t *testing.T) { 340 mkInvalid := func(s string) error { 341 return NewValueError([]byte(s)) 342 } 343 tests := []struct { 344 in string 345 out error 346 }{ 347 // invalid subtags. 348 {"ac", mkInvalid("ac")}, 349 {"AC", mkInvalid("ac")}, 350 {"aa-Uuuu", mkInvalid("Uuuu")}, 351 {"aa-AB", mkInvalid("AB")}, 352 // ill-formed wins over invalid. 353 {"ac-u", ErrSyntax}, 354 {"ac-u-ca", ErrSyntax}, 355 {"ac-u-ca-co-pinyin", ErrSyntax}, 356 {"noob", ErrSyntax}, 357 } 358 for _, tt := range tests { 359 _, err := Parse(tt.in) 360 if err != tt.out { 361 t.Errorf("%s: was %q; want %q", tt.in, err, tt.out) 362 } 363 } 364} 365