1// Copyright 2013 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:generate go run gen.go gen_common.go -output tables.go 6 7package language // import "golang.org/x/text/internal/language" 8 9// TODO: Remove above NOTE after: 10// - verifying that tables are dropped correctly (most notably matcher tables). 11 12import ( 13 "errors" 14 "fmt" 15 "strings" 16) 17 18const ( 19 // maxCoreSize is the maximum size of a BCP 47 tag without variants and 20 // extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes. 21 maxCoreSize = 12 22 23 // max99thPercentileSize is a somewhat arbitrary buffer size that presumably 24 // is large enough to hold at least 99% of the BCP 47 tags. 25 max99thPercentileSize = 32 26 27 // maxSimpleUExtensionSize is the maximum size of a -u extension with one 28 // key-type pair. Equals len("-u-") + key (2) + dash + max value (8). 29 maxSimpleUExtensionSize = 14 30) 31 32// Tag represents a BCP 47 language tag. It is used to specify an instance of a 33// specific language or locale. All language tag values are guaranteed to be 34// well-formed. The zero value of Tag is Und. 35type Tag struct { 36 // TODO: the following fields have the form TagTypeID. This name is chosen 37 // to allow refactoring the public package without conflicting with its 38 // Base, Script, and Region methods. Once the transition is fully completed 39 // the ID can be stripped from the name. 40 41 LangID Language 42 RegionID Region 43 // TODO: we will soon run out of positions for ScriptID. Idea: instead of 44 // storing lang, region, and ScriptID codes, store only the compact index and 45 // have a lookup table from this code to its expansion. This greatly speeds 46 // up table lookup, speed up common variant cases. 47 // This will also immediately free up 3 extra bytes. Also, the pVariant 48 // field can now be moved to the lookup table, as the compact index uniquely 49 // determines the offset of a possible variant. 50 ScriptID Script 51 pVariant byte // offset in str, includes preceding '-' 52 pExt uint16 // offset of first extension, includes preceding '-' 53 54 // str is the string representation of the Tag. It will only be used if the 55 // tag has variants or extensions. 56 str string 57} 58 59// Make is a convenience wrapper for Parse that omits the error. 60// In case of an error, a sensible default is returned. 61func Make(s string) Tag { 62 t, _ := Parse(s) 63 return t 64} 65 66// Raw returns the raw base language, script and region, without making an 67// attempt to infer their values. 68// TODO: consider removing 69func (t Tag) Raw() (b Language, s Script, r Region) { 70 return t.LangID, t.ScriptID, t.RegionID 71} 72 73// equalTags compares language, script and region subtags only. 74func (t Tag) equalTags(a Tag) bool { 75 return t.LangID == a.LangID && t.ScriptID == a.ScriptID && t.RegionID == a.RegionID 76} 77 78// IsRoot returns true if t is equal to language "und". 79func (t Tag) IsRoot() bool { 80 if int(t.pVariant) < len(t.str) { 81 return false 82 } 83 return t.equalTags(Und) 84} 85 86// IsPrivateUse reports whether the Tag consists solely of an IsPrivateUse use 87// tag. 88func (t Tag) IsPrivateUse() bool { 89 return t.str != "" && t.pVariant == 0 90} 91 92// RemakeString is used to update t.str in case lang, script or region changed. 93// It is assumed that pExt and pVariant still point to the start of the 94// respective parts. 95func (t *Tag) RemakeString() { 96 if t.str == "" { 97 return 98 } 99 extra := t.str[t.pVariant:] 100 if t.pVariant > 0 { 101 extra = extra[1:] 102 } 103 if t.equalTags(Und) && strings.HasPrefix(extra, "x-") { 104 t.str = extra 105 t.pVariant = 0 106 t.pExt = 0 107 return 108 } 109 var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases. 110 b := buf[:t.genCoreBytes(buf[:])] 111 if extra != "" { 112 diff := len(b) - int(t.pVariant) 113 b = append(b, '-') 114 b = append(b, extra...) 115 t.pVariant = uint8(int(t.pVariant) + diff) 116 t.pExt = uint16(int(t.pExt) + diff) 117 } else { 118 t.pVariant = uint8(len(b)) 119 t.pExt = uint16(len(b)) 120 } 121 t.str = string(b) 122} 123 124// genCoreBytes writes a string for the base languages, script and region tags 125// to the given buffer and returns the number of bytes written. It will never 126// write more than maxCoreSize bytes. 127func (t *Tag) genCoreBytes(buf []byte) int { 128 n := t.LangID.StringToBuf(buf[:]) 129 if t.ScriptID != 0 { 130 n += copy(buf[n:], "-") 131 n += copy(buf[n:], t.ScriptID.String()) 132 } 133 if t.RegionID != 0 { 134 n += copy(buf[n:], "-") 135 n += copy(buf[n:], t.RegionID.String()) 136 } 137 return n 138} 139 140// String returns the canonical string representation of the language tag. 141func (t Tag) String() string { 142 if t.str != "" { 143 return t.str 144 } 145 if t.ScriptID == 0 && t.RegionID == 0 { 146 return t.LangID.String() 147 } 148 buf := [maxCoreSize]byte{} 149 return string(buf[:t.genCoreBytes(buf[:])]) 150} 151 152// MarshalText implements encoding.TextMarshaler. 153func (t Tag) MarshalText() (text []byte, err error) { 154 if t.str != "" { 155 text = append(text, t.str...) 156 } else if t.ScriptID == 0 && t.RegionID == 0 { 157 text = append(text, t.LangID.String()...) 158 } else { 159 buf := [maxCoreSize]byte{} 160 text = buf[:t.genCoreBytes(buf[:])] 161 } 162 return text, nil 163} 164 165// UnmarshalText implements encoding.TextUnmarshaler. 166func (t *Tag) UnmarshalText(text []byte) error { 167 tag, err := Parse(string(text)) 168 *t = tag 169 return err 170} 171 172// Variants returns the part of the tag holding all variants or the empty string 173// if there are no variants defined. 174func (t Tag) Variants() string { 175 if t.pVariant == 0 { 176 return "" 177 } 178 return t.str[t.pVariant:t.pExt] 179} 180 181// VariantOrPrivateUseTags returns variants or private use tags. 182func (t Tag) VariantOrPrivateUseTags() string { 183 if t.pExt > 0 { 184 return t.str[t.pVariant:t.pExt] 185 } 186 return t.str[t.pVariant:] 187} 188 189// HasString reports whether this tag defines more than just the raw 190// components. 191func (t Tag) HasString() bool { 192 return t.str != "" 193} 194 195// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a 196// specific language are substituted with fields from the parent language. 197// The parent for a language may change for newer versions of CLDR. 198func (t Tag) Parent() Tag { 199 if t.str != "" { 200 // Strip the variants and extensions. 201 b, s, r := t.Raw() 202 t = Tag{LangID: b, ScriptID: s, RegionID: r} 203 if t.RegionID == 0 && t.ScriptID != 0 && t.LangID != 0 { 204 base, _ := addTags(Tag{LangID: t.LangID}) 205 if base.ScriptID == t.ScriptID { 206 return Tag{LangID: t.LangID} 207 } 208 } 209 return t 210 } 211 if t.LangID != 0 { 212 if t.RegionID != 0 { 213 maxScript := t.ScriptID 214 if maxScript == 0 { 215 max, _ := addTags(t) 216 maxScript = max.ScriptID 217 } 218 219 for i := range parents { 220 if Language(parents[i].lang) == t.LangID && Script(parents[i].maxScript) == maxScript { 221 for _, r := range parents[i].fromRegion { 222 if Region(r) == t.RegionID { 223 return Tag{ 224 LangID: t.LangID, 225 ScriptID: Script(parents[i].script), 226 RegionID: Region(parents[i].toRegion), 227 } 228 } 229 } 230 } 231 } 232 233 // Strip the script if it is the default one. 234 base, _ := addTags(Tag{LangID: t.LangID}) 235 if base.ScriptID != maxScript { 236 return Tag{LangID: t.LangID, ScriptID: maxScript} 237 } 238 return Tag{LangID: t.LangID} 239 } else if t.ScriptID != 0 { 240 // The parent for an base-script pair with a non-default script is 241 // "und" instead of the base language. 242 base, _ := addTags(Tag{LangID: t.LangID}) 243 if base.ScriptID != t.ScriptID { 244 return Und 245 } 246 return Tag{LangID: t.LangID} 247 } 248 } 249 return Und 250} 251 252// ParseExtension parses s as an extension and returns it on success. 253func ParseExtension(s string) (ext string, err error) { 254 scan := makeScannerString(s) 255 var end int 256 if n := len(scan.token); n != 1 { 257 return "", ErrSyntax 258 } 259 scan.toLower(0, len(scan.b)) 260 end = parseExtension(&scan) 261 if end != len(s) { 262 return "", ErrSyntax 263 } 264 return string(scan.b), nil 265} 266 267// HasVariants reports whether t has variants. 268func (t Tag) HasVariants() bool { 269 return uint16(t.pVariant) < t.pExt 270} 271 272// HasExtensions reports whether t has extensions. 273func (t Tag) HasExtensions() bool { 274 return int(t.pExt) < len(t.str) 275} 276 277// Extension returns the extension of type x for tag t. It will return 278// false for ok if t does not have the requested extension. The returned 279// extension will be invalid in this case. 280func (t Tag) Extension(x byte) (ext string, ok bool) { 281 for i := int(t.pExt); i < len(t.str)-1; { 282 var ext string 283 i, ext = getExtension(t.str, i) 284 if ext[0] == x { 285 return ext, true 286 } 287 } 288 return "", false 289} 290 291// Extensions returns all extensions of t. 292func (t Tag) Extensions() []string { 293 e := []string{} 294 for i := int(t.pExt); i < len(t.str)-1; { 295 var ext string 296 i, ext = getExtension(t.str, i) 297 e = append(e, ext) 298 } 299 return e 300} 301 302// TypeForKey returns the type associated with the given key, where key and type 303// are of the allowed values defined for the Unicode locale extension ('u') in 304// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 305// TypeForKey will traverse the inheritance chain to get the correct value. 306func (t Tag) TypeForKey(key string) string { 307 if start, end, _ := t.findTypeForKey(key); end != start { 308 return t.str[start:end] 309 } 310 return "" 311} 312 313var ( 314 errPrivateUse = errors.New("cannot set a key on a private use tag") 315 errInvalidArguments = errors.New("invalid key or type") 316) 317 318// SetTypeForKey returns a new Tag with the key set to type, where key and type 319// are of the allowed values defined for the Unicode locale extension ('u') in 320// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 321// An empty value removes an existing pair with the same key. 322func (t Tag) SetTypeForKey(key, value string) (Tag, error) { 323 if t.IsPrivateUse() { 324 return t, errPrivateUse 325 } 326 if len(key) != 2 { 327 return t, errInvalidArguments 328 } 329 330 // Remove the setting if value is "". 331 if value == "" { 332 start, end, _ := t.findTypeForKey(key) 333 if start != end { 334 // Remove key tag and leading '-'. 335 start -= 4 336 337 // Remove a possible empty extension. 338 if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' { 339 start -= 2 340 } 341 if start == int(t.pVariant) && end == len(t.str) { 342 t.str = "" 343 t.pVariant, t.pExt = 0, 0 344 } else { 345 t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:]) 346 } 347 } 348 return t, nil 349 } 350 351 if len(value) < 3 || len(value) > 8 { 352 return t, errInvalidArguments 353 } 354 355 var ( 356 buf [maxCoreSize + maxSimpleUExtensionSize]byte 357 uStart int // start of the -u extension. 358 ) 359 360 // Generate the tag string if needed. 361 if t.str == "" { 362 uStart = t.genCoreBytes(buf[:]) 363 buf[uStart] = '-' 364 uStart++ 365 } 366 367 // Create new key-type pair and parse it to verify. 368 b := buf[uStart:] 369 copy(b, "u-") 370 copy(b[2:], key) 371 b[4] = '-' 372 b = b[:5+copy(b[5:], value)] 373 scan := makeScanner(b) 374 if parseExtensions(&scan); scan.err != nil { 375 return t, scan.err 376 } 377 378 // Assemble the replacement string. 379 if t.str == "" { 380 t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1) 381 t.str = string(buf[:uStart+len(b)]) 382 } else { 383 s := t.str 384 start, end, hasExt := t.findTypeForKey(key) 385 if start == end { 386 if hasExt { 387 b = b[2:] 388 } 389 t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:]) 390 } else { 391 t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:]) 392 } 393 } 394 return t, nil 395} 396 397// findKeyAndType returns the start and end position for the type corresponding 398// to key or the point at which to insert the key-value pair if the type 399// wasn't found. The hasExt return value reports whether an -u extension was present. 400// Note: the extensions are typically very small and are likely to contain 401// only one key-type pair. 402func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) { 403 p := int(t.pExt) 404 if len(key) != 2 || p == len(t.str) || p == 0 { 405 return p, p, false 406 } 407 s := t.str 408 409 // Find the correct extension. 410 for p++; s[p] != 'u'; p++ { 411 if s[p] > 'u' { 412 p-- 413 return p, p, false 414 } 415 if p = nextExtension(s, p); p == len(s) { 416 return len(s), len(s), false 417 } 418 } 419 // Proceed to the hyphen following the extension name. 420 p++ 421 422 // curKey is the key currently being processed. 423 curKey := "" 424 425 // Iterate over keys until we get the end of a section. 426 for { 427 // p points to the hyphen preceding the current token. 428 if p3 := p + 3; s[p3] == '-' { 429 // Found a key. 430 // Check whether we just processed the key that was requested. 431 if curKey == key { 432 return start, p, true 433 } 434 // Set to the next key and continue scanning type tokens. 435 curKey = s[p+1 : p3] 436 if curKey > key { 437 return p, p, true 438 } 439 // Start of the type token sequence. 440 start = p + 4 441 // A type is at least 3 characters long. 442 p += 7 // 4 + 3 443 } else { 444 // Attribute or type, which is at least 3 characters long. 445 p += 4 446 } 447 // p points past the third character of a type or attribute. 448 max := p + 5 // maximum length of token plus hyphen. 449 if len(s) < max { 450 max = len(s) 451 } 452 for ; p < max && s[p] != '-'; p++ { 453 } 454 // Bail if we have exhausted all tokens or if the next token starts 455 // a new extension. 456 if p == len(s) || s[p+2] == '-' { 457 if curKey == key { 458 return start, p, true 459 } 460 return p, p, true 461 } 462 } 463} 464 465// ParseBase parses a 2- or 3-letter ISO 639 code. 466// It returns a ValueError if s is a well-formed but unknown language identifier 467// or another error if another error occurred. 468func ParseBase(s string) (Language, error) { 469 if n := len(s); n < 2 || 3 < n { 470 return 0, ErrSyntax 471 } 472 var buf [3]byte 473 return getLangID(buf[:copy(buf[:], s)]) 474} 475 476// ParseScript parses a 4-letter ISO 15924 code. 477// It returns a ValueError if s is a well-formed but unknown script identifier 478// or another error if another error occurred. 479func ParseScript(s string) (Script, error) { 480 if len(s) != 4 { 481 return 0, ErrSyntax 482 } 483 var buf [4]byte 484 return getScriptID(script, buf[:copy(buf[:], s)]) 485} 486 487// EncodeM49 returns the Region for the given UN M.49 code. 488// It returns an error if r is not a valid code. 489func EncodeM49(r int) (Region, error) { 490 return getRegionM49(r) 491} 492 493// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code. 494// It returns a ValueError if s is a well-formed but unknown region identifier 495// or another error if another error occurred. 496func ParseRegion(s string) (Region, error) { 497 if n := len(s); n < 2 || 3 < n { 498 return 0, ErrSyntax 499 } 500 var buf [3]byte 501 return getRegionID(buf[:copy(buf[:], s)]) 502} 503 504// IsCountry returns whether this region is a country or autonomous area. This 505// includes non-standard definitions from CLDR. 506func (r Region) IsCountry() bool { 507 if r == 0 || r.IsGroup() || r.IsPrivateUse() && r != _XK { 508 return false 509 } 510 return true 511} 512 513// IsGroup returns whether this region defines a collection of regions. This 514// includes non-standard definitions from CLDR. 515func (r Region) IsGroup() bool { 516 if r == 0 { 517 return false 518 } 519 return int(regionInclusion[r]) < len(regionContainment) 520} 521 522// Contains returns whether Region c is contained by Region r. It returns true 523// if c == r. 524func (r Region) Contains(c Region) bool { 525 if r == c { 526 return true 527 } 528 g := regionInclusion[r] 529 if g >= nRegionGroups { 530 return false 531 } 532 m := regionContainment[g] 533 534 d := regionInclusion[c] 535 b := regionInclusionBits[d] 536 537 // A contained country may belong to multiple disjoint groups. Matching any 538 // of these indicates containment. If the contained region is a group, it 539 // must strictly be a subset. 540 if d >= nRegionGroups { 541 return b&m != 0 542 } 543 return b&^m == 0 544} 545 546var errNoTLD = errors.New("language: region is not a valid ccTLD") 547 548// TLD returns the country code top-level domain (ccTLD). UK is returned for GB. 549// In all other cases it returns either the region itself or an error. 550// 551// This method may return an error for a region for which there exists a 552// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The 553// region will already be canonicalized it was obtained from a Tag that was 554// obtained using any of the default methods. 555func (r Region) TLD() (Region, error) { 556 // See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the 557 // difference between ISO 3166-1 and IANA ccTLD. 558 if r == _GB { 559 r = _UK 560 } 561 if (r.typ() & ccTLD) == 0 { 562 return 0, errNoTLD 563 } 564 return r, nil 565} 566 567// Canonicalize returns the region or a possible replacement if the region is 568// deprecated. It will not return a replacement for deprecated regions that 569// are split into multiple regions. 570func (r Region) Canonicalize() Region { 571 if cr := normRegion(r); cr != 0 { 572 return cr 573 } 574 return r 575} 576 577// Variant represents a registered variant of a language as defined by BCP 47. 578type Variant struct { 579 ID uint8 580 str string 581} 582 583// ParseVariant parses and returns a Variant. An error is returned if s is not 584// a valid variant. 585func ParseVariant(s string) (Variant, error) { 586 s = strings.ToLower(s) 587 if id, ok := variantIndex[s]; ok { 588 return Variant{id, s}, nil 589 } 590 return Variant{}, NewValueError([]byte(s)) 591} 592 593// String returns the string representation of the variant. 594func (v Variant) String() string { 595 return v.str 596} 597