1// Copyright 2013 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:generate go run gen.go gen_common.go -output tables.go 6//go:generate go run gen_index.go 7 8package language 9 10// TODO: Remove above NOTE after: 11// - verifying that tables are dropped correctly (most notably matcher tables). 12 13import ( 14 "errors" 15 "fmt" 16 "strings" 17) 18 19const ( 20 // maxCoreSize is the maximum size of a BCP 47 tag without variants and 21 // extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes. 22 maxCoreSize = 12 23 24 // max99thPercentileSize is a somewhat arbitrary buffer size that presumably 25 // is large enough to hold at least 99% of the BCP 47 tags. 26 max99thPercentileSize = 32 27 28 // maxSimpleUExtensionSize is the maximum size of a -u extension with one 29 // key-type pair. Equals len("-u-") + key (2) + dash + max value (8). 30 maxSimpleUExtensionSize = 14 31) 32 33// Tag represents a BCP 47 language tag. It is used to specify an instance of a 34// specific language or locale. All language tag values are guaranteed to be 35// well-formed. 36type Tag struct { 37 lang langID 38 region regionID 39 // TODO: we will soon run out of positions for script. Idea: instead of 40 // storing lang, region, and script codes, store only the compact index and 41 // have a lookup table from this code to its expansion. This greatly speeds 42 // up table lookup, speed up common variant cases. 43 // This will also immediately free up 3 extra bytes. Also, the pVariant 44 // field can now be moved to the lookup table, as the compact index uniquely 45 // determines the offset of a possible variant. 46 script scriptID 47 pVariant byte // offset in str, includes preceding '-' 48 pExt uint16 // offset of first extension, includes preceding '-' 49 50 // str is the string representation of the Tag. It will only be used if the 51 // tag has variants or extensions. 52 str string 53} 54 55// Make is a convenience wrapper for Parse that omits the error. 56// In case of an error, a sensible default is returned. 57func Make(s string) Tag { 58 return Default.Make(s) 59} 60 61// Make is a convenience wrapper for c.Parse that omits the error. 62// In case of an error, a sensible default is returned. 63func (c CanonType) Make(s string) Tag { 64 t, _ := c.Parse(s) 65 return t 66} 67 68// Raw returns the raw base language, script and region, without making an 69// attempt to infer their values. 70func (t Tag) Raw() (b Base, s Script, r Region) { 71 return Base{t.lang}, Script{t.script}, Region{t.region} 72} 73 74// equalTags compares language, script and region subtags only. 75func (t Tag) equalTags(a Tag) bool { 76 return t.lang == a.lang && t.script == a.script && t.region == a.region 77} 78 79// IsRoot returns true if t is equal to language "und". 80func (t Tag) IsRoot() bool { 81 if int(t.pVariant) < len(t.str) { 82 return false 83 } 84 return t.equalTags(und) 85} 86 87// private reports whether the Tag consists solely of a private use tag. 88func (t Tag) private() bool { 89 return t.str != "" && t.pVariant == 0 90} 91 92// CanonType can be used to enable or disable various types of canonicalization. 93type CanonType int 94 95const ( 96 // Replace deprecated base languages with their preferred replacements. 97 DeprecatedBase CanonType = 1 << iota 98 // Replace deprecated scripts with their preferred replacements. 99 DeprecatedScript 100 // Replace deprecated regions with their preferred replacements. 101 DeprecatedRegion 102 // Remove redundant scripts. 103 SuppressScript 104 // Normalize legacy encodings. This includes legacy languages defined in 105 // CLDR as well as bibliographic codes defined in ISO-639. 106 Legacy 107 // Map the dominant language of a macro language group to the macro language 108 // subtag. For example cmn -> zh. 109 Macro 110 // The CLDR flag should be used if full compatibility with CLDR is required. 111 // There are a few cases where language.Tag may differ from CLDR. To follow all 112 // of CLDR's suggestions, use All|CLDR. 113 CLDR 114 115 // Raw can be used to Compose or Parse without Canonicalization. 116 Raw CanonType = 0 117 118 // Replace all deprecated tags with their preferred replacements. 119 Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion 120 121 // All canonicalizations recommended by BCP 47. 122 BCP47 = Deprecated | SuppressScript 123 124 // All canonicalizations. 125 All = BCP47 | Legacy | Macro 126 127 // Default is the canonicalization used by Parse, Make and Compose. To 128 // preserve as much information as possible, canonicalizations that remove 129 // potentially valuable information are not included. The Matcher is 130 // designed to recognize similar tags that would be the same if 131 // they were canonicalized using All. 132 Default = Deprecated | Legacy 133 134 canonLang = DeprecatedBase | Legacy | Macro 135 136 // TODO: LikelyScript, LikelyRegion: suppress similar to ICU. 137) 138 139// canonicalize returns the canonicalized equivalent of the tag and 140// whether there was any change. 141func (t Tag) canonicalize(c CanonType) (Tag, bool) { 142 if c == Raw { 143 return t, false 144 } 145 changed := false 146 if c&SuppressScript != 0 { 147 if t.lang < langNoIndexOffset && uint8(t.script) == suppressScript[t.lang] { 148 t.script = 0 149 changed = true 150 } 151 } 152 if c&canonLang != 0 { 153 for { 154 if l, aliasType := normLang(t.lang); l != t.lang { 155 switch aliasType { 156 case langLegacy: 157 if c&Legacy != 0 { 158 if t.lang == _sh && t.script == 0 { 159 t.script = _Latn 160 } 161 t.lang = l 162 changed = true 163 } 164 case langMacro: 165 if c&Macro != 0 { 166 // We deviate here from CLDR. The mapping "nb" -> "no" 167 // qualifies as a typical Macro language mapping. However, 168 // for legacy reasons, CLDR maps "no", the macro language 169 // code for Norwegian, to the dominant variant "nb". This 170 // change is currently under consideration for CLDR as well. 171 // See http://unicode.org/cldr/trac/ticket/2698 and also 172 // http://unicode.org/cldr/trac/ticket/1790 for some of the 173 // practical implications. TODO: this check could be removed 174 // if CLDR adopts this change. 175 if c&CLDR == 0 || t.lang != _nb { 176 changed = true 177 t.lang = l 178 } 179 } 180 case langDeprecated: 181 if c&DeprecatedBase != 0 { 182 if t.lang == _mo && t.region == 0 { 183 t.region = _MD 184 } 185 t.lang = l 186 changed = true 187 // Other canonicalization types may still apply. 188 continue 189 } 190 } 191 } else if c&Legacy != 0 && t.lang == _no && c&CLDR != 0 { 192 t.lang = _nb 193 changed = true 194 } 195 break 196 } 197 } 198 if c&DeprecatedScript != 0 { 199 if t.script == _Qaai { 200 changed = true 201 t.script = _Zinh 202 } 203 } 204 if c&DeprecatedRegion != 0 { 205 if r := normRegion(t.region); r != 0 { 206 changed = true 207 t.region = r 208 } 209 } 210 return t, changed 211} 212 213// Canonicalize returns the canonicalized equivalent of the tag. 214func (c CanonType) Canonicalize(t Tag) (Tag, error) { 215 t, changed := t.canonicalize(c) 216 if changed { 217 t.remakeString() 218 } 219 return t, nil 220} 221 222// Confidence indicates the level of certainty for a given return value. 223// For example, Serbian may be written in Cyrillic or Latin script. 224// The confidence level indicates whether a value was explicitly specified, 225// whether it is typically the only possible value, or whether there is 226// an ambiguity. 227type Confidence int 228 229const ( 230 No Confidence = iota // full confidence that there was no match 231 Low // most likely value picked out of a set of alternatives 232 High // value is generally assumed to be the correct match 233 Exact // exact match or explicitly specified value 234) 235 236var confName = []string{"No", "Low", "High", "Exact"} 237 238func (c Confidence) String() string { 239 return confName[c] 240} 241 242// remakeString is used to update t.str in case lang, script or region changed. 243// It is assumed that pExt and pVariant still point to the start of the 244// respective parts. 245func (t *Tag) remakeString() { 246 if t.str == "" { 247 return 248 } 249 extra := t.str[t.pVariant:] 250 if t.pVariant > 0 { 251 extra = extra[1:] 252 } 253 if t.equalTags(und) && strings.HasPrefix(extra, "x-") { 254 t.str = extra 255 t.pVariant = 0 256 t.pExt = 0 257 return 258 } 259 var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases. 260 b := buf[:t.genCoreBytes(buf[:])] 261 if extra != "" { 262 diff := len(b) - int(t.pVariant) 263 b = append(b, '-') 264 b = append(b, extra...) 265 t.pVariant = uint8(int(t.pVariant) + diff) 266 t.pExt = uint16(int(t.pExt) + diff) 267 } else { 268 t.pVariant = uint8(len(b)) 269 t.pExt = uint16(len(b)) 270 } 271 t.str = string(b) 272} 273 274// genCoreBytes writes a string for the base languages, script and region tags 275// to the given buffer and returns the number of bytes written. It will never 276// write more than maxCoreSize bytes. 277func (t *Tag) genCoreBytes(buf []byte) int { 278 n := t.lang.stringToBuf(buf[:]) 279 if t.script != 0 { 280 n += copy(buf[n:], "-") 281 n += copy(buf[n:], t.script.String()) 282 } 283 if t.region != 0 { 284 n += copy(buf[n:], "-") 285 n += copy(buf[n:], t.region.String()) 286 } 287 return n 288} 289 290// String returns the canonical string representation of the language tag. 291func (t Tag) String() string { 292 if t.str != "" { 293 return t.str 294 } 295 if t.script == 0 && t.region == 0 { 296 return t.lang.String() 297 } 298 buf := [maxCoreSize]byte{} 299 return string(buf[:t.genCoreBytes(buf[:])]) 300} 301 302// MarshalText implements encoding.TextMarshaler. 303func (t Tag) MarshalText() (text []byte, err error) { 304 if t.str != "" { 305 text = append(text, t.str...) 306 } else if t.script == 0 && t.region == 0 { 307 text = append(text, t.lang.String()...) 308 } else { 309 buf := [maxCoreSize]byte{} 310 text = buf[:t.genCoreBytes(buf[:])] 311 } 312 return text, nil 313} 314 315// UnmarshalText implements encoding.TextUnmarshaler. 316func (t *Tag) UnmarshalText(text []byte) error { 317 tag, err := Raw.Parse(string(text)) 318 *t = tag 319 return err 320} 321 322// Base returns the base language of the language tag. If the base language is 323// unspecified, an attempt will be made to infer it from the context. 324// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 325func (t Tag) Base() (Base, Confidence) { 326 if t.lang != 0 { 327 return Base{t.lang}, Exact 328 } 329 c := High 330 if t.script == 0 && !(Region{t.region}).IsCountry() { 331 c = Low 332 } 333 if tag, err := addTags(t); err == nil && tag.lang != 0 { 334 return Base{tag.lang}, c 335 } 336 return Base{0}, No 337} 338 339// Script infers the script for the language tag. If it was not explicitly given, it will infer 340// a most likely candidate. 341// If more than one script is commonly used for a language, the most likely one 342// is returned with a low confidence indication. For example, it returns (Cyrl, Low) 343// for Serbian. 344// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined) 345// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks 346// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts. 347// See http://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for 348// unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified. 349// Note that an inferred script is never guaranteed to be the correct one. Latin is 350// almost exclusively used for Afrikaans, but Arabic has been used for some texts 351// in the past. Also, the script that is commonly used may change over time. 352// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 353func (t Tag) Script() (Script, Confidence) { 354 if t.script != 0 { 355 return Script{t.script}, Exact 356 } 357 sc, c := scriptID(_Zzzz), No 358 if t.lang < langNoIndexOffset { 359 if scr := scriptID(suppressScript[t.lang]); scr != 0 { 360 // Note: it is not always the case that a language with a suppress 361 // script value is only written in one script (e.g. kk, ms, pa). 362 if t.region == 0 { 363 return Script{scriptID(scr)}, High 364 } 365 sc, c = scr, High 366 } 367 } 368 if tag, err := addTags(t); err == nil { 369 if tag.script != sc { 370 sc, c = tag.script, Low 371 } 372 } else { 373 t, _ = (Deprecated | Macro).Canonicalize(t) 374 if tag, err := addTags(t); err == nil && tag.script != sc { 375 sc, c = tag.script, Low 376 } 377 } 378 return Script{sc}, c 379} 380 381// Region returns the region for the language tag. If it was not explicitly given, it will 382// infer a most likely candidate from the context. 383// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 384func (t Tag) Region() (Region, Confidence) { 385 if t.region != 0 { 386 return Region{t.region}, Exact 387 } 388 if t, err := addTags(t); err == nil { 389 return Region{t.region}, Low // TODO: differentiate between high and low. 390 } 391 t, _ = (Deprecated | Macro).Canonicalize(t) 392 if tag, err := addTags(t); err == nil { 393 return Region{tag.region}, Low 394 } 395 return Region{_ZZ}, No // TODO: return world instead of undetermined? 396} 397 398// Variant returns the variants specified explicitly for this language tag. 399// or nil if no variant was specified. 400func (t Tag) Variants() []Variant { 401 v := []Variant{} 402 if int(t.pVariant) < int(t.pExt) { 403 for x, str := "", t.str[t.pVariant:t.pExt]; str != ""; { 404 x, str = nextToken(str) 405 v = append(v, Variant{x}) 406 } 407 } 408 return v 409} 410 411// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a 412// specific language are substituted with fields from the parent language. 413// The parent for a language may change for newer versions of CLDR. 414func (t Tag) Parent() Tag { 415 if t.str != "" { 416 // Strip the variants and extensions. 417 t, _ = Raw.Compose(t.Raw()) 418 if t.region == 0 && t.script != 0 && t.lang != 0 { 419 base, _ := addTags(Tag{lang: t.lang}) 420 if base.script == t.script { 421 return Tag{lang: t.lang} 422 } 423 } 424 return t 425 } 426 if t.lang != 0 { 427 if t.region != 0 { 428 maxScript := t.script 429 if maxScript == 0 { 430 max, _ := addTags(t) 431 maxScript = max.script 432 } 433 434 for i := range parents { 435 if langID(parents[i].lang) == t.lang && scriptID(parents[i].maxScript) == maxScript { 436 for _, r := range parents[i].fromRegion { 437 if regionID(r) == t.region { 438 return Tag{ 439 lang: t.lang, 440 script: scriptID(parents[i].script), 441 region: regionID(parents[i].toRegion), 442 } 443 } 444 } 445 } 446 } 447 448 // Strip the script if it is the default one. 449 base, _ := addTags(Tag{lang: t.lang}) 450 if base.script != maxScript { 451 return Tag{lang: t.lang, script: maxScript} 452 } 453 return Tag{lang: t.lang} 454 } else if t.script != 0 { 455 // The parent for an base-script pair with a non-default script is 456 // "und" instead of the base language. 457 base, _ := addTags(Tag{lang: t.lang}) 458 if base.script != t.script { 459 return und 460 } 461 return Tag{lang: t.lang} 462 } 463 } 464 return und 465} 466 467// returns token t and the rest of the string. 468func nextToken(s string) (t, tail string) { 469 p := strings.Index(s[1:], "-") 470 if p == -1 { 471 return s[1:], "" 472 } 473 p++ 474 return s[1:p], s[p:] 475} 476 477// Extension is a single BCP 47 extension. 478type Extension struct { 479 s string 480} 481 482// String returns the string representation of the extension, including the 483// type tag. 484func (e Extension) String() string { 485 return e.s 486} 487 488// ParseExtension parses s as an extension and returns it on success. 489func ParseExtension(s string) (e Extension, err error) { 490 scan := makeScannerString(s) 491 var end int 492 if n := len(scan.token); n != 1 { 493 return Extension{}, errSyntax 494 } 495 scan.toLower(0, len(scan.b)) 496 end = parseExtension(&scan) 497 if end != len(s) { 498 return Extension{}, errSyntax 499 } 500 return Extension{string(scan.b)}, nil 501} 502 503// Type returns the one-byte extension type of e. It returns 0 for the zero 504// exception. 505func (e Extension) Type() byte { 506 if e.s == "" { 507 return 0 508 } 509 return e.s[0] 510} 511 512// Tokens returns the list of tokens of e. 513func (e Extension) Tokens() []string { 514 return strings.Split(e.s, "-") 515} 516 517// Extension returns the extension of type x for tag t. It will return 518// false for ok if t does not have the requested extension. The returned 519// extension will be invalid in this case. 520func (t Tag) Extension(x byte) (ext Extension, ok bool) { 521 for i := int(t.pExt); i < len(t.str)-1; { 522 var ext string 523 i, ext = getExtension(t.str, i) 524 if ext[0] == x { 525 return Extension{ext}, true 526 } 527 } 528 return Extension{}, false 529} 530 531// Extensions returns all extensions of t. 532func (t Tag) Extensions() []Extension { 533 e := []Extension{} 534 for i := int(t.pExt); i < len(t.str)-1; { 535 var ext string 536 i, ext = getExtension(t.str, i) 537 e = append(e, Extension{ext}) 538 } 539 return e 540} 541 542// TypeForKey returns the type associated with the given key, where key and type 543// are of the allowed values defined for the Unicode locale extension ('u') in 544// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 545// TypeForKey will traverse the inheritance chain to get the correct value. 546func (t Tag) TypeForKey(key string) string { 547 if start, end, _ := t.findTypeForKey(key); end != start { 548 return t.str[start:end] 549 } 550 return "" 551} 552 553var ( 554 errPrivateUse = errors.New("cannot set a key on a private use tag") 555 errInvalidArguments = errors.New("invalid key or type") 556) 557 558// SetTypeForKey returns a new Tag with the key set to type, where key and type 559// are of the allowed values defined for the Unicode locale extension ('u') in 560// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 561// An empty value removes an existing pair with the same key. 562func (t Tag) SetTypeForKey(key, value string) (Tag, error) { 563 if t.private() { 564 return t, errPrivateUse 565 } 566 if len(key) != 2 { 567 return t, errInvalidArguments 568 } 569 570 // Remove the setting if value is "". 571 if value == "" { 572 start, end, _ := t.findTypeForKey(key) 573 if start != end { 574 // Remove key tag and leading '-'. 575 start -= 4 576 577 // Remove a possible empty extension. 578 if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' { 579 start -= 2 580 } 581 if start == int(t.pVariant) && end == len(t.str) { 582 t.str = "" 583 t.pVariant, t.pExt = 0, 0 584 } else { 585 t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:]) 586 } 587 } 588 return t, nil 589 } 590 591 if len(value) < 3 || len(value) > 8 { 592 return t, errInvalidArguments 593 } 594 595 var ( 596 buf [maxCoreSize + maxSimpleUExtensionSize]byte 597 uStart int // start of the -u extension. 598 ) 599 600 // Generate the tag string if needed. 601 if t.str == "" { 602 uStart = t.genCoreBytes(buf[:]) 603 buf[uStart] = '-' 604 uStart++ 605 } 606 607 // Create new key-type pair and parse it to verify. 608 b := buf[uStart:] 609 copy(b, "u-") 610 copy(b[2:], key) 611 b[4] = '-' 612 b = b[:5+copy(b[5:], value)] 613 scan := makeScanner(b) 614 if parseExtensions(&scan); scan.err != nil { 615 return t, scan.err 616 } 617 618 // Assemble the replacement string. 619 if t.str == "" { 620 t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1) 621 t.str = string(buf[:uStart+len(b)]) 622 } else { 623 s := t.str 624 start, end, hasExt := t.findTypeForKey(key) 625 if start == end { 626 if hasExt { 627 b = b[2:] 628 } 629 t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:]) 630 } else { 631 t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:]) 632 } 633 } 634 return t, nil 635} 636 637// findKeyAndType returns the start and end position for the type corresponding 638// to key or the point at which to insert the key-value pair if the type 639// wasn't found. The hasExt return value reports whether an -u extension was present. 640// Note: the extensions are typically very small and are likely to contain 641// only one key-type pair. 642func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) { 643 p := int(t.pExt) 644 if len(key) != 2 || p == len(t.str) || p == 0 { 645 return p, p, false 646 } 647 s := t.str 648 649 // Find the correct extension. 650 for p++; s[p] != 'u'; p++ { 651 if s[p] > 'u' { 652 p-- 653 return p, p, false 654 } 655 if p = nextExtension(s, p); p == len(s) { 656 return len(s), len(s), false 657 } 658 } 659 // Proceed to the hyphen following the extension name. 660 p++ 661 662 // curKey is the key currently being processed. 663 curKey := "" 664 665 // Iterate over keys until we get the end of a section. 666 for { 667 // p points to the hyphen preceding the current token. 668 if p3 := p + 3; s[p3] == '-' { 669 // Found a key. 670 // Check whether we just processed the key that was requested. 671 if curKey == key { 672 return start, p, true 673 } 674 // Set to the next key and continue scanning type tokens. 675 curKey = s[p+1 : p3] 676 if curKey > key { 677 return p, p, true 678 } 679 // Start of the type token sequence. 680 start = p + 4 681 // A type is at least 3 characters long. 682 p += 7 // 4 + 3 683 } else { 684 // Attribute or type, which is at least 3 characters long. 685 p += 4 686 } 687 // p points past the third character of a type or attribute. 688 max := p + 5 // maximum length of token plus hyphen. 689 if len(s) < max { 690 max = len(s) 691 } 692 for ; p < max && s[p] != '-'; p++ { 693 } 694 // Bail if we have exhausted all tokens or if the next token starts 695 // a new extension. 696 if p == len(s) || s[p+2] == '-' { 697 if curKey == key { 698 return start, p, true 699 } 700 return p, p, true 701 } 702 } 703} 704 705// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags 706// for which data exists in the text repository. The index will change over time 707// and should not be stored in persistent storage. Extensions, except for the 708// 'va' type of the 'u' extension, are ignored. It will return 0, false if no 709// compact tag exists, where 0 is the index for the root language (Und). 710func CompactIndex(t Tag) (index int, ok bool) { 711 // TODO: perhaps give more frequent tags a lower index. 712 // TODO: we could make the indexes stable. This will excluded some 713 // possibilities for optimization, so don't do this quite yet. 714 b, s, r := t.Raw() 715 if len(t.str) > 0 { 716 if strings.HasPrefix(t.str, "x-") { 717 // We have no entries for user-defined tags. 718 return 0, false 719 } 720 if uint16(t.pVariant) != t.pExt { 721 // There are no tags with variants and an u-va type. 722 if t.TypeForKey("va") != "" { 723 return 0, false 724 } 725 t, _ = Raw.Compose(b, s, r, t.Variants()) 726 } else if _, ok := t.Extension('u'); ok { 727 // Strip all but the 'va' entry. 728 variant := t.TypeForKey("va") 729 t, _ = Raw.Compose(b, s, r) 730 t, _ = t.SetTypeForKey("va", variant) 731 } 732 if len(t.str) > 0 { 733 // We have some variants. 734 for i, s := range specialTags { 735 if s == t { 736 return i + 1, true 737 } 738 } 739 return 0, false 740 } 741 } 742 // No variants specified: just compare core components. 743 // The key has the form lllssrrr, where l, s, and r are nibbles for 744 // respectively the langID, scriptID, and regionID. 745 key := uint32(b.langID) << (8 + 12) 746 key |= uint32(s.scriptID) << 12 747 key |= uint32(r.regionID) 748 x, ok := coreTags[key] 749 return int(x), ok 750} 751 752// Base is an ISO 639 language code, used for encoding the base language 753// of a language tag. 754type Base struct { 755 langID 756} 757 758// ParseBase parses a 2- or 3-letter ISO 639 code. 759// It returns a ValueError if s is a well-formed but unknown language identifier 760// or another error if another error occurred. 761func ParseBase(s string) (Base, error) { 762 if n := len(s); n < 2 || 3 < n { 763 return Base{}, errSyntax 764 } 765 var buf [3]byte 766 l, err := getLangID(buf[:copy(buf[:], s)]) 767 return Base{l}, err 768} 769 770// Script is a 4-letter ISO 15924 code for representing scripts. 771// It is idiomatically represented in title case. 772type Script struct { 773 scriptID 774} 775 776// ParseScript parses a 4-letter ISO 15924 code. 777// It returns a ValueError if s is a well-formed but unknown script identifier 778// or another error if another error occurred. 779func ParseScript(s string) (Script, error) { 780 if len(s) != 4 { 781 return Script{}, errSyntax 782 } 783 var buf [4]byte 784 sc, err := getScriptID(script, buf[:copy(buf[:], s)]) 785 return Script{sc}, err 786} 787 788// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions. 789type Region struct { 790 regionID 791} 792 793// EncodeM49 returns the Region for the given UN M.49 code. 794// It returns an error if r is not a valid code. 795func EncodeM49(r int) (Region, error) { 796 rid, err := getRegionM49(r) 797 return Region{rid}, err 798} 799 800// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code. 801// It returns a ValueError if s is a well-formed but unknown region identifier 802// or another error if another error occurred. 803func ParseRegion(s string) (Region, error) { 804 if n := len(s); n < 2 || 3 < n { 805 return Region{}, errSyntax 806 } 807 var buf [3]byte 808 r, err := getRegionID(buf[:copy(buf[:], s)]) 809 return Region{r}, err 810} 811 812// IsCountry returns whether this region is a country or autonomous area. This 813// includes non-standard definitions from CLDR. 814func (r Region) IsCountry() bool { 815 if r.regionID == 0 || r.IsGroup() || r.IsPrivateUse() && r.regionID != _XK { 816 return false 817 } 818 return true 819} 820 821// IsGroup returns whether this region defines a collection of regions. This 822// includes non-standard definitions from CLDR. 823func (r Region) IsGroup() bool { 824 if r.regionID == 0 { 825 return false 826 } 827 return int(regionInclusion[r.regionID]) < len(regionContainment) 828} 829 830// Contains returns whether Region c is contained by Region r. It returns true 831// if c == r. 832func (r Region) Contains(c Region) bool { 833 return r.regionID.contains(c.regionID) 834} 835 836func (r regionID) contains(c regionID) bool { 837 if r == c { 838 return true 839 } 840 g := regionInclusion[r] 841 if g >= nRegionGroups { 842 return false 843 } 844 m := regionContainment[g] 845 846 d := regionInclusion[c] 847 b := regionInclusionBits[d] 848 849 // A contained country may belong to multiple disjoint groups. Matching any 850 // of these indicates containment. If the contained region is a group, it 851 // must strictly be a subset. 852 if d >= nRegionGroups { 853 return b&m != 0 854 } 855 return b&^m == 0 856} 857 858var errNoTLD = errors.New("language: region is not a valid ccTLD") 859 860// TLD returns the country code top-level domain (ccTLD). UK is returned for GB. 861// In all other cases it returns either the region itself or an error. 862// 863// This method may return an error for a region for which there exists a 864// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The 865// region will already be canonicalized it was obtained from a Tag that was 866// obtained using any of the default methods. 867func (r Region) TLD() (Region, error) { 868 // See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the 869 // difference between ISO 3166-1 and IANA ccTLD. 870 if r.regionID == _GB { 871 r = Region{_UK} 872 } 873 if (r.typ() & ccTLD) == 0 { 874 return Region{}, errNoTLD 875 } 876 return r, nil 877} 878 879// Canonicalize returns the region or a possible replacement if the region is 880// deprecated. It will not return a replacement for deprecated regions that 881// are split into multiple regions. 882func (r Region) Canonicalize() Region { 883 if cr := normRegion(r.regionID); cr != 0 { 884 return Region{cr} 885 } 886 return r 887} 888 889// Variant represents a registered variant of a language as defined by BCP 47. 890type Variant struct { 891 variant string 892} 893 894// ParseVariant parses and returns a Variant. An error is returned if s is not 895// a valid variant. 896func ParseVariant(s string) (Variant, error) { 897 s = strings.ToLower(s) 898 if _, ok := variantIndex[s]; ok { 899 return Variant{s}, nil 900 } 901 return Variant{}, mkErrInvalid([]byte(s)) 902} 903 904// String returns the string representation of the variant. 905func (v Variant) String() string { 906 return v.variant 907} 908