1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package utf8 implements functions and constants to support text encoded in 6// UTF-8. It includes functions to translate between runes and UTF-8 byte sequences. 7// See https://en.wikipedia.org/wiki/UTF-8 8package utf8 9 10// The conditions RuneError==unicode.ReplacementChar and 11// MaxRune==unicode.MaxRune are verified in the tests. 12// Defining them locally avoids this package depending on package unicode. 13 14// Numbers fundamental to the encoding. 15const ( 16 RuneError = '\uFFFD' // the "error" Rune or "Unicode replacement character" 17 RuneSelf = 0x80 // characters below RuneSelf are represented as themselves in a single byte. 18 MaxRune = '\U0010FFFF' // Maximum valid Unicode code point. 19 UTFMax = 4 // maximum number of bytes of a UTF-8 encoded Unicode character. 20) 21 22// Code points in the surrogate range are not valid for UTF-8. 23const ( 24 surrogateMin = 0xD800 25 surrogateMax = 0xDFFF 26) 27 28const ( 29 t1 = 0b00000000 30 tx = 0b10000000 31 t2 = 0b11000000 32 t3 = 0b11100000 33 t4 = 0b11110000 34 t5 = 0b11111000 35 36 maskx = 0b00111111 37 mask2 = 0b00011111 38 mask3 = 0b00001111 39 mask4 = 0b00000111 40 41 rune1Max = 1<<7 - 1 42 rune2Max = 1<<11 - 1 43 rune3Max = 1<<16 - 1 44 45 // The default lowest and highest continuation byte. 46 locb = 0b10000000 47 hicb = 0b10111111 48 49 // These names of these constants are chosen to give nice alignment in the 50 // table below. The first nibble is an index into acceptRanges or F for 51 // special one-byte cases. The second nibble is the Rune length or the 52 // Status for the special one-byte case. 53 xx = 0xF1 // invalid: size 1 54 as = 0xF0 // ASCII: size 1 55 s1 = 0x02 // accept 0, size 2 56 s2 = 0x13 // accept 1, size 3 57 s3 = 0x03 // accept 0, size 3 58 s4 = 0x23 // accept 2, size 3 59 s5 = 0x34 // accept 3, size 4 60 s6 = 0x04 // accept 0, size 4 61 s7 = 0x44 // accept 4, size 4 62) 63 64// first is information about the first byte in a UTF-8 sequence. 65var first = [256]uint8{ 66 // 1 2 3 4 5 6 7 8 9 A B C D E F 67 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x00-0x0F 68 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x10-0x1F 69 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x20-0x2F 70 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x30-0x3F 71 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x40-0x4F 72 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x50-0x5F 73 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x60-0x6F 74 as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, as, // 0x70-0x7F 75 // 1 2 3 4 5 6 7 8 9 A B C D E F 76 xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0x80-0x8F 77 xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0x90-0x9F 78 xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0xA0-0xAF 79 xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0xB0-0xBF 80 xx, xx, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, // 0xC0-0xCF 81 s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, // 0xD0-0xDF 82 s2, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s4, s3, s3, // 0xE0-0xEF 83 s5, s6, s6, s6, s7, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0xF0-0xFF 84} 85 86// acceptRange gives the range of valid values for the second byte in a UTF-8 87// sequence. 88type acceptRange struct { 89 lo uint8 // lowest value for second byte. 90 hi uint8 // highest value for second byte. 91} 92 93// acceptRanges has size 16 to avoid bounds checks in the code that uses it. 94var acceptRanges = [16]acceptRange{ 95 0: {locb, hicb}, 96 1: {0xA0, hicb}, 97 2: {locb, 0x9F}, 98 3: {0x90, hicb}, 99 4: {locb, 0x8F}, 100} 101 102// FullRune reports whether the bytes in p begin with a full UTF-8 encoding of a rune. 103// An invalid encoding is considered a full Rune since it will convert as a width-1 error rune. 104func FullRune(p []byte) bool { 105 n := len(p) 106 if n == 0 { 107 return false 108 } 109 x := first[p[0]] 110 if n >= int(x&7) { 111 return true // ASCII, invalid or valid. 112 } 113 // Must be short or invalid. 114 accept := acceptRanges[x>>4] 115 if n > 1 && (p[1] < accept.lo || accept.hi < p[1]) { 116 return true 117 } else if n > 2 && (p[2] < locb || hicb < p[2]) { 118 return true 119 } 120 return false 121} 122 123// FullRuneInString is like FullRune but its input is a string. 124func FullRuneInString(s string) bool { 125 n := len(s) 126 if n == 0 { 127 return false 128 } 129 x := first[s[0]] 130 if n >= int(x&7) { 131 return true // ASCII, invalid, or valid. 132 } 133 // Must be short or invalid. 134 accept := acceptRanges[x>>4] 135 if n > 1 && (s[1] < accept.lo || accept.hi < s[1]) { 136 return true 137 } else if n > 2 && (s[2] < locb || hicb < s[2]) { 138 return true 139 } 140 return false 141} 142 143// DecodeRune unpacks the first UTF-8 encoding in p and returns the rune and 144// its width in bytes. If p is empty it returns (RuneError, 0). Otherwise, if 145// the encoding is invalid, it returns (RuneError, 1). Both are impossible 146// results for correct, non-empty UTF-8. 147// 148// An encoding is invalid if it is incorrect UTF-8, encodes a rune that is 149// out of range, or is not the shortest possible UTF-8 encoding for the 150// value. No other validation is performed. 151func DecodeRune(p []byte) (r rune, size int) { 152 n := len(p) 153 if n < 1 { 154 return RuneError, 0 155 } 156 p0 := p[0] 157 x := first[p0] 158 if x >= as { 159 // The following code simulates an additional check for x == xx and 160 // handling the ASCII and invalid cases accordingly. This mask-and-or 161 // approach prevents an additional branch. 162 mask := rune(x) << 31 >> 31 // Create 0x0000 or 0xFFFF. 163 return rune(p[0])&^mask | RuneError&mask, 1 164 } 165 sz := int(x & 7) 166 accept := acceptRanges[x>>4] 167 if n < sz { 168 return RuneError, 1 169 } 170 b1 := p[1] 171 if b1 < accept.lo || accept.hi < b1 { 172 return RuneError, 1 173 } 174 if sz <= 2 { // <= instead of == to help the compiler eliminate some bounds checks 175 return rune(p0&mask2)<<6 | rune(b1&maskx), 2 176 } 177 b2 := p[2] 178 if b2 < locb || hicb < b2 { 179 return RuneError, 1 180 } 181 if sz <= 3 { 182 return rune(p0&mask3)<<12 | rune(b1&maskx)<<6 | rune(b2&maskx), 3 183 } 184 b3 := p[3] 185 if b3 < locb || hicb < b3 { 186 return RuneError, 1 187 } 188 return rune(p0&mask4)<<18 | rune(b1&maskx)<<12 | rune(b2&maskx)<<6 | rune(b3&maskx), 4 189} 190 191// DecodeRuneInString is like DecodeRune but its input is a string. If s is 192// empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it 193// returns (RuneError, 1). Both are impossible results for correct, non-empty 194// UTF-8. 195// 196// An encoding is invalid if it is incorrect UTF-8, encodes a rune that is 197// out of range, or is not the shortest possible UTF-8 encoding for the 198// value. No other validation is performed. 199func DecodeRuneInString(s string) (r rune, size int) { 200 n := len(s) 201 if n < 1 { 202 return RuneError, 0 203 } 204 s0 := s[0] 205 x := first[s0] 206 if x >= as { 207 // The following code simulates an additional check for x == xx and 208 // handling the ASCII and invalid cases accordingly. This mask-and-or 209 // approach prevents an additional branch. 210 mask := rune(x) << 31 >> 31 // Create 0x0000 or 0xFFFF. 211 return rune(s[0])&^mask | RuneError&mask, 1 212 } 213 sz := int(x & 7) 214 accept := acceptRanges[x>>4] 215 if n < sz { 216 return RuneError, 1 217 } 218 s1 := s[1] 219 if s1 < accept.lo || accept.hi < s1 { 220 return RuneError, 1 221 } 222 if sz <= 2 { // <= instead of == to help the compiler eliminate some bounds checks 223 return rune(s0&mask2)<<6 | rune(s1&maskx), 2 224 } 225 s2 := s[2] 226 if s2 < locb || hicb < s2 { 227 return RuneError, 1 228 } 229 if sz <= 3 { 230 return rune(s0&mask3)<<12 | rune(s1&maskx)<<6 | rune(s2&maskx), 3 231 } 232 s3 := s[3] 233 if s3 < locb || hicb < s3 { 234 return RuneError, 1 235 } 236 return rune(s0&mask4)<<18 | rune(s1&maskx)<<12 | rune(s2&maskx)<<6 | rune(s3&maskx), 4 237} 238 239// DecodeLastRune unpacks the last UTF-8 encoding in p and returns the rune and 240// its width in bytes. If p is empty it returns (RuneError, 0). Otherwise, if 241// the encoding is invalid, it returns (RuneError, 1). Both are impossible 242// results for correct, non-empty UTF-8. 243// 244// An encoding is invalid if it is incorrect UTF-8, encodes a rune that is 245// out of range, or is not the shortest possible UTF-8 encoding for the 246// value. No other validation is performed. 247func DecodeLastRune(p []byte) (r rune, size int) { 248 end := len(p) 249 if end == 0 { 250 return RuneError, 0 251 } 252 start := end - 1 253 r = rune(p[start]) 254 if r < RuneSelf { 255 return r, 1 256 } 257 // guard against O(n^2) behavior when traversing 258 // backwards through strings with long sequences of 259 // invalid UTF-8. 260 lim := end - UTFMax 261 if lim < 0 { 262 lim = 0 263 } 264 for start--; start >= lim; start-- { 265 if RuneStart(p[start]) { 266 break 267 } 268 } 269 if start < 0 { 270 start = 0 271 } 272 r, size = DecodeRune(p[start:end]) 273 if start+size != end { 274 return RuneError, 1 275 } 276 return r, size 277} 278 279// DecodeLastRuneInString is like DecodeLastRune but its input is a string. If 280// s is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, 281// it returns (RuneError, 1). Both are impossible results for correct, 282// non-empty UTF-8. 283// 284// An encoding is invalid if it is incorrect UTF-8, encodes a rune that is 285// out of range, or is not the shortest possible UTF-8 encoding for the 286// value. No other validation is performed. 287func DecodeLastRuneInString(s string) (r rune, size int) { 288 end := len(s) 289 if end == 0 { 290 return RuneError, 0 291 } 292 start := end - 1 293 r = rune(s[start]) 294 if r < RuneSelf { 295 return r, 1 296 } 297 // guard against O(n^2) behavior when traversing 298 // backwards through strings with long sequences of 299 // invalid UTF-8. 300 lim := end - UTFMax 301 if lim < 0 { 302 lim = 0 303 } 304 for start--; start >= lim; start-- { 305 if RuneStart(s[start]) { 306 break 307 } 308 } 309 if start < 0 { 310 start = 0 311 } 312 r, size = DecodeRuneInString(s[start:end]) 313 if start+size != end { 314 return RuneError, 1 315 } 316 return r, size 317} 318 319// RuneLen returns the number of bytes required to encode the rune. 320// It returns -1 if the rune is not a valid value to encode in UTF-8. 321func RuneLen(r rune) int { 322 switch { 323 case r < 0: 324 return -1 325 case r <= rune1Max: 326 return 1 327 case r <= rune2Max: 328 return 2 329 case surrogateMin <= r && r <= surrogateMax: 330 return -1 331 case r <= rune3Max: 332 return 3 333 case r <= MaxRune: 334 return 4 335 } 336 return -1 337} 338 339// EncodeRune writes into p (which must be large enough) the UTF-8 encoding of the rune. 340// It returns the number of bytes written. 341func EncodeRune(p []byte, r rune) int { 342 // Negative values are erroneous. Making it unsigned addresses the problem. 343 switch i := uint32(r); { 344 case i <= rune1Max: 345 p[0] = byte(r) 346 return 1 347 case i <= rune2Max: 348 _ = p[1] // eliminate bounds checks 349 p[0] = t2 | byte(r>>6) 350 p[1] = tx | byte(r)&maskx 351 return 2 352 case i > MaxRune, surrogateMin <= i && i <= surrogateMax: 353 r = RuneError 354 fallthrough 355 case i <= rune3Max: 356 _ = p[2] // eliminate bounds checks 357 p[0] = t3 | byte(r>>12) 358 p[1] = tx | byte(r>>6)&maskx 359 p[2] = tx | byte(r)&maskx 360 return 3 361 default: 362 _ = p[3] // eliminate bounds checks 363 p[0] = t4 | byte(r>>18) 364 p[1] = tx | byte(r>>12)&maskx 365 p[2] = tx | byte(r>>6)&maskx 366 p[3] = tx | byte(r)&maskx 367 return 4 368 } 369} 370 371// RuneCount returns the number of runes in p. Erroneous and short 372// encodings are treated as single runes of width 1 byte. 373func RuneCount(p []byte) int { 374 np := len(p) 375 var n int 376 for i := 0; i < np; { 377 n++ 378 c := p[i] 379 if c < RuneSelf { 380 // ASCII fast path 381 i++ 382 continue 383 } 384 x := first[c] 385 if x == xx { 386 i++ // invalid. 387 continue 388 } 389 size := int(x & 7) 390 if i+size > np { 391 i++ // Short or invalid. 392 continue 393 } 394 accept := acceptRanges[x>>4] 395 if c := p[i+1]; c < accept.lo || accept.hi < c { 396 size = 1 397 } else if size == 2 { 398 } else if c := p[i+2]; c < locb || hicb < c { 399 size = 1 400 } else if size == 3 { 401 } else if c := p[i+3]; c < locb || hicb < c { 402 size = 1 403 } 404 i += size 405 } 406 return n 407} 408 409// RuneCountInString is like RuneCount but its input is a string. 410func RuneCountInString(s string) (n int) { 411 ns := len(s) 412 for i := 0; i < ns; n++ { 413 c := s[i] 414 if c < RuneSelf { 415 // ASCII fast path 416 i++ 417 continue 418 } 419 x := first[c] 420 if x == xx { 421 i++ // invalid. 422 continue 423 } 424 size := int(x & 7) 425 if i+size > ns { 426 i++ // Short or invalid. 427 continue 428 } 429 accept := acceptRanges[x>>4] 430 if c := s[i+1]; c < accept.lo || accept.hi < c { 431 size = 1 432 } else if size == 2 { 433 } else if c := s[i+2]; c < locb || hicb < c { 434 size = 1 435 } else if size == 3 { 436 } else if c := s[i+3]; c < locb || hicb < c { 437 size = 1 438 } 439 i += size 440 } 441 return n 442} 443 444// RuneStart reports whether the byte could be the first byte of an encoded, 445// possibly invalid rune. Second and subsequent bytes always have the top two 446// bits set to 10. 447func RuneStart(b byte) bool { return b&0xC0 != 0x80 } 448 449// Valid reports whether p consists entirely of valid UTF-8-encoded runes. 450func Valid(p []byte) bool { 451 n := len(p) 452 for i := 0; i < n; { 453 pi := p[i] 454 if pi < RuneSelf { 455 i++ 456 continue 457 } 458 x := first[pi] 459 if x == xx { 460 return false // Illegal starter byte. 461 } 462 size := int(x & 7) 463 if i+size > n { 464 return false // Short or invalid. 465 } 466 accept := acceptRanges[x>>4] 467 if c := p[i+1]; c < accept.lo || accept.hi < c { 468 return false 469 } else if size == 2 { 470 } else if c := p[i+2]; c < locb || hicb < c { 471 return false 472 } else if size == 3 { 473 } else if c := p[i+3]; c < locb || hicb < c { 474 return false 475 } 476 i += size 477 } 478 return true 479} 480 481// ValidString reports whether s consists entirely of valid UTF-8-encoded runes. 482func ValidString(s string) bool { 483 n := len(s) 484 for i := 0; i < n; { 485 si := s[i] 486 if si < RuneSelf { 487 i++ 488 continue 489 } 490 x := first[si] 491 if x == xx { 492 return false // Illegal starter byte. 493 } 494 size := int(x & 7) 495 if i+size > n { 496 return false // Short or invalid. 497 } 498 accept := acceptRanges[x>>4] 499 if c := s[i+1]; c < accept.lo || accept.hi < c { 500 return false 501 } else if size == 2 { 502 } else if c := s[i+2]; c < locb || hicb < c { 503 return false 504 } else if size == 3 { 505 } else if c := s[i+3]; c < locb || hicb < c { 506 return false 507 } 508 i += size 509 } 510 return true 511} 512 513// ValidRune reports whether r can be legally encoded as UTF-8. 514// Code points that are out of range or a surrogate half are illegal. 515func ValidRune(r rune) bool { 516 switch { 517 case 0 <= r && r < surrogateMin: 518 return true 519 case surrogateMax < r && r <= MaxRune: 520 return true 521 } 522 return false 523} 524