1 /* Copyright 2016 The encode_unicode Developers
2 *
3 * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 * http://opensource.org/licenses/MIT>, at your option. This file may not be
6 * copied, modified, or distributed except according to those terms.
7 */
8
9 #![allow(unused_unsafe)]// explicit unsafe{} blocks in unsafe functions are a good thing.
10
11 use utf8_char::Utf8Char;
12 use utf16_char::Utf16Char;
13 use utf8_iterators::*;
14 use utf16_iterators::*;
15 use decoding_iterators::*;
16 use error::*;
17 extern crate core;
18 use self::core::{char, u32, mem};
19 use self::core::ops::{Not, Index, RangeFull};
20 use self::core::borrow::Borrow;
21 #[cfg(feature="ascii")]
22 extern crate ascii;
23 #[cfg(feature="ascii")]
24 use self::ascii::AsciiStr;
25
26 // TODO better docs and tests
27
28 /// Methods for working with `u8`s as UTF-8 bytes.
29 pub trait U8UtfExt {
30 /// How many more bytes will you need to complete this codepoint?
31 ///
32 /// # Errors
33 ///
34 /// An error is returned if the byte is not a valid start of an UTF-8
35 /// codepoint:
36 ///
37 /// * `128..192`: ContinuationByte
38 /// * `248..`: TooLongSequence
39 ///
40 /// Values in 244..248 represent a too high codepoint, but do not cause an
41 /// error.
extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte>42 fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte>;
43
44 /// How many more bytes will you need to complete this codepoint?
45 ///
46 /// This function assumes that the byte is a valid UTF-8 start, and might
47 /// return any value otherwise. (but the function is pure and safe to call
48 /// with any value).
extra_utf8_bytes_unchecked(self) -> usize49 fn extra_utf8_bytes_unchecked(self) -> usize;
50 }
51
52 impl U8UtfExt for u8 {
53 #[inline]
extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte>54 fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte> {
55 use error::InvalidUtf8FirstByte::{ContinuationByte,TooLongSeqence};
56 // the bit twiddling is explained in extra_utf8_bytes_unchecked()
57 if self < 128 {
58 return Ok(0);
59 }
60 match ((self as u32)<<25).not().leading_zeros() {
61 n @ 1...3 => Ok(n as usize),
62 0 => Err(ContinuationByte),
63 _ => Err(TooLongSeqence),
64 }
65 }
66 #[inline]
extra_utf8_bytes_unchecked(self) -> usize67 fn extra_utf8_bytes_unchecked(self) -> usize {
68 // For fun I've optimized this function (for x86 instruction count):
69 // The most straightforward implementation, that lets the compiler do
70 // the optimizing:
71 //match self {
72 // 0b0000_0000...0b0111_1111 => 0,
73 // 0b1100_0010...0b1101_1111 => 1,
74 // 0b1110_0000...0b1110_1111 => 2,
75 // 0b1111_0000...0b1111_0100 => 3,
76 // _ => whatever()
77 //}
78 // Using `unsafe{self::core::hint::unreachable_unchecked()}` for the
79 // "don't care" case is a terrible idea: while having the function
80 // non-deterministically return whatever happens to be in a register
81 // MIGHT be acceptable, it permits the function to not `ret`urn at all,
82 // but let execution fall through to whatever comes after it in the
83 // binary! (in other words completely UB).
84 // Currently unreachable_unchecked() might trap too,
85 // which is certainly not what we want.
86 // I also think `unsafe{mem::unitialized()}` is much more likely to
87 // explicitly produce whatever happens to be in a register than tell
88 // the compiler it can ignore this branch but needs to produce a value.
89 //
90 // From the bit patterns we see that for non-ASCII values the result is
91 // (number of leading set bits) - 1
92 // The standard library doesn't have a method for counting leading ones,
93 // but it has leading_zeros(), which can be used after inverting.
94 // This function can therefore be reduced to the one-liner
95 //`self.not().leading_zeros().saturating_sub(1) as usize`, which would
96 // be branchless for architectures with instructions for
97 // leading_zeros() and saturating_sub().
98
99 // Shortest version as long as ASCII-ness can be predicted: (especially
100 // if the BSR instruction which leading_zeros() uses is microcoded or
101 // doesn't exist)
102 // u8.leading_zeros() would cast to a bigger type internally, so that's
103 // free. compensating by shifting left by 24 before inverting lets the
104 // compiler know that the value passed to leading_zeros() is not zero,
105 // for which BSR's output is undefined and leading_zeros() normally has
106 // special case with a branch.
107 // Shifting one bit too many left acts as a saturating_sub(1).
108 if self<128 {0} else {((self as u32)<<25).not().leading_zeros() as usize}
109
110 // Branchless but longer version: (9 instructions)
111 // It's tempting to try (self|0x80).not().leading_zeros().wrapping_sub(1),
112 // but that produces high lengths for ASCII values 0b01xx_xxxx.
113 // If we could somehow (branchlessy) clear that bit for ASCII values...
114 // We can by masking with the value shifted right with sign extension!
115 // (any nonzero number of bits in range works)
116 //let extended = self as i8 as i32;
117 //let ascii_cleared = (extended<<25) & (extended>>25);
118 //ascii_cleared.not().leading_zeros() as usize
119
120 // cmov version: (7 instructions)
121 //(((self as u32)<<24).not().leading_zeros() as usize).saturating_sub(1)
122 }
123 }
124
125
126 /// Methods for working with `u16`s as UTF-16 units.
127 pub trait U16UtfExt {
128 /// Will you need an extra unit to complete this codepoint?
129 ///
130 /// Returns `Err` for trailing surrogates, `Ok(true)` for leading surrogates,
131 /// and `Ok(false)` for others.
utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit>132 fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit>;
133
134 /// Does this `u16` need another `u16` to complete a codepoint?
135 /// Returns `(self & 0xfc00) == 0xd800`
136 ///
137 /// Is basically an unchecked variant of `utf16_needs_extra_unit()`.
is_utf16_leading_surrogate(self) -> bool138 fn is_utf16_leading_surrogate(self) -> bool;
139 }
140 impl U16UtfExt for u16 {
141 #[inline]
utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit>142 fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit> {
143 match self {
144 // https://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF
145 0x00_00...0xd7_ff | 0xe0_00...0xff_ff => Ok(false),
146 0xd8_00...0xdb_ff => Ok(true),
147 _ => Err(InvalidUtf16FirstUnit)
148 }
149 }
150 #[inline]
is_utf16_leading_surrogate(self) -> bool151 fn is_utf16_leading_surrogate(self) -> bool {
152 (self & 0xfc00) == 0xd800// Clear the ten content bytes of a surrogate,
153 // and see if it's a leading surrogate.
154 }
155 }
156
157
158
159
160 /// Extension trait for `char` that adds methods for converting to and from UTF-8 or UTF-16.
161 pub trait CharExt: Sized {
162 /// Get the UTF-8 representation of this codepoint.
163 ///
164 /// `Utf8Char` is to `[u8;4]` what `char` is to `u32`:
165 /// a restricted type that cannot be mutated internally.
to_utf8(self) -> Utf8Char166 fn to_utf8(self) -> Utf8Char;
167
168 /// Get the UTF-16 representation of this codepoint.
169 ///
170 /// `Utf16Char` is to `[u16;2]` what `char` is to `u32`:
171 /// a restricted type that cannot be mutated internally.
to_utf16(self) -> Utf16Char172 fn to_utf16(self) -> Utf16Char;
173
174 /// Iterate over or [read](https://doc.rust-lang.org/std/io/trait.Read.html)
175 /// the one to four bytes in the UTF-8 representation of this codepoint.
176 ///
177 /// An identical alternative to the unstable `char.encode_utf8()`.
178 /// That method somehow still exist on stable, so I have to use a different name.
iter_utf8_bytes(self) -> Utf8Iterator179 fn iter_utf8_bytes(self) -> Utf8Iterator;
180
181 /// Iterate over the one or two units in the UTF-16 representation of this codepoint.
182 ///
183 /// An identical alternative to the unstable `char.encode_utf16()`.
184 /// That method somehow still exist on stable, so I have to use a different name.
iter_utf16_units(self) -> Utf16Iterator185 fn iter_utf16_units(self) -> Utf16Iterator;
186
187
188 /// Convert this char to an UTF-8 array, and also return how many bytes of
189 /// the array are used,
190 ///
191 /// The returned array is left-aligned with unused bytes set to zero.
to_utf8_array(self) -> ([u8; 4], usize)192 fn to_utf8_array(self) -> ([u8; 4], usize);
193
194 /// Convert this `char` to UTF-16.
195 ///
196 /// The second element is non-zero when a surrogate pair is required.
197 ///
198 /// # Examples
199 ///
200 /// ```
201 /// use encode_unicode::CharExt;
202 ///
203 /// assert_eq!('@'.to_utf16_array(), ['@' as u16, 0]);
204 /// assert_eq!('睷'.to_utf16_array(), ['睷' as u16, 0]);
205 /// assert_eq!('\u{abcde}'.to_utf16_array(), [0xda6f, 0xdcde]);
206 /// ```
to_utf16_array(self) -> [u16; 2]207 fn to_utf16_array(self) -> [u16; 2];
208
209 /// Convert this `char` to UTF-16.
210 /// The second item is `Some` if a surrogate pair is required.
211 ///
212 /// # Examples
213 ///
214 /// ```
215 /// use encode_unicode::CharExt;
216 ///
217 /// assert_eq!('@'.to_utf16_tuple(), ('@' as u16, None));
218 /// assert_eq!('睷'.to_utf16_tuple(), ('睷' as u16, None));
219 /// assert_eq!('\u{abcde}'.to_utf16_tuple(), (0xda6f, Some(0xdcde)));
220 /// ```
to_utf16_tuple(self) -> (u16, Option<u16>)221 fn to_utf16_tuple(self) -> (u16, Option<u16>);
222
223
224
225 /// Create a `char` from the start of an UTF-8 slice,
226 /// and also return how many bytes were used.
227 ///
228 /// # Errors
229 ///
230 /// Returns an `Err` if the slice is empty, doesn't start with a valid
231 /// UTF-8 sequence or is too short for the sequence.
232 ///
233 /// # Examples
234 ///
235 /// ```
236 /// use encode_unicode::CharExt;
237 /// use encode_unicode::error::InvalidUtf8Slice::*;
238 /// use encode_unicode::error::InvalidUtf8::*;
239 ///
240 /// assert_eq!(char::from_utf8_slice_start(&[b'A', b'B', b'C']), Ok(('A',1)));
241 /// assert_eq!(char::from_utf8_slice_start(&[0xdd, 0xbb]), Ok(('\u{77b}',2)));
242 ///
243 /// assert_eq!(char::from_utf8_slice_start(&[]), Err(TooShort(1)));
244 /// assert_eq!(char::from_utf8_slice_start(&[0xf0, 0x99]), Err(TooShort(4)));
245 /// assert_eq!(char::from_utf8_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1))));
246 /// assert_eq!(char::from_utf8_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2))));
247 /// ```
from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>248 fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>;
249
250 /// Create a `char` from the start of an UTF-16 slice,
251 /// and also return how many units were used.
252 ///
253 /// If you want to continue after an error, continue with the next `u16` unit.
from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>254 fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>;
255
256
257 /// Convert an UTF-8 sequence as returned from `.to_utf8_array()` into a `char`
258 ///
259 /// The codepoint must start at the first byte, and leftover bytes are ignored.
260 ///
261 /// # Errors
262 ///
263 /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence.
264 ///
265 /// # Examples
266 ///
267 /// ```
268 /// use encode_unicode::CharExt;
269 /// use encode_unicode::error::InvalidUtf8Array::*;
270 /// use encode_unicode::error::InvalidUtf8::*;
271 /// use encode_unicode::error::InvalidCodepoint::*;
272 ///
273 /// assert_eq!(char::from_utf8_array([b'A', 0, 0, 0]), Ok('A'));
274 /// assert_eq!(char::from_utf8_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok('\u{10befb}'));
275 /// assert_eq!(char::from_utf8_array([b'A', b'B', b'C', b'D']), Ok('A'));
276 /// assert_eq!(char::from_utf8_array([0, 0, 0xcc, 0xbb]), Ok('\0'));
277 ///
278 /// assert_eq!(char::from_utf8_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1))));
279 /// assert_eq!(char::from_utf8_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong)));
280 /// assert_eq!(char::from_utf8_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh)));
281 /// ```
from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array>282 fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array>;
283
284 /// Convert a UTF-16 pair as returned from `.to_utf16_array()` into a `char`.
285 ///
286 /// The second element is ignored when not required.
287 ///
288 /// # Examples
289 ///
290 /// ```
291 /// use encode_unicode::CharExt;
292 /// use encode_unicode::error::InvalidUtf16Array;
293 ///
294 /// assert_eq!(char::from_utf16_array(['x' as u16, 'y' as u16]), Ok('x'));
295 /// assert_eq!(char::from_utf16_array(['睷' as u16, 0]), Ok('睷'));
296 /// assert_eq!(char::from_utf16_array([0xda6f, 0xdcde]), Ok('\u{abcde}'));
297 /// assert_eq!(char::from_utf16_array([0xf111, 0xdbad]), Ok('\u{f111}'));
298 /// assert_eq!(char::from_utf16_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate));
299 /// assert_eq!(char::from_utf16_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate));
300 /// ```
from_utf16_array(utf16: [u16; 2]) -> Result<Self, InvalidUtf16Array>301 fn from_utf16_array(utf16: [u16; 2]) -> Result<Self, InvalidUtf16Array>;
302
303 /// Convert a UTF-16 pair as returned from `.to_utf16_tuple()` into a `char`.
from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple>304 fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple>;
305
306
307 /// Convert an UTF-8 sequence into a char.
308 ///
309 /// The length of the slice is taken as length of the sequence;
310 /// it should be 1,2,3 or 4.
311 ///
312 /// # Safety
313 ///
314 /// The slice must contain exactly one, valid, UTF-8 sequence.
315 ///
316 /// Passing a slice that produces an invalid codepoint is always undefined
317 /// behavior; Later checks that the codepoint is valid can be removed
318 /// by the compiler.
319 ///
320 /// # Panics
321 ///
322 /// If the slice is empty
from_utf8_exact_slice_unchecked(src: &[u8]) -> Self323 unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self;
324
325 /// Convert a UTF-16 array as returned from `.to_utf16_array()` into a
326 /// `char`.
327 ///
328 /// This function is safe because it avoids creating invalid codepoints,
329 /// but the returned value might not be what one expectedd.
330 ///
331 /// # Examples
332 ///
333 /// ```
334 /// use encode_unicode::CharExt;
335 ///
336 /// // starts with a trailing surrogate - converted as if it was a valid
337 /// // surrogate pair anyway.
338 /// assert_eq!(char::from_utf16_array_unchecked([0xdbad, 0xf19e]), '\u{fb59e}');
339 /// // missing trailing surrogate - ditto
340 /// assert_eq!(char::from_utf16_array_unchecked([0xd802, 0]), '\u{10800}');
341 /// ```
from_utf16_array_unchecked(utf16: [u16;2]) -> Self342 fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self;
343
344 /// Convert a UTF-16 tuple as returned from `.to_utf16_tuple()` into a `char`.
from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self345 unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self;
346
347
348 /// Produces more detailed errors than `char::from_u32()`
349 ///
350 /// # Errors
351 ///
352 /// This function will return an error if
353 ///
354 /// * the value is greater than 0x10ffff
355 /// * the value is between 0xd800 and 0xdfff (inclusive)
356 ///
357 /// # Examples
358 ///
359 /// ```
360 /// use encode_unicode::CharExt;
361 /// use encode_unicode::error::InvalidCodepoint;
362 ///
363 /// assert_eq!(char::from_u32_detailed(0x41), Ok('A'));
364 /// assert_eq!(char::from_u32_detailed(0x40_00_00), Err(InvalidCodepoint::TooHigh));
365 /// assert_eq!(char::from_u32_detailed(0xd951), Err(InvalidCodepoint::Utf16Reserved));
366 /// assert_eq!(char::from_u32_detailed(0xdddd), Err(InvalidCodepoint::Utf16Reserved));
367 /// assert_eq!(char::from_u32_detailed(0xdd), Ok('Ý'));
368 /// assert_eq!(char::from_u32_detailed(0x1f331), Ok(''));
369 /// ```
from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint>370 fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint>;
371 }
372
373
374
375 impl CharExt for char {
376 /////////
377 //UTF-8//
378 /////////
379
to_utf8(self) -> Utf8Char380 fn to_utf8(self) -> Utf8Char {
381 self.into()
382 }
iter_utf8_bytes(self) -> Utf8Iterator383 fn iter_utf8_bytes(self) -> Utf8Iterator {
384 self.to_utf8().into_iter()
385 }
386
to_utf8_array(self) -> ([u8; 4], usize)387 fn to_utf8_array(self) -> ([u8; 4], usize) {
388 let len = self.len_utf8();
389 let mut c = self as u32;
390 if len == 1 {// ASCII, the common case
391 ([c as u8, 0, 0, 0], 1)
392 } else {
393 let mut parts = 0;// convert to 6-bit bytes
394 parts |= c & 0x3f; c>>=6;
395 parts<<=8; parts |= c & 0x3f; c>>=6;
396 parts<<=8; parts |= c & 0x3f; c>>=6;
397 parts<<=8; parts |= c & 0x3f;
398 parts |= 0x80_80_80_80;// set the most significant bit
399 parts >>= 8*(4-len);// right-align bytes
400 // Now, unused bytes are zero, (which matters for Utf8Char.eq())
401 // and the rest are 0b10xx_xxxx
402
403 // set header on first byte
404 parts |= (0xff_00u32 >> len) & 0xff;// store length
405 parts &= Not::not(1u32 << 7-len);// clear the next bit after it
406
407 let bytes: [u8; 4] = unsafe{ mem::transmute(u32::from_le(parts)) };
408 (bytes, len)
409 }
410 }
411
412
from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>413 fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> {
414 use errors::InvalidUtf8::*;
415 use errors::InvalidUtf8Slice::*;
416 let first = match src.first() {
417 Some(first) => *first,
418 None => return Err(TooShort(1)),
419 };
420 let bytes = match first.extra_utf8_bytes() {
421 Err(e) => return Err(Utf8(FirstByte(e))),
422 Ok(0) => return Ok((first as char, 1)),
423 Ok(extra) if extra >= src.len()
424 => return Err(TooShort(extra+1)),
425 Ok(extra) => &src[..extra+1],
426 };
427 if let Some(i) = bytes.iter().skip(1).position(|&b| (b >> 6) != 0b10 ) {
428 Err(Utf8(NotAContinuationByte(i+1)))
429 } else if overlong(bytes[0], bytes[1]) {
430 Err(Utf8(OverLong))
431 } else {
432 match char::from_u32_detailed(merge_nonascii_unchecked_utf8(bytes)) {
433 Ok(c) => Ok((c, bytes.len())),
434 Err(e) => Err(Codepoint(e)),
435 }
436 }
437 }
438
from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array>439 fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array> {
440 use errors::InvalidUtf8::*;
441 use errors::InvalidUtf8Array::*;
442 let src = match utf8[0].extra_utf8_bytes() {
443 Err(error) => return Err(Utf8(FirstByte(error))),
444 Ok(0) => return Ok(utf8[0] as char),
445 Ok(extra) => &utf8[..extra+1],
446 };
447 if let Some(i) = src[1..].iter().position(|&b| (b >> 6) != 0b10 ) {
448 Err(Utf8(NotAContinuationByte(i+1)))
449 } else if overlong(utf8[0], utf8[1]) {
450 Err(Utf8(OverLong))
451 } else {
452 char::from_u32_detailed(merge_nonascii_unchecked_utf8(src))
453 .map_err(|e| Codepoint(e) )
454 }
455 }
456
from_utf8_exact_slice_unchecked(src: &[u8]) -> Self457 unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self {
458 if src.len() == 1 {
459 src[0] as char
460 } else {
461 char::from_u32_unchecked(merge_nonascii_unchecked_utf8(src))
462 }
463 }
464
465
466
467 //////////
468 //UTF-16//
469 //////////
470
to_utf16(self) -> Utf16Char471 fn to_utf16(self) -> Utf16Char {
472 Utf16Char::from(self)
473 }
iter_utf16_units(self) -> Utf16Iterator474 fn iter_utf16_units(self) -> Utf16Iterator {
475 self.to_utf16().into_iter()
476 }
477
to_utf16_array(self) -> [u16;2]478 fn to_utf16_array(self) -> [u16;2] {
479 let (first, second) = self.to_utf16_tuple();
480 [first, second.unwrap_or(0)]
481 }
to_utf16_tuple(self) -> (u16, Option<u16>)482 fn to_utf16_tuple(self) -> (u16, Option<u16>) {
483 if self <= '\u{ffff}' {// single
484 (self as u16, None)
485 } else {// double
486 let c = self as u32 - 0x_01_00_00;
487 let high = 0x_d8_00 + (c >> 10);
488 let low = 0x_dc_00 + (c & 0x_03_ff);
489 (high as u16, Some(low as u16))
490 }
491 }
492
493
from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>494 fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> {
495 use errors::InvalidUtf16Slice::*;
496 unsafe {match (src.get(0), src.get(1)) {
497 (Some(&u @ 0x00_00...0xd7_ff), _) |
498 (Some(&u @ 0xe0_00...0xff_ff), _)
499 => Ok((char::from_u32_unchecked(u as u32), 1)),
500 (Some(&0xdc_00...0xdf_ff), _) => Err(FirstLowSurrogate),
501 (None, _) => Err(EmptySlice),
502 (Some(&f @ 0xd8_00...0xdb_ff), Some(&s @ 0xdc_00...0xdf_ff))
503 => Ok((char::from_utf16_tuple_unchecked((f, Some(s))), 2)),
504 (Some(&0xd8_00...0xdb_ff), Some(_)) => Err(SecondNotLowSurrogate),
505 (Some(&0xd8_00...0xdb_ff), None) => Err(MissingSecond),
506 (Some(_), _) => unreachable!()
507 }}
508 }
509
from_utf16_array(utf16: [u16;2]) -> Result<Self, InvalidUtf16Array>510 fn from_utf16_array(utf16: [u16;2]) -> Result<Self, InvalidUtf16Array> {
511 use errors::InvalidUtf16Array::*;
512 if let Some(c) = char::from_u32(utf16[0] as u32) {
513 Ok(c) // single
514 } else if utf16[0] < 0xdc_00 && utf16[1] & 0xfc_00 == 0xdc_00 {
515 // correct surrogate pair
516 Ok(combine_surrogates(utf16[0], utf16[1]))
517 } else if utf16[0] < 0xdc_00 {
518 Err(SecondIsNotTrailingSurrogate)
519 } else {
520 Err(FirstIsTrailingSurrogate)
521 }
522 }
from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple>523 fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple> {
524 use errors::InvalidUtf16Tuple::*;
525 unsafe{ match utf16 {
526 (0x00_00...0xd7_ff, None) | // single
527 (0xe0_00...0xff_ff, None) | // single
528 (0xd8_00...0xdb_ff, Some(0xdc_00...0xdf_ff)) // correct surrogate
529 => Ok(char::from_utf16_tuple_unchecked(utf16)),
530 (0xd8_00...0xdb_ff, Some(_)) => Err(InvalidSecond),
531 (0xd8_00...0xdb_ff, None ) => Err(MissingSecond),
532 (0xdc_00...0xdf_ff, _ ) => Err(FirstIsTrailingSurrogate),
533 ( _ , Some(_)) => Err(SuperfluousSecond),
534 ( _ , None ) => unreachable!()
535 }}
536 }
537
from_utf16_array_unchecked(utf16: [u16;2]) -> Self538 fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self {
539 // treat any array with a surrogate value in [0] as a surrogate because
540 // combine_surrogates() is safe.
541 // `(utf16[0] & 0xf800) == 0xd80` might not be quite as fast as
542 // `utf16[1] != 0`, but avoiding the potential for UB is worth it
543 // since the conversion isn't zero-cost in either case.
544 char::from_u32(utf16[0] as u32)
545 .unwrap_or_else(|| combine_surrogates(utf16[0], utf16[1]) )
546 }
from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self547 unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self {
548 match utf16.1 {
549 Some(second) => combine_surrogates(utf16.0, second),
550 None => char::from_u32_unchecked(utf16.0 as u32)
551 }
552 }
553
554
from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint>555 fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint> {
556 match char::from_u32(c) {
557 Some(c) => Ok(c),
558 None if c > 0x10_ff_ff => Err(InvalidCodepoint::TooHigh),
559 None => Err(InvalidCodepoint::Utf16Reserved),
560 }
561 }
562 }
563
564 // Adapted from https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c
overlong(first: u8, second: u8) -> bool565 fn overlong(first: u8, second: u8) -> bool {
566 if first < 0x80 {
567 false
568 } else if (first & 0xe0) == 0xc0 {
569 (first & 0xfe) == 0xc0
570 } else if (first & 0xf0) == 0xe0 {
571 first == 0xe0 && (second & 0xe0) == 0x80
572 } else {
573 first == 0xf0 && (second & 0xf0) == 0x80
574 }
575 }
576
577 /// Decodes the codepoint represented by a multi-byte UTF-8 sequence.
578 ///
579 /// Does not check that the codepoint is valid,
580 /// and returns `u32` because casting invalid codepoints to `char` is insta UB.
merge_nonascii_unchecked_utf8(src: &[u8]) -> u32581 fn merge_nonascii_unchecked_utf8(src: &[u8]) -> u32 {
582 let mut c = src[0] as u32 & (0x7f >> src.len());
583 for b in &src[1..] {
584 c = (c << 6) | (b & 0b0011_1111) as u32;
585 }
586 c
587 }
588
589 /// Create a `char` from a leading and a trailing surrogate.
590 ///
591 /// This function is safe because it ignores the six most significant bits of
592 /// each arguments and always produces a codepoint in 0x01_00_00..=0x10_ff_ff.
combine_surrogates(first: u16, second: u16) -> char593 fn combine_surrogates(first: u16, second: u16) -> char {
594 unsafe {
595 let high = (first & 0x_03_ff) as u32;
596 let low = (second & 0x_03_ff) as u32;
597 let c = ((high << 10) | low) + 0x_01_00_00; // no, the constant can't be or'd in
598 char::from_u32_unchecked(c)
599 }
600 }
601
602
603
604 /// Adds `.utf8chars()` and `.utf16chars()` iterator constructors to `&str`.
605 pub trait StrExt: AsRef<str> {
606 /// Equivalent to `.chars()` but produces `Utf8Char`s.
utf8chars(&self) -> Utf8Chars607 fn utf8chars(&self) -> Utf8Chars;
608 /// Equivalent to `.chars()` but produces `Utf16Char`s.
utf16chars(&self) -> Utf16Chars609 fn utf16chars(&self) -> Utf16Chars;
610 /// Equivalent to `.char_indices()` but produces `Utf8Char`s.
utf8char_indices(&self) -> Utf8CharIndices611 fn utf8char_indices(&self) -> Utf8CharIndices;
612 /// Equivalent to `.char_indices()` but produces `Utf16Char`s.
utf16char_indices(&self) -> Utf16CharIndices613 fn utf16char_indices(&self) -> Utf16CharIndices;
614 }
615
616 impl StrExt for str {
utf8chars(&self) -> Utf8Chars617 fn utf8chars(&self) -> Utf8Chars {
618 Utf8Chars::from(self)
619 }
utf16chars(&self) -> Utf16Chars620 fn utf16chars(&self) -> Utf16Chars {
621 Utf16Chars::from(self)
622 }
utf8char_indices(&self) -> Utf8CharIndices623 fn utf8char_indices(&self) -> Utf8CharIndices {
624 Utf8CharIndices::from(self)
625 }
utf16char_indices(&self) -> Utf16CharIndices626 fn utf16char_indices(&self) -> Utf16CharIndices {
627 Utf16CharIndices::from(self)
628 }
629 }
630
631 #[cfg(feature="ascii")]
632 impl StrExt for AsciiStr {
utf8chars(&self) -> Utf8Chars633 fn utf8chars(&self) -> Utf8Chars {
634 Utf8Chars::from(self.as_str())
635 }
utf16chars(&self) -> Utf16Chars636 fn utf16chars(&self) -> Utf16Chars {
637 Utf16Chars::from(self.as_str())
638 }
utf8char_indices(&self) -> Utf8CharIndices639 fn utf8char_indices(&self) -> Utf8CharIndices {
640 Utf8CharIndices::from(self.as_str())
641 }
utf16char_indices(&self) -> Utf16CharIndices642 fn utf16char_indices(&self) -> Utf16CharIndices {
643 Utf16CharIndices::from(self.as_str())
644 }
645 }
646
647
648
649 /// Iterator methods that convert between `u8`s and `Utf8Char` or `u16`s and `Utf16Char`
650 ///
651 /// All the iterator adapters also accept iterators that produce references of
652 /// the type they convert from.
653 pub trait IterExt: Iterator+Sized {
654 /// Converts an iterator of `Utf8Char`s or `&Utf8Char`s to an iterator of
655 /// `u8`s.
656 ///
657 /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned
658 /// iterator is ~40% faster.
659 ///
660 /// The iterator also implements `Read`
661 /// (when the `std` feature isn't disabled).
662 /// Reading will never produce an error, and calls to `.read()` and `.next()`
663 /// can be mixed.
664 ///
665 /// The exact number of bytes cannot be known in advance, but `size_hint()`
666 /// gives the possible range.
667 /// (min: all remaining characters are ASCII, max: all require four bytes)
668 ///
669 /// # Examples
670 ///
671 /// From iterator of values:
672 ///
673 /// ```
674 /// use encode_unicode::{IterExt, StrExt};
675 ///
676 /// let iterator = "foo".utf8chars();
677 /// let mut bytes = [0; 4];
678 /// for (u,dst) in iterator.to_bytes().zip(&mut bytes) {*dst=u;}
679 /// assert_eq!(&bytes, b"foo\0");
680 /// ```
681 ///
682 /// From iterator of references:
683 ///
684 #[cfg_attr(feature="std", doc=" ```")]
685 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
686 /// use encode_unicode::{IterExt, StrExt, Utf8Char};
687 ///
688 /// let chars: Vec<Utf8Char> = " bomb ".utf8chars().collect();
689 /// let bytes: Vec<u8> = chars.iter().to_bytes().collect();
690 /// let flat_map: Vec<u8> = chars.iter().flat_map(|u8c| *u8c ).collect();
691 /// assert_eq!(bytes, flat_map);
692 /// ```
693 ///
694 /// `Read`ing from it:
695 ///
696 #[cfg_attr(feature="std", doc=" ```")]
697 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
698 /// use encode_unicode::{IterExt, StrExt};
699 /// use std::io::Read;
700 ///
701 /// let s = "Ååh‽";
702 /// assert_eq!(s.len(), 8);
703 /// let mut buf = [b'E'; 9];
704 /// let mut reader = s.utf8chars().to_bytes();
705 /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8);
706 /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0);
707 /// assert_eq!(&buf[..8], s.as_bytes());
708 /// assert_eq!(buf[8], b'E');
709 /// ```
to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char>710 fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char>;
711
712 /// Converts an iterator of `Utf16Char` (or `&Utf16Char`) to an iterator of
713 /// `u16`s.
714 ///
715 /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned
716 /// iterator is about twice as fast.
717 ///
718 /// The exact number of units cannot be known in advance, but `size_hint()`
719 /// gives the possible range.
720 ///
721 /// # Examples
722 ///
723 /// From iterator of values:
724 ///
725 /// ```
726 /// use encode_unicode::{IterExt, StrExt};
727 ///
728 /// let iterator = "foo".utf16chars();
729 /// let mut units = [0; 4];
730 /// for (u,dst) in iterator.to_units().zip(&mut units) {*dst=u;}
731 ///
732 /// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]);
733 /// ```
734 ///
735 /// From iterator of references:
736 ///
737 #[cfg_attr(feature="std", doc=" ```")]
738 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
739 /// use encode_unicode::{IterExt, StrExt, Utf16Char};
740 ///
741 /// // ( takes two units)
742 /// let chars: Vec<Utf16Char> = " bomb ".utf16chars().collect();
743 /// let units: Vec<u16> = chars.iter().to_units().collect();
744 /// let flat_map: Vec<u16> = chars.iter().flat_map(|u16c| *u16c ).collect();
745 ///
746 /// assert_eq!(units, flat_map);
747 /// ```
to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char>748 fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char>;
749
750 /// Decodes bytes as UTF-8 and groups them into `Utf8Char`s
751 ///
752 /// When errors (invalid values or sequences) are encountered,
753 /// it continues with the byte right after the start of the error sequence.
754 /// This is neither the most intelligent choiche (sometimes it is guaranteed to
755 /// produce another error), nor the easiest to implement, but I believe it to
756 /// be the most predictable.
757 /// It also means that ASCII characters are never hidden by errors.
758 ///
759 /// # Examples
760 ///
761 /// Replace all errors with u+FFFD REPLACEMENT_CHARACTER:
762 /// ```
763 /// use encode_unicode::{Utf8Char, IterExt};
764 ///
765 /// let mut buf = [b'\0'; 255];
766 /// let len = b"foo\xCFbar".iter()
767 /// .to_utf8chars()
768 /// .flat_map(|r| r.unwrap_or(Utf8Char::from('\u{FFFD}')).into_iter() )
769 /// .zip(&mut buf[..])
770 /// .map(|(byte, dst)| *dst = byte )
771 /// .count();
772 ///
773 /// assert_eq!(&buf[..len], "foo\u{FFFD}bar".as_bytes());
774 /// ```
775 ///
776 /// Collect everything up until the first error into a string:
777 #[cfg_attr(feature="std", doc=" ```")]
778 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
779 /// use encode_unicode::iterator::Utf8CharMerger;
780 /// let mut good = String::new();
781 /// for r in Utf8CharMerger::from(b"foo\xcc\xbbbar\xcc\xddbaz") {
782 /// if let Ok(uc) = r {
783 /// good.push_str(uc.as_str());
784 /// } else {
785 /// break;
786 /// }
787 /// }
788 /// assert_eq!(good, "foo̻bar");
789 /// ```
790 ///
791 /// Abort decoding on error:
792 #[cfg_attr(feature="std", doc=" ```")]
793 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
794 /// use encode_unicode::{IterExt, Utf8Char};
795 /// use encode_unicode::error::{InvalidUtf8Slice, InvalidUtf8};
796 ///
797 /// let result = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77".iter()
798 /// .to_utf8chars()
799 /// .collect::<Result<String,InvalidUtf8Slice>>();
800 ///
801 /// assert_eq!(result, Err(InvalidUtf8Slice::Utf8(InvalidUtf8::NotAContinuationByte(2))));
802 /// ```
to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8>803 fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8>;
804
805 /// Decodes bytes as UTF-16 and groups them into `Utf16Char`s
806 ///
807 /// When errors (unmatched leading surrogates or unexpected trailing surrogates)
808 /// are encountered, an error is produced for every unit.
809 ///
810 /// # Examples
811 ///
812 /// Replace errors with '�':
813 #[cfg_attr(feature="std", doc=" ```")]
814 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
815 /// use encode_unicode::{IterExt, Utf16Char};
816 ///
817 /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..];
818 /// let string = slice.iter()
819 /// .to_utf16chars()
820 /// .map(|r| r.unwrap_or(Utf16Char::from('\u{fffd}')) ) // REPLACEMENT_CHARACTER
821 /// .collect::<String>();
822 ///
823 /// assert_eq!(string, "a�");
824 /// ```
825 ///
826 /// ```
827 /// use encode_unicode::{IterExt, Utf16Char};
828 /// use encode_unicode::error::Utf16PairError::*;
829 ///
830 /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'Y' as u16, 0xdab1, 0xdab1];
831 /// let mut iter = slice.iter().to_utf16chars();
832 /// assert_eq!(iter.size_hint(), (3, Some(7)));
833 /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate)));
834 /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate)));
835 /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('\u{3faee}'))));
836 /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('Y'))));
837 /// assert_eq!(iter.next(), Some(Err(UnmatchedLeadingSurrogate)));
838 /// assert_eq!(iter.next(), Some(Err(Incomplete)));
839 /// assert_eq!(iter.into_remaining_units().next(), None);
840 /// ```
841 ///
842 /// Search for a codepoint and return the codepoint index of the first match:
843 /// ```
844 /// use encode_unicode::{IterExt, Utf16Char};
845 ///
846 /// let position = [0xd875, 0xdd4f, '≈' as u16, '2' as u16].iter()
847 /// .to_utf16chars()
848 /// .position(|r| r == Ok(Utf16Char::from('≈')) );
849 ///
850 /// assert_eq!(position, Some(1));
851 /// ```
to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16>852 fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16>;
853 }
854
855 impl<I:Iterator> IterExt for I {
to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char>856 fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char> {
857 iter_bytes(self)
858 }
to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char>859 fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char> {
860 iter_units(self)
861 }
to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8>862 fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8> {
863 Utf8CharMerger::from(self)
864 }
to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16>865 fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16> {
866 Utf16CharMerger::from(self)
867 }
868 }
869
870
871 /// Methods for iterating over `u8` and `u16` slices as UTF-8 or UTF-16 characters.
872 ///
873 /// The iterators are slightly faster than the similar methods in [`IterExt`](trait.IterExt.html)
874 /// because they con "push back" items for free after errors and don't need a
875 /// separate buffer that must be checked on every call to `.next()`.
876 pub trait SliceExt: Index<RangeFull> {
877 /// Decode `u8` slices as UTF-8 and iterate over the codepoints as `Utf8Char`s,
878 ///
879 /// # Examples
880 ///
881 /// Get the index and error type of the first error:
882 #[cfg_attr(feature="std", doc=" ```")]
883 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
884 /// use encode_unicode::{SliceExt, Utf8Char};
885 /// use encode_unicode::error::InvalidUtf8Slice;
886 ///
887 /// let slice = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77";
888 /// let result = slice.utf8char_indices()
889 /// .map(|(offset,r,length)| r.map_err(|e| (offset,e,length) ) )
890 /// .collect::<Result<String,(usize,InvalidUtf8Slice,usize)>>();
891 ///
892 /// assert_eq!(result, Err((7, InvalidUtf8Slice::TooShort(4), 1)));
893 /// ```
894 ///
895 /// ```
896 /// use encode_unicode::{SliceExt, Utf8Char};
897 /// use std::error::Error;
898 ///
899 /// let slice = b"\xf0\xbf\xbf\xbfXY\xdd\xbb\xe1\x80\x99quux123";
900 /// let mut fixed_size = [Utf8Char::default(); 8];
901 /// for (cp_i, (byte_index, r, _)) in slice.utf8char_indices().enumerate().take(8) {
902 /// match r {
903 /// Ok(u8c) => fixed_size[cp_i] = u8c,
904 /// Err(e) => panic!("Invalid codepoint at index {} ({})", cp_i, e.description()),
905 /// }
906 /// }
907 /// let chars = ['\u{3ffff}', 'X', 'Y', '\u{77b}', '\u{1019}', 'q', 'u', 'u'];
908 /// assert_eq!(fixed_size, chars);
909 /// ```
910 ///
911 #[cfg_attr(feature="std", doc=" ```")]
912 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
913 /// use encode_unicode::{SliceExt, Utf8Char};
914 /// use encode_unicode::error::InvalidUtf8Slice::*;
915 /// use encode_unicode::error::{InvalidUtf8, InvalidUtf8FirstByte, InvalidCodepoint};
916 ///
917 /// let bytes = b"\xfa-\xf4\x8f\xee\xa1\x8f-\xed\xa9\x87\xf0\xcc\xbb";
918 /// let mut errors = Vec::new();
919 /// let mut lengths = Vec::new();
920 /// let mut string = String::new();
921 /// for (offset,result,length) in bytes.utf8char_indices() {
922 /// lengths.push((offset,length));
923 /// let c = result.unwrap_or_else(|error| {
924 /// errors.push((offset,error));
925 /// Utf8Char::from('\u{fffd}') // replacement character
926 /// });
927 /// string.push_str(c.as_str());
928 /// }
929 ///
930 /// assert_eq!(string, "�-��\u{e84f}-����\u{33b}");
931 /// assert_eq!(lengths, [(0,1), (1,1), (2,1), (3,1), (4,3), (7,1),
932 /// (8,1), (9,1), (10,1), (11,1), (12,2)]);
933 /// assert_eq!(errors, [
934 /// ( 0, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::TooLongSeqence))),
935 /// ( 2, Utf8(InvalidUtf8::NotAContinuationByte(2))),
936 /// ( 3, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
937 /// ( 8, Codepoint(InvalidCodepoint::Utf16Reserved)),
938 /// ( 9, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
939 /// (10, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
940 /// (11, TooShort(4)), // (but it was not the last element returned!)
941 /// ]);
942 /// ```
utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>943 fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>;
944
945
946 /// Decode `u16` slices as UTF-16 and iterate over the codepoints as `Utf16Char`s,
947 ///
948 /// The iterator produces `(usize,Result<Utf16Char,Utf16Error>,usize)`,
949 /// and the slice is validated as you go.
950 ///
951 /// The first `usize` contains the offset from the start of the slice and
952 /// the last `usize` contains the length of the codepoint or error.
953 /// The length is either 1 or 2, and always 1 for errors.
954 ///
955 /// # Examples
956 ///
957 #[cfg_attr(feature="std", doc=" ```")]
958 #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
959 /// use encode_unicode::{SliceExt, Utf8Char};
960 ///
961 /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..];
962 /// let mut errors = Vec::new();
963 /// let string = slice.utf16char_indices().map(|(offset,r,_)| match r {
964 /// Ok(u16c) => Utf8Char::from(u16c),
965 /// Err(_) => {
966 /// errors.push(offset);
967 /// Utf8Char::from('\u{fffd}') // REPLACEMENT_CHARACTER
968 /// }
969 /// }).collect::<String>();
970 ///
971 /// assert_eq!(string, "a�");
972 /// assert_eq!(errors, [1]);
973 /// ```
974 ///
975 /// Search for a codepoint and return its unit and codepoint index.
976 /// ```
977 /// use encode_unicode::{SliceExt, Utf16Char};
978 ///
979 /// let slice = [0xd875,/*''*/ 0xdd4f, '≈' as u16, '2' as u16];
980 /// let position = slice.utf16char_indices()
981 /// .enumerate()
982 /// .find(|&(_,(_,r,_))| r == Ok(Utf16Char::from('≈')) )
983 /// .map(|(codepoint, (offset, _, _))| (codepoint, offset) );
984 ///
985 /// assert_eq!(position, Some((1,2)));
986 /// ```
987 ///
988 /// Error types:
989 /// ```
990 /// use encode_unicode::{SliceExt, Utf16Char};
991 /// use encode_unicode::error::Utf16PairError::*;
992 ///
993 /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'λ' as u16, 0xdab1, 0xdab1];
994 /// let mut iter = slice.utf16char_indices();
995 /// assert_eq!(iter.next(), Some((0, Err(UnexpectedTrailingSurrogate), 1)));
996 /// assert_eq!(iter.next(), Some((1, Err(UnexpectedTrailingSurrogate), 1)));
997 /// assert_eq!(iter.next(), Some((2, Ok(Utf16Char::from('\u{3faee}')), 2)));
998 /// assert_eq!(iter.next(), Some((4, Ok(Utf16Char::from('λ')), 1)));
999 /// assert_eq!(iter.next(), Some((5, Err(UnmatchedLeadingSurrogate), 1)));
1000 /// assert_eq!(iter.next(), Some((6, Err(Incomplete), 1)));
1001 /// assert_eq!(iter.next(), None);
1002 /// assert_eq!(iter.as_slice(), [])
1003 /// ```
utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>1004 fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>;
1005 }
1006
1007 impl<S: ?Sized+Index<RangeFull>> SliceExt for S {
utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>1008 fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]> {
1009 Utf8CharDecoder::from(self[..].borrow())
1010 }
utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>1011 fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]> {
1012 Utf16CharDecoder::from(self[..].borrow())
1013 }
1014 }
1015