1 // Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT 2 // file at the top-level directory of this distribution and at 3 // http://rust-lang.org/COPYRIGHT. 4 // 5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or 6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license 7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your 8 // option. This file may not be copied, modified, or distributed 9 // except according to those terms. 10 11 //! Iterators which split strings on Grapheme Cluster, Word or Sentence boundaries, according 12 //! to the [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) rules. 13 //! 14 //! ```rust 15 //! extern crate unicode_segmentation; 16 //! 17 //! use unicode_segmentation::UnicodeSegmentation; 18 //! 19 //! fn main() { 20 //! let s = "a̐éö̲\r\n"; 21 //! let g = UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>(); 22 //! let b: &[_] = &["a̐", "é", "ö̲", "\r\n"]; 23 //! assert_eq!(g, b); 24 //! 25 //! let s = "The quick (\"brown\") fox can't jump 32.3 feet, right?"; 26 //! let w = s.unicode_words().collect::<Vec<&str>>(); 27 //! let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"]; 28 //! assert_eq!(w, b); 29 //! 30 //! let s = "The quick (\"brown\") fox"; 31 //! let w = s.split_word_bounds().collect::<Vec<&str>>(); 32 //! let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", " ", "fox"]; 33 //! assert_eq!(w, b); 34 //! } 35 //! ``` 36 //! 37 //! # no_std 38 //! 39 //! unicode-segmentation does not depend on libstd, so it can be used in crates 40 //! with the `#![no_std]` attribute. 41 //! 42 //! # crates.io 43 //! 44 //! You can use this package in your project by adding the following 45 //! to your `Cargo.toml`: 46 //! 47 //! ```toml 48 //! [dependencies] 49 //! unicode-segmentation = "1.7.1" 50 //! ``` 51 52 #![deny(missing_docs, unsafe_code)] 53 #![doc(html_logo_url = "https://unicode-rs.github.io/unicode-rs_sm.png", 54 html_favicon_url = "https://unicode-rs.github.io/unicode-rs_sm.png")] 55 56 #![no_std] 57 58 #[cfg(test)] 59 #[macro_use] 60 extern crate std; 61 62 #[cfg(test)] 63 #[macro_use] 64 extern crate quickcheck; 65 66 pub use grapheme::{Graphemes, GraphemeIndices}; 67 pub use grapheme::{GraphemeCursor, GraphemeIncomplete}; 68 pub use tables::UNICODE_VERSION; 69 pub use word::{UWordBounds, UWordBoundIndices, UnicodeWords, UnicodeWordIndices}; 70 pub use sentence::{USentenceBounds, USentenceBoundIndices, UnicodeSentences}; 71 72 mod grapheme; 73 mod tables; 74 mod word; 75 mod sentence; 76 77 #[cfg(test)] 78 mod test; 79 #[cfg(test)] 80 mod testdata; 81 82 /// Methods for segmenting strings according to 83 /// [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/). 84 pub trait UnicodeSegmentation { 85 /// Returns an iterator over the [grapheme clusters][graphemes] of `self`. 86 /// 87 /// [graphemes]: http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries 88 /// 89 /// If `is_extended` is true, the iterator is over the 90 /// *extended grapheme clusters*; 91 /// otherwise, the iterator is over the *legacy grapheme clusters*. 92 /// [UAX#29](http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) 93 /// recommends extended grapheme cluster boundaries for general processing. 94 /// 95 /// # Examples 96 /// 97 /// ``` 98 /// # use self::unicode_segmentation::UnicodeSegmentation; 99 /// let gr1 = UnicodeSegmentation::graphemes("a\u{310}e\u{301}o\u{308}\u{332}", true) 100 /// .collect::<Vec<&str>>(); 101 /// let b: &[_] = &["a\u{310}", "e\u{301}", "o\u{308}\u{332}"]; 102 /// 103 /// assert_eq!(&gr1[..], b); 104 /// 105 /// let gr2 = UnicodeSegmentation::graphemes("a\r\nb", true).collect::<Vec<&str>>(); 106 /// let b: &[_] = &["a", "\r\n", "b", "", ""]; 107 /// 108 /// assert_eq!(&gr2[..], b); 109 /// ``` 110 fn graphemes<'a>(&'a self, is_extended: bool) -> Graphemes<'a>; 111 112 /// Returns an iterator over the grapheme clusters of `self` and their 113 /// byte offsets. See `graphemes()` for more information. 114 /// 115 /// # Examples 116 /// 117 /// ``` 118 /// # use self::unicode_segmentation::UnicodeSegmentation; 119 /// let gr_inds = UnicodeSegmentation::grapheme_indices("a̐éö̲\r\n", true) 120 /// .collect::<Vec<(usize, &str)>>(); 121 /// let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")]; 122 /// 123 /// assert_eq!(&gr_inds[..], b); 124 /// ``` 125 fn grapheme_indices<'a>(&'a self, is_extended: bool) -> GraphemeIndices<'a>; 126 127 /// Returns an iterator over the words of `self`, separated on 128 /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries). 129 /// 130 /// Here, "words" are just those substrings which, after splitting on 131 /// UAX#29 word boundaries, contain any alphanumeric characters. That is, the 132 /// substring must contain at least one character with the 133 /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic) 134 /// property, or with 135 /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values). 136 /// 137 /// # Example 138 /// 139 /// ``` 140 /// # use self::unicode_segmentation::UnicodeSegmentation; 141 /// let uws = "The quick (\"brown\") fox can't jump 32.3 feet, right?"; 142 /// let uw1 = uws.unicode_words().collect::<Vec<&str>>(); 143 /// let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"]; 144 /// 145 /// assert_eq!(&uw1[..], b); 146 /// ``` 147 fn unicode_words<'a>(&'a self) -> UnicodeWords<'a>; 148 149 /// Returns an iterator over the words of `self`, separated on 150 /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries), and their 151 /// offsets. 152 /// 153 /// Here, "words" are just those substrings which, after splitting on 154 /// UAX#29 word boundaries, contain any alphanumeric characters. That is, the 155 /// substring must contain at least one character with the 156 /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic) 157 /// property, or with 158 /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values). 159 /// 160 /// # Example 161 /// 162 /// ``` 163 /// # use self::unicode_segmentation::UnicodeSegmentation; 164 /// let uwis = "The quick (\"brown\") fox can't jump 32.3 feet, right?"; 165 /// let uwi1 = uwis.unicode_word_indices().collect::<Vec<(usize, &str)>>(); 166 /// let b: &[_] = &[(0, "The"), (4, "quick"), (12, "brown"), (20, "fox"), (24, "can't"), 167 /// (30, "jump"), (35, "32.3"), (40, "feet"), (46, "right")]; 168 /// 169 /// assert_eq!(&uwi1[..], b); 170 /// ``` 171 fn unicode_word_indices<'a>(&'a self) -> UnicodeWordIndices<'a>; 172 173 /// Returns an iterator over substrings of `self` separated on 174 /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries). 175 /// 176 /// The concatenation of the substrings returned by this function is just the original string. 177 /// 178 /// # Example 179 /// 180 /// ``` 181 /// # use self::unicode_segmentation::UnicodeSegmentation; 182 /// let swu1 = "The quick (\"brown\") fox".split_word_bounds().collect::<Vec<&str>>(); 183 /// let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", " ", "fox"]; 184 /// 185 /// assert_eq!(&swu1[..], b); 186 /// ``` 187 fn split_word_bounds<'a>(&'a self) -> UWordBounds<'a>; 188 189 /// Returns an iterator over substrings of `self`, split on UAX#29 word boundaries, 190 /// and their offsets. See `split_word_bounds()` for more information. 191 /// 192 /// # Example 193 /// 194 /// ``` 195 /// # use self::unicode_segmentation::UnicodeSegmentation; 196 /// let swi1 = "Brr, it's 29.3°F!".split_word_bound_indices().collect::<Vec<(usize, &str)>>(); 197 /// let b: &[_] = &[(0, "Brr"), (3, ","), (4, " "), (5, "it's"), (9, " "), (10, "29.3"), 198 /// (14, "°"), (16, "F"), (17, "!")]; 199 /// 200 /// assert_eq!(&swi1[..], b); 201 /// ``` 202 fn split_word_bound_indices<'a>(&'a self) -> UWordBoundIndices<'a>; 203 204 /// Returns an iterator over substrings of `self` separated on 205 /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries). 206 /// 207 /// Here, "sentences" are just those substrings which, after splitting on 208 /// UAX#29 sentence boundaries, contain any alphanumeric characters. That is, the 209 /// substring must contain at least one character with the 210 /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic) 211 /// property, or with 212 /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values). 213 /// 214 /// # Example 215 /// 216 /// ``` 217 /// # use self::unicode_segmentation::UnicodeSegmentation; 218 /// let uss = "Mr. Fox jumped. [...] The dog was too lazy."; 219 /// let us1 = uss.unicode_sentences().collect::<Vec<&str>>(); 220 /// let b: &[_] = &["Mr. ", "Fox jumped. ", "The dog was too lazy."]; 221 /// 222 /// assert_eq!(&us1[..], b); 223 /// ``` 224 fn unicode_sentences<'a>(&'a self) -> UnicodeSentences<'a>; 225 226 /// Returns an iterator over substrings of `self` separated on 227 /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries). 228 /// 229 /// The concatenation of the substrings returned by this function is just the original string. 230 /// 231 /// # Example 232 /// 233 /// ``` 234 /// # use self::unicode_segmentation::UnicodeSegmentation; 235 /// let ssbs = "Mr. Fox jumped. [...] The dog was too lazy."; 236 /// let ssb1 = ssbs.split_sentence_bounds().collect::<Vec<&str>>(); 237 /// let b: &[_] = &["Mr. ", "Fox jumped. ", "[...] ", "The dog was too lazy."]; 238 /// 239 /// assert_eq!(&ssb1[..], b); 240 /// ``` 241 fn split_sentence_bounds<'a>(&'a self) -> USentenceBounds<'a>; 242 243 /// Returns an iterator over substrings of `self`, split on UAX#29 sentence boundaries, 244 /// and their offsets. See `split_sentence_bounds()` for more information. 245 /// 246 /// # Example 247 /// 248 /// ``` 249 /// # use self::unicode_segmentation::UnicodeSegmentation; 250 /// let ssis = "Mr. Fox jumped. [...] The dog was too lazy."; 251 /// let ssi1 = ssis.split_sentence_bound_indices().collect::<Vec<(usize, &str)>>(); 252 /// let b: &[_] = &[(0, "Mr. "), (4, "Fox jumped. "), (16, "[...] "), 253 /// (22, "The dog was too lazy.")]; 254 /// 255 /// assert_eq!(&ssi1[..], b); 256 /// ``` 257 fn split_sentence_bound_indices<'a>(&'a self) -> USentenceBoundIndices<'a>; 258 } 259 260 impl UnicodeSegmentation for str { 261 #[inline] 262 fn graphemes(&self, is_extended: bool) -> Graphemes { 263 grapheme::new_graphemes(self, is_extended) 264 } 265 266 #[inline] 267 fn grapheme_indices(&self, is_extended: bool) -> GraphemeIndices { 268 grapheme::new_grapheme_indices(self, is_extended) 269 } 270 271 #[inline] 272 fn unicode_words(&self) -> UnicodeWords { 273 word::new_unicode_words(self) 274 } 275 276 #[inline] 277 fn unicode_word_indices(&self) -> UnicodeWordIndices { 278 word::new_unicode_word_indices(self) 279 } 280 281 #[inline] 282 fn split_word_bounds(&self) -> UWordBounds { 283 word::new_word_bounds(self) 284 } 285 286 #[inline] 287 fn split_word_bound_indices(&self) -> UWordBoundIndices { 288 word::new_word_bound_indices(self) 289 } 290 291 #[inline] 292 fn unicode_sentences(&self) -> UnicodeSentences { 293 sentence::new_unicode_sentences(self) 294 } 295 296 #[inline] 297 fn split_sentence_bounds(&self) -> USentenceBounds { 298 sentence::new_sentence_bounds(self) 299 } 300 301 #[inline] 302 fn split_sentence_bound_indices(&self) -> USentenceBoundIndices { 303 sentence::new_sentence_bound_indices(self) 304 } 305 } 306