1 use UnicodeNormalization;
2 use stream_safe;
3 use tables;
4
5 /// The QuickCheck algorithm can quickly determine if a text is or isn't
6 /// normalized without any allocations in many cases, but it has to be able to
7 /// return `Maybe` when a full decomposition and recomposition is necessary.
8 #[derive(Debug, Eq, PartialEq)]
9 pub enum IsNormalized {
10 /// The text is definitely normalized.
11 Yes,
12 /// The text is definitely not normalized.
13 No,
14 /// The text may be normalized.
15 Maybe,
16 }
17
18 // https://unicode.org/reports/tr15/#Detecting_Normalization_Forms
19 #[inline]
quick_check<F, I>(s: I, is_allowed: F, stream_safe: bool) -> IsNormalized where I: Iterator<Item=char>, F: Fn(char) -> IsNormalized20 fn quick_check<F, I>(s: I, is_allowed: F, stream_safe: bool) -> IsNormalized
21 where I: Iterator<Item=char>, F: Fn(char) -> IsNormalized
22 {
23 let mut last_cc = 0u8;
24 let mut nonstarter_count = 0;
25 let mut result = IsNormalized::Yes;
26 for ch in s {
27 // For ASCII we know it's always allowed and a starter
28 if ch <= '\x7f' {
29 last_cc = 0;
30 nonstarter_count = 0;
31 continue;
32 }
33
34 // Otherwise, lookup the combining class and QC property
35 let cc = tables::canonical_combining_class(ch);
36 if last_cc > cc && cc != 0 {
37 return IsNormalized::No;
38 }
39 match is_allowed(ch) {
40 IsNormalized::Yes => (),
41 IsNormalized::No => return IsNormalized::No,
42 IsNormalized::Maybe => {
43 result = IsNormalized::Maybe;
44 },
45 }
46 if stream_safe {
47 let decomp = stream_safe::classify_nonstarters(ch);
48
49 // If we're above `MAX_NONSTARTERS`, we're definitely *not*
50 // stream-safe normalized.
51 if nonstarter_count + decomp.leading_nonstarters > stream_safe::MAX_NONSTARTERS {
52 return IsNormalized::No;
53 }
54 if decomp.leading_nonstarters == decomp.decomposition_len {
55 nonstarter_count += decomp.decomposition_len;
56 } else {
57 nonstarter_count = decomp.trailing_nonstarters;
58 }
59 }
60 last_cc = cc;
61 }
62 result
63 }
64
65 /// Quickly check if a string is in NFC, potentially returning
66 /// `IsNormalized::Maybe` if further checks are necessary. In this case a check
67 /// like `s.chars().nfc().eq(s.chars())` should suffice.
68 #[inline]
is_nfc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized69 pub fn is_nfc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
70 quick_check(s, tables::qc_nfc, false)
71 }
72
73
74 /// Quickly check if a string is in NFKC.
75 #[inline]
is_nfkc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized76 pub fn is_nfkc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
77 quick_check(s, tables::qc_nfkc, false)
78 }
79
80 /// Quickly check if a string is in NFD.
81 #[inline]
is_nfd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized82 pub fn is_nfd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
83 quick_check(s, tables::qc_nfd, false)
84 }
85
86 /// Quickly check if a string is in NFKD.
87 #[inline]
is_nfkd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized88 pub fn is_nfkd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
89 quick_check(s, tables::qc_nfkd, false)
90 }
91
92 /// Quickly check if a string is Stream-Safe NFC.
93 #[inline]
is_nfc_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized94 pub fn is_nfc_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
95 quick_check(s, tables::qc_nfc, true)
96 }
97
98 /// Quickly check if a string is Stream-Safe NFD.
99 #[inline]
is_nfd_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized100 pub fn is_nfd_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
101 quick_check(s, tables::qc_nfd, true)
102 }
103
104 /// Authoritatively check if a string is in NFC.
105 #[inline]
is_nfc(s: &str) -> bool106 pub fn is_nfc(s: &str) -> bool {
107 match is_nfc_quick(s.chars()) {
108 IsNormalized::Yes => true,
109 IsNormalized::No => false,
110 IsNormalized::Maybe => s.chars().eq(s.chars().nfc()),
111 }
112 }
113
114 /// Authoritatively check if a string is in NFKC.
115 #[inline]
is_nfkc(s: &str) -> bool116 pub fn is_nfkc(s: &str) -> bool {
117 match is_nfkc_quick(s.chars()) {
118 IsNormalized::Yes => true,
119 IsNormalized::No => false,
120 IsNormalized::Maybe => s.chars().eq(s.chars().nfkc()),
121 }
122 }
123
124 /// Authoritatively check if a string is in NFD.
125 #[inline]
is_nfd(s: &str) -> bool126 pub fn is_nfd(s: &str) -> bool {
127 match is_nfd_quick(s.chars()) {
128 IsNormalized::Yes => true,
129 IsNormalized::No => false,
130 IsNormalized::Maybe => s.chars().eq(s.chars().nfd()),
131 }
132 }
133
134 /// Authoritatively check if a string is in NFKD.
135 #[inline]
is_nfkd(s: &str) -> bool136 pub fn is_nfkd(s: &str) -> bool {
137 match is_nfkd_quick(s.chars()) {
138 IsNormalized::Yes => true,
139 IsNormalized::No => false,
140 IsNormalized::Maybe => s.chars().eq(s.chars().nfkd()),
141 }
142 }
143
144 /// Authoritatively check if a string is Stream-Safe NFC.
145 #[inline]
is_nfc_stream_safe(s: &str) -> bool146 pub fn is_nfc_stream_safe(s: &str) -> bool {
147 match is_nfc_stream_safe_quick(s.chars()) {
148 IsNormalized::Yes => true,
149 IsNormalized::No => false,
150 IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfc()),
151 }
152 }
153
154 /// Authoritatively check if a string is Stream-Safe NFD.
155 #[inline]
is_nfd_stream_safe(s: &str) -> bool156 pub fn is_nfd_stream_safe(s: &str) -> bool {
157 match is_nfd_stream_safe_quick(s.chars()) {
158 IsNormalized::Yes => true,
159 IsNormalized::No => false,
160 IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfd()),
161 }
162 }
163
164 #[cfg(test)]
165 mod tests {
166 use super::{
167 IsNormalized,
168 is_nfc_stream_safe_quick,
169 is_nfd_stream_safe_quick,
170 };
171
172 #[test]
test_stream_safe_nfd()173 fn test_stream_safe_nfd() {
174 let okay = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone";
175 assert_eq!(is_nfd_stream_safe_quick(okay.chars()), IsNormalized::Yes);
176
177 let too_much = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone";
178 assert_eq!(is_nfd_stream_safe_quick(too_much.chars()), IsNormalized::No);
179 }
180
181 #[test]
test_stream_safe_nfc()182 fn test_stream_safe_nfc() {
183 let okay = "ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y";
184 assert_eq!(is_nfc_stream_safe_quick(okay.chars()), IsNormalized::Maybe);
185
186 let too_much = "not ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y";
187 assert_eq!(is_nfc_stream_safe_quick(too_much.chars()), IsNormalized::No);
188 }
189 }
190