1 // After editing this file, run "go generate" in the ../data directory.
2
3 // Copyright 2020 The Wuffs Authors.
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // https://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16
17 // ---------------- Unicode and UTF-8
18
19 WUFFS_BASE__MAYBE_STATIC size_t //
wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst,uint32_t code_point)20 wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point) {
21 if (code_point <= 0x7F) {
22 if (dst.len >= 1) {
23 dst.ptr[0] = (uint8_t)(code_point);
24 return 1;
25 }
26
27 } else if (code_point <= 0x07FF) {
28 if (dst.len >= 2) {
29 dst.ptr[0] = (uint8_t)(0xC0 | ((code_point >> 6)));
30 dst.ptr[1] = (uint8_t)(0x80 | ((code_point >> 0) & 0x3F));
31 return 2;
32 }
33
34 } else if (code_point <= 0xFFFF) {
35 if ((dst.len >= 3) && ((code_point < 0xD800) || (0xDFFF < code_point))) {
36 dst.ptr[0] = (uint8_t)(0xE0 | ((code_point >> 12)));
37 dst.ptr[1] = (uint8_t)(0x80 | ((code_point >> 6) & 0x3F));
38 dst.ptr[2] = (uint8_t)(0x80 | ((code_point >> 0) & 0x3F));
39 return 3;
40 }
41
42 } else if (code_point <= 0x10FFFF) {
43 if (dst.len >= 4) {
44 dst.ptr[0] = (uint8_t)(0xF0 | ((code_point >> 18)));
45 dst.ptr[1] = (uint8_t)(0x80 | ((code_point >> 12) & 0x3F));
46 dst.ptr[2] = (uint8_t)(0x80 | ((code_point >> 6) & 0x3F));
47 dst.ptr[3] = (uint8_t)(0x80 | ((code_point >> 0) & 0x3F));
48 return 4;
49 }
50 }
51
52 return 0;
53 }
54
55 // wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a
56 // UTF-8 encoded code point, based on the encoding's initial byte.
57 // - 0x00 is 1-byte UTF-8 (ASCII).
58 // - 0x01 is the start of 2-byte UTF-8.
59 // - 0x02 is the start of 3-byte UTF-8.
60 // - 0x03 is the start of 4-byte UTF-8.
61 // - 0x40 is a UTF-8 tail byte.
62 // - 0x80 is invalid UTF-8.
63 //
64 // RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
65 // UTF8-1 = %x00-7F
66 // UTF8-2 = %xC2-DF UTF8-tail
67 // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
68 // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
69 // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
70 // %xF4 %x80-8F 2( UTF8-tail )
71 // UTF8-tail = %x80-BF
72 static const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = {
73 // 0 1 2 3 4 5 6 7
74 // 8 9 A B C D E F
75 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
76 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
77 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
78 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
79 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.
80 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.
81 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
82 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
83
84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
86 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
87 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.
88 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67.
89 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.
90 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77.
91 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
92
93 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87.
94 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F.
95 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97.
96 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F.
97 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7.
98 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF.
99 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7.
100 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF.
101
102 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7.
103 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF.
104 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7.
105 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF.
106 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7.
107 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF.
108 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7.
109 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF.
110 // 0 1 2 3 4 5 6 7
111 // 8 9 A B C D E F
112 };
113
114 WUFFS_BASE__MAYBE_STATIC wuffs_base__utf_8__next__output //
wuffs_base__utf_8__next(const uint8_t * s_ptr,size_t s_len)115 wuffs_base__utf_8__next(const uint8_t* s_ptr, size_t s_len) {
116 if (s_len == 0) {
117 return wuffs_base__make_utf_8__next__output(0, 0);
118 }
119 uint32_t c = s_ptr[0];
120 switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) {
121 case 0:
122 return wuffs_base__make_utf_8__next__output(c, 1);
123
124 case 1:
125 if (s_len < 2) {
126 break;
127 }
128 c = wuffs_base__load_u16le__no_bounds_check(s_ptr);
129 if ((c & 0xC000) != 0x8000) {
130 break;
131 }
132 c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8));
133 return wuffs_base__make_utf_8__next__output(c, 2);
134
135 case 2:
136 if (s_len < 3) {
137 break;
138 }
139 c = wuffs_base__load_u24le__no_bounds_check(s_ptr);
140 if ((c & 0xC0C000) != 0x808000) {
141 break;
142 }
143 c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) |
144 (0x00003F & (c >> 16));
145 if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) {
146 break;
147 }
148 return wuffs_base__make_utf_8__next__output(c, 3);
149
150 case 3:
151 if (s_len < 4) {
152 break;
153 }
154 c = wuffs_base__load_u32le__no_bounds_check(s_ptr);
155 if ((c & 0xC0C0C000) != 0x80808000) {
156 break;
157 }
158 c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) |
159 (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24));
160 if ((c <= 0xFFFF) || (0x110000 <= c)) {
161 break;
162 }
163 return wuffs_base__make_utf_8__next__output(c, 4);
164 }
165
166 return wuffs_base__make_utf_8__next__output(
167 WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);
168 }
169
170 WUFFS_BASE__MAYBE_STATIC wuffs_base__utf_8__next__output //
wuffs_base__utf_8__next_from_end(const uint8_t * s_ptr,size_t s_len)171 wuffs_base__utf_8__next_from_end(const uint8_t* s_ptr, size_t s_len) {
172 if (s_len == 0) {
173 return wuffs_base__make_utf_8__next__output(0, 0);
174 }
175 const uint8_t* ptr = &s_ptr[s_len - 1];
176 if (*ptr < 0x80) {
177 return wuffs_base__make_utf_8__next__output(*ptr, 1);
178
179 } else if (*ptr < 0xC0) {
180 const uint8_t* too_far = &s_ptr[(s_len > 4) ? (s_len - 4) : 0];
181 uint32_t n = 1;
182 while (ptr != too_far) {
183 ptr--;
184 n++;
185 if (*ptr < 0x80) {
186 break;
187 } else if (*ptr < 0xC0) {
188 continue;
189 }
190 wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(ptr, n);
191 if (o.byte_length != n) {
192 break;
193 }
194 return o;
195 }
196 }
197
198 return wuffs_base__make_utf_8__next__output(
199 WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);
200 }
201
202 WUFFS_BASE__MAYBE_STATIC size_t //
wuffs_base__utf_8__longest_valid_prefix(const uint8_t * s_ptr,size_t s_len)203 wuffs_base__utf_8__longest_valid_prefix(const uint8_t* s_ptr, size_t s_len) {
204 // TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time).
205 //
206 // TODO: possibly optimize this by manually inlining the
207 // wuffs_base__utf_8__next calls.
208 size_t original_len = s_len;
209 while (s_len > 0) {
210 wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s_ptr, s_len);
211 if ((o.code_point > 0x7F) && (o.byte_length == 1)) {
212 break;
213 }
214 s_ptr += o.byte_length;
215 s_len -= o.byte_length;
216 }
217 return original_len - s_len;
218 }
219
220 WUFFS_BASE__MAYBE_STATIC size_t //
wuffs_base__ascii__longest_valid_prefix(const uint8_t * s_ptr,size_t s_len)221 wuffs_base__ascii__longest_valid_prefix(const uint8_t* s_ptr, size_t s_len) {
222 // TODO: possibly optimize this by checking 4 or 8 bytes at a time.
223 const uint8_t* original_ptr = s_ptr;
224 const uint8_t* p = s_ptr;
225 const uint8_t* q = s_ptr + s_len;
226 for (; (p != q) && ((*p & 0x80) == 0); p++) {
227 }
228 return (size_t)(p - original_ptr);
229 }
230