1 /*
2 * Copyright 1998-2015 University Corporation for Atmospheric Research/Unidata
3 * See the LICENSE file for more information.
4 */
5
6 #include <config.h>
7 #include <stdlib.h>
8 #include <stdio.h>
9 #include <string.h>
10 #include "netcdf.h"
11 #include "ncutf8.h"
12 /*
13
14 This test is taken from the UTF-8 decoder
15 capability and stress test file created by
16
17 Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2015-08-28 - CC BY 4.0
18
19 This test file can help you examine, how your UTF-8 decoder handles
20 various types of correct, malformed, or otherwise interesting UTF-8
21 sequences. This file is not meant to be a conformance test. It does
22 not prescribe any particular outcome. Therefore, there is no way to
23 "pass" or "fail" this test file, even though the text does suggest a
24 preferable decoder behaviour at some places. Its aim is, instead, to
25 help you think about, and test, the behaviour of your UTF-8 decoder on a
26 systematic collection of unusual inputs. Experience so far suggests
27 that most first-time authors of UTF-8 decoders find at least one
28 serious problem in their decoder using this file.
29
30 The test lines below cover boundary conditions, malformed UTF-8
31 sequences, as well as correctly encoded UTF-8 sequences of Unicode code
32 points that should never occur in a correct UTF-8 file.
33
34 According to ISO 10646-1:2000, sections D.7 and 2.3c, a device
35 receiving UTF-8 shall interpret a "malformed sequence in the same way
36 that it interprets a character that is outside the adopted subset" and
37 "characters that are not within the adopted subset shall be indicated
38 to the user" by a receiving device. One commonly used approach in
39 UTF-8 decoders is to replace any malformed UTF-8 sequence by a
40 replacement character (U+FFFD), which looks a bit like an inverted
41 question mark, or a similar symbol. It might be a good idea to
42 visually distinguish a malformed UTF-8 sequence from a correctly
43 encoded Unicode character that is just not available in the current
44 font but otherwise fully legal, even though ISO 10646-1 doesn't
45 mandate this. In any case, just ignoring malformed sequences or
46 unavailable characters does not conform to ISO 10646, will make
47 debugging more difficult, and can lead to user confusion.
48
49 Please check, whether a malformed UTF-8 sequence is (1) represented at
50 all, (2) represented by exactly one single replacement character (or
51 equivalent signal), and (3) the following quotation mark after an
52 illegal UTF-8 sequence is correctly displayed, i.e. proper
53 resynchronization takes place immediately after any malformed
54 sequence. This file says "THE END" in the last line, so if you don't
55 see that, your decoder crashed somehow before, which should always be
56 cause for concern.
57
58 All lines in this file are exactly 79 characters long (plus the line
59 feed). In addition, all lines end with "|", except for the two test
60 lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls
61 U+0000 and U+007F. If you display this file with a fixed-width font,
62 these "|" characters should all line up in column 79 (right margin).
63 This allows you to test quickly, whether your UTF-8 decoder finds the
64 correct number of characters in every line, that is whether each
65 malformed sequences is replaced by a single replacement character.
66
67 Note that, as an alternative to the notion of malformed sequence used
68 here, it is also a perfectly acceptable (and in some situations even
69 preferable) solution to represent each individual byte of a malformed
70 sequence with a replacement character. If you follow this strategy in
71 your decoder, then please ignore the "|" column.
72 */
73
74
75 struct Test {
76 int xfail;
77 const char* id;
78 const char* description;
79 const char* data;
80 };
81 #define NULLTEST {0,NULL,NULL,NULL}
82
83 /* The following tests are in envv form */
84
85 /*1 Some correct UTF-8 text
86 You should see the Greek word 'kosme':
87 */
88 static const struct Test utf8ok[] = {
89 {0, "1.1.1", "Greek word 'kosme'",
90 "κόσμε"},
91 NULLTEST
92 };
93
94 static const struct Test utf8boundary[] = {
95
96 /*2 Boundary condition test */
97 /*2.1 First possible sequence of a certain length */
98 {0,"2.1.1", "1 byte (U-00000000)", "\000"},
99 {0,"2.1.2", "2 bytes (U-00000080)", "€"},
100 {0,"2.1.3", "3 bytes (U-00000800)", "ࠀ"},
101 {0,"2.1.4", "4 bytes (U-00010000)", "��"},
102 {1,"2.1.5", "5 bytes (U-00200000)", "�����"},
103 {1,"2.1.6", "6 bytes (U-04000000)", "������"},
104
105 /*2.2 Last possible sequence of a certain length*/
106 {0,"2.2.1", "1 byte (U-0000007F)", ""},
107 {0,"2.2.2", "2 bytes (U-000007FF)", "߿"},
108 {0,"2.2.3", "3 bytes (U-0000FFFF)", ""}, /*See 5.3.2 */
109 {1,"2.2.4", "4 bytes (U-001FFFFF)", "����"},
110 {1,"2.2.5", "5 bytes (U-03FFFFFF)", "�����"},
111 {1,"2.2.6", "6 bytes (U-7FFFFFFF)", "������"},
112
113 /*2.3 Other boundary conditions*/
114
115 {0,"2.3.1", "U-0000D7FF = ed 9f bf", ""},
116 {0,"2.3.2", "U-0000E000 = ee 80 80", ""},
117 {0,"2.3.3", "U-0000FFFD = ef bf bd", "�"},
118 {0,"2.3.4", "U-0010FFFF = f4 8f bf bf", "��"},
119 {1,"2.3.5", "U-00110000 = f4 90 80 80", "����"},
120 NULLTEST
121 };
122
123 static const struct Test utf8bad[] = {
124
125 /*3 Malformed sequences*/
126
127 /*3.1 Unexpected continuation bytes
128 Each unexpected continuation byte should be separately signalled
129 as a malformed sequence of its own.
130 */
131 {1,"3.1.1", "First continuation byte 0x80", "�"},
132 {1,"3.1.2", "Last continuation byte 0xbf", "�"},
133
134 {1,"3.1.3", "2 continuation bytes", "��"},
135 {1,"3.1.4", "3 continuation bytes", "���"},
136 {1,"3.1.5", "4 continuation bytes", "����"},
137 {1,"3.1.6", "5 continuation bytes", "�����"},
138 {1,"3.1.7", "6 continuation bytes", "������"},
139 {1,"3.1.8", "7 continuation bytes", "�������"},
140 {1,"3.1.9", "Sequence of all 64 possible continuation bytes (0x80-0xbf)",
141 "����������������������������������������������������������������"
142 },
143
144 /*3.2 Lonely start characters*/
145
146 /*3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf),
147 each followed by a space character*/
148
149 {1,"3.2.1", "All 32 first bytes of 2-byte sequences",
150 "� � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � "
151 },
152
153 /*3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef),
154 each followed by a space character:*/
155 {1,"3.2.2", "All 16 first bytes of 3-byte sequences",
156 "� � � � � � � � � � � � � � � � "
157 },
158
159 /*3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7),
160 each followed by a space character:*/
161 {1,"3.2.3", "All 8 first bytes of 4-byte sequences",
162 "� � � � � � � � "
163 },
164
165 /*3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb),
166 each followed by a space character:*/
167 {1,"3.2.4", "All 4 first bytes of 5-byte sequences",
168 "� � � � "
169 },
170
171 /*3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd),
172 each followed by a space character:*/
173 {1,"3.2.5", "All 2 first bytes of 6-byte sequences",
174 "� � "
175 },
176
177 /*3.3 Sequences with last continuation byte missing
178 All bytes of an incomplete sequence should be signalled as a single
179 malformed sequence, i.e., you should see only a single replacement
180 character in each of the next 10 tests. (Characters as in section 2)
181 */
182 {1,"3.3.1", "2-byte sequence with last byte missing (U+0000)", "�"},
183 {1,"3.3.2", "3-byte sequence with last byte missing (U+0000)", "��"},
184 {1,"3.3.3", "4-byte sequence with last byte missing (U+0000)", "���"},
185 {1,"3.3.4", "5-byte sequence with last byte missing (U+0000)", "����"},
186 {1,"3.3.5", "6-byte sequence with last byte missing (U+0000)", "�����"},
187 {1,"3.3.6", "2-byte sequence with last byte missing (U-000007FF)", "�"},
188 {1,"3.3.7", "3-byte sequence with last byte missing (U-0000FFFF)", "�"},
189 {1,"3.3.8", "4-byte sequence with last byte missing (U-001FFFFF)", "���"},
190 {1,"3.3.9", "5-byte sequence with last byte missing (U-03FFFFFF)", "����"},
191 {1,"3.3.10", "6-byte sequence with last byte missing (U-7FFFFFFF)", "�����"},
192
193 /*3.4 Concatenation of incomplete sequences
194 All the 10 sequences of 3.3 concatenated; you should see 10 malformed
195 sequences being signalled:
196 */
197 {1, "3.4.1", "All the 10 sequences of 3.3 concatenated",
198 "�����������������������������"
199 },
200
201 /*3.5 Impossible bytes
202 The following two bytes cannot appear in a correct UTF-8 string
203 */
204
205 {1,"3.5.1", "fe", "�"},
206 {1,"3.5.2", "ff", "�"},
207 {1,"3.5.3", "fe fe ff ff", "����"},
208
209 /*
210 4 Overlong sequences
211
212 The following sequences are not malformed according to the letter of
213 the Unicode 2.0 standard. However, they are longer then necessary and
214 a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8
215 decoder" should reject them just like malformed sequences for two
216 reasons: (1) It helps to debug applications if overlong sequences are
217 not treated as valid representations of characters, because this helps
218 to spot problems more quickly. (2) Overlong sequences provide
219 alternative representations of characters, that could maliciously be
220 used to bypass filters that check only for ASCII characters. For
221 instance, a 2-byte encoded line feed (LF) would not be caught by a
222 line counter that counts only 0x0a bytes, but it would still be
223 processed as a line feed by an unsafe UTF-8 decoder later in the
224 pipeline. From a security point of view, ASCII compatibility of UTF-8
225 sequences means also, that ASCII characters are *only* allowed to be
226 represented by ASCII bytes in the range 0x00-0x7f. To ensure this
227 aspect of ASCII compatibility, use only "safe UTF-8 decoders" that
228 reject overlong UTF-8 sequences for which a shorter encoding exists.
229 */
230
231 /*4.1 Examples of an overlong ASCII character
232
233 With a safe UTF-8 decoder, all of the following five overlong
234 representations of the ASCII character slash ("/") should be rejected
235 like a malformed UTF-8 sequence, for instance by substituting it with
236 a replacement character. If you see a slash below, you do not have a
237 safe UTF-8 decoder!
238 */
239
240 {1,"4.1.1", "U+002F = c0 af ", "��"},
241 {1,"4.1.2", "U+002F = e0 80 af ", "���"},
242 {1,"4.1.3", "U+002F = f0 80 80 af ", "����"},
243 {1,"4.1.4", "U+002F = f8 80 80 80 af ", "�����"},
244 {1,"4.1.5", "U+002F = fc 80 80 80 80 af ", "������"},
245
246 /*4.2 Maximum overlong sequences
247
248 Below you see the highest Unicode value that is still resulting in an
249 overlong sequence if represented with the given number of bytes. This
250 is a boundary test for safe UTF-8 decoders. All five characters should
251 be rejected like malformed UTF-8 sequences.
252 */
253
254 {1,"4.2.1", "U-0000007F = c1 bf ", "��"},
255 {1,"4.2.2", "U-000007FF = e0 9f bf ", "���"},
256 {1,"4.2.3", "U-0000FFFF = f0 8f bf bf ", "����"},
257 {1,"4.2.4", "U-001FFFFF = f8 87 bf bf bf ", "�����"},
258 {1,"4.2.5", "U-03FFFFFF = fc 83 bf bf bf bf ", "������"},
259
260 /*
261 4.3 Overlong representation of the NUL character
262
263 The following five sequences should also be rejected like malformed
264 UTF-8 sequences and should not be treated like the ASCII NUL
265 character.
266 */
267
268 {1,"4.3.1", "U+0000 = c0 80 ", "��"},
269 {1,"4.3.2", "U+0000 = e0 80 80 ", "���"},
270 {1,"4.3.3", "U+0000 = f0 80 80 80 ", "����"},
271 {1,"4.3.4", "U+0000 = f8 80 80 80 80 ", "�����"},
272 {1,"4.3.5", "U+0000 = fc 80 80 80 80 80 ", "������"},
273
274 /*
275 5 Illegal code positions
276
277 The following UTF-8 sequences should be rejected like malformed
278 sequences, because they never represent valid ISO 10646 characters and
279 a UTF-8 decoder that accepts them might introduce security problems
280 comparable to overlong UTF-8 sequences.
281 */
282 /*5.1 Single UTF-16 surrogates*/
283
284 {1,"5.1.1", "U+D800 = ed a0 80 ", "�"},
285 {1,"5.1.2", "U+DB7F = ed ad bf ", "�"},
286 {1,"5.1.3", "U+DB80 = ed ae 80 ", "�"},
287 {1,"5.1.4", "U+DBFF = ed af bf ", "�"},
288 {1,"5.1.5", "U+DC00 = ed b0 80 ", "�"},
289 {1,"5.1.6", "U+DF80 = ed be 80 ", "�"},
290 {1,"5.1.7", "U+DFFF = ed bf bf ", "�"},
291
292 /*5.2 Paired UTF-16 surrogates */
293
294 {1,"5.2.1", "U+D800 U+DC00 = ed a0 80 ed b0 80 ", "��"},
295 {1,"5.2.2", "U+D800 U+DFFF = ed a0 80 ed bf bf ", "��"},
296 {1,"5.2.3", "U+DB7F U+DC00 = ed ad bf ed b0 80 ", "��"},
297 {1,"5.2.4", "U+DB7F U+DFFF = ed ad bf ed bf bf ", "��"},
298 {1,"5.2.5", "U+DB80 U+DC00 = ed ae 80 ed b0 80 ", "��"},
299 {1,"5.2.6", "U+DB80 U+DFFF = ed ae 80 ed bf bf ", "��"},
300 {1,"5.2.7", "U+DBFF U+DC00 = ed af bf ed b0 80 ", "��"},
301 {1,"5.2.8", "U+DBFF U+DFFF = ed af bf ed bf bf ", "��"},
302 NULLTEST
303 };
304
305 /*5.3 Noncharacter code positions
306
307 The following "noncharacters" are "reserved for internal use" by
308 applications, and according to older versions of the Unicode Standard
309 "should never be interchanged". Unicode Corrigendum #9 dropped the
310 latter restriction. Nevertheless, their presence in incoming UTF-8 data
311 can remain a potential security risk, depending on what use is made of
312 these codes subsequently. Examples of such internal use:
313
314 - Some file APIs with 16-bit characters may use the integer value -1
315 = U+FFFF to signal an end-of-file (EOF) or error condition.
316
317 - In some UTF-16 receivers, code point U+FFFE might trigger a
318 byte-swap operation (to convert between UTF-16LE and UTF-16BE).
319
320 With such internal use of noncharacters, it may be desirable and safer
321 to block those code points in UTF-8 decoders, as they should never
322 occur legitimately in incoming UTF-8 data, and could trigger unsafe
323 behaviour in subsequent processing.
324
325 Particularly problematic noncharacters in 16-bit applications:
326 */
327
328 static const struct Test utf8problematic[] = {
329 {0,"5.3.1", "U+FFFE = ef bf be ", ""},
330 {0,"5.3.2", "U+FFFF = ef bf bf ", ""},
331 NULLTEST
332 };
333
334 /* Other (utf16) noncharacters: */
335 static const struct Test utf8nonchars[] = {
336 {0,"5.3.3", "U+FDD0 .. U+FDEF ",
337 ""
338 },
339
340 /* Do not understand this test; it passes, but should it? */
341 {0,"5.3.4", "U+nFFFE U+nFFFF (for n = 1..10)",
342 "����������������������������������������������������������������"
343 },
344 NULLTEST
345 };
346
347 static char*
trim(const char * s)348 trim(const char* s)
349 {
350 int i;
351 size_t l = strlen(s);
352 char* t = strdup(s);
353 for(i=l-1;i >= 0; i--) {
354 if(t[i] != ' ') break;
355 }
356 t[i+1] = '\0';
357 return t;
358 }
359
360 static int
test(const struct Test * tests,const char * title)361 test(const struct Test* tests, const char* title)
362 {
363 int status = NC_NOERR;
364 int failures = 0;
365 const struct Test* p;
366
367 fprintf(stderr,"Testing %s...\n",title);
368 for(p=tests;p->id;p++) {
369 char* id;
370 char* description;
371 const char* pf;
372 id = trim(p->id);
373 description = trim(p->description);
374 status = nc_utf8_validate((const unsigned char*)p->data);
375 if(status == NC_NOERR && p->xfail) {pf = "Fail"; failures++;}
376 else if(status != NC_NOERR && p->xfail) pf = "Pass";
377 else if(status == NC_NOERR && !p->xfail) pf = "Pass";
378 else if(status != NC_NOERR && !p->xfail) {pf = "Fail"; failures++;}
379 fprintf(stderr,"%s: %s %s\n",pf,id,description);
380 fflush(stderr);
381 free(id);
382 free(description);
383 }
384 return failures;
385 }
386
387 int
main(int argc,char ** argv)388 main(int argc, char** argv)
389 {
390 int failures = 0;
391
392 printf("\n Testing UTF-8 sequences.\n");
393 failures += test(utf8ok,"Correct Sequences");
394 failures += test(utf8boundary,"Boundary Tests");
395 failures += test(utf8bad,"Invalid strings");
396 failures += test(utf8problematic,"Problematic strings");
397 failures += test(utf8nonchars,"Other non-characters");
398 fprintf(stderr,"No. of failures = %d\n",failures);
399 exit(failures == 0 ? 0 : 1);
400 }
401