1 /*
2  * Copyright (c) Ian F. Darwin 1986-1995.
3  * Software written by Ian F. Darwin and others;
4  * maintained 1995-present by Christos Zoulas and others.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice immediately at the beginning of the file, without modification,
11  *    this list of conditions, and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * Encoding -- determine the character encoding of a text file.
30  *
31  * Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
32  * international characters.
33  */
34 
35 #include "file.h"
36 
37 #ifndef	lint
38 FILE_RCSID("@(#)$File: encoding.c,v 1.27 2021/02/05 21:33:49 christos Exp $")
39 #endif	/* lint */
40 
41 #include "magic.h"
42 #include <string.h>
43 #include <stdlib.h>
44 
45 
46 private int looks_ascii(const unsigned char *, size_t, file_unichar_t *,
47     size_t *);
48 private int looks_utf8_with_BOM(const unsigned char *, size_t, file_unichar_t *,
49     size_t *);
50 private int looks_utf7(const unsigned char *, size_t, file_unichar_t *,
51     size_t *);
52 private int looks_ucs16(const unsigned char *, size_t, file_unichar_t *,
53     size_t *);
54 private int looks_ucs32(const unsigned char *, size_t, file_unichar_t *,
55     size_t *);
56 private int looks_latin1(const unsigned char *, size_t, file_unichar_t *,
57     size_t *);
58 private int looks_extended(const unsigned char *, size_t, file_unichar_t *,
59     size_t *);
60 private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
61 
62 #ifdef DEBUG_ENCODING
63 #define DPRINTF(a) printf a
64 #else
65 #define DPRINTF(a)
66 #endif
67 
68 /*
69  * Try to determine whether text is in some character code we can
70  * identify.  Each of these tests, if it succeeds, will leave
71  * the text converted into one-file_unichar_t-per-character Unicode in
72  * ubuf, and the number of characters converted in ulen.
73  */
74 protected int
file_encoding(struct magic_set * ms,const struct buffer * b,file_unichar_t ** ubuf,size_t * ulen,const char ** code,const char ** code_mime,const char ** type)75 file_encoding(struct magic_set *ms, const struct buffer *b,
76     file_unichar_t **ubuf, size_t *ulen, const char **code,
77     const char **code_mime, const char **type)
78 {
79 	const unsigned char *buf = CAST(const unsigned char *, b->fbuf);
80 	size_t nbytes = b->flen;
81 	size_t mlen;
82 	int rv = 1, ucs_type;
83 	unsigned char *nbuf = NULL;
84 	file_unichar_t *udefbuf;
85 	size_t udeflen;
86 
87 	if (ubuf == NULL)
88 		ubuf = &udefbuf;
89 	if (ulen == NULL)
90 		ulen = &udeflen;
91 
92 	*type = "text";
93 	*ulen = 0;
94 	*code = "unknown";
95 	*code_mime = "binary";
96 
97 	if (nbytes > ms->encoding_max)
98 		nbytes = ms->encoding_max;
99 
100 	mlen = (nbytes + 1) * sizeof((*ubuf)[0]);
101 	*ubuf = CAST(file_unichar_t *, ecalloc(CAST(size_t, 1), mlen));
102 	if (*ubuf == NULL) {
103 		file_oomem(ms, mlen);
104 		goto done;
105 	}
106 	mlen = (nbytes + 1) * sizeof(nbuf[0]);
107 	if ((nbuf = CAST(unsigned char *,
108 	    ecalloc(CAST(size_t, 1), mlen))) == NULL) {
109 		file_oomem(ms, mlen);
110 		goto done;
111 	}
112 
113 	if (looks_ascii(buf, nbytes, *ubuf, ulen)) {
114 		if (looks_utf7(buf, nbytes, *ubuf, ulen) > 0) {
115 			DPRINTF(("utf-7 %" SIZE_T_FORMAT "u\n", *ulen));
116 			*code = "Unicode text, UTF-7";
117 			*code_mime = "utf-7";
118 		} else {
119 			DPRINTF(("ascii %" SIZE_T_FORMAT "u\n", *ulen));
120 			*code = "ASCII";
121 			*code_mime = "us-ascii";
122 		}
123 	} else if (looks_utf8_with_BOM(buf, nbytes, *ubuf, ulen) > 0) {
124 		DPRINTF(("utf8/bom %" SIZE_T_FORMAT "u\n", *ulen));
125 		*code = "Unicode text, UTF-8 (with BOM)";
126 		*code_mime = "utf-8";
127 	} else if (file_looks_utf8(buf, nbytes, *ubuf, ulen) > 1) {
128 		DPRINTF(("utf8 %" SIZE_T_FORMAT "u\n", *ulen));
129 		*code = "Unicode text, UTF-8";
130 		*code_mime = "utf-8";
131 	} else if ((ucs_type = looks_ucs32(buf, nbytes, *ubuf, ulen)) != 0) {
132 		if (ucs_type == 1) {
133 			*code = "Unicode text, UTF-32, little-endian";
134 			*code_mime = "utf-32le";
135 		} else {
136 			*code = "Unicode text, UTF-32, big-endian";
137 			*code_mime = "utf-32be";
138 		}
139 		DPRINTF(("ucs32 %" SIZE_T_FORMAT "u\n", *ulen));
140 	} else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {
141 		if (ucs_type == 1) {
142 			*code = "Unicode text, UTF-16, little-endian";
143 			*code_mime = "utf-16le";
144 		} else {
145 			*code = "Unicode text, UTF-16, big-endian";
146 			*code_mime = "utf-16be";
147 		}
148 		DPRINTF(("ucs16 %" SIZE_T_FORMAT "u\n", *ulen));
149 	} else if (looks_latin1(buf, nbytes, *ubuf, ulen)) {
150 		DPRINTF(("latin1 %" SIZE_T_FORMAT "u\n", *ulen));
151 		*code = "ISO-8859";
152 		*code_mime = "iso-8859-1";
153 	} else if (looks_extended(buf, nbytes, *ubuf, ulen)) {
154 		DPRINTF(("extended %" SIZE_T_FORMAT "u\n", *ulen));
155 		*code = "Non-ISO extended-ASCII";
156 		*code_mime = "unknown-8bit";
157 	} else {
158 		from_ebcdic(buf, nbytes, nbuf);
159 
160 		if (looks_ascii(nbuf, nbytes, *ubuf, ulen)) {
161 			DPRINTF(("ebcdic %" SIZE_T_FORMAT "u\n", *ulen));
162 			*code = "EBCDIC";
163 			*code_mime = "ebcdic";
164 		} else if (looks_latin1(nbuf, nbytes, *ubuf, ulen)) {
165 			DPRINTF(("ebcdic/international %" SIZE_T_FORMAT "u\n",
166 			    *ulen));
167 			*code = "International EBCDIC";
168 			*code_mime = "ebcdic";
169 		} else { /* Doesn't look like text at all */
170 			DPRINTF(("binary\n"));
171 			rv = 0;
172 			*type = "binary";
173 		}
174 	}
175 
176  done:
177 	efree(nbuf);
178 	if (ubuf == &udefbuf)
179 		efree(udefbuf);
180 
181 	return rv;
182 }
183 
184 /*
185  * This table reflects a particular philosophy about what constitutes
186  * "text," and there is room for disagreement about it.
187  *
188  * Version 3.31 of the file command considered a file to be ASCII if
189  * each of its characters was approved by either the isascii() or
190  * isalpha() function.  On most systems, this would mean that any
191  * file consisting only of characters in the range 0x00 ... 0x7F
192  * would be called ASCII text, but many systems might reasonably
193  * consider some characters outside this range to be alphabetic,
194  * so the file command would call such characters ASCII.  It might
195  * have been more accurate to call this "considered textual on the
196  * local system" than "ASCII."
197  *
198  * It considered a file to be "International language text" if each
199  * of its characters was either an ASCII printing character (according
200  * to the real ASCII standard, not the above test), a character in
201  * the range 0x80 ... 0xFF, or one of the following control characters:
202  * backspace, tab, line feed, vertical tab, form feed, carriage return,
203  * escape.  No attempt was made to determine the language in which files
204  * of this type were written.
205  *
206  *
207  * The table below considers a file to be ASCII if all of its characters
208  * are either ASCII printing characters (again, according to the X3.4
209  * standard, not isascii()) or any of the following controls: bell,
210  * backspace, tab, line feed, form feed, carriage return, esc, nextline.
211  *
212  * I include bell because some programs (particularly shell scripts)
213  * use it literally, even though it is rare in normal text.  I exclude
214  * vertical tab because it never seems to be used in real text.  I also
215  * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
216  * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
217  * character to.  It might be more appropriate to include it in the 8859
218  * set instead of the ASCII set, but it's got to be included in *something*
219  * we recognize or EBCDIC files aren't going to be considered textual.
220  * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
221  * and Latin characters, so these should possibly be allowed.  But they
222  * make a real mess on VT100-style displays if they're not paired properly,
223  * so we are probably better off not calling them text.
224  *
225  * A file is considered to be ISO-8859 text if its characters are all
226  * either ASCII, according to the above definition, or printing characters
227  * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
228  *
229  * Finally, a file is considered to be international text from some other
230  * character code if its characters are all either ISO-8859 (according to
231  * the above definition) or characters in the range 0x80 ... 0x9F, which
232  * ISO-8859 considers to be control characters but the IBM PC and Macintosh
233  * consider to be printing characters.
234  */
235 
236 #define F 0   /* character never appears in text */
237 #define T 1   /* character appears in plain ASCII text */
238 #define I 2   /* character appears in ISO-8859 text */
239 #define X 3   /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
240 
241 private char text_chars[256] = {
242 	/*                  BEL BS HT LF VT FF CR    */
243 	F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F,  /* 0x0X */
244 	/*                              ESC          */
245 	F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F,  /* 0x1X */
246 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x2X */
247 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x3X */
248 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x4X */
249 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x5X */
250 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x6X */
251 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F,  /* 0x7X */
252 	/*            NEL                            */
253 	X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X,  /* 0x8X */
254 	X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,  /* 0x9X */
255 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xaX */
256 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xbX */
257 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xcX */
258 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xdX */
259 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xeX */
260 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I   /* 0xfX */
261 };
262 
263 #define LOOKS(NAME, COND) \
264 private int \
265 looks_ ## NAME(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf, \
266     size_t *ulen) \
267 { \
268 	size_t i, u; \
269 	unsigned char dist[256]; \
270 	memset(dist, 0, sizeof(dist)); \
271 \
272 	*ulen = 0; \
273 \
274 	for (i = 0; i < nbytes; i++) { \
275 		int t = text_chars[buf[i]]; \
276 \
277 		if (COND) \
278 			return 0; \
279 \
280 		ubuf[(*ulen)++] = buf[i]; \
281 		dist[buf[i]]++; \
282 	} \
283 	u = 0; \
284 	for (i = 0; i < __arraycount(dist); i++) { \
285 		if (dist[i]) \
286 			u += dist[i]; \
287 	} \
288 	if (u < 3) \
289 		return 0; \
290 \
291 	return 1; \
292 }
293 
294 LOOKS(ascii, t != T)
295 LOOKS(latin1, t != T && t != I)
296 LOOKS(extended, t != T && t != I && t != X)
297 
298 /*
299  * Decide whether some text looks like UTF-8. Returns:
300  *
301  *     -1: invalid UTF-8
302  *      0: uses odd control characters, so doesn't look like text
303  *      1: 7-bit text
304  *      2: definitely UTF-8 text (valid high-bit set bytes)
305  *
306  * If ubuf is non-NULL on entry, text is decoded into ubuf, *ulen;
307  * ubuf must be big enough!
308  */
309 
310 // from: https://golang.org/src/unicode/utf8/utf8.go
311 
312 #define	XX 0xF1 // invalid: size 1
313 #define	AS 0xF0 // ASCII: size 1
314 #define	S1 0x02 // accept 0, size 2
315 #define	S2 0x13 // accept 1, size 3
316 #define	S3 0x03 // accept 0, size 3
317 #define	S4 0x23 // accept 2, size 3
318 #define	S5 0x34 // accept 3, size 4
319 #define	S6 0x04 // accept 0, size 4
320 #define	S7 0x44 // accept 4, size 4
321 
322 #define LOCB 0x80
323 #define HICB 0xBF
324 
325 // first is information about the first byte in a UTF-8 sequence.
326 static const uint8_t first[] = {
327     //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
328     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x00-0x0F
329     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x10-0x1F
330     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x20-0x2F
331     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x30-0x3F
332     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x40-0x4F
333     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x50-0x5F
334     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x60-0x6F
335     AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x70-0x7F
336     //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
337     XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x80-0x8F
338     XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x90-0x9F
339     XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xA0-0xAF
340     XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xB0-0xBF
341     XX, XX, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xC0-0xCF
342     S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xD0-0xDF
343     S2, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S4, S3, S3, // 0xE0-0xEF
344     S5, S6, S6, S6, S7, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xF0-0xFF
345 };
346 
347 // acceptRange gives the range of valid values for the second byte in a UTF-8
348 // sequence.
349 struct accept_range {
350 	uint8_t lo; // lowest value for second byte.
351 	uint8_t hi; // highest value for second byte.
352 } accept_ranges[16] = {
353 // acceptRanges has size 16 to avoid bounds checks in the code that uses it.
354 	{ LOCB, HICB },
355 	{ 0xA0, HICB },
356 	{ LOCB, 0x9F },
357 	{ 0x90, HICB },
358 	{ LOCB, 0x8F },
359 };
360 
361 protected int
file_looks_utf8(const unsigned char * buf,size_t nbytes,file_unichar_t * ubuf,size_t * ulen)362 file_looks_utf8(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf,
363     size_t *ulen)
364 {
365 	size_t i;
366 	int n;
367 	file_unichar_t c;
368 	int gotone = 0, ctrl = 0;
369 
370 	if (ubuf)
371 		*ulen = 0;
372 
373 	for (i = 0; i < nbytes; i++) {
374 		if ((buf[i] & 0x80) == 0) {	   /* 0xxxxxxx is plain ASCII */
375 			/*
376 			 * Even if the whole file is valid UTF-8 sequences,
377 			 * still reject it if it uses weird control characters.
378 			 */
379 
380 			if (text_chars[buf[i]] != T)
381 				ctrl = 1;
382 
383 			if (ubuf)
384 				ubuf[(*ulen)++] = buf[i];
385 		} else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
386 			return -1;
387 		} else {			   /* 11xxxxxx begins UTF-8 */
388 			int following;
389 			uint8_t x = first[buf[i]];
390 			const struct accept_range *ar = &accept_ranges[x >> 4];
391 			if (x == XX)
392 				return -1;
393 
394 			if ((buf[i] & 0x20) == 0) {		/* 110xxxxx */
395 				c = buf[i] & 0x1f;
396 				following = 1;
397 			} else if ((buf[i] & 0x10) == 0) {	/* 1110xxxx */
398 				c = buf[i] & 0x0f;
399 				following = 2;
400 			} else if ((buf[i] & 0x08) == 0) {	/* 11110xxx */
401 				c = buf[i] & 0x07;
402 				following = 3;
403 			} else if ((buf[i] & 0x04) == 0) {	/* 111110xx */
404 				c = buf[i] & 0x03;
405 				following = 4;
406 			} else if ((buf[i] & 0x02) == 0) {	/* 1111110x */
407 				c = buf[i] & 0x01;
408 				following = 5;
409 			} else
410 				return -1;
411 
412 			for (n = 0; n < following; n++) {
413 				i++;
414 				if (i >= nbytes)
415 					goto done;
416 
417 				if (n == 0 &&
418 				     (buf[i] < ar->lo || buf[i] > ar->hi))
419 					return -1;
420 
421 				if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
422 					return -1;
423 
424 				c = (c << 6) + (buf[i] & 0x3f);
425 			}
426 
427 			if (ubuf)
428 				ubuf[(*ulen)++] = c;
429 			gotone = 1;
430 		}
431 	}
432 done:
433 	return ctrl ? 0 : (gotone ? 2 : 1);
434 }
435 
436 /*
437  * Decide whether some text looks like UTF-8 with BOM. If there is no
438  * BOM, return -1; otherwise return the result of looks_utf8 on the
439  * rest of the text.
440  */
441 private int
looks_utf8_with_BOM(const unsigned char * buf,size_t nbytes,file_unichar_t * ubuf,size_t * ulen)442 looks_utf8_with_BOM(const unsigned char *buf, size_t nbytes,
443     file_unichar_t *ubuf, size_t *ulen)
444 {
445 	if (nbytes > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf)
446 		return file_looks_utf8(buf + 3, nbytes - 3, ubuf, ulen);
447 	else
448 		return -1;
449 }
450 
451 private int
looks_utf7(const unsigned char * buf,size_t nbytes,file_unichar_t * ubuf,size_t * ulen)452 looks_utf7(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf,
453     size_t *ulen)
454 {
455 	if (nbytes > 4 && buf[0] == '+' && buf[1] == '/' && buf[2] == 'v')
456 		switch (buf[3]) {
457 		case '8':
458 		case '9':
459 		case '+':
460 		case '/':
461 			if (ubuf)
462 				*ulen = 0;
463 			return 1;
464 		default:
465 			return -1;
466 		}
467 	else
468 		return -1;
469 }
470 
471 private int
looks_ucs16(const unsigned char * bf,size_t nbytes,file_unichar_t * ubf,size_t * ulen)472 looks_ucs16(const unsigned char *bf, size_t nbytes, file_unichar_t *ubf,
473     size_t *ulen)
474 {
475 	int bigend;
476 	size_t i;
477 
478 	if (nbytes < 2)
479 		return 0;
480 
481 	if (bf[0] == 0xff && bf[1] == 0xfe)
482 		bigend = 0;
483 	else if (bf[0] == 0xfe && bf[1] == 0xff)
484 		bigend = 1;
485 	else
486 		return 0;
487 
488 	*ulen = 0;
489 
490 	for (i = 2; i + 1 < nbytes; i += 2) {
491 		/* XXX fix to properly handle chars > 65536 */
492 
493 		if (bigend)
494 			ubf[(*ulen)++] = bf[i + 1]
495 			    | (CAST(file_unichar_t, bf[i]) << 8);
496 		else
497 			ubf[(*ulen)++] = bf[i]
498 			    | (CAST(file_unichar_t, bf[i + 1]) << 8);
499 
500 		if (ubf[*ulen - 1] == 0xfffe)
501 			return 0;
502 		if (ubf[*ulen - 1] < 128 &&
503 		    text_chars[CAST(size_t, ubf[*ulen - 1])] != T)
504 			return 0;
505 	}
506 
507 	return 1 + bigend;
508 }
509 
510 private int
looks_ucs32(const unsigned char * bf,size_t nbytes,file_unichar_t * ubf,size_t * ulen)511 looks_ucs32(const unsigned char *bf, size_t nbytes, file_unichar_t *ubf,
512     size_t *ulen)
513 {
514 	int bigend;
515 	size_t i;
516 
517 	if (nbytes < 4)
518 		return 0;
519 
520 	if (bf[0] == 0xff && bf[1] == 0xfe && bf[2] == 0 && bf[3] == 0)
521 		bigend = 0;
522 	else if (bf[0] == 0 && bf[1] == 0 && bf[2] == 0xfe && bf[3] == 0xff)
523 		bigend = 1;
524 	else
525 		return 0;
526 
527 	*ulen = 0;
528 
529 	for (i = 4; i + 3 < nbytes; i += 4) {
530 		/* XXX fix to properly handle chars > 65536 */
531 
532 		if (bigend)
533 			ubf[(*ulen)++] = CAST(file_unichar_t, bf[i + 3])
534 			    | (CAST(file_unichar_t, bf[i + 2]) << 8)
535 			    | (CAST(file_unichar_t, bf[i + 1]) << 16)
536 			    | (CAST(file_unichar_t, bf[i]) << 24);
537 		else
538 			ubf[(*ulen)++] = CAST(file_unichar_t, bf[i + 0])
539 			    | (CAST(file_unichar_t, bf[i + 1]) << 8)
540 			    | (CAST(file_unichar_t, bf[i + 2]) << 16)
541 			    | (CAST(file_unichar_t, bf[i + 3]) << 24);
542 
543 		if (ubf[*ulen - 1] == 0xfffe)
544 			return 0;
545 		if (ubf[*ulen - 1] < 128 &&
546 		    text_chars[CAST(size_t, ubf[*ulen - 1])] != T)
547 			return 0;
548 	}
549 
550 	return 1 + bigend;
551 }
552 #undef F
553 #undef T
554 #undef I
555 #undef X
556 
557 /*
558  * This table maps each EBCDIC character to an (8-bit extended) ASCII
559  * character, as specified in the rationale for the dd(1) command in
560  * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
561  *
562  * Unfortunately it does not seem to correspond exactly to any of the
563  * five variants of EBCDIC documented in IBM's _Enterprise Systems
564  * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
565  * Edition, July, 1999, pp. I-1 - I-4.
566  *
567  * Fortunately, though, all versions of EBCDIC, including this one, agree
568  * on most of the printing characters that also appear in (7-bit) ASCII.
569  * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
570  *
571  * Fortunately too, there is general agreement that codes 0x00 through
572  * 0x3F represent control characters, 0x41 a nonbreaking space, and the
573  * remainder printing characters.
574  *
575  * This is sufficient to allow us to identify EBCDIC text and to distinguish
576  * between old-style and internationalized examples of text.
577  */
578 
579 private unsigned char ebcdic_to_ascii[] = {
580   0,   1,   2,   3, 156,   9, 134, 127, 151, 141, 142,  11,  12,  13,  14,  15,
581  16,  17,  18,  19, 157, 133,   8, 135,  24,  25, 146, 143,  28,  29,  30,  31,
582 128, 129, 130, 131, 132,  10,  23,  27, 136, 137, 138, 139, 140,   5,   6,   7,
583 144, 145,  22, 147, 148, 149, 150,   4, 152, 153, 154, 155,  20,  21, 158,  26,
584 ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
585 '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
586 '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
587 186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
588 195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
589 202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
590 209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
591 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
592 '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
593 '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
594 '\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
595 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
596 };
597 
598 #ifdef notdef
599 /*
600  * The following EBCDIC-to-ASCII table may relate more closely to reality,
601  * or at least to modern reality.  It comes from
602  *
603  *   http://ftp.s390.ibm.com/products/oe/bpxqp9.html
604  *
605  * and maps the characters of EBCDIC code page 1047 (the code used for
606  * Unix-derived software on IBM's 390 systems) to the corresponding
607  * characters from ISO 8859-1.
608  *
609  * If this table is used instead of the above one, some of the special
610  * cases for the NEL character can be taken out of the code.
611  */
612 
613 private unsigned char ebcdic_1047_to_8859[] = {
614 0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
615 0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
616 0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
617 0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
618 0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
619 0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
620 0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
621 0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
622 0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
623 0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
624 0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
625 0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
626 0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
627 0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
628 0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
629 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
630 };
631 #endif
632 
633 /*
634  * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
635  */
636 private void
from_ebcdic(const unsigned char * buf,size_t nbytes,unsigned char * out)637 from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
638 {
639 	size_t i;
640 
641 	for (i = 0; i < nbytes; i++) {
642 		out[i] = ebcdic_to_ascii[buf[i]];
643 	}
644 }
645