1 /*
2 * "streamable kanji code filter and converter"
3 * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4 *
5 * LICENSE NOTICES
6 *
7 * This file is part of "streamable kanji code filter and converter",
8 * which is distributed under the terms of GNU Lesser General Public
9 * License (version 2) as published by the Free Software Foundation.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with "streamable kanji code filter and converter";
18 * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19 * Suite 330, Boston, MA 02111-1307 USA
20 *
21 * The author of this file:
22 *
23 */
24 /*
25 * the source code included in this files was separated from mbfilter_cp936.c
26 * by rui hirokawa <hirokawa@php.net> on 11 Aug 2011.
27 *
28 */
29
30 #include "mbfilter.h"
31 #include "mbfilter_gb18030.h"
32
33 #include "unicode_table_cp936.h"
34 #include "unicode_table_gb18030.h"
35
36 static int mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter *filter);
37
38 static const char *mbfl_encoding_gb18030_aliases[] = {"gb-18030", "gb-18030-2000", NULL};
39
40 const mbfl_encoding mbfl_encoding_gb18030 = {
41 mbfl_no_encoding_gb18030,
42 "GB18030",
43 "GB18030",
44 mbfl_encoding_gb18030_aliases,
45 NULL,
46 MBFL_ENCTYPE_GL_UNSAFE,
47 &vtbl_gb18030_wchar,
48 &vtbl_wchar_gb18030
49 };
50
51 const struct mbfl_convert_vtbl vtbl_gb18030_wchar = {
52 mbfl_no_encoding_gb18030,
53 mbfl_no_encoding_wchar,
54 mbfl_filt_conv_common_ctor,
55 NULL,
56 mbfl_filt_conv_gb18030_wchar,
57 mbfl_filt_conv_gb18030_wchar_flush,
58 NULL,
59 };
60
61 const struct mbfl_convert_vtbl vtbl_wchar_gb18030 = {
62 mbfl_no_encoding_wchar,
63 mbfl_no_encoding_gb18030,
64 mbfl_filt_conv_common_ctor,
65 NULL,
66 mbfl_filt_conv_wchar_gb18030,
67 mbfl_filt_conv_common_flush,
68 NULL,
69 };
70
71 #define CK(statement) do { if ((statement) < 0) return (-1); } while (0)
72
73 /* `tbl` contains inclusive ranges, each represented by a pair of unsigned shorts */
mbfl_bisec_srch(int w,const unsigned short * tbl,int n)74 int mbfl_bisec_srch(int w, const unsigned short *tbl, int n)
75 {
76 int l = 0, r = n-1;
77 while (l <= r) {
78 int probe = (l + r) >> 1;
79 unsigned short lo = tbl[2 * probe], hi = tbl[(2 * probe) + 1];
80 if (w < lo) {
81 r = probe - 1;
82 } else if (w > hi) {
83 l = probe + 1;
84 } else {
85 return probe;
86 }
87 }
88 return -1;
89 }
90
91 /* `tbl` contains single values, not ranges */
mbfl_bisec_srch2(int w,const unsigned short tbl[],int n)92 int mbfl_bisec_srch2(int w, const unsigned short tbl[], int n)
93 {
94 int l = 0, r = n-1;
95 while (l <= r) {
96 int probe = (l + r) >> 1;
97 unsigned short val = tbl[probe];
98 if (w < val) {
99 r = probe - 1;
100 } else if (w > val) {
101 l = probe + 1;
102 } else {
103 return probe;
104 }
105 }
106 return -1;
107 }
108
mbfl_filt_conv_gb18030_wchar(int c,mbfl_convert_filter * filter)109 int mbfl_filt_conv_gb18030_wchar(int c, mbfl_convert_filter *filter)
110 {
111 int k;
112 int c1, c2, c3, w = -1;
113
114 switch (filter->status) {
115 case 0:
116 if (c >= 0 && c < 0x80) { /* latin */
117 CK((*filter->output_function)(c, filter->data));
118 } else if (c > 0x80 && c < 0xff) { /* dbcs/qbcs lead byte */
119 filter->status = 1;
120 filter->cache = c;
121 } else {
122 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
123 }
124 break;
125
126 case 1: /* dbcs/qbcs second byte */
127 c1 = filter->cache;
128 filter->status = 0;
129
130 if (c1 >= 0x81 && c1 <= 0x84 && c >= 0x30 && c <= 0x39) {
131 /* 4 byte range: Unicode BMP */
132 filter->status = 2;
133 filter->cache = (c1 << 8) | c;
134 return 0;
135 } else if (c1 >= 0x90 && c1 <= 0xe3 && c >= 0x30 && c <= 0x39) {
136 /* 4 byte range: Unicode 16 planes */
137 filter->status = 2;
138 filter->cache = (c1 << 8) | c;
139 return 0;
140 } else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) && (c >= 0xa1 && c <= 0xfe)) {
141 /* UDA part 1,2: U+E000-U+E4C5 */
142 w = 94*(c1 >= 0xf8 ? c1 - 0xf2 : c1 - 0xaa) + (c - 0xa1) + 0xe000;
143 CK((*filter->output_function)(w, filter->data));
144 } else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
145 /* UDA part3 : U+E4C6-U+E765*/
146 w = 96*(c1 - 0xa1) + c - (c >= 0x80 ? 0x41 : 0x40) + 0xe4c6;
147 CK((*filter->output_function)(w, filter->data));
148 }
149
150 c2 = (c1 << 8) | c;
151
152 if (w <= 0 &&
153 ((c2 >= 0xa2ab && c2 <= 0xa9f0 + (0xe80f-0xe801)) ||
154 (c2 >= 0xd7fa && c2 <= 0xd7fa + (0xe814-0xe810)) ||
155 (c2 >= 0xfe50 && c2 <= 0xfe80 + (0xe864-0xe844)))) {
156 for (k = 0; k < mbfl_gb18030_pua_tbl_max; k++) {
157 if (c2 >= mbfl_gb18030_pua_tbl[k][2] && c2 <= mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][1] - mbfl_gb18030_pua_tbl[k][0]) {
158 w = c2 - mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][0];
159 CK((*filter->output_function)(w, filter->data));
160 break;
161 }
162 }
163 }
164
165 if (w <= 0) {
166 if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
167 (c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
168 (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
169 (c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
170 (c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
171 w = (c1 - 0x81)*192 + (c - 0x40);
172 if (w >= 0 && w < cp936_ucs_table_size) {
173 w = cp936_ucs_table[w];
174 if (!w)
175 w = MBFL_BAD_INPUT;
176 } else {
177 w = MBFL_BAD_INPUT;
178 }
179 CK((*filter->output_function)(w, filter->data));
180 } else {
181 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
182 }
183 }
184 break;
185
186 case 2: /* qbcs third byte */
187 c1 = (filter->cache >> 8) & 0xff;
188 c2 = filter->cache & 0xff;
189 filter->status = filter->cache = 0;
190 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c2 >= 0x30 && c2 <= 0x39 && c >= 0x81 && c <= 0xfe) {
191 filter->cache = (c1 << 16) | (c2 << 8) | c;
192 filter->status = 3;
193 } else {
194 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
195 }
196 break;
197
198 case 3: /* qbcs fourth byte */
199 c1 = (filter->cache >> 16) & 0xff;
200 c2 = (filter->cache >> 8) & 0xff;
201 c3 = filter->cache & 0xff;
202 filter->status = filter->cache = 0;
203 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c2 >= 0x30 && c2 <= 0x39 && c3 >= 0x81 && c3 <= 0xfe && c >= 0x30 && c <= 0x39) {
204 if (c1 >= 0x90 && c1 <= 0xe3) {
205 w = ((((c1 - 0x90)*10 + (c2 - 0x30))*126 + (c3 - 0x81)))*10 + (c - 0x30) + 0x10000;
206 if (w > 0x10FFFF) {
207 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
208 return 0;
209 }
210 } else { /* Unicode BMP */
211 w = (((c1 - 0x81)*10 + (c2 - 0x30))*126 + (c3 - 0x81))*10 + (c - 0x30);
212 if (w >= 0 && w <= 39419) {
213 k = mbfl_bisec_srch(w, mbfl_gb2uni_tbl, mbfl_gb_uni_max);
214 if (k < 0) {
215 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
216 return 0;
217 }
218 w += mbfl_gb_uni_ofst[k];
219 } else {
220 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
221 return 0;
222 }
223 }
224 CK((*filter->output_function)(w, filter->data));
225 } else {
226 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
227 }
228 break;
229
230 default:
231 filter->status = 0;
232 break;
233 }
234
235 return 0;
236 }
237
mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter * filter)238 static int mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter *filter)
239 {
240 if (filter->status) {
241 /* multi-byte character was truncated */
242 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
243 }
244
245 if (filter->flush_function) {
246 (*filter->flush_function)(filter->data);
247 }
248
249 return 0;
250 }
251
mbfl_filt_conv_wchar_gb18030(int c,mbfl_convert_filter * filter)252 int mbfl_filt_conv_wchar_gb18030(int c, mbfl_convert_filter *filter)
253 {
254 int k, k1, k2;
255 int c1, s = 0, s1 = 0;
256
257 if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
258 if (c == 0x01f9) {
259 s = 0xa8bf;
260 } else {
261 s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
262 }
263 } else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
264 if (c == 0x20ac) { /* euro-sign */
265 s = 0xa2e3;
266 } else {
267 s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
268 }
269 } else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
270 s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
271 } else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
272 s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
273 } else if (c >= ucs_ci_cp936_table_min && c < ucs_ci_cp936_table_max) {
274 /* U+F900-FA2F CJK Compatibility Ideographs */
275 if (c == 0xf92c) {
276 s = 0xfd9c;
277 } else if (c == 0xf979) {
278 s = 0xfd9d;
279 } else if (c == 0xf995) {
280 s = 0xfd9e;
281 } else if (c == 0xf9e7) {
282 s = 0xfd9f;
283 } else if (c == 0xf9f1) {
284 s = 0xfda0;
285 } else if (c >= 0xfa0c && c <= 0xfa29) {
286 s = ucs_ci_s_cp936_table[c - 0xfa0c];
287 }
288 } else if (c >= ucs_cf_cp936_table_min && c < ucs_cf_cp936_table_max) {
289 /* FE30h CJK Compatibility Forms */
290 s = ucs_cf_cp936_table[c - ucs_cf_cp936_table_min];
291 } else if (c >= ucs_sfv_cp936_table_min && c < ucs_sfv_cp936_table_max) {
292 /* U+FE50-FE6F Small Form Variants */
293 s = ucs_sfv_cp936_table[c - ucs_sfv_cp936_table_min];
294 } else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
295 /* U+FF00-FFFF HW/FW Forms */
296 if (c == 0xff04) {
297 s = 0xa1e7;
298 } else if (c == 0xff5e) {
299 s = 0xa1ab;
300 } else if (c >= 0xff01 && c <= 0xff5d) {
301 s = c - 0xff01 + 0xa3a1;
302 } else if (c >= 0xffe0 && c <= 0xffe5) {
303 s = ucs_hff_s_cp936_table[c-0xffe0];
304 }
305 }
306
307 /* While GB18030 and CP936 are very similar, some mappings are different between these encodings;
308 * do a binary search in a table of differing codepoints to see if we have one */
309 if (s <= 0 && c >= mbfl_gb18030_c_tbl_key[0] && c <= mbfl_gb18030_c_tbl_key[mbfl_gb18030_c_tbl_max-1]) {
310 k1 = mbfl_bisec_srch2(c, mbfl_gb18030_c_tbl_key, mbfl_gb18030_c_tbl_max);
311 if (k1 >= 0) {
312 s = mbfl_gb18030_c_tbl_val[k1];
313 }
314 }
315
316 if (c >= 0xe000 && c <= 0xe864) { /* PUA */
317 if (c < 0xe766) {
318 if (c < 0xe4c6) {
319 c1 = c - 0xe000;
320 s = (c1 % 94) + 0xa1;
321 c1 /= 94;
322 s |= (c1 < 0x06 ? c1 + 0xaa : c1 + 0xf2) << 8;
323 } else {
324 c1 = c - 0xe4c6;
325 s = ((c1 / 96) + 0xa1) << 8;
326 c1 %= 96;
327 s |= c1 + (c1 >= 0x3f ? 0x41 : 0x40);
328 }
329 } else {
330 /* U+E766..U+E864 */
331 k1 = 0;
332 k2 = mbfl_gb18030_pua_tbl_max;
333 while (k1 < k2) {
334 k = (k1 + k2) >> 1;
335 if (c < mbfl_gb18030_pua_tbl[k][0]) {
336 k2 = k;
337 } else if (c > mbfl_gb18030_pua_tbl[k][1]) {
338 k1 = k + 1;
339 } else {
340 s = c - mbfl_gb18030_pua_tbl[k][0] + mbfl_gb18030_pua_tbl[k][2];
341 break;
342 }
343 }
344 }
345 }
346
347 /* If we have not yet found a suitable mapping for this codepoint, it requires a 4-byte code */
348 if (s <= 0 && c >= 0x0080 && c <= 0xffff) {
349 /* BMP */
350 s = mbfl_bisec_srch(c, mbfl_uni2gb_tbl, mbfl_gb_uni_max);
351 if (s >= 0) {
352 c1 = c - mbfl_gb_uni_ofst[s];
353 s = (c1 % 10) + 0x30;
354 c1 /= 10;
355 s |= ((c1 % 126) + 0x81) << 8;
356 c1 /= 126;
357 s |= ((c1 % 10) + 0x30) << 16;
358 c1 /= 10;
359 s1 = c1 + 0x81;
360 }
361 } else if (c >= 0x10000 && c <= 0x10ffff) {
362 /* Code set 3: Unicode U+10000..U+10FFFF */
363 c1 = c - 0x10000;
364 s = (c1 % 10) + 0x30;
365 c1 /= 10;
366 s |= ((c1 % 126) + 0x81) << 8;
367 c1 /= 126;
368 s |= ((c1 % 10) + 0x30) << 16;
369 c1 /= 10;
370 s1 = c1 + 0x90;
371 }
372
373 if (s <= 0) {
374 if (c == 0) {
375 s = 0;
376 } else {
377 s = -1;
378 }
379 }
380
381 if (s >= 0) {
382 if (s <= 0x80) { /* latin */
383 CK((*filter->output_function)(s, filter->data));
384 } else if (s1 > 0) { /* qbcs */
385 CK((*filter->output_function)(s1 & 0xff, filter->data));
386 CK((*filter->output_function)((s >> 16) & 0xff, filter->data));
387 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
388 CK((*filter->output_function)(s & 0xff, filter->data));
389 } else { /* dbcs */
390 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
391 CK((*filter->output_function)(s & 0xff, filter->data));
392 }
393 } else {
394 CK(mbfl_filt_conv_illegal_output(c, filter));
395 }
396
397 return 0;
398 }
399