1 /*
2  * Copyright © 2009  Red Hat, Inc.
3  * Copyright © 2011  Codethink Limited
4  * Copyright © 2010,2011,2012  Google, Inc.
5  *
6  *  This is part of HarfBuzz, a text shaping library.
7  *
8  * Permission is hereby granted, without written agreement and without
9  * license or royalty fees, to use, copy, modify, and distribute this
10  * software and its documentation for any purpose, provided that the
11  * above copyright notice and the following two paragraphs appear in
12  * all copies of this software.
13  *
14  * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
15  * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
16  * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
17  * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
18  * DAMAGE.
19  *
20  * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
21  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
22  * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
23  * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
24  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
25  *
26  * Red Hat Author(s): Behdad Esfahbod
27  * Codethink Author(s): Ryan Lortie
28  * Google Author(s): Behdad Esfahbod
29  */
30 
31 #ifndef HB_UNICODE_HH
32 #define HB_UNICODE_HH
33 
34 #include "hb.hh"
35 
36 
37 extern HB_INTERNAL const uint8_t _hb_modified_combining_class[256];
38 
39 /*
40  * hb_unicode_funcs_t
41  */
42 
43 #define HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS \
44   HB_UNICODE_FUNC_IMPLEMENT (combining_class) \
45   HB_IF_NOT_DEPRECATED (HB_UNICODE_FUNC_IMPLEMENT (eastasian_width)) \
46   HB_UNICODE_FUNC_IMPLEMENT (general_category) \
47   HB_UNICODE_FUNC_IMPLEMENT (mirroring) \
48   HB_UNICODE_FUNC_IMPLEMENT (script) \
49   HB_UNICODE_FUNC_IMPLEMENT (compose) \
50   HB_UNICODE_FUNC_IMPLEMENT (decompose) \
51   HB_IF_NOT_DEPRECATED (HB_UNICODE_FUNC_IMPLEMENT (decompose_compatibility)) \
52   /* ^--- Add new callbacks here */
53 
54 /* Simple callbacks are those taking a hb_codepoint_t and returning a hb_codepoint_t */
55 #define HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS_SIMPLE \
56   HB_UNICODE_FUNC_IMPLEMENT (hb_unicode_combining_class_t, combining_class) \
57   HB_IF_NOT_DEPRECATED (HB_UNICODE_FUNC_IMPLEMENT (unsigned int, eastasian_width)) \
58   HB_UNICODE_FUNC_IMPLEMENT (hb_unicode_general_category_t, general_category) \
59   HB_UNICODE_FUNC_IMPLEMENT (hb_codepoint_t, mirroring) \
60   HB_UNICODE_FUNC_IMPLEMENT (hb_script_t, script) \
61   /* ^--- Add new simple callbacks here */
62 
63 struct hb_unicode_funcs_t
64 {
65   hb_object_header_t header;
66 
67   hb_unicode_funcs_t *parent;
68 
69 #define HB_UNICODE_FUNC_IMPLEMENT(return_type, name) \
70   return_type name (hb_codepoint_t unicode) { return func.name (this, unicode, user_data.name); }
71 HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS_SIMPLE
72 #undef HB_UNICODE_FUNC_IMPLEMENT
73 
composehb_unicode_funcs_t74   hb_bool_t compose (hb_codepoint_t a, hb_codepoint_t b,
75 		     hb_codepoint_t *ab)
76   {
77     *ab = 0;
78     if (unlikely (!a || !b)) return false;
79     return func.compose (this, a, b, ab, user_data.compose);
80   }
81 
decomposehb_unicode_funcs_t82   hb_bool_t decompose (hb_codepoint_t ab,
83 		       hb_codepoint_t *a, hb_codepoint_t *b)
84   {
85     *a = ab; *b = 0;
86     return func.decompose (this, ab, a, b, user_data.decompose);
87   }
88 
decompose_compatibilityhb_unicode_funcs_t89   unsigned int decompose_compatibility (hb_codepoint_t  u,
90 					hb_codepoint_t *decomposed)
91   {
92 #ifdef HB_DISABLE_DEPRECATED
93     unsigned int ret  = 0;
94 #else
95     unsigned int ret = func.decompose_compatibility (this, u, decomposed, user_data.decompose_compatibility);
96 #endif
97     if (ret == 1 && u == decomposed[0]) {
98       decomposed[0] = 0;
99       return 0;
100     }
101     decomposed[ret] = 0;
102     return ret;
103   }
104 
105   unsigned int
modified_combining_classhb_unicode_funcs_t106   modified_combining_class (hb_codepoint_t u)
107   {
108     /* XXX This hack belongs to the USE shaper (for Tai Tham):
109      * Reorder SAKOT to ensure it comes after any tone marks. */
110     if (unlikely (u == 0x1A60u)) return 254;
111 
112     /* XXX This hack belongs to the Tibetan shaper:
113      * Reorder PADMA to ensure it comes after any vowel marks. */
114     if (unlikely (u == 0x0FC6u)) return 254;
115     /* Reorder TSA -PHRU to reorder before U+0F74 */
116     if (unlikely (u == 0x0F39u)) return 127;
117 
118     return _hb_modified_combining_class[combining_class (u)];
119   }
120 
121   static hb_bool_t
is_variation_selectorhb_unicode_funcs_t122   is_variation_selector (hb_codepoint_t unicode)
123   {
124     /* U+180B..180D MONGOLIAN FREE VARIATION SELECTORs are handled in the
125      * Arabic shaper.  No need to match them here. */
126     return unlikely (hb_in_ranges<hb_codepoint_t> (unicode,
127 						   0xFE00u, 0xFE0Fu, /* VARIATION SELECTOR-1..16 */
128 						   0xE0100u, 0xE01EFu));  /* VARIATION SELECTOR-17..256 */
129   }
130 
131   /* Default_Ignorable codepoints:
132    *
133    * Note: While U+115F, U+1160, U+3164 and U+FFA0 are Default_Ignorable,
134    * we do NOT want to hide them, as the way Uniscribe has implemented them
135    * is with regular spacing glyphs, and that's the way fonts are made to work.
136    * As such, we make exceptions for those four.
137    * Also ignoring U+1BCA0..1BCA3. https://github.com/harfbuzz/harfbuzz/issues/503
138    *
139    * Unicode 7.0:
140    * $ grep '; Default_Ignorable_Code_Point ' DerivedCoreProperties.txt | sed 's/;.*#/#/'
141    * 00AD          # Cf       SOFT HYPHEN
142    * 034F          # Mn       COMBINING GRAPHEME JOINER
143    * 061C          # Cf       ARABIC LETTER MARK
144    * 115F..1160    # Lo   [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER
145    * 17B4..17B5    # Mn   [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA
146    * 180B..180D    # Mn   [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE
147    * 180E          # Cf       MONGOLIAN VOWEL SEPARATOR
148    * 200B..200F    # Cf   [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK
149    * 202A..202E    # Cf   [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE
150    * 2060..2064    # Cf   [5] WORD JOINER..INVISIBLE PLUS
151    * 2065          # Cn       <reserved-2065>
152    * 2066..206F    # Cf  [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES
153    * 3164          # Lo       HANGUL FILLER
154    * FE00..FE0F    # Mn  [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16
155    * FEFF          # Cf       ZERO WIDTH NO-BREAK SPACE
156    * FFA0          # Lo       HALFWIDTH HANGUL FILLER
157    * FFF0..FFF8    # Cn   [9] <reserved-FFF0>..<reserved-FFF8>
158    * 1BCA0..1BCA3  # Cf   [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP
159    * 1D173..1D17A  # Cf   [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE
160    * E0000         # Cn       <reserved-E0000>
161    * E0001         # Cf       LANGUAGE TAG
162    * E0002..E001F  # Cn  [30] <reserved-E0002>..<reserved-E001F>
163    * E0020..E007F  # Cf  [96] TAG SPACE..CANCEL TAG
164    * E0080..E00FF  # Cn [128] <reserved-E0080>..<reserved-E00FF>
165    * E0100..E01EF  # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
166    * E01F0..E0FFF  # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
167    */
168   static hb_bool_t
is_default_ignorablehb_unicode_funcs_t169   is_default_ignorable (hb_codepoint_t ch)
170   {
171     hb_codepoint_t plane = ch >> 16;
172     if (likely (plane == 0))
173     {
174       /* BMP */
175       hb_codepoint_t page = ch >> 8;
176       switch (page) {
177 	case 0x00: return unlikely (ch == 0x00ADu);
178 	case 0x03: return unlikely (ch == 0x034Fu);
179 	case 0x06: return unlikely (ch == 0x061Cu);
180 	case 0x17: return hb_in_range<hb_codepoint_t> (ch, 0x17B4u, 0x17B5u);
181 	case 0x18: return hb_in_range<hb_codepoint_t> (ch, 0x180Bu, 0x180Eu);
182 	case 0x20: return hb_in_ranges<hb_codepoint_t> (ch, 0x200Bu, 0x200Fu,
183 					    0x202Au, 0x202Eu,
184 					    0x2060u, 0x206Fu);
185 	case 0xFE: return hb_in_range<hb_codepoint_t> (ch, 0xFE00u, 0xFE0Fu) || ch == 0xFEFFu;
186 	case 0xFF: return hb_in_range<hb_codepoint_t> (ch, 0xFFF0u, 0xFFF8u);
187 	default: return false;
188       }
189     }
190     else
191     {
192       /* Other planes */
193       switch (plane) {
194 	case 0x01: return hb_in_range<hb_codepoint_t> (ch, 0x1D173u, 0x1D17Au);
195 	case 0x0E: return hb_in_range<hb_codepoint_t> (ch, 0xE0000u, 0xE0FFFu);
196 	default: return false;
197       }
198     }
199   }
200 
201   /* Space estimates based on:
202    * https://unicode.org/charts/PDF/U2000.pdf
203    * https://docs.microsoft.com/en-us/typography/develop/character-design-standards/whitespace
204    */
205   enum space_t {
206     NOT_SPACE = 0,
207     SPACE_EM   = 1,
208     SPACE_EM_2 = 2,
209     SPACE_EM_3 = 3,
210     SPACE_EM_4 = 4,
211     SPACE_EM_5 = 5,
212     SPACE_EM_6 = 6,
213     SPACE_EM_16 = 16,
214     SPACE_4_EM_18,	/* 4/18th of an EM! */
215     SPACE,
216     SPACE_FIGURE,
217     SPACE_PUNCTUATION,
218     SPACE_NARROW,
219   };
220   static space_t
space_fallback_typehb_unicode_funcs_t221   space_fallback_type (hb_codepoint_t u)
222   {
223     switch (u)
224     {
225       /* All GC=Zs chars that can use a fallback. */
226       default:	    return NOT_SPACE;	/* U+1680 OGHAM SPACE MARK */
227       case 0x0020u: return SPACE;	/* U+0020 SPACE */
228       case 0x00A0u: return SPACE;	/* U+00A0 NO-BREAK SPACE */
229       case 0x2000u: return SPACE_EM_2;	/* U+2000 EN QUAD */
230       case 0x2001u: return SPACE_EM;	/* U+2001 EM QUAD */
231       case 0x2002u: return SPACE_EM_2;	/* U+2002 EN SPACE */
232       case 0x2003u: return SPACE_EM;	/* U+2003 EM SPACE */
233       case 0x2004u: return SPACE_EM_3;	/* U+2004 THREE-PER-EM SPACE */
234       case 0x2005u: return SPACE_EM_4;	/* U+2005 FOUR-PER-EM SPACE */
235       case 0x2006u: return SPACE_EM_6;	/* U+2006 SIX-PER-EM SPACE */
236       case 0x2007u: return SPACE_FIGURE;	/* U+2007 FIGURE SPACE */
237       case 0x2008u: return SPACE_PUNCTUATION;	/* U+2008 PUNCTUATION SPACE */
238       case 0x2009u: return SPACE_EM_5;		/* U+2009 THIN SPACE */
239       case 0x200Au: return SPACE_EM_16;		/* U+200A HAIR SPACE */
240       case 0x202Fu: return SPACE_NARROW;	/* U+202F NARROW NO-BREAK SPACE */
241       case 0x205Fu: return SPACE_4_EM_18;	/* U+205F MEDIUM MATHEMATICAL SPACE */
242       case 0x3000u: return SPACE_EM;		/* U+3000 IDEOGRAPHIC SPACE */
243     }
244   }
245 
246   struct {
247 #define HB_UNICODE_FUNC_IMPLEMENT(name) hb_unicode_##name##_func_t name;
248     HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS
249 #undef HB_UNICODE_FUNC_IMPLEMENT
250   } func;
251 
252   struct {
253 #define HB_UNICODE_FUNC_IMPLEMENT(name) void *name;
254     HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS
255 #undef HB_UNICODE_FUNC_IMPLEMENT
256   } user_data;
257 
258   struct {
259 #define HB_UNICODE_FUNC_IMPLEMENT(name) hb_destroy_func_t name;
260     HB_UNICODE_FUNCS_IMPLEMENT_CALLBACKS
261 #undef HB_UNICODE_FUNC_IMPLEMENT
262   } destroy;
263 };
264 DECLARE_NULL_INSTANCE (hb_unicode_funcs_t);
265 
266 
267 /*
268  * Modified combining marks
269  */
270 
271 /* Hebrew
272  *
273  * We permute the "fixed-position" classes 10-26 into the order
274  * described in the SBL Hebrew manual:
275  *
276  * https://www.sbl-site.org/Fonts/SBLHebrewUserManual1.5x.pdf
277  *
278  * (as recommended by:
279  *  https://forum.fontlab.com/archive-old-microsoft-volt-group/vista-and-diacritic-ordering/msg22823/)
280  *
281  * More details here:
282  * https://bugzilla.mozilla.org/show_bug.cgi?id=662055
283  */
284 #define HB_MODIFIED_COMBINING_CLASS_CCC10 22 /* sheva */
285 #define HB_MODIFIED_COMBINING_CLASS_CCC11 15 /* hataf segol */
286 #define HB_MODIFIED_COMBINING_CLASS_CCC12 16 /* hataf patah */
287 #define HB_MODIFIED_COMBINING_CLASS_CCC13 17 /* hataf qamats */
288 #define HB_MODIFIED_COMBINING_CLASS_CCC14 23 /* hiriq */
289 #define HB_MODIFIED_COMBINING_CLASS_CCC15 18 /* tsere */
290 #define HB_MODIFIED_COMBINING_CLASS_CCC16 19 /* segol */
291 #define HB_MODIFIED_COMBINING_CLASS_CCC17 20 /* patah */
292 #define HB_MODIFIED_COMBINING_CLASS_CCC18 21 /* qamats */
293 #define HB_MODIFIED_COMBINING_CLASS_CCC19 14 /* holam */
294 #define HB_MODIFIED_COMBINING_CLASS_CCC20 24 /* qubuts */
295 #define HB_MODIFIED_COMBINING_CLASS_CCC21 12 /* dagesh */
296 #define HB_MODIFIED_COMBINING_CLASS_CCC22 25 /* meteg */
297 #define HB_MODIFIED_COMBINING_CLASS_CCC23 13 /* rafe */
298 #define HB_MODIFIED_COMBINING_CLASS_CCC24 10 /* shin dot */
299 #define HB_MODIFIED_COMBINING_CLASS_CCC25 11 /* sin dot */
300 #define HB_MODIFIED_COMBINING_CLASS_CCC26 26 /* point varika */
301 
302 /*
303  * Arabic
304  *
305  * Modify to move Shadda (ccc=33) before other marks.  See:
306  * https://unicode.org/faq/normalization.html#8
307  * https://unicode.org/faq/normalization.html#9
308  */
309 #define HB_MODIFIED_COMBINING_CLASS_CCC27 28 /* fathatan */
310 #define HB_MODIFIED_COMBINING_CLASS_CCC28 29 /* dammatan */
311 #define HB_MODIFIED_COMBINING_CLASS_CCC29 30 /* kasratan */
312 #define HB_MODIFIED_COMBINING_CLASS_CCC30 31 /* fatha */
313 #define HB_MODIFIED_COMBINING_CLASS_CCC31 32 /* damma */
314 #define HB_MODIFIED_COMBINING_CLASS_CCC32 33 /* kasra */
315 #define HB_MODIFIED_COMBINING_CLASS_CCC33 27 /* shadda */
316 #define HB_MODIFIED_COMBINING_CLASS_CCC34 34 /* sukun */
317 #define HB_MODIFIED_COMBINING_CLASS_CCC35 35 /* superscript alef */
318 
319 /* Syriac */
320 #define HB_MODIFIED_COMBINING_CLASS_CCC36 36 /* superscript alaph */
321 
322 /* Telugu
323  *
324  * Modify Telugu length marks (ccc=84, ccc=91).
325  * These are the only matras in the main Indic scripts range that have
326  * a non-zero ccc.  That makes them reorder with the Halant (ccc=9).
327  * Assign 5 and 6, which are otherwise unassigned.
328  */
329 #define HB_MODIFIED_COMBINING_CLASS_CCC84 5 /* length mark */
330 #define HB_MODIFIED_COMBINING_CLASS_CCC91 6 /* ai length mark */
331 
332 /* Thai
333  *
334  * Modify U+0E38 and U+0E39 (ccc=103) to be reordered before U+0E3A (ccc=9).
335  * Assign 3, which is unassigned otherwise.
336  * Uniscribe does this reordering too.
337  */
338 #define HB_MODIFIED_COMBINING_CLASS_CCC103 3 /* sara u / sara uu */
339 #define HB_MODIFIED_COMBINING_CLASS_CCC107 107 /* mai * */
340 
341 /* Lao */
342 #define HB_MODIFIED_COMBINING_CLASS_CCC118 118 /* sign u / sign uu */
343 #define HB_MODIFIED_COMBINING_CLASS_CCC122 122 /* mai * */
344 
345 /* Tibetan
346  *
347  * In case of multiple vowel-signs, use u first (but after achung)
348  * this allows Dzongkha multi-vowel shortcuts to render correctly
349  */
350 #define HB_MODIFIED_COMBINING_CLASS_CCC129 129 /* sign aa */
351 #define HB_MODIFIED_COMBINING_CLASS_CCC130 132 /* sign i */
352 #define HB_MODIFIED_COMBINING_CLASS_CCC132 131 /* sign u */
353 
354 /* Misc */
355 
356 #define HB_UNICODE_GENERAL_CATEGORY_IS_MARK(gen_cat) \
357 	(FLAG_UNSAFE (gen_cat) & \
358 	 (FLAG (HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK) | \
359 	  FLAG (HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK) | \
360 	  FLAG (HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK)))
361 
362 
363 /*
364  * Ranges, used for bsearch tables.
365  */
366 
367 struct hb_unicode_range_t
368 {
369   static int
cmphb_unicode_range_t370   cmp (const void *_key, const void *_item)
371   {
372     hb_codepoint_t cp = *((hb_codepoint_t *) _key);
373     const hb_unicode_range_t *range = (hb_unicode_range_t *) _item;
374 
375     if (cp < range->start)
376       return -1;
377     else if (cp <= range->end)
378       return 0;
379     else
380       return +1;
381   }
382 
383   hb_codepoint_t start;
384   hb_codepoint_t end;
385 };
386 
387 /*
388  * Emoji.
389  */
390 
391 HB_INTERNAL bool
392 _hb_unicode_is_emoji_Extended_Pictographic (hb_codepoint_t cp);
393 
394 
395 extern "C" HB_INTERNAL hb_unicode_funcs_t *hb_ucd_get_unicode_funcs ();
396 
397 
398 #endif /* HB_UNICODE_HH */
399