1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 * Copyright (c) 1996-2016, International Business Machines Corporation
6 *               and others. All Rights Reserved.
7 *******************************************************************************
8 * File unorm.h
9 *
10 * Created by: Vladimir Weinstein 12052000
11 *
12 * Modification history :
13 *
14 * Date        Name        Description
15 * 02/01/01    synwee      Added normalization quickcheck enum and method.
16 */
17 #ifndef UNORM_H
18 #define UNORM_H
19 
20 #include "unicode/utypes.h"
21 
22 #if !UCONFIG_NO_NORMALIZATION
23 
24 #include "unicode/uiter.h"
25 #include "unicode/unorm2.h"
26 
27 /**
28  * \file
29  * \brief C API: Unicode Normalization
30  *
31  * Old Unicode normalization API.
32  *
33  * This API has been replaced by the unorm2.h API and is only available
34  * for backward compatibility. The functions here simply delegate to the
35  * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
36  * There is one exception: The new API does not provide a replacement for unorm_compare().
37  * Its declaration has been moved to unorm2.h.
38  *
39  * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
40  * decomposed form, allowing for easier sorting and searching of text.
41  * <code>unorm_normalize</code> supports the standard normalization forms described in
42  * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
43  * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
44  *
45  * Characters with accents or other adornments can be encoded in
46  * several different ways in Unicode.  For example, take the character A-acute.
47  * In Unicode, this can be encoded as a single character (the
48  * "composed" form):
49  *
50  * \code
51  *      00C1    LATIN CAPITAL LETTER A WITH ACUTE
52  * \endcode
53  *
54  * or as two separate characters (the "decomposed" form):
55  *
56  * \code
57  *      0041    LATIN CAPITAL LETTER A
58  *      0301    COMBINING ACUTE ACCENT
59  * \endcode
60  *
61  * To a user of your program, however, both of these sequences should be
62  * treated as the same "user-level" character "A with acute accent".  When you are searching or
63  * comparing text, you must ensure that these two sequences are treated
64  * equivalently.  In addition, you must handle characters with more than one
65  * accent.  Sometimes the order of a character's combining accents is
66  * significant, while in other cases accent sequences in different orders are
67  * really equivalent.
68  *
69  * Similarly, the string "ffi" can be encoded as three separate letters:
70  *
71  * \code
72  *      0066    LATIN SMALL LETTER F
73  *      0066    LATIN SMALL LETTER F
74  *      0069    LATIN SMALL LETTER I
75  * \endcode
76  *
77  * or as the single character
78  *
79  * \code
80  *      FB03    LATIN SMALL LIGATURE FFI
81  * \endcode
82  *
83  * The ffi ligature is not a distinct semantic character, and strictly speaking
84  * it shouldn't be in Unicode at all, but it was included for compatibility
85  * with existing character sets that already provided it.  The Unicode standard
86  * identifies such characters by giving them "compatibility" decompositions
87  * into the corresponding semantic characters.  When sorting and searching, you
88  * will often want to use these mappings.
89  *
90  * <code>unorm_normalize</code> helps solve these problems by transforming text into the
91  * canonical composed and decomposed forms as shown in the first example above.
92  * In addition, you can have it perform compatibility decompositions so that
93  * you can treat compatibility characters the same as their equivalents.
94  * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
95  * order, so that you do not have to worry about accent rearrangement on your
96  * own.
97  *
98  * Form FCD, "Fast C or D", is also designed for collation.
99  * It allows to work on strings that are not necessarily normalized
100  * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
101  * characters and their decomposed equivalents the same.
102  *
103  * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
104  * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
105  * themselves.
106  *
107  * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
108  * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
109  * as their decompositions do not need canonical reordering.
110  *
111  * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
112  * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
113  * return UNORM_YES for most strings in practice.
114  *
115  * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
116  *
117  * For more details on FCD see the collation design document:
118  * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
119  *
120  * ICU collation performs either NFD or FCD normalization automatically if normalization
121  * is turned on for the collator object.
122  * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
123  * transliteration/transcription, unique representations, etc.
124  *
125  * The W3C generally recommends to exchange texts in NFC.
126  * Note also that most legacy character encodings use only precomposed forms and often do not
127  * encode any combining marks by themselves. For conversion to such character encodings the
128  * Unicode text needs to be normalized to NFC.
129  * For more usage examples, see the Unicode Standard Annex.
130  */
131 
132 // Do not conditionalize the following enum with #ifndef U_HIDE_DEPRECATED_API,
133 // it is needed for layout of Normalizer object.
134 #ifndef U_FORCE_HIDE_DEPRECATED_API
135 
136 /**
137  * Constants for normalization modes.
138  * @deprecated ICU 56 Use unorm2.h instead.
139  */
140 typedef enum {
141   /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */
142   UNORM_NONE = 1,
143   /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */
144   UNORM_NFD = 2,
145   /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */
146   UNORM_NFKD = 3,
147   /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
148   UNORM_NFC = 4,
149   /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */
150   UNORM_DEFAULT = UNORM_NFC,
151   /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
152   UNORM_NFKC =5,
153   /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */
154   UNORM_FCD = 6,
155 
156   /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */
157   UNORM_MODE_COUNT
158 } UNormalizationMode;
159 
160 #endif  // U_FORCE_HIDE_DEPRECATED_API
161 
162 #ifndef U_HIDE_DEPRECATED_API
163 
164 /**
165  * Constants for options flags for normalization.
166  * Use 0 for default options,
167  * including normalization according to the Unicode version
168  * that is currently supported by ICU (see u_getUnicodeVersion).
169  * @deprecated ICU 56 Use unorm2.h instead.
170  */
171 enum {
172     /**
173      * Options bit set value to select Unicode 3.2 normalization
174      * (except NormalizationCorrections).
175      * At most one Unicode version can be selected at a time.
176      * @deprecated ICU 56 Use unorm2.h instead.
177      */
178     UNORM_UNICODE_3_2=0x20
179 };
180 
181 /**
182  * Lowest-order bit number of unorm_compare() options bits corresponding to
183  * normalization options bits.
184  *
185  * The options parameter for unorm_compare() uses most bits for
186  * itself and for various comparison and folding flags.
187  * The most significant bits, however, are shifted down and passed on
188  * to the normalization implementation.
189  * (That is, from unorm_compare(..., options, ...),
190  * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
191  * internal normalization functions.)
192  *
193  * @see unorm_compare
194  * @deprecated ICU 56 Use unorm2.h instead.
195  */
196 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
197 
198 /**
199  * Normalize a string.
200  * The string will be normalized according the specified normalization mode
201  * and options.
202  * The source and result buffers must not be the same, nor overlap.
203  *
204  * @param source The string to normalize.
205  * @param sourceLength The length of source, or -1 if NUL-terminated.
206  * @param mode The normalization mode; one of UNORM_NONE,
207  *             UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
208  * @param options The normalization options, ORed together (0 for no options).
209  * @param result A pointer to a buffer to receive the result string.
210  *               The result string is NUL-terminated if possible.
211  * @param resultLength The maximum size of result.
212  * @param status A pointer to a UErrorCode to receive any errors.
213  * @return The total buffer size needed; if greater than resultLength,
214  *         the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
215  * @deprecated ICU 56 Use unorm2.h instead.
216  */
217 U_DEPRECATED int32_t U_EXPORT2
218 unorm_normalize(const UChar *source, int32_t sourceLength,
219                 UNormalizationMode mode, int32_t options,
220                 UChar *result, int32_t resultLength,
221                 UErrorCode *status);
222 
223 /**
224  * Performing quick check on a string, to quickly determine if the string is
225  * in a particular normalization format.
226  * Three types of result can be returned UNORM_YES, UNORM_NO or
227  * UNORM_MAYBE. Result UNORM_YES indicates that the argument
228  * string is in the desired normalized format, UNORM_NO determines that
229  * argument string is not in the desired normalized format. A
230  * UNORM_MAYBE result indicates that a more thorough check is required,
231  * the user may have to put the string in its normalized form and compare the
232  * results.
233  *
234  * @param source       string for determining if it is in a normalized format
235  * @param sourcelength length of source to test, or -1 if NUL-terminated
236  * @param mode         which normalization form to test for
237  * @param status       a pointer to a UErrorCode to receive any errors
238  * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
239  *
240  * @see unorm_isNormalized
241  * @deprecated ICU 56 Use unorm2.h instead.
242  */
243 U_DEPRECATED UNormalizationCheckResult U_EXPORT2
244 unorm_quickCheck(const UChar *source, int32_t sourcelength,
245                  UNormalizationMode mode,
246                  UErrorCode *status);
247 
248 /**
249  * Performing quick check on a string; same as unorm_quickCheck but
250  * takes an extra options parameter like most normalization functions.
251  *
252  * @param src        String that is to be tested if it is in a normalization format.
253  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
254  * @param mode       Which normalization form to test for.
255  * @param options    The normalization options, ORed together (0 for no options).
256  * @param pErrorCode ICU error code in/out parameter.
257  *                   Must fulfill U_SUCCESS before the function call.
258  * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
259  *
260  * @see unorm_quickCheck
261  * @see unorm_isNormalized
262  * @deprecated ICU 56 Use unorm2.h instead.
263  */
264 U_DEPRECATED UNormalizationCheckResult U_EXPORT2
265 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
266                             UNormalizationMode mode, int32_t options,
267                             UErrorCode *pErrorCode);
268 
269 /**
270  * Test if a string is in a given normalization form.
271  * This is semantically equivalent to source.equals(normalize(source, mode)) .
272  *
273  * Unlike unorm_quickCheck(), this function returns a definitive result,
274  * never a "maybe".
275  * For NFD, NFKD, and FCD, both functions work exactly the same.
276  * For NFC and NFKC where quickCheck may return "maybe", this function will
277  * perform further tests to arrive at a true/false result.
278  *
279  * @param src        String that is to be tested if it is in a normalization format.
280  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
281  * @param mode       Which normalization form to test for.
282  * @param pErrorCode ICU error code in/out parameter.
283  *                   Must fulfill U_SUCCESS before the function call.
284  * @return Boolean value indicating whether the source string is in the
285  *         "mode" normalization form.
286  *
287  * @see unorm_quickCheck
288  * @deprecated ICU 56 Use unorm2.h instead.
289  */
290 U_DEPRECATED UBool U_EXPORT2
291 unorm_isNormalized(const UChar *src, int32_t srcLength,
292                    UNormalizationMode mode,
293                    UErrorCode *pErrorCode);
294 
295 /**
296  * Test if a string is in a given normalization form; same as unorm_isNormalized but
297  * takes an extra options parameter like most normalization functions.
298  *
299  * @param src        String that is to be tested if it is in a normalization format.
300  * @param srcLength  Length of source to test, or -1 if NUL-terminated.
301  * @param mode       Which normalization form to test for.
302  * @param options    The normalization options, ORed together (0 for no options).
303  * @param pErrorCode ICU error code in/out parameter.
304  *                   Must fulfill U_SUCCESS before the function call.
305  * @return Boolean value indicating whether the source string is in the
306  *         "mode/options" normalization form.
307  *
308  * @see unorm_quickCheck
309  * @see unorm_isNormalized
310  * @deprecated ICU 56 Use unorm2.h instead.
311  */
312 U_DEPRECATED UBool U_EXPORT2
313 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
314                               UNormalizationMode mode, int32_t options,
315                               UErrorCode *pErrorCode);
316 
317 /**
318  * Iterative normalization forward.
319  * This function (together with unorm_previous) is somewhat
320  * similar to the C++ Normalizer class (see its non-static functions).
321  *
322  * Iterative normalization is useful when only a small portion of a longer
323  * string/text needs to be processed.
324  *
325  * For example, the likelihood may be high that processing the first 10% of some
326  * text will be sufficient to find certain data.
327  * Another example: When one wants to concatenate two normalized strings and get a
328  * normalized result, it is much more efficient to normalize just a small part of
329  * the result around the concatenation place instead of re-normalizing everything.
330  *
331  * The input text is an instance of the C character iteration API UCharIterator.
332  * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
333  * other kind of text object.
334  *
335  * If a buffer overflow occurs, then the caller needs to reset the iterator to the
336  * old index and call the function again with a larger buffer - if the caller cares
337  * for the actual output.
338  * Regardless of the output buffer, the iterator will always be moved to the next
339  * normalization boundary.
340  *
341  * This function (like unorm_previous) serves two purposes:
342  *
343  * 1) To find the next boundary so that the normalization of the part of the text
344  * from the current position to that boundary does not affect and is not affected
345  * by the part of the text beyond that boundary.
346  *
347  * 2) To normalize the text up to the boundary.
348  *
349  * The second step is optional, per the doNormalize parameter.
350  * It is omitted for operations like string concatenation, where the two adjacent
351  * string ends need to be normalized together.
352  * In such a case, the output buffer will just contain a copy of the text up to the
353  * boundary.
354  *
355  * pNeededToNormalize is an output-only parameter. Its output value is only defined
356  * if normalization was requested (doNormalize) and successful (especially, no
357  * buffer overflow).
358  * It is useful for operations like a normalizing transliterator, where one would
359  * not want to replace a piece of text if it is not modified.
360  *
361  * If doNormalize==true and pNeededToNormalize!=NULL then *pNeeded... is set true
362  * if the normalization was necessary.
363  *
364  * If doNormalize==false then *pNeededToNormalize will be set to false.
365  *
366  * If the buffer overflows, then *pNeededToNormalize will be undefined;
367  * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
368  * will be undefined.
369  *
370  * @param src The input text in the form of a C character iterator.
371  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
372  * @param destCapacity The number of UChars that fit into dest.
373  * @param mode The normalization mode.
374  * @param options The normalization options, ORed together (0 for no options).
375  * @param doNormalize Indicates if the source text up to the next boundary
376  *                    is to be normalized (true) or just copied (false).
377  * @param pNeededToNormalize Output flag indicating if the normalization resulted in
378  *                           different text from the input.
379  *                           Not defined if an error occurs including buffer overflow.
380  *                           Always false if !doNormalize.
381  * @param pErrorCode ICU error code in/out parameter.
382  *                   Must fulfill U_SUCCESS before the function call.
383  * @return Length of output (number of UChars) when successful or buffer overflow.
384  *
385  * @see unorm_previous
386  * @see unorm_normalize
387  *
388  * @deprecated ICU 56 Use unorm2.h instead.
389  */
390 U_DEPRECATED int32_t U_EXPORT2
391 unorm_next(UCharIterator *src,
392            UChar *dest, int32_t destCapacity,
393            UNormalizationMode mode, int32_t options,
394            UBool doNormalize, UBool *pNeededToNormalize,
395            UErrorCode *pErrorCode);
396 
397 /**
398  * Iterative normalization backward.
399  * This function (together with unorm_next) is somewhat
400  * similar to the C++ Normalizer class (see its non-static functions).
401  * For all details see unorm_next.
402  *
403  * @param src The input text in the form of a C character iterator.
404  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
405  * @param destCapacity The number of UChars that fit into dest.
406  * @param mode The normalization mode.
407  * @param options The normalization options, ORed together (0 for no options).
408  * @param doNormalize Indicates if the source text up to the next boundary
409  *                    is to be normalized (true) or just copied (false).
410  * @param pNeededToNormalize Output flag indicating if the normalization resulted in
411  *                           different text from the input.
412  *                           Not defined if an error occurs including buffer overflow.
413  *                           Always false if !doNormalize.
414  * @param pErrorCode ICU error code in/out parameter.
415  *                   Must fulfill U_SUCCESS before the function call.
416  * @return Length of output (number of UChars) when successful or buffer overflow.
417  *
418  * @see unorm_next
419  * @see unorm_normalize
420  *
421  * @deprecated ICU 56 Use unorm2.h instead.
422  */
423 U_DEPRECATED int32_t U_EXPORT2
424 unorm_previous(UCharIterator *src,
425                UChar *dest, int32_t destCapacity,
426                UNormalizationMode mode, int32_t options,
427                UBool doNormalize, UBool *pNeededToNormalize,
428                UErrorCode *pErrorCode);
429 
430 /**
431  * Concatenate normalized strings, making sure that the result is normalized as well.
432  *
433  * If both the left and the right strings are in
434  * the normalization form according to "mode/options",
435  * then the result will be
436  *
437  * \code
438  *     dest=normalize(left+right, mode, options)
439  * \endcode
440  *
441  * With the input strings already being normalized,
442  * this function will use unorm_next() and unorm_previous()
443  * to find the adjacent end pieces of the input strings.
444  * Only the concatenation of these end pieces will be normalized and
445  * then concatenated with the remaining parts of the input strings.
446  *
447  * It is allowed to have dest==left to avoid copying the entire left string.
448  *
449  * @param left Left source string, may be same as dest.
450  * @param leftLength Length of left source string, or -1 if NUL-terminated.
451  * @param right Right source string. Must not be the same as dest, nor overlap.
452  * @param rightLength Length of right source string, or -1 if NUL-terminated.
453  * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
454  * @param destCapacity The number of UChars that fit into dest.
455  * @param mode The normalization mode.
456  * @param options The normalization options, ORed together (0 for no options).
457  * @param pErrorCode ICU error code in/out parameter.
458  *                   Must fulfill U_SUCCESS before the function call.
459  * @return Length of output (number of UChars) when successful or buffer overflow.
460  *
461  * @see unorm_normalize
462  * @see unorm_next
463  * @see unorm_previous
464  *
465  * @deprecated ICU 56 Use unorm2.h instead.
466  */
467 U_DEPRECATED int32_t U_EXPORT2
468 unorm_concatenate(const UChar *left, int32_t leftLength,
469                   const UChar *right, int32_t rightLength,
470                   UChar *dest, int32_t destCapacity,
471                   UNormalizationMode mode, int32_t options,
472                   UErrorCode *pErrorCode);
473 
474 #endif  /* U_HIDE_DEPRECATED_API */
475 #endif /* #if !UCONFIG_NO_NORMALIZATION */
476 #endif
477