1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2010-2015, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ******************************************************************************* 8 * collationdata.h 9 * 10 * created on: 2010oct27 11 * created by: Markus W. Scherer 12 */ 13 14 #ifndef __COLLATIONDATA_H__ 15 #define __COLLATIONDATA_H__ 16 17 #include "unicode/utypes.h" 18 19 #if !UCONFIG_NO_COLLATION 20 21 #include "unicode/ucol.h" 22 #include "unicode/uniset.h" 23 #include "collation.h" 24 #include "normalizer2impl.h" 25 #include "utrie2.h" 26 27 struct UDataMemory; 28 29 U_NAMESPACE_BEGIN 30 31 class UVector32; 32 33 /** 34 * Collation data container. 35 * Immutable data created by a CollationDataBuilder, or loaded from a file, 36 * or deserialized from API-provided binary data. 37 * 38 * Includes data for the collation base (root/default), aliased if this is not the base. 39 */ 40 struct U_I18N_API CollationData : public UMemory { 41 // Note: The ucadata.icu loader could discover the reserved ranges by setting an array 42 // parallel with the ranges, and resetting ranges that are indexed. 43 // The reordering builder code could clone the resulting template array. 44 enum { 45 REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14, 46 REORDER_RESERVED_AFTER_LATIN 47 }; 48 49 enum { 50 MAX_NUM_SPECIAL_REORDER_CODES = 8, 51 /** C++ only, data reader check scriptStartsLength. */ 52 MAX_NUM_SCRIPT_RANGES = 256 53 }; 54 CollationDataCollationData55 CollationData(const Normalizer2Impl &nfc) 56 : trie(NULL), 57 ce32s(NULL), ces(NULL), contexts(NULL), base(NULL), 58 jamoCE32s(NULL), 59 nfcImpl(nfc), 60 numericPrimary(0x12000000), 61 ce32sLength(0), cesLength(0), contextsLength(0), 62 compressibleBytes(NULL), 63 unsafeBackwardSet(NULL), 64 fastLatinTable(NULL), fastLatinTableLength(0), 65 numScripts(0), scriptsIndex(NULL), scriptStarts(NULL), scriptStartsLength(0), 66 rootElements(NULL), rootElementsLength(0) {} 67 getCE32CollationData68 uint32_t getCE32(UChar32 c) const { 69 return UTRIE2_GET32(trie, c); 70 } 71 getCE32FromSupplementaryCollationData72 uint32_t getCE32FromSupplementary(UChar32 c) const { 73 return UTRIE2_GET32_FROM_SUPP(trie, c); 74 } 75 isDigitCollationData76 UBool isDigit(UChar32 c) const { 77 return c < 0x660 ? c <= 0x39 && 0x30 <= c : 78 Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG); 79 } 80 isUnsafeBackwardCollationData81 UBool isUnsafeBackward(UChar32 c, UBool numeric) const { 82 return unsafeBackwardSet->contains(c) || (numeric && isDigit(c)); 83 } 84 isCompressibleLeadByteCollationData85 UBool isCompressibleLeadByte(uint32_t b) const { 86 return compressibleBytes[b]; 87 } 88 isCompressiblePrimaryCollationData89 inline UBool isCompressiblePrimary(uint32_t p) const { 90 return isCompressibleLeadByte(p >> 24); 91 } 92 93 /** 94 * Returns the CE32 from two contexts words. 95 * Access to the defaultCE32 for contraction and prefix matching. 96 */ readCE32CollationData97 static uint32_t readCE32(const UChar *p) { 98 return ((uint32_t)p[0] << 16) | p[1]; 99 } 100 101 /** 102 * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG). 103 * Requires that ce32 is special. 104 */ 105 uint32_t getIndirectCE32(uint32_t ce32) const; 106 /** 107 * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG), 108 * if ce32 is special. 109 */ 110 uint32_t getFinalCE32(uint32_t ce32) const; 111 112 /** 113 * Computes a CE from c's ce32 which has the OFFSET_TAG. 114 */ getCEFromOffsetCE32CollationData115 int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const { 116 int64_t dataCE = ces[Collation::indexFromCE32(ce32)]; 117 return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE)); 118 } 119 120 /** 121 * Returns the single CE that c maps to. 122 * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE. 123 */ 124 int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const; 125 126 /** 127 * Returns the FCD16 value for code point c. c must be >= 0. 128 */ getFCD16CollationData129 uint16_t getFCD16(UChar32 c) const { 130 return nfcImpl.getFCD16(c); 131 } 132 133 /** 134 * Returns the first primary for the script's reordering group. 135 * @return the primary with only the first primary lead byte of the group 136 * (not necessarily an actual root collator primary weight), 137 * or 0 if the script is unknown 138 */ 139 uint32_t getFirstPrimaryForGroup(int32_t script) const; 140 141 /** 142 * Returns the last primary for the script's reordering group. 143 * @return the last primary of the group 144 * (not an actual root collator primary weight), 145 * or 0 if the script is unknown 146 */ 147 uint32_t getLastPrimaryForGroup(int32_t script) const; 148 149 /** 150 * Finds the reordering group which contains the primary weight. 151 * @return the first script of the group, or -1 if the weight is beyond the last group 152 */ 153 int32_t getGroupForPrimary(uint32_t p) const; 154 155 int32_t getEquivalentScripts(int32_t script, 156 int32_t dest[], int32_t capacity, UErrorCode &errorCode) const; 157 158 /** 159 * Writes the permutation of primary-weight ranges 160 * for the given reordering of scripts and groups. 161 * The caller checks for illegal arguments and 162 * takes care of [DEFAULT] and memory allocation. 163 * 164 * Each list element will be a (limit, offset) pair as described 165 * for the CollationSettings::reorderRanges. 166 * The list will be empty if no ranges are reordered. 167 */ 168 void makeReorderRanges(const int32_t *reorder, int32_t length, 169 UVector32 &ranges, UErrorCode &errorCode) const; 170 171 /** @see jamoCE32s */ 172 static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27; 173 174 /** Main lookup trie. */ 175 const UTrie2 *trie; 176 /** 177 * Array of CE32 values. 178 * At index 0 there must be CE32(U+0000) 179 * to support U+0000's special-tag for NUL-termination handling. 180 */ 181 const uint32_t *ce32s; 182 /** Array of CE values for expansions and OFFSET_TAG. */ 183 const int64_t *ces; 184 /** Array of prefix and contraction-suffix matching data. */ 185 const UChar *contexts; 186 /** Base collation data, or NULL if this data itself is a base. */ 187 const CollationData *base; 188 /** 189 * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T. 190 * They are normally simple CE32s, rarely expansions. 191 * For fast handling of HANGUL_TAG. 192 */ 193 const uint32_t *jamoCE32s; 194 const Normalizer2Impl &nfcImpl; 195 /** The single-byte primary weight (xx000000) for numeric collation. */ 196 uint32_t numericPrimary; 197 198 int32_t ce32sLength; 199 int32_t cesLength; 200 int32_t contextsLength; 201 202 /** 256 flags for which primary-weight lead bytes are compressible. */ 203 const UBool *compressibleBytes; 204 /** 205 * Set of code points that are unsafe for starting string comparison after an identical prefix, 206 * or in backwards CE iteration. 207 */ 208 const UnicodeSet *unsafeBackwardSet; 209 210 /** 211 * Fast Latin table for common-Latin-text string comparisons. 212 * Data structure see class CollationFastLatin. 213 */ 214 const uint16_t *fastLatinTable; 215 int32_t fastLatinTableLength; 216 217 /** 218 * Data for scripts and reordering groups. 219 * Uses include building a reordering permutation table and 220 * providing script boundaries to AlphabeticIndex. 221 */ 222 int32_t numScripts; 223 /** 224 * The length of scriptsIndex is numScripts+16. 225 * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts. 226 * 16 special reorder codes (not all used) are mapped starting at numScripts. 227 * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit. 228 * There are special codes at the end for reorder-reserved primary ranges. 229 * 230 * Multiple scripts may share a range and index, for example Hira & Kana. 231 */ 232 const uint16_t *scriptsIndex; 233 /** 234 * Start primary weight (top 16 bits only) for a group/script/reserved range 235 * indexed by scriptsIndex. 236 * The first range (separators & terminators) and the last range (trailing weights) 237 * are not reorderable, and no scriptsIndex entry points to them. 238 */ 239 const uint16_t *scriptStarts; 240 int32_t scriptStartsLength; 241 242 /** 243 * Collation elements in the root collator. 244 * Used by the CollationRootElements class. The data structure is described there. 245 * NULL in a tailoring. 246 */ 247 const uint32_t *rootElements; 248 int32_t rootElementsLength; 249 250 private: 251 int32_t getScriptIndex(int32_t script) const; 252 void makeReorderRanges(const int32_t *reorder, int32_t length, 253 UBool latinMustMove, 254 UVector32 &ranges, UErrorCode &errorCode) const; 255 int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const; 256 int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const; 257 }; 258 259 U_NAMESPACE_END 260 261 #endif // !UCONFIG_NO_COLLATION 262 #endif // __COLLATIONDATA_H__ 263