1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ********************************************************************** 5 * Copyright (C) 1999-2011, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ********************************************************************** 8 * 9 * ucnv_cnv.h: 10 * Definitions for converter implementations. 11 * 12 * Modification History: 13 * 14 * Date Name Description 15 * 05/09/00 helena Added implementation to handle fallback mappings. 16 * 06/29/2000 helena Major rewrite of the callback APIs. 17 */ 18 19 #ifndef UCNV_CNV_H 20 #define UCNV_CNV_H 21 22 #include "unicode/utypes.h" 23 24 #if !UCONFIG_NO_CONVERSION 25 26 #include "unicode/ucnv.h" 27 #include "unicode/ucnv_err.h" 28 #include "unicode/uset.h" 29 #include "uset_imp.h" 30 31 U_CDECL_BEGIN 32 33 /* this is used in fromUnicode DBCS tables as an "unassigned" marker */ 34 #define missingCharMarker 0xFFFF 35 36 /* 37 * #define missingUCharMarker 0xfffe 38 * 39 * commented out because there are actually two values used in toUnicode tables: 40 * U+fffe "unassigned" 41 * U+ffff "illegal" 42 */ 43 44 /** Forward declaration, see ucnv_bld.h */ 45 struct UConverterSharedData; 46 typedef struct UConverterSharedData UConverterSharedData; 47 48 /* function types for UConverterImpl ---------------------------------------- */ 49 50 /* struct with arguments for UConverterLoad and ucnv_load() */ 51 typedef struct { 52 int32_t size; /* sizeof(UConverterLoadArgs) */ 53 int32_t nestedLoads; /* count nested ucnv_load() calls */ 54 UBool onlyTestIsLoadable; /* input: don't actually load */ 55 UBool reserved0; /* reserved - for good alignment of the pointers */ 56 int16_t reserved; /* reserved - for good alignment of the pointers */ 57 uint32_t options; 58 const char *pkg, *name, *locale; 59 } UConverterLoadArgs; 60 61 #define UCNV_LOAD_ARGS_INITIALIZER \ 62 { (int32_t)sizeof(UConverterLoadArgs), 0, FALSE, FALSE, 0, 0, NULL, NULL, NULL } 63 64 typedef void (*UConverterLoad) (UConverterSharedData *sharedData, 65 UConverterLoadArgs *pArgs, 66 const uint8_t *raw, UErrorCode *pErrorCode); 67 typedef void (*UConverterUnload) (UConverterSharedData *sharedData); 68 69 typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode); 70 typedef void (*UConverterClose) (UConverter *cnv); 71 72 typedef enum UConverterResetChoice { 73 UCNV_RESET_BOTH, 74 UCNV_RESET_TO_UNICODE, 75 UCNV_RESET_FROM_UNICODE 76 } UConverterResetChoice; 77 78 typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice); 79 80 /* 81 * Converter implementation function(s) for ucnv_toUnicode(). 82 * If the toUnicodeWithOffsets function pointer is NULL, 83 * then the toUnicode function will be used and the offsets will be set to -1. 84 * 85 * Must maintain state across buffers. Use toUBytes[toULength] for partial input 86 * sequences; it will be checked in ucnv.c at the end of the input stream 87 * to detect truncated input. 88 * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND. 89 * 90 * The toUnicodeWithOffsets must write exactly as many offset values as target 91 * units. Write offset values of -1 for when the source index corresponding to 92 * the output unit is not known (e.g., the character started in an earlier buffer). 93 * The pArgs->offsets pointer need not be moved forward. 94 * 95 * At function return, either one of the following conditions must be true: 96 * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit 97 * - another error code with toUBytes[toULength] set to the offending input 98 * - no error, and the source is consumed: source==sourceLimit 99 * 100 * The ucnv.c code will handle the end of the input (reset) 101 * (reset, and truncation detection) and callbacks. 102 */ 103 typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *); 104 105 /* 106 * Same rules as for UConverterToUnicode. 107 * A lead surrogate is kept in fromUChar32 across buffers, and if an error 108 * occurs, then the offending input code point must be put into fromUChar32 109 * as well. 110 */ 111 typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *); 112 113 /* 114 * Converter implementation function for ucnv_convertEx(), for direct conversion 115 * between two charsets without pivoting through UTF-16. 116 * The rules are the same as for UConverterToUnicode and UConverterFromUnicode. 117 * In addition, 118 * - The toUnicode side must behave and keep state exactly like the 119 * UConverterToUnicode implementation for the same source charset. 120 * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back 121 * to pivoting. When this function is called, the conversion framework makes 122 * sure that this warning is not set on input. 123 * - Continuing a partial match and flushing the toUnicode replay buffer 124 * are handled by pivoting, using the toUnicode and fromUnicode functions. 125 */ 126 typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs, 127 UConverterToUnicodeArgs *pToUArgs, 128 UErrorCode *pErrorCode); 129 130 /* 131 * Converter implementation function for ucnv_getNextUChar(). 132 * If the function pointer is NULL, then the toUnicode function will be used. 133 * 134 * Will be called at a character boundary (toULength==0). 135 * May return with 136 * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input 137 * (the return value will be ignored) 138 * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!) 139 * with toUBytes[toULength] set to the offending input 140 * (the return value will be ignored) 141 * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer, 142 * to indicate that the ucnv.c code shall call the toUnicode function instead 143 * - return a real code point result 144 * 145 * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed. 146 * 147 * The ucnv.c code will handle the end of the input (reset) 148 * (except for truncation detection!) and callbacks. 149 */ 150 typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *); 151 152 typedef void (*UConverterGetStarters)(const UConverter* converter, 153 UBool starters[256], 154 UErrorCode *pErrorCode); 155 156 /* If this function pointer is null or if the function returns null 157 * the name field in static data struct should be returned by 158 * ucnv_getName() API function 159 */ 160 typedef const char * (*UConverterGetName) (const UConverter *cnv); 161 162 /** 163 * Write the codepage substitution character. 164 * If this function is not set, then ucnv_cbFromUWriteSub() writes 165 * the substitution character from UConverter. 166 * For stateful converters, it is typically necessary to handle this 167 * specificially for the converter in order to properly maintain the state. 168 */ 169 typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode); 170 171 /** 172 * For converter-specific safeClone processing 173 * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes 174 * after the converter is done opening. 175 * If this function is set, then it is called just after a memcpy() of 176 * converter data to the new, empty converter, and is expected to set up 177 * the initial state of the converter. It is not expected to increment the 178 * reference counts of the standard data types such as the shared data. 179 */ 180 typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv, 181 void *stackBuffer, 182 int32_t *pBufferSize, 183 UErrorCode *status); 184 185 /** 186 * Filters for some ucnv_getUnicodeSet() implementation code. 187 */ 188 typedef enum UConverterSetFilter { 189 UCNV_SET_FILTER_NONE, 190 UCNV_SET_FILTER_DBCS_ONLY, 191 UCNV_SET_FILTER_2022_CN, 192 UCNV_SET_FILTER_SJIS, 193 UCNV_SET_FILTER_GR94DBCS, 194 UCNV_SET_FILTER_HZ, 195 UCNV_SET_FILTER_COUNT 196 } UConverterSetFilter; 197 198 /** 199 * Fills the set of Unicode code points that can be converted by an ICU converter. 200 * The API function ucnv_getUnicodeSet() clears the USet before calling 201 * the converter's getUnicodeSet() implementation; the converter should only 202 * add the appropriate code points to allow recursive use. 203 * For example, the ISO-2022-JP converter will call each subconverter's 204 * getUnicodeSet() implementation to consecutively add code points to 205 * the same USet, which will result in a union of the sets of all subconverters. 206 * 207 * For more documentation, see ucnv_getUnicodeSet() in ucnv.h. 208 */ 209 typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv, 210 const USetAdder *sa, 211 UConverterUnicodeSet which, 212 UErrorCode *pErrorCode); 213 214 UBool CONVERSION_U_SUCCESS (UErrorCode err); 215 216 /** 217 * UConverterImpl contains all the data and functions for a converter type. 218 * Its function pointers work much like a C++ vtable. 219 * Many converter types need to define only a subset of the functions; 220 * when a function pointer is NULL, then a default action will be performed. 221 * 222 * Every converter type must implement toUnicode, fromUnicode, and getNextUChar, 223 * otherwise the converter may crash. 224 * Every converter type that has variable-length codepage sequences should 225 * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for 226 * correct offset handling. 227 * All other functions may or may not be implemented - it depends only on 228 * whether the converter type needs them. 229 * 230 * When open() fails, then close() will be called, if present. 231 */ 232 struct UConverterImpl { 233 UConverterType type; 234 235 UConverterLoad load; 236 UConverterUnload unload; 237 238 UConverterOpen open; 239 UConverterClose close; 240 UConverterReset reset; 241 242 UConverterToUnicode toUnicode; 243 UConverterToUnicode toUnicodeWithOffsets; 244 UConverterFromUnicode fromUnicode; 245 UConverterFromUnicode fromUnicodeWithOffsets; 246 UConverterGetNextUChar getNextUChar; 247 248 UConverterGetStarters getStarters; 249 UConverterGetName getName; 250 UConverterWriteSub writeSub; 251 UConverterSafeClone safeClone; 252 UConverterGetUnicodeSet getUnicodeSet; 253 254 UConverterConvert toUTF8; 255 UConverterConvert fromUTF8; 256 }; 257 258 extern const UConverterSharedData 259 _MBCSData, _Latin1Data, 260 _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData, 261 _ISO2022Data, 262 _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6, 263 _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19, 264 _HZData,_ISCIIData, _SCSUData, _ASCIIData, 265 _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData, _CompoundTextData; 266 267 U_CDECL_END 268 269 /** Always use fallbacks from codepage to Unicode */ 270 #define TO_U_USE_FALLBACK(useFallback) TRUE 271 #define UCNV_TO_U_USE_FALLBACK(cnv) TRUE 272 273 /** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */ 274 #define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000) 275 #define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c)) 276 #define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c) 277 278 /** 279 * Magic number for ucnv_getNextUChar(), returned by a 280 * getNextUChar() implementation to indicate to use the converter's toUnicode() 281 * instead of the native function. 282 * @internal 283 */ 284 #define UCNV_GET_NEXT_UCHAR_USE_TO_U -9 285 286 U_CFUNC void 287 ucnv_getCompleteUnicodeSet(const UConverter *cnv, 288 const USetAdder *sa, 289 UConverterUnicodeSet which, 290 UErrorCode *pErrorCode); 291 292 U_CFUNC void 293 ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv, 294 const USetAdder *sa, 295 UConverterUnicodeSet which, 296 UErrorCode *pErrorCode); 297 298 U_CFUNC void 299 ucnv_fromUWriteBytes(UConverter *cnv, 300 const char *bytes, int32_t length, 301 char **target, const char *targetLimit, 302 int32_t **offsets, 303 int32_t sourceIndex, 304 UErrorCode *pErrorCode); 305 U_CFUNC void 306 ucnv_toUWriteUChars(UConverter *cnv, 307 const UChar *uchars, int32_t length, 308 UChar **target, const UChar *targetLimit, 309 int32_t **offsets, 310 int32_t sourceIndex, 311 UErrorCode *pErrorCode); 312 313 U_CFUNC void 314 ucnv_toUWriteCodePoint(UConverter *cnv, 315 UChar32 c, 316 UChar **target, const UChar *targetLimit, 317 int32_t **offsets, 318 int32_t sourceIndex, 319 UErrorCode *pErrorCode); 320 321 #endif 322 323 #endif /* UCNV_CNV */ 324