tools/gennorm2/extradata.cpp

// © 2017 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html

// extradata.cpp
// created: 2017jun04 Markus W. Scherer
// (pulled out of n2builder.cpp)

#include "unicode/utypes.h"

#if !UCONFIG_NO_NORMALIZATION

#include <stdio.h>
#include <stdlib.h>
#include "unicode/errorcode.h"
#include "unicode/unistr.h"
#include "unicode/utf16.h"
#include "extradata.h"
#include "normalizer2impl.h"
#include "norms.h"
#include "toolutil.h"
#include "utrie2.h"
#include "uvectr32.h"

U_NAMESPACE_BEGIN

ExtraData::ExtraData(Norms &n, UBool fast) :
        Norms::Enumerator(n),
        yesYesCompositions(1000, (UChar32)0xffff, 2),  // 0=inert, 1=Jamo L, 2=start of compositions
        yesNoMappingsAndCompositions(1000, (UChar32)0, 1),  // 0=Hangul LV, 1=start of normal data
        yesNoMappingsOnly(1000, (UChar32)0, 1),  // 0=Hangul LVT, 1=start of normal data
        optimizeFast(fast) {
    // Hangul LV algorithmically decomposes to two Jamo.
    // Some code may harmlessly read this firstUnit.
    yesNoMappingsAndCompositions.setCharAt(0, 2);
    // Hangul LVT algorithmically decomposes to three Jamo.
    // Some code may harmlessly read this firstUnit.
    yesNoMappingsOnly.setCharAt(0, 3);
}

int32_t ExtraData::writeMapping(UChar32 c, const Norm &norm, UnicodeString &dataString) {
    UnicodeString &m=*norm.mapping;
    int32_t length=m.length();
    // Write the mapping & raw mapping extraData.
    int32_t firstUnit=length|(norm.trailCC<<8);
    int32_t preMappingLength=0;
    if(norm.rawMapping!=NULL) {
        UnicodeString &rm=*norm.rawMapping;
        int32_t rmLength=rm.length();
        if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) {
            fprintf(stderr,
                    "gennorm2 error: "
                    "raw mapping for U+%04lX longer than maximum of %d\n",
                    (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
            exit(U_INVALID_FORMAT_ERROR);
        }
        UChar rm0=rm.charAt(0);
        if( rmLength==length-1 &&
            // 99: overlong substring lengths get pinned to remainder lengths anyway
            0==rm.compare(1, 99, m, 2, 99) &&
            rm0>Normalizer2Impl::MAPPING_LENGTH_MASK
        ) {
            // Compression:
            // rawMapping=rm0+mapping.substring(2) -> store only rm0
            //
            // The raw mapping is the same as the final mapping after replacing
            // the final mapping's first two code units with the raw mapping's first one.
            // In this case, we store only that first unit, rm0.
            // This helps with a few hundred mappings.
            dataString.append(rm0);
            preMappingLength=1;
        } else {
            // Store the raw mapping with its length.
            dataString.append(rm);
            dataString.append((UChar)rmLength);
            preMappingLength=rmLength+1;
        }
        firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING;
    }
    int32_t cccLccc=norm.cc|(norm.leadCC<<8);
    if(cccLccc!=0) {
        dataString.append((UChar)cccLccc);
        ++preMappingLength;
        firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD;
    }
    dataString.append((UChar)firstUnit);
    dataString.append(m);
    return preMappingLength;
}

int32_t ExtraData::writeNoNoMapping(UChar32 c, const Norm &norm,
                                    UnicodeString &dataString,
                                    Hashtable &previousMappings) {
    UnicodeString newMapping;
    int32_t offset=writeMapping(c, norm, newMapping);
    UBool found=false;
    int32_t previousOffset=previousMappings.getiAndFound(newMapping, found);
    if(found) {
        // Duplicate, point to the identical mapping that has already been stored.
        offset=previousOffset;
    } else {
        // Append this new mapping and
        // enter it into the hashtable, avoiding value 0 which is "not found".
        offset=dataString.length()+offset;
        dataString.append(newMapping);
        IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.putiAllowZero()");
        previousMappings.putiAllowZero(newMapping, offset, errorCode);
    }
    return offset;
}

UBool ExtraData::setNoNoDelta(UChar32 c, Norm &norm) const {
    // Try a compact, algorithmic encoding to a single compYesAndZeroCC code point.
    // Do not map from ASCII to non-ASCII.
    if(norm.mappingCP>=0 &&
            !(c<=0x7f && norm.mappingCP>0x7f) &&
            norms.getNormRef(norm.mappingCP).type<Norm::NO_NO_COMP_YES) {
        int32_t delta=norm.mappingCP-c;
        if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) {
            norm.type=Norm::NO_NO_DELTA;
            norm.offset=delta;
            return TRUE;
        }
    }
    return FALSE;
}

void ExtraData::writeCompositions(UChar32 c, const Norm &norm, UnicodeString &dataString) {
    if(norm.cc!=0) {
        fprintf(stderr,
                "gennorm2 error: "
                "U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n",
                (long)c);
        exit(U_INVALID_FORMAT_ERROR);
    }
    int32_t length;
    const CompositionPair *pairs=norm.getCompositionPairs(length);
    for(int32_t i=0; i<length; ++i) {
        const CompositionPair &pair=pairs[i];
        // 22 bits for the composite character and whether it combines forward.
        UChar32 compositeAndFwd=pair.composite<<1;
        if(norms.getNormRef(pair.composite).compositions!=NULL) {
            compositeAndFwd|=1;  // The composite character also combines-forward.
        }
        // Encode most pairs in two units and some in three.
        int32_t firstUnit, secondUnit, thirdUnit;
        if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) {
            if(compositeAndFwd<=0xffff) {
                firstUnit=pair.trail<<1;
                secondUnit=compositeAndFwd;
                thirdUnit=-1;
            } else {
                firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE;
                secondUnit=compositeAndFwd>>16;
                thirdUnit=compositeAndFwd;
            }
        } else {
            firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+
                       (pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))|
                      Normalizer2Impl::COMP_1_TRIPLE;
            secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)|
                       (compositeAndFwd>>16);
            thirdUnit=compositeAndFwd;
        }
        // Set the high bit of the first unit if this is the last composition pair.
        if(i==(length-1)) {
            firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE;
        }
        dataString.append((UChar)firstUnit).append((UChar)secondUnit);
        if(thirdUnit>=0) {
            dataString.append((UChar)thirdUnit);
        }
    }
}

void ExtraData::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
    if(start!=end) {
        fprintf(stderr,
                "gennorm2 error: unexpected shared data for "
                "multiple code points U+%04lX..U+%04lX\n",
                (long)start, (long)end);
        exit(U_INTERNAL_PROGRAM_ERROR);
    }
    if(norm.error!=nullptr) {
        fprintf(stderr, "gennorm2 error: U+%04lX %s\n", (long)start, norm.error);
        exit(U_INVALID_FORMAT_ERROR);
    }
    writeExtraData(start, norm);
}

//  Ticket #13342 - Disable optimizations on MSVC for this function as a workaround.
#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210))
#pragma optimize( "", off )
#endif

void ExtraData::writeExtraData(UChar32 c, Norm &norm) {
    switch(norm.type) {
    case Norm::INERT:
        break;  // no extra data
    case Norm::YES_YES_COMBINES_FWD:
        norm.offset=yesYesCompositions.length();
        writeCompositions(c, norm, yesYesCompositions);
        break;
    case Norm::YES_NO_COMBINES_FWD:
        norm.offset=yesNoMappingsAndCompositions.length()+
                writeMapping(c, norm, yesNoMappingsAndCompositions);
        writeCompositions(c, norm, yesNoMappingsAndCompositions);
        break;
    case Norm::YES_NO_MAPPING_ONLY:
        norm.offset=yesNoMappingsOnly.length()+
                writeMapping(c, norm, yesNoMappingsOnly);
        break;
    case Norm::NO_NO_COMP_YES:
        if(!optimizeFast && setNoNoDelta(c, norm)) {
            break;
        }
        norm.offset=writeNoNoMapping(c, norm, noNoMappingsCompYes, previousNoNoMappingsCompYes);
        break;
    case Norm::NO_NO_COMP_BOUNDARY_BEFORE:
        if(!optimizeFast && setNoNoDelta(c, norm)) {
            break;
        }
        norm.offset=writeNoNoMapping(
            c, norm, noNoMappingsCompBoundaryBefore, previousNoNoMappingsCompBoundaryBefore);
        break;
    case Norm::NO_NO_COMP_NO_MAYBE_CC:
        norm.offset=writeNoNoMapping(
            c, norm, noNoMappingsCompNoMaybeCC, previousNoNoMappingsCompNoMaybeCC);
        break;
    case Norm::NO_NO_EMPTY:
        // There can be multiple extra data entries for mappings to the empty string
        // if they have different raw mappings.
        norm.offset=writeNoNoMapping(c, norm, noNoMappingsEmpty, previousNoNoMappingsEmpty);
        break;
    case Norm::MAYBE_YES_COMBINES_FWD:
        norm.offset=maybeYesCompositions.length();
        writeCompositions(c, norm, maybeYesCompositions);
        break;
    case Norm::MAYBE_YES_SIMPLE:
        break;  // no extra data
    case Norm::YES_YES_WITH_CC:
        break;  // no extra data
    default:  // Should not occur.
        exit(U_INTERNAL_PROGRAM_ERROR);
    }
}

// Ticket #13342 - Turn optimization back on.
#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210))
#pragma optimize( "", on )
#endif

U_NAMESPACE_END

#endif // #if !UCONFIG_NO_NORMALIZATION