tools/gennorm2/n2builder.cpp

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
*
*   Copyright (C) 2009-2016, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
*******************************************************************************
*   file name:  n2builder.cpp
*   encoding:   UTF-8
*   tab size:   8 (not used)
*   indentation:4
*
*   created on: 2009nov25
*   created by: Markus W. Scherer
*
* Builds Normalizer2 data and writes a binary .nrm file.
* For the file format see source/common/normalizer2impl.h.
*/

#include "unicode/utypes.h"
#include "n2builder.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <vector>
#include "unicode/errorcode.h"
#include "unicode/localpointer.h"
#include "unicode/putil.h"
#include "unicode/ucptrie.h"
#include "unicode/udata.h"
#include "unicode/umutablecptrie.h"
#include "unicode/uniset.h"
#include "unicode/unistr.h"
#include "unicode/usetiter.h"
#include "unicode/ustring.h"
#include "charstr.h"
#include "extradata.h"
#include "hash.h"
#include "normalizer2impl.h"
#include "norms.h"
#include "toolutil.h"
#include "unewdata.h"
#include "uvectr32.h"
#include "writesrc.h"

#if !UCONFIG_NO_NORMALIZATION

/* UDataInfo cf. udata.h */
static UDataInfo dataInfo={
    sizeof(UDataInfo),
    0,

    U_IS_BIG_ENDIAN,
    U_CHARSET_FAMILY,
    U_SIZEOF_UCHAR,
    0,

    { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */
    { 4, 0, 0, 0 },             /* formatVersion */
    { 11, 0, 0, 0 }             /* dataVersion (Unicode version) */
};

U_NAMESPACE_BEGIN

class HangulIterator {
public:
    struct Range {
        UChar32 start, end;
    };

    HangulIterator() : rangeIndex(0) {}
    const Range *nextRange() {
        if(rangeIndex<UPRV_LENGTHOF(ranges)) {
            return ranges+rangeIndex++;
        } else {
            return NULL;
        }
    }
private:
    static const Range ranges[4];
    int32_t rangeIndex;
};

const HangulIterator::Range HangulIterator::ranges[4]={
    { Hangul::JAMO_L_BASE, Hangul::JAMO_L_END },
    { Hangul::JAMO_V_BASE, Hangul::JAMO_V_END },
    // JAMO_T_BASE+1: not U+11A7
    { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END },
    { Hangul::HANGUL_BASE, Hangul::HANGUL_END },
};

Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) :
        norms(errorCode),
        phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL),
        norm16TrieBytes(nullptr), norm16TrieLength(0) {
    memset(unicodeVersion, 0, sizeof(unicodeVersion));
    memset(indexes, 0, sizeof(indexes));
    memset(smallFCD, 0, sizeof(smallFCD));
}

Normalizer2DataBuilder::~Normalizer2DataBuilder() {
    delete[] norm16TrieBytes;
}

void
Normalizer2DataBuilder::setUnicodeVersion(const char *v) {
    UVersionInfo nullVersion={ 0, 0, 0, 0 };
    UVersionInfo version;
    u_versionFromString(version, v);
    if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) &&
        0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH)
    ) {
        char buffer[U_MAX_VERSION_STRING_LENGTH];
        u_versionToString(unicodeVersion, buffer);
        fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n",
                buffer, v);
        exit(U_ILLEGAL_ARGUMENT_ERROR);
    }
    memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH);
}

Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) {
    if(p!=NULL) {
        if(p->mappingType!=Norm::NONE) {
            if( overrideHandling==OVERRIDE_NONE ||
                (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase)
            ) {
                fprintf(stderr,
                        "error in gennorm2 phase %d: "
                        "not permitted to override mapping for U+%04lX from phase %d\n",
                        (int)phase, (long)c, (int)p->mappingPhase);
                exit(U_INVALID_FORMAT_ERROR);
            }
            delete p->mapping;
            p->mapping=NULL;
        }
        p->mappingPhase=phase;
    }
    return p;
}

void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) {
    overrideHandling=oh;
    ++phase;
}

void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) {
    norms.createNorm(c)->cc=cc;
    norms.ccSet.add(c);
}

static UBool isWellFormed(const UnicodeString &s) {
    UErrorCode errorCode=U_ZERO_ERROR;
    u_strToUTF8(NULL, 0, NULL, toUCharPtr(s.getBuffer()), s.length(), &errorCode);
    return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR;
}

void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) {
    if(!isWellFormed(m)) {
        fprintf(stderr,
                "error in gennorm2 phase %d: "
                "illegal one-way mapping from U+%04lX to malformed string\n",
                (int)phase, (long)c);
        exit(U_INVALID_FORMAT_ERROR);
    }
    Norm *p=checkNormForMapping(norms.createNorm(c), c);
    p->mapping=new UnicodeString(m);
    p->mappingType=Norm::ONE_WAY;
    p->setMappingCP();
    norms.mappingSet.add(c);
}

void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) {
    if(U_IS_SURROGATE(c)) {
        fprintf(stderr,
                "error in gennorm2 phase %d: "
                "illegal round-trip mapping from surrogate code point U+%04lX\n",
                (int)phase, (long)c);
        exit(U_INVALID_FORMAT_ERROR);
    }
    if(!isWellFormed(m)) {
        fprintf(stderr,
                "error in gennorm2 phase %d: "
                "illegal round-trip mapping from U+%04lX to malformed string\n",
                (int)phase, (long)c);
        exit(U_INVALID_FORMAT_ERROR);
    }
    int32_t numCP=u_countChar32(toUCharPtr(m.getBuffer()), m.length());
    if(numCP!=2) {
        fprintf(stderr,
                "error in gennorm2 phase %d: "
                "illegal round-trip mapping from U+%04lX to %d!=2 code points\n",
                (int)phase, (long)c, (int)numCP);
        exit(U_INVALID_FORMAT_ERROR);
    }
    Norm *p=checkNormForMapping(norms.createNorm(c), c);
    p->mapping=new UnicodeString(m);
    p->mappingType=Norm::ROUND_TRIP;
    p->mappingCP=U_SENTINEL;
    norms.mappingSet.add(c);
}

void Normalizer2DataBuilder::removeMapping(UChar32 c) {
    // createNorm(c), not getNorm(c), to record a non-mapping and detect conflicting data.
    Norm *p=checkNormForMapping(norms.createNorm(c), c);
    p->mappingType=Norm::REMOVED;
    norms.mappingSet.add(c);
}

UBool Normalizer2DataBuilder::mappingHasCompBoundaryAfter(const BuilderReorderingBuffer &buffer,
                                                          Norm::MappingType mappingType) const {
    if(buffer.isEmpty()) {
        return FALSE;  // Maps-to-empty-string is no boundary of any kind.
    }
    int32_t lastStarterIndex=buffer.lastStarterIndex();
    if(lastStarterIndex<0) {
        return FALSE;  // no starter
    }
    const int32_t lastIndex=buffer.length()-1;
    if(mappingType==Norm::ONE_WAY && lastStarterIndex<lastIndex && buffer.ccAt(lastIndex)>1) {
        // One-way mapping where after the last starter is at least one combining mark
        // with a combining class greater than 1,
        // which means that another combining mark can reorder before it.
        // By contrast, in a round-trip mapping this does not prevent a boundary as long as
        // the starter or composite does not combine-forward with a following combining mark.
        return FALSE;
    }
    UChar32 starter=buffer.charAt(lastStarterIndex);
    if(lastStarterIndex==0 && norms.combinesBack(starter)) {
        // The last starter is at the beginning of the mapping and combines backward.
        return FALSE;
    }
    if(Hangul::isJamoL(starter) ||
            (Hangul::isJamoV(starter) &&
            0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1)))) {
        // A Jamo leading consonant or an LV pair combines-forward if it is at the end,
        // otherwise it is blocked.
        return lastStarterIndex!=lastIndex;
    }
    // Note: There can be no Hangul syllable in the fully decomposed mapping.

    // Multiple starters can combine into one.
    // Look for the first of the last sequence of starters, excluding Jamos.
    int32_t i=lastStarterIndex;
    UChar32 c;
    while(0<i && buffer.ccAt(i-1)==0 && !Hangul::isJamo(c=buffer.charAt(i-1))) {
        starter=c;
        --i;
    }
    // Compose as far as possible, and see if further compositions with
    // characters following this mapping are possible.
    const Norm *starterNorm=norms.getNorm(starter);
    if(i==lastStarterIndex &&
            (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
        return TRUE;  // The last starter does not combine forward.
    }
    uint8_t prevCC=0;
    while(++i<buffer.length()) {
        uint8_t cc=buffer.ccAt(i);  // !=0 if after last starter
        if(i>lastStarterIndex && norms.combinesWithCCBetween(*starterNorm, prevCC, cc)) {
            // The starter combines with a mark that reorders before the current one.
            return FALSE;
        }
        UChar32 c=buffer.charAt(i);
        if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) &&
                norms.getNormRef(c).combinesBack && (starter=starterNorm->combine(c))>=0) {
            // The starter combines with c into a composite replacement starter.
            starterNorm=norms.getNorm(starter);
            if(i>=lastStarterIndex &&
                    (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
                return TRUE;  // The composite does not combine further.
            }
            // Keep prevCC because we "removed" the combining mark.
        } else if(cc==0) {
            starterNorm=norms.getNorm(c);
            if(i==lastStarterIndex &&
                    (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
                return TRUE;  // The new starter does not combine forward.
            }
            prevCC=0;
        } else {
            prevCC=cc;
        }
    }
    if(prevCC==0) {
        return FALSE;  // forward-combining starter at the very end
    }
    if(norms.combinesWithCCBetween(*starterNorm, prevCC, 256)) {
        // The starter combines with another mark.
        return FALSE;
    }
    return TRUE;
}

UBool Normalizer2DataBuilder::mappingRecomposes(const BuilderReorderingBuffer &buffer) const {
    if(buffer.lastStarterIndex()<0) {
        return FALSE;  // no starter
    }
    const Norm *starterNorm=nullptr;
    uint8_t prevCC=0;
    for(int32_t i=0; i<buffer.length(); ++i) {
        UChar32 c=buffer.charAt(i);
        uint8_t cc=buffer.ccAt(i);
        if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) &&
                norms.getNormRef(c).combinesBack && starterNorm->combine(c)>=0) {
            return TRUE;  // normal composite
        } else if(cc==0) {
            if(Hangul::isJamoL(c)) {
                if((i+1)<buffer.length() && Hangul::isJamoV(buffer.charAt(i+1))) {
                    return TRUE;  // Hangul syllable
                }
                starterNorm=nullptr;
            } else {
                starterNorm=norms.getNorm(c);
            }
        }
        prevCC=cc;
    }
    return FALSE;
}

void Normalizer2DataBuilder::postProcess(Norm &norm) {
    // Prerequisites: Compositions are built, mappings are recursively decomposed.
    // Mappings are not yet in canonical order.
    //
    // This function works on a Norm struct. We do not know which code point(s) map(s) to it.
    // Therefore, we cannot compute algorithmic mapping deltas here.
    // Error conditions are checked, but printed later when we do know the offending code point.
    if(norm.hasMapping()) {
        if(norm.mapping->length()>Normalizer2Impl::MAPPING_LENGTH_MASK) {
            norm.error="mapping longer than maximum of 31";
            return;
        }
        // Ensure canonical order.
        BuilderReorderingBuffer buffer;
        if(norm.rawMapping!=nullptr) {
            norms.reorder(*norm.rawMapping, buffer);
            buffer.reset();
        }
        norms.reorder(*norm.mapping, buffer);
        if(buffer.isEmpty()) {
            // A character that is deleted (maps to an empty string) must
            // get the worst-case lccc and tccc values because arbitrary
            // characters on both sides will become adjacent.
            norm.leadCC=1;
            norm.trailCC=0xff;
        } else {
            norm.leadCC=buffer.ccAt(0);
            norm.trailCC=buffer.ccAt(buffer.length()-1);
        }

        norm.hasCompBoundaryBefore=
            !buffer.isEmpty() && norm.leadCC==0 && !norms.combinesBack(buffer.charAt(0));
        norm.hasCompBoundaryAfter=
            norm.compositions==nullptr && mappingHasCompBoundaryAfter(buffer, norm.mappingType);

        if(norm.combinesBack) {
            norm.error="combines-back and decomposes, not possible in Unicode normalization";
        } else if(norm.mappingType==Norm::ROUND_TRIP) {
            if(norm.compositions!=NULL) {
                norm.type=Norm::YES_NO_COMBINES_FWD;
            } else {
                norm.type=Norm::YES_NO_MAPPING_ONLY;
            }
        } else {  // one-way mapping
            if(norm.compositions!=NULL) {
                norm.error="combines-forward and has a one-way mapping, "
                           "not possible in Unicode normalization";
            } else if(buffer.isEmpty()) {
                norm.type=Norm::NO_NO_EMPTY;
            } else if(!norm.hasCompBoundaryBefore) {
                norm.type=Norm::NO_NO_COMP_NO_MAYBE_CC;
            } else if(mappingRecomposes(buffer)) {
                norm.type=Norm::NO_NO_COMP_BOUNDARY_BEFORE;
            } else {
                // The mapping is comp-normalized.
                norm.type=Norm::NO_NO_COMP_YES;
            }
        }
    } else {  // no mapping
        norm.leadCC=norm.trailCC=norm.cc;

        norm.hasCompBoundaryBefore=
            norm.cc==0 && !norm.combinesBack;
        norm.hasCompBoundaryAfter=
            norm.cc==0 && !norm.combinesBack && norm.compositions==nullptr;

        if(norm.combinesBack) {
            if(norm.compositions!=nullptr) {
                // Earlier code checked ccc=0.
                norm.type=Norm::MAYBE_YES_COMBINES_FWD;
            } else {
                norm.type=Norm::MAYBE_YES_SIMPLE;  // any ccc
            }
        } else if(norm.compositions!=nullptr) {
            // Earlier code checked ccc=0.
            norm.type=Norm::YES_YES_COMBINES_FWD;
        } else if(norm.cc!=0) {
            norm.type=Norm::YES_YES_WITH_CC;
        } else {
            norm.type=Norm::INERT;
        }
    }
}

class Norm16Writer : public Norms::Enumerator {
public:
    Norm16Writer(UMutableCPTrie *trie, Norms &n, Normalizer2DataBuilder &b) :
            Norms::Enumerator(n), builder(b), norm16Trie(trie) {}
    void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE {
        builder.writeNorm16(norm16Trie, start, end, norm);
    }
    Normalizer2DataBuilder &builder;
    UMutableCPTrie *norm16Trie;
};

void Normalizer2DataBuilder::setSmallFCD(UChar32 c) {
    UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
    smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
}

void Normalizer2DataBuilder::writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm) {
    if((norm.leadCC|norm.trailCC)!=0) {
        for(UChar32 c=start; c<=end; ++c) {
            setSmallFCD(c);
        }
    }

    int32_t norm16;
    switch(norm.type) {
    case Norm::INERT:
        norm16=Normalizer2Impl::INERT;
        break;
    case Norm::YES_YES_COMBINES_FWD:
        norm16=norm.offset*2;
        break;
    case Norm::YES_NO_COMBINES_FWD:
        norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+norm.offset*2;
        break;
    case Norm::YES_NO_MAPPING_ONLY:
        norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+norm.offset*2;
        break;
    case Norm::NO_NO_COMP_YES:
        norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+norm.offset*2;
        break;
    case Norm::NO_NO_COMP_BOUNDARY_BEFORE:
        norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]+norm.offset*2;
        break;
    case Norm::NO_NO_COMP_NO_MAYBE_CC:
        norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]+norm.offset*2;
        break;
    case Norm::NO_NO_EMPTY:
        norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]+norm.offset*2;
        break;
    case Norm::NO_NO_DELTA:
        {
            // Positive offset from minNoNoDelta, shifted left for additional bits.
            int32_t offset=(norm.offset+Normalizer2Impl::MAX_DELTA)<<Normalizer2Impl::DELTA_SHIFT;
            if(norm.trailCC==0) {
                // DELTA_TCCC_0==0
            } else if(norm.trailCC==1) {
                offset|=Normalizer2Impl::DELTA_TCCC_1;
            } else {
                offset|=Normalizer2Impl::DELTA_TCCC_GT_1;
            }
            norm16=getMinNoNoDelta()+offset;
            break;
        }
    case Norm::MAYBE_YES_COMBINES_FWD:
        norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+norm.offset*2;
        break;
    case Norm::MAYBE_YES_SIMPLE:
        norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+norm.cc*2;  // ccc=0..255
        break;
    case Norm::YES_YES_WITH_CC:
        U_ASSERT(norm.cc!=0);
        norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-2+norm.cc*2;  // ccc=1..255
        break;
    default:  // Should not occur.
        exit(U_INTERNAL_PROGRAM_ERROR);
    }
    U_ASSERT((norm16&1)==0);
    if(norm.hasCompBoundaryAfter) {
        norm16|=Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER;
    }
    IcuToolErrorCode errorCode("gennorm2/writeNorm16()");
    umutablecptrie_setRange(norm16Trie, start, end, (uint32_t)norm16, errorCode);

    // Set the minimum code points for real data lookups in the quick check loops.
    UBool isDecompNo=
            (Norm::YES_NO_COMBINES_FWD<=norm.type && norm.type<=Norm::NO_NO_DELTA) ||
            norm.cc!=0;
    if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
        indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start;
    }
    UBool isCompNoMaybe= norm.type>=Norm::NO_NO_COMP_YES;
    if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
        indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start;
    }
    if(norm.leadCC!=0 && start<indexes[Normalizer2Impl::IX_MIN_LCCC_CP]) {
        indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=start;
    }
}

void Normalizer2DataBuilder::setHangulData(UMutableCPTrie *norm16Trie) {
    HangulIterator hi;
    const HangulIterator::Range *range;
    // Check that none of the Hangul/Jamo code points have data.
    while((range=hi.nextRange())!=NULL) {
        for(UChar32 c=range->start; c<=range->end; ++c) {
            if(umutablecptrie_get(norm16Trie, c)>Normalizer2Impl::INERT) {
                fprintf(stderr,
                        "gennorm2 error: "
                        "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n",
                        (long)c);
                exit(U_INVALID_FORMAT_ERROR);
            }
        }
    }
    // Set data for algorithmic runtime handling.
    IcuToolErrorCode errorCode("gennorm2/setHangulData()");

    // Jamo V/T are maybeYes
    if(Hangul::JAMO_V_BASE<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
        indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=Hangul::JAMO_V_BASE;
    }
    umutablecptrie_setRange(norm16Trie, Hangul::JAMO_L_BASE, Hangul::JAMO_L_END,
                            Normalizer2Impl::JAMO_L, errorCode);
    umutablecptrie_setRange(norm16Trie, Hangul::JAMO_V_BASE, Hangul::JAMO_V_END,
                            Normalizer2Impl::JAMO_VT, errorCode);
    // JAMO_T_BASE+1: not U+11A7
    umutablecptrie_setRange(norm16Trie, Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END,
                            Normalizer2Impl::JAMO_VT, errorCode);

    // Hangul LV encoded as minYesNo
    uint32_t lv=indexes[Normalizer2Impl::IX_MIN_YES_NO];
    // Hangul LVT encoded as minYesNoMappingsOnly|HAS_COMP_BOUNDARY_AFTER
    uint32_t lvt=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]|
        Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER;
    if(Hangul::HANGUL_BASE<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
        indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=Hangul::HANGUL_BASE;
    }
    // Set the first LV, then write all other Hangul syllables as LVT,
    // then overwrite the remaining LV.
    umutablecptrie_set(norm16Trie, Hangul::HANGUL_BASE, lv, errorCode);
    umutablecptrie_setRange(norm16Trie, Hangul::HANGUL_BASE+1, Hangul::HANGUL_END, lvt, errorCode);
    UChar32 c=Hangul::HANGUL_BASE;
    while((c+=Hangul::JAMO_T_COUNT)<=Hangul::HANGUL_END) {
        umutablecptrie_set(norm16Trie, c, lv, errorCode);
    }
    errorCode.assertSuccess();
}

LocalUCPTriePointer Normalizer2DataBuilder::processData() {
    // Build composition lists before recursive decomposition,
    // so that we still have the raw, pair-wise mappings.
    CompositionBuilder compBuilder(norms);
    norms.enumRanges(compBuilder);

    // Recursively decompose all mappings.
    Decomposer decomposer(norms);
    do {
        decomposer.didDecompose=FALSE;
        norms.enumRanges(decomposer);
    } while(decomposer.didDecompose);

    // Set the Norm::Type and other properties.
    int32_t normsLength=norms.length();
    for(int32_t i=1; i<normsLength; ++i) {
        postProcess(norms.getNormRefByIndex(i));
    }

    // Write the properties, mappings and composition lists to
    // appropriate parts of the "extra data" array.
    ExtraData extra(norms, optimization==OPTIMIZE_FAST);
    norms.enumRanges(extra);

    extraData=extra.yesYesCompositions;
    indexes[Normalizer2Impl::IX_MIN_YES_NO]=extraData.length()*2;
    extraData.append(extra.yesNoMappingsAndCompositions);
    indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]=extraData.length()*2;
    extraData.append(extra.yesNoMappingsOnly);
    indexes[Normalizer2Impl::IX_MIN_NO_NO]=extraData.length()*2;
    extraData.append(extra.noNoMappingsCompYes);
    indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]=extraData.length()*2;
    extraData.append(extra.noNoMappingsCompBoundaryBefore);
    indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]=extraData.length()*2;
    extraData.append(extra.noNoMappingsCompNoMaybeCC);
    indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]=extraData.length()*2;
    extraData.append(extra.noNoMappingsEmpty);
    indexes[Normalizer2Impl::IX_LIMIT_NO_NO]=extraData.length()*2;

    // Pad the maybeYesCompositions length to a multiple of 4,
    // so that NO_NO_DELTA bits 2..1 can be used without subtracting the center.
    while(extra.maybeYesCompositions.length()&3) {
        extra.maybeYesCompositions.append((UChar)0);
    }
    extraData.insert(0, extra.maybeYesCompositions);
    indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]=
        Normalizer2Impl::MIN_NORMAL_MAYBE_YES-
        extra.maybeYesCompositions.length()*2;

    // Pad to even length for 4-byte alignment of following data.
    if(extraData.length()&1) {
        extraData.append((UChar)0);
    }

    int32_t minNoNoDelta=getMinNoNoDelta();
    U_ASSERT((minNoNoDelta&7)==0);
    if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) {
        fprintf(stderr,
                "gennorm2 error: "
                "data structure overflow, too much mapping composition data\n");
        exit(U_BUFFER_OVERFLOW_ERROR);
    }

    // writeNorm16() and setHangulData() reduce these as needed.
    indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000;
    indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000;
    indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=0x110000;

    IcuToolErrorCode errorCode("gennorm2/processData()");
    UMutableCPTrie *norm16Trie = umutablecptrie_open(
        Normalizer2Impl::INERT, Normalizer2Impl::INERT, errorCode);
    errorCode.assertSuccess();

    // Map each code point to its norm16 value,
    // including the properties that fit directly,
    // and the offset to the "extra data" if necessary.
    Norm16Writer norm16Writer(norm16Trie, norms, *this);
    norms.enumRanges(norm16Writer);
    // TODO: iterate via getRange() instead of callback?

    setHangulData(norm16Trie);

    // Look for the "worst" norm16 value of any supplementary code point
    // corresponding to a lead surrogate, and set it as that surrogate's value.
    // Enables UTF-16 quick check inner loops to look at only code units.
    //
    // We could be more sophisticated:
    // We could collect a bit set for whether there are values in the different
    // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.)
    // and select the best value that only breaks the composition and/or decomposition
    // inner loops if necessary.
    // However, that seems like overkill for an optimization for supplementary characters.
    //
    // First check that surrogate code *points* are inert.
    // The parser should have rejected values/mappings for them.
    uint32_t value;
    UChar32 end = umutablecptrie_getRange(norm16Trie, 0xd800, UCPMAP_RANGE_NORMAL, 0,
                                          nullptr, nullptr, &value);
    if (value != Normalizer2Impl::INERT || end < 0xdfff) {
        fprintf(stderr,
                "gennorm2 error: not all surrogate code points are inert: U+d800..U+%04x=%lx\n",
                (int)end, (long)value);
        exit(U_INTERNAL_PROGRAM_ERROR);
    }
    uint32_t maxNorm16 = 0;
    // ANDing values yields 0 bits where any value has a 0.
    // Used for worst-case HAS_COMP_BOUNDARY_AFTER.
    uint32_t andedNorm16 = 0;
    end = 0;
    for (UChar32 start = 0x10000;;) {
        if (start > end) {
            end = umutablecptrie_getRange(norm16Trie, start, UCPMAP_RANGE_NORMAL, 0,
                                          nullptr, nullptr, &value);
            if (end < 0) { break; }
        }
        if ((start & 0x3ff) == 0) {
            // Data for a new lead surrogate.
            maxNorm16 = andedNorm16 = value;
        } else {
            if (value > maxNorm16) {
                maxNorm16 = value;
            }
            andedNorm16 &= value;
        }
        // Intersect each range with the code points for one lead surrogate.
        UChar32 leadEnd = start | 0x3ff;
        if (leadEnd <= end) {
            // End of the supplementary block for a lead surrogate.
            if (maxNorm16 >= (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]) {
                // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0.
                // Otherwise it might end up at something like JAMO_VT which stays in
                // the inner decomposition quick check loop.
                maxNorm16 = (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO];
            }
            maxNorm16 =
                (maxNorm16 & ~Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER)|
                (andedNorm16 & Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER);
            if (maxNorm16 != Normalizer2Impl::INERT) {
                umutablecptrie_set(norm16Trie, U16_LEAD(start), maxNorm16, errorCode);
            }
            if (value == Normalizer2Impl::INERT) {
                // Potentially skip inert supplementary blocks for several lead surrogates.
                start = (end + 1) & ~0x3ff;
            } else {
                start = leadEnd + 1;
            }
        } else {
            start = end + 1;
        }
    }

    // Adjust supplementary minimum code points to break quick check loops at their lead surrogates.
    // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate)
    // which is harmless.
    // As a result, the minimum code points are always BMP code points.
    int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP];
    if(minCP>=0x10000) {
        indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP);
    }
    minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP];
    if(minCP>=0x10000) {
        indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP);
    }
    minCP=indexes[Normalizer2Impl::IX_MIN_LCCC_CP];
    if(minCP>=0x10000) {
        indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=U16_LEAD(minCP);
    }

    LocalUCPTriePointer builtTrie(
        umutablecptrie_buildImmutable(norm16Trie, UCPTRIE_TYPE_FAST, UCPTRIE_VALUE_BITS_16, errorCode));
    norm16TrieLength=ucptrie_toBinary(builtTrie.getAlias(), nullptr, 0, errorCode);
    if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) {
        fprintf(stderr, "gennorm2 error: unable to build/serialize the normalization trie - %s\n",
                errorCode.errorName());
        exit(errorCode.reset());
    }
    umutablecptrie_close(norm16Trie);
    errorCode.reset();
    norm16TrieBytes=new uint8_t[norm16TrieLength];
    ucptrie_toBinary(builtTrie.getAlias(), norm16TrieBytes, norm16TrieLength, errorCode);
    errorCode.assertSuccess();

    int32_t offset=(int32_t)sizeof(indexes);
    indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset;
    offset+=norm16TrieLength;
    indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset;
    offset+=extraData.length()*2;
    indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset;
    offset+=sizeof(smallFCD);
    int32_t totalSize=offset;
    for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) {
        indexes[i]=totalSize;
    }

    if(beVerbose) {
        printf("size of normalization trie:         %5ld bytes\n", (long)norm16TrieLength);
        printf("size of 16-bit extra data:          %5ld uint16_t\n", (long)extraData.length());
        printf("size of small-FCD data:             %5ld bytes\n", (long)sizeof(smallFCD));
        printf("size of binary data file contents:  %5ld bytes\n", (long)totalSize);
        printf("minDecompNoCodePoint:              U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]);
        printf("minCompNoMaybeCodePoint:           U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]);
        printf("minLcccCodePoint:                  U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_LCCC_CP]);
        printf("minYesNo: (with compositions)      0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]);
        printf("minYesNoMappingsOnly:              0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]);
        printf("minNoNo: (comp-normalized)         0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]);
        printf("minNoNoCompBoundaryBefore:         0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]);
        printf("minNoNoCompNoMaybeCC:              0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]);
        printf("minNoNoEmpty:                      0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]);
        printf("limitNoNo:                         0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]);
        printf("minNoNoDelta:                      0x%04x\n", (int)minNoNoDelta);
        printf("minMaybeYes:                       0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]);
    }

    UVersionInfo nullVersion={ 0, 0, 0, 0 };
    if(0==memcmp(nullVersion, unicodeVersion, 4)) {
        u_versionFromString(unicodeVersion, U_UNICODE_VERSION);
    }
    memcpy(dataInfo.dataVersion, unicodeVersion, 4);
    return builtTrie;
}

void Normalizer2DataBuilder::writeBinaryFile(const char *filename) {
    processData();

    IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()");
    UNewDataMemory *pData=
        udata_create(NULL, NULL, filename, &dataInfo,
                     haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode);
    if(errorCode.isFailure()) {
        fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n",
                filename, errorCode.errorName());
        exit(errorCode.reset());
    }
    udata_writeBlock(pData, indexes, sizeof(indexes));
    udata_writeBlock(pData, norm16TrieBytes, norm16TrieLength);
    udata_writeUString(pData, toUCharPtr(extraData.getBuffer()), extraData.length());
    udata_writeBlock(pData, smallFCD, sizeof(smallFCD));
    int32_t writtenSize=udata_finish(pData, errorCode);
    if(errorCode.isFailure()) {
        fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName());
        exit(errorCode.reset());
    }
    int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
    if(writtenSize!=totalSize) {
        fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n",
            (long)writtenSize, (long)totalSize);
        exit(U_INTERNAL_PROGRAM_ERROR);
    }
}

void
Normalizer2DataBuilder::writeCSourceFile(const char *filename) {
    LocalUCPTriePointer norm16Trie = processData();

    IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()");
    const char *basename=findBasename(filename);
    CharString path(filename, (int32_t)(basename-filename), errorCode);
    CharString dataName(basename, errorCode);
    const char *extension=strrchr(basename, '.');
    if(extension!=NULL) {
        dataName.truncate((int32_t)(extension-basename));
    }
    const char *name=dataName.data();
    errorCode.assertSuccess();

    FILE *f=usrc_create(path.data(), basename, 2016, "icu/source/tools/gennorm2/n2builder.cpp");
    if(f==NULL) {
        fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n",
                filename);
        exit(U_FILE_ACCESS_ERROR);
    }
    fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f);

    char line[100];
    sprintf(line, "static const UVersionInfo %s_formatVersion={", name);
    usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n");
    sprintf(line, "static const UVersionInfo %s_dataVersion={", name);
    usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n");
    sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", name);
    usrc_writeArray(f, line, indexes, 32, Normalizer2Impl::IX_COUNT, "\n};\n\n");

    usrc_writeUCPTrie(f, name, norm16Trie.getAlias());

    sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", name);
    usrc_writeArray(f, line, extraData.getBuffer(), 16, extraData.length(), "\n};\n\n");
    sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", name);
    usrc_writeArray(f, line, smallFCD, 8, sizeof(smallFCD), "\n};\n\n");

    fputs("#endif  // INCLUDED_FROM_NORMALIZER2_CPP\n", f);
    fclose(f);
}

namespace {

bool equalStrings(const UnicodeString *s1, const UnicodeString *s2) {
    if(s1 == nullptr) {
        return s2 == nullptr;
    } else if(s2 == nullptr) {
        return false;
    } else {
        return *s1 == *s2;
    }
}

const char *typeChars = "?-=>";

void writeMapping(FILE *f, const UnicodeString *m) {
    if(m != nullptr && !m->isEmpty()) {
        int32_t i = 0;
        UChar32 c = m->char32At(i);
        fprintf(f, "%04lX", (long)c);
        while((i += U16_LENGTH(c)) < m->length()) {
            c = m->char32At(i);
            fprintf(f, " %04lX", (long)c);
        }
    }
    fputs("\n", f);
}

}  // namespace

void
Normalizer2DataBuilder::writeDataFile(const char *filename, bool writeRemoved) const {
    // Do not processData() before writing the input-syntax data file.
    FILE *f = fopen(filename, "w");
    if(f == nullptr) {
        fprintf(stderr, "gennorm2/writeDataFile() error: unable to create the output file %s\n",
                filename);
        exit(U_FILE_ACCESS_ERROR);
        return;
    }

    if(unicodeVersion[0] != 0 || unicodeVersion[1] != 0 ||
            unicodeVersion[2] != 0 || unicodeVersion[3] != 0) {
        char uv[U_MAX_VERSION_STRING_LENGTH];
        u_versionToString(unicodeVersion, uv);
        fprintf(f, "* Unicode %s\n\n", uv);
    }

    UnicodeSetIterator ccIter(norms.ccSet);
    UChar32 start = U_SENTINEL;
    UChar32 end = U_SENTINEL;
    uint8_t prevCC = 0;
    bool done = false;
    bool didWrite = false;
    do {
        UChar32 c;
        uint8_t cc;
        if(ccIter.next() && !ccIter.isString()) {
            c = ccIter.getCodepoint();
            cc = norms.getCC(c);
        } else {
            c = 0x110000;
            cc = 0;
            done = true;
        }
        if(cc == prevCC && c == (end + 1)) {
            end = c;
        } else {
            if(prevCC != 0) {
                if(start == end) {
                    fprintf(f, "%04lX:%d\n", (long)start, (int)prevCC);
                } else {
                    fprintf(f, "%04lX..%04lX:%d\n", (long)start, (long)end, (int)prevCC);
                }
                didWrite = true;
            }
            start = end = c;
            prevCC = cc;
        }
    } while(!done);
    if(didWrite) {
        fputs("\n", f);
    }

    UnicodeSetIterator mIter(norms.mappingSet);
    start = U_SENTINEL;
    end = U_SENTINEL;
    const UnicodeString *prevMapping = nullptr;
    Norm::MappingType prevType = Norm::NONE;
    done = false;
    do {
        UChar32 c;
        const Norm *norm;
        if(mIter.next() && !mIter.isString()) {
            c = mIter.getCodepoint();
            norm = norms.getNorm(c);
        } else {
            c = 0x110000;
            norm = nullptr;
            done = true;
        }
        const UnicodeString *mapping;
        Norm::MappingType type;
        if(norm == nullptr) {
            mapping = nullptr;
            type = Norm::NONE;
        } else {
            type = norm->mappingType;
            if(type == Norm::NONE) {
                mapping = nullptr;
            } else {
                mapping = norm->mapping;
            }
        }
        if(type == prevType && equalStrings(mapping, prevMapping) && c == (end + 1)) {
            end = c;
        } else {
            if(writeRemoved ? prevType != Norm::NONE : prevType > Norm::REMOVED) {
                if(start == end) {
                    fprintf(f, "%04lX%c", (long)start, typeChars[prevType]);
                } else {
                    fprintf(f, "%04lX..%04lX%c", (long)start, (long)end, typeChars[prevType]);
                }
                writeMapping(f, prevMapping);
            }
            start = end = c;
            prevMapping = mapping;
            prevType = type;
        }
    } while(!done);

    fclose(f);
}

void
Normalizer2DataBuilder::computeDiff(const Normalizer2DataBuilder &b1,
                                    const Normalizer2DataBuilder &b2,
                                    Normalizer2DataBuilder &diff) {
    // Compute diff = b1 - b2
    // so that we should be able to get b1 = b2 + diff.
    if(0 != memcmp(b1.unicodeVersion, b2.unicodeVersion, U_MAX_VERSION_LENGTH)) {
        memcpy(diff.unicodeVersion, b1.unicodeVersion, U_MAX_VERSION_LENGTH);
    }

    UnicodeSet ccSet(b1.norms.ccSet);
    ccSet.addAll(b2.norms.ccSet);
    UnicodeSetIterator ccIter(ccSet);
    while(ccIter.next() && !ccIter.isString()) {
        UChar32 c = ccIter.getCodepoint();
        uint8_t cc1 = b1.norms.getCC(c);
        uint8_t cc2 = b2.norms.getCC(c);
        if(cc1 != cc2) {
            diff.setCC(c, cc1);
        }
    }

    UnicodeSet mSet(b1.norms.mappingSet);
    mSet.addAll(b2.norms.mappingSet);
    UnicodeSetIterator mIter(mSet);
    while(mIter.next() && !mIter.isString()) {
        UChar32 c = mIter.getCodepoint();
        const Norm *norm1 = b1.norms.getNorm(c);
        const Norm *norm2 = b2.norms.getNorm(c);
        const UnicodeString *mapping1;
        Norm::MappingType type1;
        if(norm1 == nullptr || !norm1->hasMapping()) {
            mapping1 = nullptr;
            type1 = Norm::NONE;
        } else {
            mapping1 = norm1->mapping;
            type1 = norm1->mappingType;
        }
        const UnicodeString *mapping2;
        Norm::MappingType type2;
        if(norm2 == nullptr || !norm2->hasMapping()) {
            mapping2 = nullptr;
            type2 = Norm::NONE;
        } else {
            mapping2 = norm2->mapping;
            type2 = norm2->mappingType;
        }
        if(type1 == type2 && equalStrings(mapping1, mapping2)) {
            // Nothing to do.
        } else if(type1 == Norm::NONE) {
            diff.removeMapping(c);
        } else if(type1 == Norm::ROUND_TRIP) {
            diff.setRoundTripMapping(c, *mapping1);
        } else if(type1 == Norm::ONE_WAY) {
            diff.setOneWayMapping(c, *mapping1);
        }
    }
}

U_NAMESPACE_END

#endif /* #if !UCONFIG_NO_NORMALIZATION */

/*
 * Hey, Emacs, please set the following:
 *
 * Local Variables:
 * indent-tabs-mode: nil
 * End:
 */