tools/uni2characterwidth/uni2characterwidth.cpp

/*
    This file is part of Konsole, a terminal emulator for KDE.

    SPDX-FileCopyrightText: 2018 Mariusz Glebocki <mglb@arccos-1.net>

    SPDX-License-Identifier: GPL-2.0-or-later
*/

#include "template.h"
#include <QCommandLineParser>
#include <QCoreApplication>
#include <QEventLoop>
#include <QFile>
#include <QFileInfo>
#include <QLoggingCategory>
#include <QMap>
#include <QRegularExpression>
#include <QRegularExpressionMatch>
#include <QString>
#include <QTextStream>

#include <KIO/Job>

static constexpr unsigned int CODE_POINTS_NUM = 0x110000;
static constexpr unsigned int LAST_CODE_POINT = CODE_POINTS_NUM - 1;

struct UcdEntry {
    struct {
        uint first;
        uint last;
    } cp;
    QStringList fields;
};

class UcdParserBase
{
public:
    ~UcdParserBase()
    {
        _source->close();
    }

    bool hasNext()
    {
        bool hadNext = _hasNext;
        if (!_nextFetched) {
            _hasNext = fetchNext();
            _nextFetched = true;
        }
        return hadNext;
    }

protected:
    UcdParserBase(QIODevice *source, UcdEntry *entry)
        : _source(source)
        , _nextFetched(false)
        , _hasNext(true)
        , _lineNo(0)
        , _entry(entry)
    {
        Q_ASSERT(_source);
        Q_ASSERT(_entry);
    }

    bool fetchNext()
    {
        Q_ASSERT(_source->isOpen());
        if (!_source->isOpen())
            return false;

        static const QRegularExpression ENTRY_RE = QRegularExpression(QStringLiteral(
            // Match 1: "cp1" - first CP / "cp2" (optional) - last CP
            R"#((?:^(?<cp1>[[:xdigit:]]+)(?:\.\.(?<cp2>[[:xdigit:]]+))?[ \t]*;)#"
            // Match 1: "field0" - first data field"
            //          "udRangeInd" (UnicodeData.txt only) - if present, the line is either first or last line of a range
            R"#([ \t]*(?<field0>[^#;\n]*?(?:, (?<udRangeInd>First|Last)>)?)[ \t]*(?:;|(?:\#.*)?$))|)#"
            // Match 2..n: "field" - n-th field
            R"#((?:\G(?<=;)[ \t]*(?<field>[^#;\n]*?)[ \t]*(?:;|(?:#.*)?$)))#"));
        static const QRegularExpression UD_RANGE_IND_RE(QStringLiteral(", (First|Last)"));
        static const QRegularExpression COMMENT_RE(QStringLiteral("^[ \t]*(#.*)?$"));

        QString line;
        bool ok;
        _entry->fields.clear();
        while (!_source->atEnd()) {
            line = QString::fromUtf8(_source->readLine());
            _lineNo++;
            auto mit = ENTRY_RE.globalMatch(line);
            if (!mit.hasNext()) {
                // Do not complain about comments and empty lines
                if (!COMMENT_RE.match(line).hasMatch())
                    qDebug() << QStringLiteral("Line %1: does not match - skipping").arg(_lineNo);
                continue;
            }

            auto match = mit.next();
            _entry->cp.first = match.captured(QStringLiteral("cp1")).toUInt(&ok, 16);
            if (!ok) {
                qDebug() << QStringLiteral("Line %d Invalid cp1 - skipping").arg(_lineNo);
                continue;
            }
            _entry->cp.last = match.captured(QStringLiteral("cp2")).toUInt(&ok, 16);
            if (!ok) {
                _entry->cp.last = _entry->cp.first;
            }
            QString field0 = match.captured(QStringLiteral("field0"));
            if (field0.isNull()) {
                qDebug() << QStringLiteral("Line %d: Missing field0 - skipping").arg(_lineNo);
                continue;
            }
            if (!match.captured(QStringLiteral("udRangeInd")).isNull()) {
                if (match.captured(QStringLiteral("udRangeInd")) == QStringLiteral("First")) {
                    // Fetch next valid line, as it pairs with the current one to form a range
                    QRegularExpressionMatch nlMatch;
                    int firstLineNo = _lineNo;
                    while (!_source->atEnd() && !nlMatch.hasMatch()) {
                        line = QString::fromUtf8(_source->readLine());
                        _lineNo++;
                        nlMatch = ENTRY_RE.match(line);
                        if (!nlMatch.hasMatch()) {
                            qDebug() << QStringLiteral("Line %d: does not match - skipping").arg(_lineNo);
                        }
                    }
                    if (nlMatch.hasMatch()) {
                        _entry->cp.last = nlMatch.captured(QStringLiteral("cp1")).toUInt(&ok, 16);
                        if (!ok) {
                            qDebug() << QStringLiteral("Line %1-%2: Missing or invalid second cp1 (\"Last\" entry) - skipping").arg(firstLineNo).arg(_lineNo);
                            continue;
                        }
                    }
                }
                field0.remove(UD_RANGE_IND_RE);
            }
            _entry->fields.append(field0);

            while (mit.hasNext()) {
                _entry->fields.append(mit.next().captured(QStringLiteral("field")));
            }

            return !_source->atEnd();
        }
        return false;
    }

    QIODevice *_source;
    bool _nextFetched;
    bool _hasNext;

private:
    int _lineNo;
    UcdEntry *_entry;
};

template<class EntryType>
class UcdParser : public UcdParserBase
{
public:
    static_assert(std::is_base_of<UcdEntry, EntryType>::value, "'EntryType' has to be derived from UcdParser::Entry");

    UcdParser(QIODevice *source)
        : UcdParserBase(source, &_typedEntry)
    {
    }

    inline const EntryType &next()
    {
        if (!_nextFetched)
            fetchNext();
        _nextFetched = false;
        return _typedEntry;
    }

private:
    EntryType _typedEntry;
};

class KIODevice : public QIODevice
{
public:
    enum Error {
        NoError,
        UnknownError,
        TimeoutError,
        UnknownHostError,
        MalformedUrlError,
        NotFoundError,
    };

    KIODevice(const QUrl &url)
        : _url(url)
        , _job(nullptr)
        , _error(NoError)
    {
    }

    ~KIODevice()
    {
        close();
    }

    bool open()
    {
        if (_job)
            return false;

        _job = KIO::storedGet(_url);
        QObject::connect(_job, &KIO::StoredTransferJob::result, _job, [&](KJob *) {
            if (_job->isErrorPage())
                _eventLoop.exit(KIO::ERR_DOES_NOT_EXIST);
            else if (_job->error() != KJob::NoError)
                _eventLoop.exit(_job->error());
            else
                _data = _job->data();

            _eventLoop.exit(KJob::NoError);
        });

        _eventLoop.exec();
        switch (_job->error()) {
        case KJob::NoError:
            _error = NoError;
            setErrorString(QStringLiteral(""));
            QIODevice::open(QIODevice::ReadOnly | QIODevice::Unbuffered);
            break;
        case KJob::KilledJobError:
            _error = TimeoutError;
            break;
        case KIO::ERR_UNKNOWN_HOST:
            _error = UnknownHostError;
            break;
        case KIO::ERR_DOES_NOT_EXIST:
            _error = NotFoundError;
            break;
        case KIO::ERR_MALFORMED_URL:
            _error = MalformedUrlError;
            break;
        default:
            _error = UnknownError;
            break;
        }
        if (_error != NoError) {
            setErrorString(QStringLiteral("KIO: ") + _job->errorString());
            delete _job;
            _job = nullptr;
            _data.clear();
        }
        return _error == NoError;
    }
    bool open(OpenMode mode) override
    {
        Q_ASSERT(mode == QIODevice::ReadOnly);
        return open();
    }
    void close() override
    {
        if (_job) {
            delete _job;
            _job = nullptr;
            _error = NoError;
            setErrorString(QStringLiteral(""));
            _data.clear();
            QIODevice::close();
        }
    }

    qint64 size() const override
    {
        return _data.size();
    }

    int error() const
    {
        return _error;
    }
    void unsetError()
    {
        _error = NoError;
    }

protected:
    qint64 writeData(const char *, qint64) override
    {
        return -1;
    }
    qint64 readData(char *data, qint64 maxSize) override
    {
        Q_UNUSED(maxSize);
        Q_ASSERT(_job);
        Q_ASSERT(_job->error() == NoError);
        Q_ASSERT(data != nullptr);
        if (maxSize == 0 || pos() >= _data.length()) {
            return 0;
        } else if (pos() < _data.length()) {
            qint64 bytesToCopy = qMin(maxSize, _data.length() - pos());
            memcpy(data, _data.data() + pos(), bytesToCopy);
            return bytesToCopy;
        } else {
            return -1;
        }
    }

private:
    QUrl _url;
    KIO::StoredTransferJob *_job;
    Error _error;
    QEventLoop _eventLoop;
    QByteArray _data;
};

struct CategoryProperty {
    enum Flag : uint32_t {
        Invalid = 0,
#define CATEGORY_PROPERTY_VALUE(val, sym, intVal) sym = intVal,
#include "properties.h"
    };
    enum Group : uint32_t {
#define CATEGORY_PROPERTY_GROUP(val, sym, intVal) sym = intVal,
#include "properties.h"
    };

    CategoryProperty(uint32_t value = Unassigned)
        : _value(value)
    {
    }
    CategoryProperty(const QString &string)
        : _value(fromString(string))
    {
    }
    operator uint32_t &()
    {
        return _value;
    }
    operator const uint32_t &() const
    {
        return _value;
    }
    bool isValid() const
    {
        return _value != Invalid;
    }

private:
    static uint32_t fromString(const QString &string)
    {
        static const QMap<QString, uint32_t> map = {
#define CATEGORY_PROPERTY_VALUE(val, sym, intVal) {QStringLiteral(#val), sym},
#include "properties.h"
        };
        return map.contains(string) ? map[string] : uint8_t(Invalid);
    }
    uint32_t _value;
};

struct EastAsianWidthProperty {
    enum Value : uint8_t {
        Invalid = 0x80,
#define EAST_ASIAN_WIDTH_PROPERTY_VALUE(val, sym, intVal) sym = intVal,
#include "properties.h"
    };

    EastAsianWidthProperty(uint8_t value = Neutral)
        : _value(value)
    {
    }
    EastAsianWidthProperty(const QString &string)
        : _value(fromString(string))
    {
    }
    operator uint8_t &()
    {
        return _value;
    }
    operator const uint8_t &() const
    {
        return _value;
    }
    bool isValid() const
    {
        return _value != Invalid;
    }

private:
    static uint8_t fromString(const QString &string)
    {
        static const QMap<QString, Value> map = {
#define EAST_ASIAN_WIDTH_PROPERTY_VALUE(val, sym, intVal) {QStringLiteral(#val), Value::sym},
#include "properties.h"
        };
        return map.contains(string) ? map[string] : Invalid;
    }
    uint8_t _value;
};

struct EmojiProperty {
    enum Flag : uint8_t {
        Invalid = 0x80,
#define EMOJI_PROPERTY_VALUE(val, sym, intVal) sym = intVal,
#include "properties.h"
    };

    EmojiProperty(uint8_t value = None)
        : _value(value)
    {
    }
    EmojiProperty(const QString &string)
        : _value(fromString(string))
    {
    }
    operator uint8_t &()
    {
        return _value;
    }
    operator const uint8_t &() const
    {
        return _value;
    }
    bool isValid() const
    {
        return !(_value & Invalid);
    }

private:
    static uint8_t fromString(const QString &string)
    {
        static const QMap<QString, uint8_t> map = {
#define EMOJI_PROPERTY_VALUE(val, sym, intVal) {QStringLiteral(#val), sym},
#include "properties.h"
        };
        return map.contains(string) ? map[string] : uint8_t(Invalid);
    }
    uint8_t _value;
};

struct CharacterWidth {
    enum Width : int8_t {
        Invalid = SCHAR_MIN,
        _VALID_START = -3,
        Ambiguous = -2,
        NonPrintable = -1,
        // 0
        // 1
        Unassigned = 1,
        // 2
        _VALID_END = 3,
    };

    CharacterWidth(const CharacterWidth &other)
        : _width(other._width)
    {
    }
    CharacterWidth(int8_t width = Invalid)
        : _width(width)
    {
    }
    CharacterWidth &operator=(const CharacterWidth &other)
    {
        _width = other._width;
        return *this;
    }
    int operator=(const int8_t width)
    {
        _width = width;
        return _width;
    }
    int width() const
    {
        return _width;
    }
    operator int() const
    {
        return width();
    }

    const QString toString() const
    {
        switch (_width) {
        case Ambiguous:
            return QStringLiteral("Ambiguous");
        case NonPrintable:
            return QStringLiteral("NonPrintable");
        case 0:
            return QStringLiteral("0");
        case 1:
            return QStringLiteral("1");
        case 2:
            return QStringLiteral("2");
        default:
        case Invalid:
            return QStringLiteral("Invalid");
        }
    }

    bool isValid() const
    {
        return (_width > _VALID_START && _width < _VALID_END);
    };

private:
    int8_t _width;
};

struct CharacterProperties {
    CategoryProperty category;
    EastAsianWidthProperty eastAsianWidth;
    EmojiProperty emoji;
    CharacterWidth customWidth;
    // For debug purposes in "details" output generator
    uint8_t widthFromPropsRule;
};

struct UnicodeDataEntry : public UcdEntry {
    enum FieldId {
        NameId = 0,
        CategoryId = 1,
    };
    CategoryProperty category() const
    {
        return CategoryProperty(this->fields.value(CategoryId));
    }
};

struct EastAsianWidthEntry : public UcdEntry {
    enum FieldId {
        WidthId = 0,
    };
    EastAsianWidthProperty eastAsianWidth() const
    {
        return EastAsianWidthProperty(this->fields.value(WidthId));
    }
};

struct EmojiDataEntry : public UcdEntry {
    enum FieldId {
        EmojiId = 0,
    };
    EmojiProperty emoji() const
    {
        return EmojiProperty(this->fields.value(EmojiId));
    }
};

struct GenericWidthEntry : public UcdEntry {
    enum FieldId {
        WidthId = 0,
    };
    CharacterWidth width() const
    {
        bool ok;
        CharacterWidth w = this->fields.value(WidthId).toInt(&ok, 10);
        return (ok && w.isValid()) ? w : CharacterWidth::Invalid;
    }
};

struct WidthsRange {
    struct {
        uint first;
        uint last;
    } cp;
    CharacterWidth width;
};

QVector<WidthsRange> rangesFromWidths(const QVector<CharacterWidth> &widths, QPair<uint, uint> ucsRange = {0, CODE_POINTS_NUM})
{
    QVector<WidthsRange> ranges;

    if (ucsRange.second >= CODE_POINTS_NUM)
        ucsRange.second = widths.size() - 1;

    uint first = ucsRange.first;
    for (uint cp = first + 1; cp <= uint(ucsRange.second); ++cp) {
        if (widths[first] != widths[cp]) {
            ranges.append({{first, cp - 1}, widths[cp - 1]});
            first = cp;
        }
    }
    ranges.append({{first, uint(ucsRange.second)}, widths[ucsRange.second]});

    return ranges;
}

// Real ranges look like this (each continuous letter sequence is a range):
//
//     D    D D D   D D        D D                   8 ranges
//         C C   C C C C     CC C CC                 9 ranges
//  BBB BBB       B     B BBB       BBBBBB           6 ranges
// A           A         A                A          4 ranges
//                                               ∑: 27 ranges
//
// To reduce total ranges count, the holes in groups can be filled with ranges
// from groups above them:
//
//     D    D D D   D D        D D                   8 ranges
//         CCC   C CCCCC     CCCCCCC                 4 ranges
//  BBBBBBB       BBBBBBB BBBBBBBBBBBBBBBB           3 ranges
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA          1 ranges
//                                               ∑: 16 ranges
//
// First range is always without change. Last range (A) can be dropped
// (it always contains everything). Search should be done in order: D, C, B (A).
// For simplicity the function returns all ranges, including first and last.
QMap<CharacterWidth, QVector<QPair<uint, uint>>>
mergedRangesFromWidths(const QVector<CharacterWidth> &widths, const QVector<CharacterWidth> widthsSortOrder, QPair<uint, uint> ucsRange = {0, CODE_POINTS_NUM})
{
    if (ucsRange.second >= CODE_POINTS_NUM)
        ucsRange.second = widths.size() - 1;
    QVector<WidthsRange> ranges = rangesFromWidths(widths, ucsRange);
    QMap<CharacterWidth, QVector<QPair<uint, uint>>> mergedRanges;

    int cmwi; // Currently Merged Width Index
    int sri = -1; // Start Range Index (for current width)
    int cri; // Current Range Index

    // First width ranges are without change. Last one has one range spanning everything, so we can skip this
    for (cmwi = 1; cmwi < widthsSortOrder.size() - 1; ++cmwi) {
        const CharacterWidth &cmw = widthsSortOrder[cmwi]; // Currently Merged Width
        for (cri = 0; cri < ranges.size(); ++cri) {
            WidthsRange &cr = ranges[cri]; // Current Range
            if (cr.width == cmw) {
                // Range is suitable for merge
                if (sri < 0) {
                    // First one, just remember it
                    sri = cri;
                } else {
                    // Merge
                    ranges[sri].cp.last = cr.cp.last;
                    cr.width = CharacterWidth::Invalid;
                }
            } else {
                // Current range has another width - can we continue merging?
                if (sri >= 0) {
                    const int crwi = widthsSortOrder.indexOf(cr.width); // Current Range Width Index
                    if (!(crwi < cmwi && crwi >= 0)) {
                        // current range is not above currently merged width - stop merging
                        sri = -1;
                    }
                }
            }
        }
    }

    for (const auto &range : qAsConst(ranges)) {
        if (range.width.isValid() && range.width != widthsSortOrder.last())
            mergedRanges[range.width].append({range.cp.first, range.cp.last});
    }
    mergedRanges[widthsSortOrder.last()].append({ucsRange.first, ucsRange.second});

    return mergedRanges;
}

namespace generators
{
using GeneratorFunc = bool (*)(QTextStream &, const QVector<CharacterProperties> &, const QVector<CharacterWidth> &, const QMap<QString, QString> &);

bool code(QTextStream &out, const QVector<CharacterProperties> &props, const QVector<CharacterWidth> &widths, const QMap<QString, QString> &args)
{
    static constexpr int DIRECT_LUT_SIZE = 256;

    Q_UNUSED(props);
    QTextStream eout(stderr, QIODevice::WriteOnly);

    if (args.value(QStringLiteral("param")).isEmpty()) {
        eout << QStringLiteral("Template file not specified.") << Qt::endl << Qt::endl;
        return false;
    }
    QFile templateFile(args.value(QStringLiteral("param")));
    if (!templateFile.open(QIODevice::ReadOnly)) {
        eout << QStringLiteral("Could not open file ") << templateFile.fileName() << ": " << templateFile.errorString();
        exit(1);
    }

    const QString templateText = QString::fromUtf8(templateFile.readAll());
    templateFile.close();

    Var::Map data = {
        {QStringLiteral("gen-file-warning"), QStringLiteral("THIS IS A GENERATED FILE. DO NOT EDIT.")},
        {QStringLiteral("cmdline"), args.value(QStringLiteral("cmdline"))},
        {QStringLiteral("direct-lut"), Var::Vector(DIRECT_LUT_SIZE)},
        {QStringLiteral("direct-lut-size"), DIRECT_LUT_SIZE},
        {QStringLiteral("ranges-luts"), Var::Vector()},
        {QStringLiteral("ranges-lut-list"), Var::Vector()},
        {QStringLiteral("ranges-lut-list-size"), 0},
    };

    // Fill direct-lut with widths of 0x00-0xFF
    for (unsigned i = 0; i < DIRECT_LUT_SIZE; ++i) {
        Q_ASSERT(widths[i].isValid());
        data[QStringLiteral("direct-lut")].vec[i] = int(widths[i]);
    }

    static const QVector<CharacterWidth> widthsSortOrder = {CharacterWidth::NonPrintable, 2, CharacterWidth::Ambiguous, 0, 1};
    const QMap<CharacterWidth, QVector<QPair<uint, uint>>> mergedRanges = mergedRangesFromWidths(widths, widthsSortOrder, {DIRECT_LUT_SIZE, CODE_POINTS_NUM});

    // Find last non-empty ranges lut
    int lastWidthId = 0;
    for (int wi = widthsSortOrder.size() - 1; wi > 0; --wi) {
        if (mergedRanges.contains(widthsSortOrder[wi])) {
            lastWidthId = wi;
            break;
        }
    }
    // Create ranges-luts for all widths except last non-empty one and empty ones
    for (int wi = 0; lastWidthId != 0 && wi < lastWidthId; ++wi) {
        const CharacterWidth width = widthsSortOrder[wi];
        auto currentMergedRangesIt = mergedRanges.find(width);
        if (currentMergedRangesIt == mergedRanges.end() || currentMergedRangesIt.value().isEmpty())
            continue;
        const int size = mergedRanges[width].size();
        const QString name = QString(QStringLiteral("LUT_%1")).arg(width.toString().toUpper());
        data[QStringLiteral("ranges-luts")].vec.append(Var::Map{
            {QStringLiteral("name"), name},
            {QStringLiteral("ranges"), Var::Vector()},
            {QStringLiteral("size"), size},
        });
        data[QStringLiteral("ranges-lut-list")].vec.append(Var::Map{
            {QStringLiteral("width"), int(width)},
            {QStringLiteral("name"), name},
            {QStringLiteral("size"), size},
        });
        auto &currentLut = data[QStringLiteral("ranges-luts")].vec.last()[QStringLiteral("ranges")].vec;
        for (const auto &range : *currentMergedRangesIt) {
            Q_ASSERT(range.first <= LAST_CODE_POINT);
            Q_ASSERT(range.second <= LAST_CODE_POINT);
            currentLut.append(Var(Var::Map{{QStringLiteral("first"), range.first}, {QStringLiteral("last"), range.second}}));
        }
    }
    data[QStringLiteral("ranges-lut-list")].vec.append(Var::Map{
        {QStringLiteral("width"), widthsSortOrder[lastWidthId].width()},
        {QStringLiteral("name"), QStringLiteral("nullptr")},
        {QStringLiteral("size"), 1},
    });
    data[QStringLiteral("ranges-lut-list-size")] = mergedRanges.size();

    Template t(templateText);
    t.parse();
    out << t.generate(data);

    return true;
}

bool list(QTextStream &out, const QVector<CharacterProperties> &props, const QVector<CharacterWidth> &widths, const QMap<QString, QString> &args)
{
    Q_UNUSED(props);

    out << QStringLiteral("# generated with: ") << args.value(QStringLiteral("cmdline")) << QStringLiteral("\n");
    for (uint cp = 1; cp <= LAST_CODE_POINT; ++cp) {
        out << QString::asprintf("%06X ; %2d\n", cp, int(widths[cp]));
    }

    return true;
}

bool ranges(QTextStream &out, const QVector<CharacterProperties> &props, const QVector<CharacterWidth> &widths, const QMap<QString, QString> &args)
{
    Q_UNUSED(props);
    const auto ranges = rangesFromWidths(widths);

    out << QStringLiteral("# generated with: ") << args.value(QStringLiteral("cmdline")) << QStringLiteral("\n");
    for (const WidthsRange &range : ranges) {
        if (range.cp.first != range.cp.last)
            out << QString::asprintf("%06X..%06X ; %2d\n", range.cp.first, range.cp.last, int(range.width));
        else
            out << QString::asprintf("%06X         ; %2d\n", range.cp.first, int(range.width));
    }

    return true;
}

bool compactRanges(QTextStream &out, const QVector<CharacterProperties> &props, const QVector<CharacterWidth> &widths, const QMap<QString, QString> &args)
{
    Q_UNUSED(props);
    static const QVector<CharacterWidth> widthsSortOrder = {CharacterWidth::NonPrintable, 2, CharacterWidth::Ambiguous, 0, 1};
    const auto mergedRanges = mergedRangesFromWidths(widths, widthsSortOrder);

    out << QStringLiteral("# generated with: ") << args.value(QStringLiteral("cmdline")) << QStringLiteral("\n");
    for (const int width : qAsConst(widthsSortOrder)) {
        const auto currentMergedRangesIt = mergedRanges.find(width);
        if (currentMergedRangesIt == mergedRanges.end() || currentMergedRangesIt.value().isEmpty())
            continue;
        for (const auto &range : currentMergedRangesIt.value()) {
            if (range.first != range.second)
                out << QString::asprintf("%06X..%06X ; %2d\n", range.first, range.second, int(width));
            else
                out << QString::asprintf("%06X         ; %2d\n", range.first, int(width));
        }
    }

    return true;
}

bool details(QTextStream &out, const QVector<CharacterProperties> &props, const QVector<CharacterWidth> &widths, const QMap<QString, QString> &args)
{
    out.setFieldAlignment(QTextStream::AlignLeft);

    out << QStringLiteral("# generated with: ") << args.value(QStringLiteral("cmdline")) << QStringLiteral("\n");
    out << QString::asprintf("#%-5s ; %-4s ; %-8s ; %-3s ; %-2s ; %-4s ; %-4s\n", "CP", "Wdth", "Cat", "EAW", "EM", "CstW", "Rule");
    QMap<CharacterWidth, uint> widthStats;
    for (uint cp = 0; cp <= LAST_CODE_POINT; ++cp) {
        out << QString::asprintf("%06X ; %4d ; %08X ;  %02X ; %02X ; %4d ; %d\n",
                                 cp,
                                 int8_t(widths[cp]),
                                 uint32_t(props[cp].category),
                                 uint8_t(props[cp].eastAsianWidth),
                                 uint8_t(props[cp].emoji),
                                 int8_t(props[cp].customWidth),
                                 props[cp].widthFromPropsRule);
        if (!widthStats.contains(widths[cp]))
            widthStats.insert(widths[cp], 0);
        widthStats[widths[cp]]++;
    }
    QMap<CharacterWidth, uint> rangesStats;
    const auto ranges = rangesFromWidths(widths);
    for (const auto &range : ranges) {
        if (!rangesStats.contains(range.width))
            rangesStats.insert(range.width, 0);
        rangesStats[range.width]++;
    }
    out << QStringLiteral("# STATS") << Qt::endl;
    out << QStringLiteral("#") << Qt::endl;
    out << QStringLiteral("# Characters count for each width:") << Qt::endl;
    for (auto wi = widthStats.constBegin(); wi != widthStats.constEnd(); ++wi) {
        out << QString::asprintf("# %2d: %7d\n", int(wi.key()), widthStats[wi.key()]);
    }
    out << QStringLiteral("#") << Qt::endl;
    out << QStringLiteral("# Ranges count for each width:") << Qt::endl;
    int howmany = 0;
    for (auto wi = rangesStats.constBegin(); wi != rangesStats.constEnd(); ++wi) {
        if (howmany >= 20)
            break;
        howmany++;
        out << QString::asprintf("# %2d: %7d\n", int(wi.key()), rangesStats[wi.key()]);
    }

    return true;
}
} // namespace generators

template<class EntryType>
static void processInputFiles(QVector<CharacterProperties> &props,
                              const QStringList &files,
                              const QString &fileTypeName,
                              void (*cb)(CharacterProperties &prop, const EntryType &entry))
{
    static const QRegularExpression PROTOCOL_RE(QStringLiteral(R"#(^[a-z]+://)#"));
    for (const QString &fileName : files) {
        qInfo().noquote() << QStringLiteral("Parsing as %1: %2").arg(fileTypeName).arg(fileName);
        QSharedPointer<QIODevice> source = nullptr;
        if (PROTOCOL_RE.match(fileName).hasMatch()) {
            source.reset(new KIODevice(QUrl(fileName)));
        } else {
            source.reset(new QFile(fileName));
        }

        if (!source->open(QIODevice::ReadOnly)) {
            qCritical() << QStringLiteral("Could not open %1: %2").arg(fileName).arg(source->errorString());
            exit(1);
        }
        UcdParser<EntryType> p(source.data());
        while (p.hasNext()) {
            const auto &e = p.next();
            for (uint cp = e.cp.first; cp <= e.cp.last; ++cp) {
                cb(props[cp], e);
            }
        }
    }
}

static const QString escapeCmdline(const QStringList &args)
{
    static QString cmdline = QString();
    if (!cmdline.isEmpty())
        return cmdline;

    QTextStream stream(&cmdline, QIODevice::WriteOnly);

    // basename for command name
    stream << QFileInfo(args[0]).baseName();
    for (auto it = args.begin() + 1; it != args.end(); ++it) {
        if (!it->startsWith(QLatin1Char('-')))
            stream << QStringLiteral(" \"") << QString(*it).replace(QRegularExpression(QStringLiteral(R"(["`$\\])")), QStringLiteral(R"(\\\1)")) << '"';
        else
            stream << ' ' << *it;
    }
    stream.flush();
    return cmdline;
}

enum ConvertOptions {
    AmbiguousWidthOpt = 0,
    EmojiOpt = 1,
};

// Character width assignment
//
// Rules (from highest to lowest priority):
//
// * Local overlay
// * (not implemented) Character unique properties described in The Unicode Standard, Version 10.0
// * Unicode category Cc, Cs: -1
// * Emoji: 2
// * Unicode category Mn, Me, Cf: 0
// * East Asian Width W, F: 2
// * East Asian Width H, N, Na: 1
// * East Asian Width A: (varies)
// * Unassigned/Undefined/Private Use: 1
//
// The list is loosely based on character width implementations in Vim 8.1
// and glibc 2.27. There are a few cases which could look better
// (decomposed Hangul, emoji with modifiers, etc) with different widths,
// but interactive terminal programs (at least vim, zsh, everything based
// on glibc's wcwidth) would see their width as it is implemented now.
static inline CharacterWidth widthFromProps(const CharacterProperties &props, uint cp, const QMap<ConvertOptions, int> &convertOpts)
{
    CharacterWidth cw;
    auto &widthFromPropsRule = const_cast<uint8_t &>(props.widthFromPropsRule);
    if (props.customWidth.isValid()) {
        widthFromPropsRule = 1;
        cw = props.customWidth;

    } else if ((CategoryProperty::Control | CategoryProperty::Surrogate) & props.category) {
        widthFromPropsRule = 2;
        cw = CharacterWidth::NonPrintable;

    } else if (convertOpts[EmojiOpt] & props.emoji && !(EmojiProperty::EmojiComponent & props.emoji)) {
        widthFromPropsRule = 3;
        cw = 2;

    } else if ((CategoryProperty::NonspacingMark | CategoryProperty::EnclosingMark | CategoryProperty::Format) & props.category) {
        widthFromPropsRule = 4;
        cw = 0;

    } else if ((EastAsianWidthProperty::Wide | EastAsianWidthProperty::Fullwidth) & props.eastAsianWidth) {
        widthFromPropsRule = 5;
        cw = 2;

    } else if ((EastAsianWidthProperty::Halfwidth | EastAsianWidthProperty::Neutral | EastAsianWidthProperty::Narrow) & props.eastAsianWidth) {
        widthFromPropsRule = 6;
        cw = 1;

    } else if ((CategoryProperty::Unassigned | CategoryProperty::PrivateUse) & props.category) {
        widthFromPropsRule = 7;
        cw = CharacterWidth::Unassigned;

    } else if ((EastAsianWidthProperty::Ambiguous)&props.eastAsianWidth) {
        widthFromPropsRule = 8;
        cw = convertOpts[AmbiguousWidthOpt];

    } else if (!props.category.isValid()) {
        widthFromPropsRule = 9;
        qWarning() << QStringLiteral("Code point U+%1 has invalid category - this should not happen. Assuming \"unassigned\"").arg(cp, 4, 16, QLatin1Char('0'));
        cw = CharacterWidth::Unassigned;

    } else {
        widthFromPropsRule = 10;
        qWarning()
            << QStringLiteral("Code point U+%1 not classified - this should not happen. Assuming non-printable character").arg(cp, 4, 16, QLatin1Char('0'));
        cw = CharacterWidth::NonPrintable;
    }

    return cw;
}

int main(int argc, char *argv[])
{
    static const QMap<QString, generators::GeneratorFunc> GENERATOR_FUNCS_MAP = {
        {QStringLiteral("code"), generators::code},
        {QStringLiteral("compact-ranges"), generators::compactRanges},
        {QStringLiteral("ranges"), generators::ranges},
        {QStringLiteral("list"), generators::list},
        {QStringLiteral("details"), generators::details},
        {QStringLiteral("dummy"),
         [](QTextStream &, const QVector<CharacterProperties> &, const QVector<CharacterWidth> &, const QMap<QString, QString> &) -> bool {
             return true;
         }},
    };
    qSetMessagePattern(QStringLiteral("%{message}"));

    QCoreApplication app(argc, argv);
    QCommandLineParser parser;
    parser.setApplicationDescription(QStringLiteral("\nUCD files to characters widths converter.\n"));
    parser.addHelpOption();
    parser.addOptions({
        {{QStringLiteral("U"), QStringLiteral("unicode-data")}, QStringLiteral("Path or URL to UnicodeData.txt."), QStringLiteral("URL|file")},
        {{QStringLiteral("A"), QStringLiteral("east-asian-width")}, QStringLiteral("Path or URL to EastAsianWidth.txt."), QStringLiteral("URL|file")},
        {{QStringLiteral("E"), QStringLiteral("emoji-data")}, QStringLiteral("Path or URL to emoji-data.txt."), QStringLiteral("URL|file")},
        {{QStringLiteral("W"), QStringLiteral("generic-width")},
         QStringLiteral("Path or URL to generic file with width data. Accepts output from compact-ranges, ranges, list and details generator."),
         QStringLiteral("URL|file")},

        {QStringLiteral("ambiguous-width"),
         QStringLiteral("Ambiguous characters width."),
         QStringLiteral("separate|1|2"),
         QString(QStringLiteral("%1")).arg(CharacterWidth::Ambiguous)},
        {QStringLiteral("emoji"),
         QStringLiteral("Which emoji emoji subset is treated as emoji."),
         QStringLiteral("all|presentation"),
         QStringLiteral("presentation")},

        {{QStringLiteral("g"), QStringLiteral("generator")},
         QStringLiteral("Output generator (use \"-\" to list available generators). The code generator requires path to a template file."),
         QStringLiteral("generator[:template]"),
         QStringLiteral("details")},
    });
    parser.addPositionalArgument(QStringLiteral("output"), QStringLiteral("Output file (leave empty for stdout)."));
    parser.process(app);

    const QStringList unicodeDataFiles = parser.values(QStringLiteral("unicode-data"));
    const QStringList eastAsianWidthFiles = parser.values(QStringLiteral("east-asian-width"));
    const QStringList emojiDataFiles = parser.values(QStringLiteral("emoji-data"));
    const QStringList genericWidthFiles = parser.values(QStringLiteral("generic-width"));
    const QString ambiguousWidthStr = parser.value(QStringLiteral("ambiguous-width"));
    const QString emojiStr = parser.value(QStringLiteral("emoji"));
    const QString generator = parser.value(QStringLiteral("generator"));
    const QString outputFileName = parser.positionalArguments().value(0);

    QTextStream eout(stderr, QIODevice::WriteOnly);
    if (unicodeDataFiles.isEmpty() && eastAsianWidthFiles.isEmpty() && emojiDataFiles.isEmpty() && genericWidthFiles.isEmpty()) {
        eout << QStringLiteral("Input files not specified.") << Qt::endl << Qt::endl;
        parser.showHelp(1);
    }

    static QMap<ConvertOptions, int> convertOpts = {
        {AmbiguousWidthOpt, CharacterWidth::Ambiguous},
        {EmojiOpt, EmojiProperty::EmojiPresentation},
    };

    if (emojiStr == QStringLiteral("presentation"))
        convertOpts[EmojiOpt] = EmojiProperty::EmojiPresentation;
    else if (emojiStr == QStringLiteral("all"))
        convertOpts[EmojiOpt] = EmojiProperty::Emoji;
    else {
        convertOpts[EmojiOpt] = EmojiProperty::EmojiPresentation;
        qWarning() << QStringLiteral("invalid emoji option value: %1. Assuming \"presentation\".").arg(emojiStr);
    }

    if (ambiguousWidthStr == QStringLiteral("separate"))
        convertOpts[AmbiguousWidthOpt] = CharacterWidth::Ambiguous;
    else if (ambiguousWidthStr == QStringLiteral("1"))
        convertOpts[AmbiguousWidthOpt] = 1;
    else if (ambiguousWidthStr == QStringLiteral("2"))
        convertOpts[AmbiguousWidthOpt] = 2;
    else {
        convertOpts[AmbiguousWidthOpt] = CharacterWidth::Ambiguous;
        qWarning() << QStringLiteral("Invalid ambiguous-width option value: %1. Assuming \"separate\".").arg(emojiStr);
    }

    const int sepPos = generator.indexOf(QLatin1Char(':'));
    const auto generatorName = generator.left(sepPos);
    const auto generatorParam = sepPos >= 0 ? generator.mid(sepPos + 1) : QString();

    if (!GENERATOR_FUNCS_MAP.contains(generatorName)) {
        int status = 0;
        if (generatorName != QStringLiteral("-")) {
            status = 1;
            eout << QStringLiteral("Invalid output generator. Available generators:") << Qt::endl;
        }

        for (auto it = GENERATOR_FUNCS_MAP.constBegin(); it != GENERATOR_FUNCS_MAP.constEnd(); ++it) {
            eout << it.key() << Qt::endl;
        }
        exit(status);
    }
    auto generatorFunc = GENERATOR_FUNCS_MAP[generatorName];

    QFile outFile;
    if (!outputFileName.isEmpty()) {
        outFile.setFileName(outputFileName);
        if (!outFile.open(QIODevice::WriteOnly)) {
            eout << QStringLiteral("Could not open file ") << outputFileName << QStringLiteral(": ") << outFile.errorString() << Qt::endl;
            exit(1);
        }
    } else {
        outFile.open(stdout, QIODevice::WriteOnly);
    }
    QTextStream out(&outFile);

    QVector<CharacterProperties> props(CODE_POINTS_NUM);

    processInputFiles<UnicodeDataEntry>(props,
                                        unicodeDataFiles,
                                        QStringLiteral("UnicodeData.txt"),
                                        [](CharacterProperties &prop, const UnicodeDataEntry &entry) {
                                            prop.category = entry.category();
                                        });

    processInputFiles<EastAsianWidthEntry>(props,
                                           eastAsianWidthFiles,
                                           QStringLiteral("EastAsianWidth.txt"),
                                           [](CharacterProperties &prop, const EastAsianWidthEntry &entry) {
                                               prop.eastAsianWidth = entry.eastAsianWidth();
                                           });

    processInputFiles<EmojiDataEntry>(props, emojiDataFiles, QStringLiteral("emoji-data.txt"), [](CharacterProperties &prop, const EmojiDataEntry &entry) {
        prop.emoji |= entry.emoji();
    });

    processInputFiles<GenericWidthEntry>(props,
                                         genericWidthFiles,
                                         QStringLiteral("generic width data"),
                                         [](CharacterProperties &prop, const GenericWidthEntry &entry) {
                                             prop.customWidth = entry.width();
                                         });

    qInfo() << "Generating character width data";
    QVector<CharacterWidth> widths(CODE_POINTS_NUM);
    widths[0] = 0; // NULL character always has width 0
    for (uint cp = 1; cp <= LAST_CODE_POINT; ++cp) {
        widths[cp] = widthFromProps(props[cp], cp, convertOpts);
    }

    const QMap<QString, QString> generatorArgs = {
        {QStringLiteral("cmdline"), escapeCmdline(app.arguments())},
        {QStringLiteral("param"), generatorParam},
        {QStringLiteral("output"), outputFileName.isEmpty() ? QStringLiteral("<stdout>") : outputFileName},
    };

    qInfo() << "Generating output";
    if (!generatorFunc(out, props, widths, generatorArgs)) {
        parser.showHelp(1);
    }

    return 0;
}