xref: /dports/textproc/py-ufal.udpipe/ufal.udpipe-1.2.0.3/udpipe/udpipe.cpp

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// This file is a bundle of all sources and headers of UDPipe library.
// Comments and copyrights of all individual files are kept.

#include <algorithm>
#include <atomic>
#include <cassert>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <initializer_list>
#include <iomanip>
#include <iostream>
#include <iterator>
#include <limits>
#include <list>
#include <map>
#include <memory>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

namespace ufal {
namespace udpipe {

/////////
// File: utils/common.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Headers available in all sources

namespace utils {

using namespace std;

// Assert that int is at least 4B
static_assert(sizeof(int) >= sizeof(int32_t), "Int must be at least 4B wide!");

// Assert that we are on a little endian system
#ifdef __BYTE_ORDER__
static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Only little endian systems are supported!");
#endif

#define runtime_failure(message) exit((cerr << message << endl, 1))

} // namespace utils

/////////
// File: utils/string_piece.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

struct string_piece {
  const char* str;
  size_t len;

  string_piece() : str(nullptr), len(0) {}
  string_piece(const char* str) : str(str), len(strlen(str)) {}
  string_piece(const char* str, size_t len) : str(str), len(len) {}
  string_piece(const string& str) : str(str.c_str()), len(str.size()) {}
};

inline ostream& operator<<(ostream& os, const string_piece& str) {
  return os.write(str.str, str.len);
}

inline bool operator==(const string_piece& a, const string_piece& b) {
  return a.len == b.len && memcmp(a.str, b.str, a.len) == 0;
}

inline bool operator!=(const string_piece& a, const string_piece& b) {
  return a.len != b.len || memcmp(a.str, b.str, a.len) != 0;
}

} // namespace utils

/////////
// File: common.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

using namespace utils;

/////////
// File: sentence/empty_node.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class empty_node {
 public:
  int id;         // 0 is root, >0 is sentence word, <0 is undefined
  int index;      // index for the current id, should be numbered from 1, 0=undefined
  string form;    // form
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  string deps;    // secondary dependencies
  string misc;    // miscellaneous information

  empty_node(int id = -1, int index = 0) : id(id), index(index) {}
};

/////////
// File: sentence/token.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2017 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class token {
 public:
  string form;
  string misc;

  token(string_piece form = string_piece(), string_piece misc = string_piece());

  // CoNLL-U defined SpaceAfter=No feature
  bool get_space_after() const;
  void set_space_after(bool space_after);

  // UDPipe-specific all-spaces-preserving SpacesBefore and SpacesAfter features
  void get_spaces_before(string& spaces_before) const;
  void set_spaces_before(string_piece spaces_before);
  void get_spaces_after(string& spaces_after) const;
  void set_spaces_after(string_piece spaces_after);
  void get_spaces_in_token(string& spaces_in_token) const;
  void set_spaces_in_token(string_piece spaces_in_token);

  // UDPipe-specific TokenRange feature
  bool get_token_range(size_t& start, size_t& end) const;
  void set_token_range(size_t start, size_t end);

 private:
  bool get_misc_field(string_piece name, string_piece& value) const;
  void remove_misc_field(string_piece name);
  string& start_misc_field(string_piece name);

  void append_escaped_spaces(string_piece spaces, string& escaped_spaces) const;
  void unescape_spaces(string_piece escaped_spaces, string& spaces) const;
};

/////////
// File: sentence/multiword_token.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_token : public token {
 public:
  // form and misc are inherited from token
  int id_first, id_last;

  multiword_token(int id_first = -1, int id_last = -1, string_piece form = string_piece(), string_piece misc = string_piece())
      : token(form, misc), id_first(id_first), id_last(id_last) {}
};

/////////
// File: sentence/word.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class word : public token {
 public:
  // form and misc are inherited from token
  int id;         // 0 is root, >0 is sentence word, <0 is undefined
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  int head;       // head, 0 is root, <0 is undefined
  string deprel;  // dependency relation to the head
  string deps;    // secondary dependencies

  vector<int> children;

  word(int id = -1, string_piece form = string_piece()) : token(form), id(id), head(-1) {}
};

/////////
// File: sentence/sentence.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class sentence {
 public:
  sentence();

  vector<word> words;
  vector<multiword_token> multiword_tokens;
  vector<empty_node> empty_nodes;
  vector<string> comments;
  static const string root_form;

  // Basic sentence modifications
  bool empty();
  void clear();
  word& add_word(string_piece form = string_piece());
  void set_head(int id, int head, const string& deprel);
  void unlink_all_words();

  // CoNLL-U defined comments
  bool get_new_doc(string* id = nullptr) const;
  void set_new_doc(bool new_doc, string_piece id = string_piece());
  bool get_new_par(string* id = nullptr) const;
  void set_new_par(bool new_par, string_piece id = string_piece());
  bool get_sent_id(string& id) const;
  void set_sent_id(string_piece id);
  bool get_text(string& text) const;
  void set_text(string_piece text);

 private:
  bool get_comment(string_piece name, string* value) const;
  void remove_comment(string_piece name);
  void set_comment(string_piece name, string_piece value = string_piece());
};

/////////
// File: sentence/input_format.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class input_format {
 public:
  virtual ~input_format() {}

  virtual bool read_block(istream& is, string& block) const = 0;
  virtual void reset_document(string_piece id = string_piece()) = 0;
  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_sentence(sentence& s, string& error) = 0;

  // Static factory methods
  static input_format* new_input_format(const string& name);
  static input_format* new_conllu_input_format(const string& options = string());
  static input_format* new_generic_tokenizer_input_format(const string& options = string());
  static input_format* new_horizontal_input_format(const string& options = string());
  static input_format* new_vertical_input_format(const string& options = string());

  static input_format* new_presegmented_tokenizer(input_format* tokenizer);

  static const string CONLLU_V1;
  static const string CONLLU_V2;
  static const string GENERIC_TOKENIZER_NORMALIZED_SPACES;
  static const string GENERIC_TOKENIZER_PRESEGMENTED;
  static const string GENERIC_TOKENIZER_RANGES;
};

/////////
// File: model/model.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class model {
 public:
  virtual ~model() {}

  static model* load(const char* fname);
  static model* load(istream& is);

  virtual input_format* new_tokenizer(const string& options) const = 0;
  virtual bool tag(sentence& s, const string& options, string& error) const = 0;
  virtual bool parse(sentence& s, const string& options, string& error) const = 0;

  static const string DEFAULT;
  static const string TOKENIZER_NORMALIZED_SPACES;
  static const string TOKENIZER_PRESEGMENTED;
  static const string TOKENIZER_RANGES;
};

/////////
// File: model/evaluator.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class evaluator {
 public:
  evaluator(const model* m, const string& tokenizer, const string& tagger, const string& parser);

  void set_model(const model* m);
  void set_tokenizer(const string& tokenizer);
  void set_tagger(const string& tagger);
  void set_parser(const string& parser);

  bool evaluate(istream& is, ostream& os, string& error) const;

  static const string DEFAULT;
  static const string NONE;

 private:
  const model* m;
  string tokenizer, tagger, parser;

  struct f1_info { size_t total_system, total_gold; double precision, recall, f1; };
  template <class T>
  static f1_info evaluate_f1(const vector<pair<size_t, T>>& system, const vector<pair<size_t, T>>& gold);

  class evaluation_data {
   public:
    struct word_data {
      size_t start, end;
      bool is_multiword;
      word w;

      word_data(size_t start, size_t end, int id, bool is_multiword, const word& w);
    };

    void add_sentence(const sentence& s);

    u32string chars;
    vector<pair<size_t, size_t>> sentences, tokens;
    vector<pair<size_t, string>> multiwords;
    vector<word_data> words;
  };

  class word_alignment {
   public:
    struct pair_system_gold {
      word system; const word& gold;
      pair_system_gold(const word& system, const word& gold) : system(system), gold(gold) {}
    };
    vector<pair_system_gold> matched;
    size_t total_system, total_gold;

    template <class Equals>
    f1_info evaluate_f1(Equals equals);

    static bool perfect_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment);
    static void best_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment);
  };
};

/////////
// File: unilib/unicode.h
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

class unicode {
  enum : uint8_t {
    _Lu = 1, _Ll = 2, _Lt = 3, _Lm = 4, _Lo = 5,
    _Mn = 6, _Mc = 7, _Me = 8,
    _Nd = 9, _Nl = 10, _No = 11,
    _Pc = 12, _Pd = 13, _Ps = 14, _Pe = 15, _Pi = 16, _Pf = 17, _Po = 18,
    _Sm = 19, _Sc = 20, _Sk = 21, _So = 22,
    _Zs = 23, _Zl = 24, _Zp = 25,
    _Cc = 26, _Cf = 27, _Cs = 28, _Co = 29, _Cn = 30
  };

 public:
  typedef uint32_t category_t;
  enum : category_t {
    Lu = 1 << _Lu, Ll = 1 << _Ll, Lt = 1 << _Lt, Lut = Lu | Lt, LC = Lu | Ll | Lt,
      Lm = 1 << _Lm, Lo = 1 << _Lo, L = Lu | Ll | Lt | Lm | Lo,
    Mn = 1 << _Mn, Mc = 1 << _Mc, Me = 1 << _Me, M = Mn | Mc | Me,
    Nd = 1 << _Nd, Nl = 1 << _Nl, No = 1 << _No, N = Nd | Nl | No,
    Pc = 1 << _Pc, Pd = 1 << _Pd, Ps = 1 << _Ps, Pe = 1 << _Pe, Pi = 1 << _Pi,
      Pf = 1 << _Pf, Po = 1 << _Po, P = Pc | Pd | Ps | Pe | Pi | Pf | Po,
    Sm = 1 << _Sm, Sc = 1 << _Sc, Sk = 1 << _Sk, So = 1 << _So, S = Sm | Sc | Sk | So,
    Zs = 1 << _Zs, Zl = 1 << _Zl, Zp = 1 << _Zp, Z = Zs | Zl | Zp,
    Cc = 1 << _Cc, Cf = 1 << _Cf, Cs = 1 << _Cs, Co = 1 << _Co, Cn = 1 << _Cn, C = Cc | Cf | Cs | Co | Cn
  };

  static inline category_t category(char32_t chr);

  static inline char32_t lowercase(char32_t chr);
  static inline char32_t uppercase(char32_t chr);
  static inline char32_t titlecase(char32_t chr);

 private:
  static const char32_t CHARS = 0x110000;
  static const int32_t DEFAULT_CAT = Cn;

  static const uint8_t category_index[CHARS >> 8];
  static const uint8_t category_block[][256];
  static const uint8_t othercase_index[CHARS >> 8];
  static const char32_t othercase_block[][256];

  enum othercase_type { LOWER_ONLY = 1, UPPERTITLE_ONLY = 2, LOWER_THEN_UPPER = 3, UPPER_THEN_TITLE = 4, TITLE_THEN_LOWER = 5 };
};

unicode::category_t unicode::category(char32_t chr) {
  return chr < CHARS ? 1 << category_block[category_index[chr >> 8]][chr & 0xFF] : DEFAULT_CAT;
}

char32_t unicode::lowercase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::LOWER_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::LOWER_THEN_UPPER) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::TITLE_THEN_LOWER) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

char32_t unicode::uppercase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::UPPERTITLE_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::UPPER_THEN_TITLE) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::LOWER_THEN_UPPER) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

char32_t unicode::titlecase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::UPPERTITLE_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::TITLE_THEN_LOWER) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::UPPER_THEN_TITLE) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

} // namespace unilib

/////////
// File: unilib/utf8.h
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

class utf8 {
 public:
  static bool valid(const char* str);
  static bool valid(const char* str, size_t len);
  static inline bool valid(const std::string& str);

  static inline char32_t decode(const char*& str);
  static inline char32_t decode(const char*& str, size_t& len);
  static inline char32_t first(const char* str);
  static inline char32_t first(const char* str, size_t len);
  static inline char32_t first(const std::string& str);

  static void decode(const char* str, std::u32string& decoded);
  static void decode(const char* str, size_t len, std::u32string& decoded);
  static inline void decode(const std::string& str, std::u32string& decoded);

  class string_decoder {
   public:
    class iterator;
    inline iterator begin();
    inline iterator end();
   private:
    inline string_decoder(const char* str);
    const char* str;
    friend class utf8;
  };
  static inline string_decoder decoder(const char* str);
  static inline string_decoder decoder(const std::string& str);

  class buffer_decoder {
   public:
    class iterator;
    inline iterator begin();
    inline iterator end();
   private:
    inline buffer_decoder(const char* str, size_t len);
    const char* str;
    size_t len;
    friend class utf8;
  };
  static inline buffer_decoder decoder(const char* str, size_t len);

  static inline void append(char*& str, char32_t chr);
  static inline void append(std::string& str, char32_t chr);
  static void encode(const std::u32string& str, std::string& encoded);

  template<class F> static void map(F f, const char* str, std::string& result);
  template<class F> static void map(F f, const char* str, size_t len, std::string& result);
  template<class F> static void map(F f, const std::string& str, std::string& result);

 private:
  static const char REPLACEMENT_CHAR = '?';
};

bool utf8::valid(const std::string& str) {
  return valid(str.c_str());
}

char32_t utf8::decode(const char*& str) {
  if (((unsigned char)*str) < 0x80) return (unsigned char)*str++;
  else if (((unsigned char)*str) < 0xC0) return ++str, REPLACEMENT_CHAR;
  else if (((unsigned char)*str) < 0xE0) {
    char32_t res = (((unsigned char)*str++) & 0x1F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else if (((unsigned char)*str) < 0xF0) {
    char32_t res = (((unsigned char)*str++) & 0x0F) << 12;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else if (((unsigned char)*str) < 0xF8) {
    char32_t res = (((unsigned char)*str++) & 0x07) << 18;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 12;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else return ++str, REPLACEMENT_CHAR;
}

char32_t utf8::decode(const char*& str, size_t& len) {
  if (!len) return 0;
  --len;
  if (((unsigned char)*str) < 0x80) return (unsigned char)*str++;
  else if (((unsigned char)*str) < 0xC0) return ++str, REPLACEMENT_CHAR;
  else if (((unsigned char)*str) < 0xE0) {
    char32_t res = (((unsigned char)*str++) & 0x1F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else if (((unsigned char)*str) < 0xF0) {
    char32_t res = (((unsigned char)*str++) & 0x0F) << 12;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else if (((unsigned char)*str) < 0xF8) {
    char32_t res = (((unsigned char)*str++) & 0x07) << 18;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 12;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else return ++str, REPLACEMENT_CHAR;
}

char32_t utf8::first(const char* str) {
  return decode(str);
}

char32_t utf8::first(const char* str, size_t len) {
  return decode(str, len);
}

char32_t utf8::first(const std::string& str) {
  return first(str.c_str());
}

void utf8::decode(const std::string& str, std::u32string& decoded) {
  decode(str.c_str(), decoded);
}

class utf8::string_decoder::iterator : public std::iterator<std::input_iterator_tag, char32_t> {
 public:
  iterator(const char* str) : codepoint(0), next(str) { operator++(); }
  iterator(const iterator& it) : codepoint(it.codepoint), next(it.next) {}
  iterator& operator++() { if (next) { codepoint = decode(next); if (!codepoint) next = nullptr; } return *this; }
  iterator operator++(int) { iterator tmp(*this); operator++(); return tmp; }
  bool operator==(const iterator& other) const { return next == other.next; }
  bool operator!=(const iterator& other) const { return next != other.next; }
  const char32_t& operator*() { return codepoint; }
 private:
  char32_t codepoint;
  const char* next;
};

utf8::string_decoder::string_decoder(const char* str) : str(str) {}

utf8::string_decoder::iterator utf8::string_decoder::begin() {
  return iterator(str);
}

utf8::string_decoder::iterator utf8::string_decoder::end() {
  return iterator(nullptr);
}

utf8::string_decoder utf8::decoder(const char* str) {
  return string_decoder(str);
}

utf8::string_decoder utf8::decoder(const std::string& str) {
  return string_decoder(str.c_str());
}

class utf8::buffer_decoder::iterator : public std::iterator<std::input_iterator_tag, char32_t> {
 public:
  iterator(const char* str, size_t len) : codepoint(0), next(str), len(len) { operator++(); }
  iterator(const iterator& it) : codepoint(it.codepoint), next(it.next), len(it.len) {}
  iterator& operator++() { if (!len) next = nullptr; if (next) codepoint = decode(next, len); return *this; }
  iterator operator++(int) { iterator tmp(*this); operator++(); return tmp; }
  bool operator==(const iterator& other) const { return next == other.next; }
  bool operator!=(const iterator& other) const { return next != other.next; }
  const char32_t& operator*() { return codepoint; }
 private:
  char32_t codepoint;
  const char* next;
  size_t len;
};

utf8::buffer_decoder::buffer_decoder(const char* str, size_t len) : str(str), len(len) {}

utf8::buffer_decoder::iterator utf8::buffer_decoder::begin() {
  return iterator(str, len);
}

utf8::buffer_decoder::iterator utf8::buffer_decoder::end() {
  return iterator(nullptr, 0);
}

utf8::buffer_decoder utf8::decoder(const char* str, size_t len) {
  return buffer_decoder(str, len);
}

void utf8::append(char*& str, char32_t chr) {
  if (chr < 0x80) *str++ = chr;
  else if (chr < 0x800) { *str++ = 0xC0 + (chr >> 6); *str++ = 0x80 + (chr & 0x3F); }
  else if (chr < 0x10000) { *str++ = 0xE0 + (chr >> 12); *str++ = 0x80 + ((chr >> 6) & 0x3F); *str++ = 0x80 + (chr & 0x3F); }
  else if (chr < 0x200000) { *str++ = 0xF0 + (chr >> 18); *str++ = 0x80 + ((chr >> 12) & 0x3F); *str++ = 0x80 + ((chr >> 6) & 0x3F); *str++ = 0x80 + (chr & 0x3F); }
  else *str++ = REPLACEMENT_CHAR;
}

void utf8::append(std::string& str, char32_t chr) {
  if (chr < 0x80) str += chr;
  else if (chr < 0x800) { str += 0xC0 + (chr >> 6); str += 0x80 + (chr & 0x3F); }
  else if (chr < 0x10000) { str += 0xE0 + (chr >> 12); str += 0x80 + ((chr >> 6) & 0x3F); str += 0x80 + (chr & 0x3F); }
  else if (chr < 0x200000) { str += 0xF0 + (chr >> 18); str += 0x80 + ((chr >> 12) & 0x3F); str += 0x80 + ((chr >> 6) & 0x3F); str += 0x80 + (chr & 0x3F); }
  else str += REPLACEMENT_CHAR;
}

template<class F> void utf8::map(F f, const char* str, std::string& result) {
  result.clear();

  for (char32_t chr; (chr = decode(str)); )
    append(result, f(chr));
}

template<class F> void utf8::map(F f, const char* str, size_t len, std::string& result) {
  result.clear();

  while (len)
    append(result, f(decode(str, len)));
}

template<class F> void utf8::map(F f, const std::string& str, std::string& result) {
  map(f, str.c_str(), result);
}

} // namespace unilib

/////////
// File: model/evaluator.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string evaluator::DEFAULT;
const string evaluator::NONE = "none";

evaluator::evaluator(const model* m, const string& tokenizer, const string& tagger, const string& parser) {
  set_model(m);
  set_tokenizer(tokenizer);
  set_tagger(tagger);
  set_parser(parser);
}

void evaluator::set_model(const model* m) {
  this->m = m;
}

void evaluator::set_tokenizer(const string& tokenizer) {
  this->tokenizer = tokenizer;
}

void evaluator::set_tagger(const string& tagger) {
  this->tagger = tagger;
}

void evaluator::set_parser(const string& parser) {
  this->parser = parser;
}

bool evaluator::evaluate(istream& is, ostream& os, string& error) const {
  error.clear();

  unique_ptr<input_format> conllu_input(input_format::new_conllu_input_format());
  if (!conllu_input) return error.assign("Cannot allocate CoNLL-U input format instance!"), false;

  vector<string> plain_text_paragraphs(1); unsigned space_after_nos = 0;
  sentence system, gold;
  evaluation_data gold_data, system_goldtok_data, system_goldtok_goldtags_data, system_plaintext_data;

  string block;
  while (conllu_input->read_block(is, block)) {
    conllu_input->set_text(block);
    while (conllu_input->next_sentence(gold, error)) {
      gold_data.add_sentence(gold);

      // Detokenize the input when tokenizing
      if (tokenizer != NONE) {
        if (gold.get_new_doc() || gold.get_new_par()) {
          plain_text_paragraphs.back().append("\n\n");
          plain_text_paragraphs.emplace_back();
        }

        for (size_t i = 1, j = 0; i < gold.words.size(); i++) {
          const token& tok = j < gold.multiword_tokens.size() && gold.multiword_tokens[j].id_first == int(i) ? (const token&)gold.multiword_tokens[j] : (const token&)gold.words[i];
          plain_text_paragraphs.back().append(tok.form);
          if (tok.get_space_after())
            plain_text_paragraphs.back().push_back(' ');
          else
            space_after_nos += 1;
          if (j < gold.multiword_tokens.size() && gold.multiword_tokens[j].id_first == int(i))
            i = gold.multiword_tokens[j++].id_last;
        }
      }

      // Goldtok data
      if (tokenizer == NONE && tagger != NONE) {
        system.clear();
        for (size_t i = 1; i < gold.words.size(); i++)
          system.add_word(gold.words[i].form);

        if (tagger != NONE) {
          if (!m->tag(system, tagger, error))
            return false;
          if (parser != NONE)
            if (!m->parse(system, parser, error))
              return false;
        }
        system_goldtok_data.add_sentence(system);
      }

      // Goldtok_goldtags data
      if (tokenizer == NONE && tagger == NONE && parser != NONE) {
        system.clear();
        for (size_t i = 1; i < gold.words.size(); i++) {
          system.add_word(gold.words[i].form);
          system.words[i].upostag = gold.words[i].upostag;
          system.words[i].xpostag = gold.words[i].xpostag;
          system.words[i].feats = gold.words[i].feats;
          system.words[i].lemma = gold.words[i].lemma;
        }
        if (parser != NONE)
          if (!m->parse(system, parser, error))
            return false;
        system_goldtok_goldtags_data.add_sentence(system);
      }
    }
    if (!error.empty()) return false;
  }

  // Tokenize, tag and parse plaintext input
  if (tokenizer != NONE) {
    unique_ptr<input_format> t(m->new_tokenizer(tokenizer));
    if (!t) return error.assign("Cannot allocate new tokenizer!"), false;

    for (auto&& plain_text : plain_text_paragraphs) {
      t->set_text(plain_text);
      while (t->next_sentence(system, error)) {
        if (tagger != NONE) {
          if (!m->tag(system, tagger, error))
            return false;

          if (parser != NONE)
            if (!m->parse(system, parser, error))
              return false;
        }
        system_plaintext_data.add_sentence(system);
      }
      if (!error.empty()) return false;
    }
  }

  // Evaluate from plain text
  if (tokenizer != NONE) {
    if (system_plaintext_data.chars != gold_data.chars) {
      os << "Cannot evaluate tokenizer, it returned different sequence of token characters!" << endl;
    } else {
      word_alignment plaintext_alignment;
      word_alignment::best_alignment(system_plaintext_data, gold_data, plaintext_alignment);

      os << "Number of SpaceAfter=No features in gold data: " << space_after_nos << endl;

      auto tokens = evaluate_f1(system_plaintext_data.tokens, gold_data.tokens);
      auto multiwords = evaluate_f1(system_plaintext_data.multiwords, gold_data.multiwords);
      auto sentences = evaluate_f1(system_plaintext_data.sentences, gold_data.sentences);
      auto words = plaintext_alignment.evaluate_f1([](const word&, const word&) {return true;});
      if (multiwords.total_gold || multiwords.total_system)
        os << "Tokenizer tokens - system: " << tokens.total_system << ", gold: " << tokens.total_gold
           << ", precision: " << fixed << setprecision(2) << 100. * tokens.precision
           << "%, recall: " << 100. * tokens.recall << "%, f1: " << 100. * tokens.f1 << "%" << endl
           << "Tokenizer multiword tokens - system: " << multiwords.total_system << ", gold: " << multiwords.total_gold
           << ", precision: " << fixed << setprecision(2) << 100. * multiwords.precision
           << "%, recall: " << 100. * multiwords.recall << "%, f1: " << 100. * multiwords.f1 << "%" << endl;
      os << "Tokenizer words - system: " << words.total_system << ", gold: " << words.total_gold
         << ", precision: " << fixed << setprecision(2) << 100. * words.precision
         << "%, recall: " << 100. * words.recall << "%, f1: " << 100. * words.f1 << "%" << endl
         << "Tokenizer sentences - system: " << sentences.total_system << ", gold: " << sentences.total_gold
         << ", precision: " << fixed << setprecision(2) << 100. * sentences.precision
         << "%, recall: " << 100. * sentences.recall << "%, f1: " << 100. * sentences.f1 << "%" << endl;

      if (tagger != NONE) {
        auto upostags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag; });
        auto xpostags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.xpostag == u.xpostag; });
        auto feats = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.feats == u.feats; });
        auto alltags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag && w.xpostag == u.xpostag && w.feats == u.feats; });
        auto lemmas = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.lemma == u.lemma; });
        os << "Tagging from plain text (CoNLL17 F1 score) - gold forms: " << upostags.total_gold << ", upostag: "
           << fixed << setprecision(2) << 100. * upostags.f1 << "%, xpostag: "
           << 100. * xpostags.f1 << "%, feats: " << 100. * feats.f1 << "%, alltags: "
           << 100. * alltags.f1 << "%, lemmas: " << 100. * lemmas.f1 << '%' << endl;
      }

      if (tagger != NONE && parser != NONE) {
        auto uas = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
        auto las = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
        os << "Parsing from plain text with computed tags (CoNLL17 F1 score) - gold forms: " << uas.total_gold
           << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
      }
    }
  }

  // Evaluate tagger from gold tokenization
  if (tokenizer == NONE && tagger != NONE) {
    word_alignment goldtok_alignment;
    if (!word_alignment::perfect_alignment(system_goldtok_data, gold_data, goldtok_alignment))
      return error.assign("Internal UDPipe error (the words of the gold data do not match)!"), false;

    auto upostags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag; });
    auto xpostags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.xpostag == u.xpostag; });
    auto feats = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.feats == u.feats; });
    auto alltags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag && w.xpostag == u.xpostag && w.feats == u.feats; });
    auto lemmas = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.lemma == u.lemma; });
    os << "Tagging from gold tokenization - forms: " << upostags.total_gold << ", upostag: "
       << fixed << setprecision(2) << 100. * upostags.f1 << "%, xpostag: "
       << 100. * xpostags.f1 << "%, feats: " << 100. * feats.f1 << "%, alltags: "
       << 100. * alltags.f1 << "%, lemmas: " << 100. * lemmas.f1 << '%' << endl;

    if (parser != NONE) {
      auto uas = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
      auto las = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
      os << "Parsing from gold tokenization with computed tags - forms: " << uas.total_gold
         << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
    }
  }

  // Evaluate parser from gold tokenization
  if (tokenizer == NONE && tagger == NONE && parser != NONE) {
    word_alignment goldtok_goldtags_alignment;
    if (!word_alignment::perfect_alignment(system_goldtok_goldtags_data, gold_data, goldtok_goldtags_alignment))
      return error.assign("Internal UDPipe error (the words of the goldtok data do not match)!"), false;

    auto uas = goldtok_goldtags_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
    auto las = goldtok_goldtags_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
    os << "Parsing from gold tokenization with gold tags - forms: " << uas.total_gold
       << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
  }

  return true;
}

template <class T>
evaluator::f1_info evaluator::evaluate_f1(const vector<pair<size_t, T>>& system, const vector<pair<size_t, T>>& gold) {
  size_t both = 0;
  for (size_t si = 0, gi = 0; si < system.size() || gi < gold.size(); )
    if (si < system.size() && (gi == gold.size() || system[si].first < gold[gi].first))
      si++;
    else if (gi < gold.size() && (si == system.size() || gold[gi].first < system[si].first))
      gi++;
    else
      both += system[si++].second == gold[gi++].second;

  return {system.size(), gold.size(), system.size() ? both / double(system.size()) : 0.,
    gold.size() ? both / double(gold.size()) : 0., system.size()+gold.size() ? 2 * both / double(system.size() + gold.size()) : 0. };
}

evaluator::evaluation_data::word_data::word_data(size_t start, size_t end, int id, bool is_multiword, const word& w)
  : start(start), end(end), is_multiword(is_multiword), w(w)
{
  // Use absolute ids for words and heads
  this->w.id = id;
  this->w.head = w.head ? id + (w.head - w.id) : 0;

  // Forms in MWTs are compares case-insensitively in LCS, therefore
  // we lowercase them here.
  unilib::utf8::map(unilib::unicode::lowercase, w.form, this->w.form);

  // During evaluation, only universal part of DEPREL (up to a colon) is used.
  auto colon = w.deprel.find(':');
  if (colon != string::npos)
    this->w.deprel.erase(colon);
}

void evaluator::evaluation_data::add_sentence(const sentence& s) {
  sentences.emplace_back(chars.size(), chars.size());
  for (size_t i = 1, j = 0; i < s.words.size(); i++) {
    tokens.emplace_back(chars.size(), chars.size());
    const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;
    for (auto&& chr : unilib::utf8::decoder(form))
      if (chr != ' ')
        chars.push_back(chr);
    tokens.back().second = chars.size();

    if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i)) {
      multiwords.emplace_back(tokens.back().first, form);
      for (size_t k = i; int(k) <= s.multiword_tokens[j].id_last; k++) {
        words.emplace_back(tokens.back().first, tokens.back().second, words.size() + 1, true, s.words[k]);
        multiwords.back().second.append(" ").append(words.back().w.form);
      }
      i = s.multiword_tokens[j++].id_last;
    } else {
      words.emplace_back(tokens.back().first, tokens.back().second, words.size() + 1, false, s.words[i]);
    }
  }
  sentences.back().second = chars.size();
}

template <class Equals>
evaluator::f1_info evaluator::word_alignment::evaluate_f1(Equals equals) {
  size_t both = 0;
  for (auto&& match : matched)
    if (equals(match.system, match.gold))
      both++;

  return {total_system, total_gold, total_system ? both / double(total_system) : 0.,
    total_gold ? both / double(total_gold) : 0., total_system+total_gold ? 2 * both / double(total_system + total_gold) : 0. };
}

bool evaluator::word_alignment::perfect_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment) {
  alignment.total_system = system.words.size();
  alignment.total_gold = gold.words.size();
  if (alignment.total_system != alignment.total_gold) return false;

  alignment.matched.clear();
  alignment.matched.reserve(alignment.total_system);
  for (size_t i = 0; i < system.words.size(); i++) {
    if (system.words[i].w.form != gold.words[i].w.form)
      return false;
    alignment.matched.emplace_back(system.words[i].w, gold.words[i].w);
  }

  return true;
}

void evaluator::word_alignment::best_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment) {
  alignment.total_system = system.words.size();
  alignment.total_gold = gold.words.size();
  alignment.matched.clear();

  for (size_t si = 0, gi = 0; si < system.words.size() && gi < gold.words.size(); )
    if ((system.words[si].start > gold.words[gi].start || !system.words[si].is_multiword) &&
        (gold.words[gi].start > system.words[si].start || !gold.words[gi].is_multiword)) {
      // No multiword, align using start+end indices
      if (system.words[si].start == gold.words[gi].start && system.words[si].end == gold.words[gi].end)
        alignment.matched.emplace_back(system.words[si++].w, gold.words[gi++].w);
      else if (system.words[si].start <= gold.words[gi].start)
        si++;
      else
        gi++;
    } else {
      // We have a multiword
      size_t ss = si, gs = gi, multiword_range_end = system.words[si].is_multiword ? system.words[si].end : gold.words[gi].end;

      // Find all words in the multiword range
      while ((si < system.words.size() && (system.words[si].is_multiword ? system.words[si].start < multiword_range_end :
                                           system.words[si].end <= multiword_range_end)) ||
             (gi < gold.words.size() && (gold.words[gi].is_multiword ? gold.words[gi].start < multiword_range_end :
                                         gold.words[gi].end <= multiword_range_end))) {
        // Extend the multiword range
        if (si < system.words.size() && (gi >= gold.words.size() || system.words[si].start <= gold.words[gi].start)) {
          if (system.words[si].is_multiword) multiword_range_end = max(multiword_range_end, system.words[si].end);
          si++;
        } else {
          if (gold.words[gi].is_multiword) multiword_range_end = max(multiword_range_end, gold.words[gi].end);
          gi++;
        }
      }

      // LCS on the chosen words
      vector<vector<unsigned>> lcs(si - ss);
      for (unsigned s = si - ss; s--; ) {
        lcs[s].resize(gi - gs);
        for (unsigned g = gi - gs; g--; ) {
          lcs[s][g] = max(lcs[s][g], s+1 < lcs.size() ? lcs[s+1][g] : 0);
          lcs[s][g] = max(lcs[s][g], g+1 < lcs[s].size() ? lcs[s][g+1] : 0);
          if (system.words[ss + s].w.form == gold.words[gs + g].w.form)
            lcs[s][g] = max(lcs[s][g], 1 + (s+1 < lcs.size() && g+1 < lcs[s].size() ? lcs[s+1][g+1] : 0));
        }
      }

      for (unsigned s = 0, g = 0; s < si - ss && g < gi - gs; ) {
        if (system.words[ss + s].w.form == gold.words[gs + g].w.form)
          alignment.matched.emplace_back(system.words[ss + s++].w, gold.words[gs + g++].w);
        else if (lcs[s][g] == (s+1 < lcs.size() ? lcs[s+1][g] : 0))
          s++;
        else /* if (lcs[s][g] == (g+1 < lcs[s].size() ? lcs[s][g+1] : 0)) */
          g++;
      }
    }

  // Reindex HEAD pointers in system to use gold indices
  vector<int> gold_aligned(system.words.size(), -1);
  for (auto&& match : alignment.matched)
    gold_aligned[match.system.id - 1] = match.gold.id;
  for (auto&& match : alignment.matched)
    if (match.system.head)
      match.system.head = gold_aligned[match.system.head - 1];
}

/////////
// File: morphodita/tokenizer/tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Range of a token, measured in Unicode characters, not UTF8 bytes.
struct token_range {
  size_t start;
  size_t length;

  token_range() {}
  token_range(size_t start, size_t length) : start(start), length(length) {}
};

class tokenizer {
 public:
  virtual ~tokenizer() {}

  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_sentence(vector<string_piece>* forms, vector<token_range>* tokens) = 0;

  // Static factory methods
  static tokenizer* new_vertical_tokenizer();

  static tokenizer* new_czech_tokenizer();
  static tokenizer* new_english_tokenizer();
  static tokenizer* new_generic_tokenizer();
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_factory.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tokenizer_factory {
 public:
  virtual ~tokenizer_factory() {}

  static tokenizer_factory* load(istream& is);
  static tokenizer_factory* load(const char* fname);

  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer() const = 0;
};

} // namespace morphodita

/////////
// File: morphodita/derivator/derivator.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct derivated_lemma {
  string lemma;
};

class derivator {
 public:
  virtual ~derivator() {}

  // For given lemma, return the parent in the derivation graph.
  // The lemma is assumed to be lemma id and any lemma comments are ignored.
  virtual bool parent(string_piece lemma, derivated_lemma& parent) const = 0;

  // For given lemma, return the children in the derivation graph.
  // The lemma is assumed to be lemma id and any lemma comments are ignored.
  virtual bool children(string_piece lemma, vector<derivated_lemma>& children) const = 0;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct tagged_form {
  string form;
  string tag;

  tagged_form() {}
  tagged_form(const string& form, const string& tag) : form(form), tag(tag) {}
};

struct tagged_lemma {
  string lemma;
  string tag;

  tagged_lemma() {}
  tagged_lemma(const string& lemma, const string& tag) : lemma(lemma), tag(tag) {}
};

struct tagged_lemma_forms {
  string lemma;
  vector<tagged_form> forms;

  tagged_lemma_forms() {}
  tagged_lemma_forms(const string& lemma) : lemma(lemma) {}
};

class morpho {
 public:
  virtual ~morpho() {}

  static morpho* load(istream& is);
  static morpho* load(const char* fname);

  enum guesser_mode { NO_GUESSER = 0, GUESSER = 1 };

  // Perform morphologic analysis of a form. The form is given by a pointer and
  // length and therefore does not need to be '\0' terminated.  The guesser
  // parameter specifies whether a guesser can be used if the form is not found
  // in the dictionary. Output is assigned to the lemmas vector.
  //
  // If the form is found in the dictionary, analyses are assigned to lemmas
  // and NO_GUESSER returned. If guesser == GUESSER and the form analyses are
  // found using a guesser, they are assigned to lemmas and GUESSER is
  // returned.  Otherwise <0 is returned and lemmas are filled with one
  // analysis containing given form as lemma and a tag for unknown word.
  virtual int analyze(string_piece form, guesser_mode guesser, vector<tagged_lemma>& lemmas) const = 0;

  // Perform morphologic generation of a lemma. The lemma is given by a pointer
  // and length and therefore does not need to be '\0' terminated. Optionally
  // a tag_wildcard can be specified (or be NULL) and if so, results are
  // filtered using this wildcard. The guesser parameter speficies whether
  // a guesser can be used if the lemma is not found in the dictionary. Output
  // is assigned to the forms vector.
  //
  // Tag_wildcard can be either NULL or a wildcard applied to the results.
  // A ? in the wildcard matches any character, [bytes] matches any of the
  // bytes and [^bytes] matches any byte different from the specified ones.
  // A - has no special meaning inside the bytes and if ] is first in bytes, it
  // does not end the bytes group.
  //
  // If the given lemma is only a raw lemma, all lemma ids with this raw lemma
  // are returned. Otherwise only matching lemma ids are returned, ignoring any
  // lemma comments. For every found lemma, matching forms are filtered using
  // the tag_wildcard. If at least one lemma is found in the dictionary,
  // NO_GUESSER is returned. If guesser == GUESSER and the lemma is found by
  // the guesser, GUESSER is returned. Otherwise, forms are cleared and <0 is
  // returned.
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector<tagged_lemma_forms>& forms) const = 0;

  // Rawlemma and lemma id identification
  virtual int raw_lemma_len(string_piece lemma) const = 0;
  virtual int lemma_id_len(string_piece lemma) const = 0;

  // Rawform identification
  virtual int raw_form_len(string_piece form) const = 0;

  // Construct a new tokenizer instance appropriate for this morphology.
  // Can return NULL if no such tokenizer exists.
  virtual tokenizer* new_tokenizer() const = 0;

  // Return a derivator for this morphology, or NULL if it does not exist.
  // The returned instance is owned by the morphology and should not be deleted.
  virtual const derivator* get_derivator() const;

 protected:
  unique_ptr<derivator> derinet;
};

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagger {
 public:
  virtual ~tagger() {}

  static tagger* load(const char* fname);
  static tagger* load(istream& is);

  // Return morpho associated with the tagger. Do not delete the pointer, it is
  // owned by the tagger instance and deleted in the tagger destructor.
  virtual const morpho* get_morpho() const = 0;

  // Perform morphologic analysis and subsequent disambiguation.
  virtual void tag(const vector<string_piece>& forms, vector<tagged_lemma>& tags, morpho::guesser_mode guesser = morpho::guesser_mode(-1)) const = 0;

  // Perform disambiguation only on given analyses.
  virtual void tag_analyzed(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<int>& tags) const = 0;

  // Construct a new tokenizer instance appropriate for this tagger.
  // Can return NULL if no such tokenizer exists.
  // Is equal to get_morpho()->new_tokenizer.
  tokenizer* new_tokenizer() const;
};

} // namespace morphodita

/////////
// File: parsito/tree/node.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class node {
 public:
  int id;         // 0 is root, >0 is sentence node, <0 is undefined
  string form;    // form
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  int head;       // head, 0 is root, <0 is without parent
  string deprel;  // dependency relation to the head
  string deps;    // secondary dependencies
  string misc;    // miscellaneous information

  vector<int> children;

  node(int id = -1, const string& form = string()) : id(id), form(form), head(-1) {}
};

} // namespace parsito

/////////
// File: parsito/tree/tree.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class tree {
 public:
  tree();

  vector<node> nodes;

  bool empty();
  void clear();
  node& add_node(const string& form);
  void set_head(int id, int head, const string& deprel);
  void unlink_all_nodes();

  static const string root_form;
};

} // namespace parsito

/////////
// File: parsito/configuration/configuration.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class configuration {
 public:
  configuration(bool single_root) : single_root(single_root) {}

  void init(tree* t);
  bool final();

  tree* t;
  vector<int> stack;
  vector<int> buffer;

  bool single_root;
};

} // namespace parsito

/////////
// File: utils/binary_decoder.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class binary_decoder_error : public runtime_error {
 public:
  explicit binary_decoder_error(const char* description) : runtime_error(description) {}
};

class binary_decoder {
 public:
  inline unsigned char* fill(unsigned len);

  inline unsigned next_1B() throw (binary_decoder_error);
  inline unsigned next_2B() throw (binary_decoder_error);
  inline unsigned next_4B() throw (binary_decoder_error);
  inline void next_str(string& str) throw (binary_decoder_error);
  template <class T> inline const T* next(unsigned elements) throw (binary_decoder_error);

  inline bool is_end();
  inline unsigned tell();
  inline void seek(unsigned pos) throw (binary_decoder_error);

 private:
  vector<unsigned char> buffer;
  const unsigned char* data;
  const unsigned char* data_end;
};

//
// Definitions
//

unsigned char* binary_decoder::fill(unsigned len) {
  buffer.resize(len);
  data = buffer.data();
  data_end = buffer.data() + len;

  return buffer.data();
}

unsigned binary_decoder::next_1B() throw (binary_decoder_error) {
  if (data + 1 > data_end) throw binary_decoder_error("No more data in binary_decoder");
  return *data++;
}

unsigned binary_decoder::next_2B() throw (binary_decoder_error) {
  if (data + sizeof(uint16_t) > data_end) throw binary_decoder_error("No more data in binary_decoder");
  unsigned result = *(uint16_t*)data;
  data += sizeof(uint16_t);
  return result;
}

unsigned binary_decoder::next_4B() throw (binary_decoder_error) {
  if (data + sizeof(uint32_t) > data_end) throw binary_decoder_error("No more data in binary_decoder");
  unsigned result = *(uint32_t*)data;
  data += sizeof(uint32_t);
  return result;
}

void binary_decoder::next_str(string& str) throw (binary_decoder_error) {
  unsigned len = next_1B();
  if (len == 255) len = next_4B();
  str.assign(next<char>(len), len);
}

template <class T> const T* binary_decoder::next(unsigned elements) throw (binary_decoder_error) {
  if (data + sizeof(T) * elements > data_end) throw binary_decoder_error("No more data in binary_decoder");
  const T* result = (const T*) data;
  data += sizeof(T) * elements;
  return result;
}

bool binary_decoder::is_end() {
  return data >= data_end;
}

unsigned binary_decoder::tell() {
  return data - buffer.data();
}

void binary_decoder::seek(unsigned pos) throw (binary_decoder_error) {
  if (pos > buffer.size()) throw binary_decoder_error("Cannot seek past end of binary_decoder");
  data = buffer.data() + pos;
}

} // namespace utils

/////////
// File: parsito/parser/parser.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Parser
class parser {
 public:
  virtual ~parser() {};

  virtual void parse(tree& t, unsigned beam_size = 0, double* cost = nullptr) const = 0;

  enum { NO_CACHE = 0, FULL_CACHE = 2147483647};
  static parser* load(const char* file, unsigned cache = 1000);
  static parser* load(istream& in, unsigned cache = 1000);

 protected:
  virtual void load(binary_decoder& data, unsigned cache) = 0;
  static parser* create(const string& name);
};

} // namespace parsito

/////////
// File: tokenizer/multiword_splitter.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_splitter {
 public:
  void append_token(string_piece token, string_piece misc, sentence& s) const;

  static multiword_splitter* load(istream& is);

 private:
  multiword_splitter(unsigned version) : version(version) {}
  unsigned version;
  enum { VERSION_LATEST = 2 };
  friend class multiword_splitter_trainer;

  struct suffix_info {
    vector<string> words;
  };
  unordered_map<string, suffix_info> full_rules, suffix_rules;
};

/////////
// File: utils/parse_int.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Try to parse an int from given string. If the int cannot be parsed or does
// not fit into int, false is returned and the error string is filled using the
// value_name argument.
inline bool parse_int(string_piece str, const char* value_name, int& value, string& error);

// Try to parse an int from given string. If the int cannot be parsed or does
// not fit into int, an error is displayed and program exits.
inline int parse_int(string_piece str, const char* value_name);

//
// Definitions
//

bool parse_int(string_piece str, const char* value_name, int& value, string& error) {
  string_piece original = str;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Allow minus
  bool positive = true;
  if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
    positive = str.str[0] == '+';
    str.str++, str.len--;
  }

  // Parse value, checking for overflow/underflow
  if (!str.len) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': empty string."), false;
  if (!(str.str[0] >= '0' || str.str[0] <= '9')) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': non-digit character found."), false;

  value = 0;
  while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
    if (positive) {
      if (value > (numeric_limits<int>::max() - (str.str[0] - '0')) / 10)
        return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': overflow occured."), false;
      value = 10 * value + (str.str[0] - '0');
    } else {
      if (value < (numeric_limits<int>::min() + (str.str[0] - '0')) / 10)
        return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': underflow occured."), false;
      value = 10 * value - (str.str[0] - '0');
    }
    str.str++, str.len--;
  }

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Check for remaining characters
  if (str.len) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': non-digit character found."), false;

  return true;
}

int parse_int(string_piece str, const char* value_name) {
  int result;
  string error;

  if (!parse_int(str, value_name, result, error))
    runtime_failure(error);

  return result;
}

} // namespace utils

/////////
// File: utils/named_values.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class named_values {
 public:
  typedef unordered_map<string, string> map;

  inline static bool parse(const string& values, map& parsed_values, string& error);
};

//
// Definitions
//

bool named_values::parse(const string& values, map& parsed_values, string& error) {
  error.clear();
  parsed_values.clear();

  string name, file;
  for (size_t start = 0; start < values.size(); ) {
    while (start < values.size() && values[start] == ';') start++;
    if (start >= values.size()) break;

    size_t name_end = values.find_first_of("=;", start);
    name.assign(values, start, name_end - start);
    string& value = parsed_values[name];

    if (name_end == string::npos) {
      start = name_end;
    } else if (values[name_end] == ';') {
      start = name_end + 1;
    } else /* if (values[name_end] == '=') */ {
      size_t equal_sign = name_end;

      if (equal_sign + 1 + 5 <= values.size() && values.compare(equal_sign + 1, 5, "file:") == 0) {
        // Value of type file:
        size_t file_name = equal_sign + 1 + 5;
        size_t semicolon = min(values.find(';', file_name), values.size());

        file.assign(values, file_name, semicolon - file_name);
        ifstream is(file.c_str());
        if (!is.is_open()) return error.assign("Cannot open file '").append(file).append("'!"), false;

        char buffer[1024];
        for (value.clear(); is.read(buffer, sizeof(buffer)); )
          value.append(buffer, sizeof(buffer));
        value.append(buffer, is.gcount());

        start = semicolon + 1;
      } else if (equal_sign + 1 + 5 <= values.size() && values.compare(equal_sign + 1, 5, "data:") == 0) {
        // Value of type data:
        size_t data_size_start = equal_sign + 1 + 5;
        size_t data_size_end = values.find(':', data_size_start);
        if (data_size_end == string::npos) return error.assign("Cannot parse named values, data size of value '").append(name).append("' not terminated!"), false;

        int data_size;
        if (!parse_int(string_piece(values.c_str() + data_size_start, data_size_end - data_size_start), "data_size", data_size, error)) return false;

        size_t data_start = data_size_end + 1, data_end = data_start + data_size;
        if (data_end > values.size()) return error.assign("Cannot parse named values, value '").append(name).append("' shorter than specified length!"), false;
        if (data_end < values.size() && values[data_end] != ';') return error.assign("Cannot parse named values, value '").append(name).append("' not terminated by semicolon!"), false;

        value.assign(values, data_start, data_end - data_start);
        start = data_end + 1;
      } else {
        // Value of string type
        size_t semicolon = min(values.find(';', equal_sign), values.size());
        value.assign(values, equal_sign + 1, semicolon - equal_sign - 1);
        start = semicolon + 1;
      }
    }
  }

  return true;
}

} // namespace utils

/////////
// File: utils/threadsafe_stack.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

template <class T>
class threadsafe_stack {
 public:
  inline void push(T* t);
  inline T* pop();

 private:
  vector<unique_ptr<T>> stack;
  atomic_flag lock = ATOMIC_FLAG_INIT;
};

//
// Definitions
//

template <class T>
void threadsafe_stack<T>::push(T* t) {
  while (lock.test_and_set(memory_order_acquire)) {}
  stack.emplace_back(t);
  lock.clear(memory_order_release);
}

template <class T>
T* threadsafe_stack<T>::pop() {
  T* res = nullptr;

  while (lock.test_and_set(memory_order_acquire)) {}
  if (!stack.empty()) {
    res = stack.back().release();
    stack.pop_back();
  }
  lock.clear(memory_order_release);

  return res;
}

} // namespace utils

/////////
// File: model/model_morphodita_parsito.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class model_morphodita_parsito : public model {
 public:
  virtual input_format* new_tokenizer(const string& options) const override;
  virtual bool tag(sentence& s, const string& options, string& error) const override;
  virtual bool parse(sentence& s, const string& options, string& error) const override;

  static model* load(istream& is);

 private:
  model_morphodita_parsito(unsigned version);
  unsigned version;
  enum { VERSION_LATEST = 3 };

  unique_ptr<morphodita::tokenizer_factory> tokenizer_factory;
  unique_ptr<multiword_splitter> splitter;
  struct tagger_model {
    bool upostag; int lemma; bool xpostag, feats;
    unique_ptr<morphodita::tagger> tagger;

    tagger_model(bool upostag, int lemma, bool xpostag, bool feats, morphodita::tagger* tagger)
        : upostag(upostag), lemma(lemma), xpostag(xpostag), feats(feats), tagger(tagger) {}
  };
  vector<tagger_model> taggers;
  unique_ptr<parsito::parser> parser;

  struct tagger_cache {
    vector<string> forms_normalized;
    vector<string_piece> forms_string_pieces;
    vector<morphodita::tagged_lemma> lemmas;
  };
  mutable threadsafe_stack<tagger_cache> tagger_caches;

  struct parser_cache {
    parsito::tree tree;
    named_values::map options;
  };
  mutable threadsafe_stack<parser_cache> parser_caches;

  bool parse(sentence& s, const string& options, string& error, double* cost) const;

  class joint_with_parsing_tokenizer : public input_format {
   public:
    joint_with_parsing_tokenizer(input_format* tokenizer, const model_morphodita_parsito& model,
                                 int max_sentence_len, double change_boundary_logprob, double sentence_logprob)
        : tokenizer(tokenizer), model(model), max_sentence_len(max_sentence_len),
          change_boundary_logprob(change_boundary_logprob), sentence_logprob(sentence_logprob) {}

    virtual bool read_block(istream& is, string& block) const override;
    virtual void reset_document(string_piece id) override;
    virtual void set_text(string_piece text, bool make_copy = false) override;
    virtual bool next_sentence(sentence& s, string& error) override;

   private:
    bool parse_paragraph(vector<sentence>& paragraph, string& error);

    unique_ptr<input_format> tokenizer;
    const model_morphodita_parsito& model;
    int max_sentence_len;
    double change_boundary_logprob;
    double sentence_logprob;

    string_piece text;
    string text_copy;
    bool new_document = true;
    string document_id;
    unsigned sentence_id = 1;
    vector<sentence> sentences;
    size_t sentences_index = 0;
  };

  void fill_word_analysis(const morphodita::tagged_lemma& analysis, bool upostag, int lemma, bool xpostag, bool feats, word& word) const;
  const string& normalize_form(string_piece form, string& output) const;
  const string& normalize_lemma(string_piece lemma, string& output) const;
  friend class trainer_morphodita_parsito;
};

/////////
// File: model/model.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string model::DEFAULT;
const string model::TOKENIZER_NORMALIZED_SPACES = "normalized_spaces";
const string model::TOKENIZER_PRESEGMENTED = "presegmented";
const string model::TOKENIZER_RANGES = "ranges";

model* model::load(const char* fname) {
  ifstream in(fname, ifstream::in | ifstream::binary);
  if (!in.is_open()) return nullptr;
  return load(in);
}

model* model::load(istream& is) {
  char len;
  if (!is.get(len)) return nullptr;
  string name(len, ' ');
  if (!is.read(&name[0], len)) return nullptr;

  if (name == "morphodita_parsito") return model_morphodita_parsito::load(is);

  return nullptr;
}

/////////
// File: tokenizer/morphodita_tokenizer_wrapper.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class morphodita_tokenizer_wrapper : public input_format {
 public:
  morphodita_tokenizer_wrapper(morphodita::tokenizer* tokenizer, const multiword_splitter* splitter, bool normalized_spaces, bool token_ranges);

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unique_ptr<morphodita::tokenizer> tokenizer;
  const multiword_splitter* splitter;
  bool normalized_spaces, token_ranges;

  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;

  string_piece text;
  string text_copy;
  size_t unicode_offset = 0, text_unicode_length = 0;
  string saved_spaces;
  vector<string_piece> forms;
  vector<morphodita::token_range> tokens;
  token tok;
};

/////////
// File: utils/getpara.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Read paragraph until EOF or end line. All encountered \n are stored.
inline istream& getpara(istream& is, string& para);

//
// Definitions
//

istream& getpara(istream& is, string& para) {
  para.clear();

  for (string line; getline(is, line); ) {
    para.append(line);
    para.push_back('\n');

    if (line.empty()) break;
  }

  if (is.eof() && !para.empty()) is.clear(istream::eofbit);
  return is;
}

} // namespace utils

/////////
// File: utils/parse_double.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Try to parse an double from given string. If the double cannot be parsed or does
// not fit doubleo double, false is returned and the error string is filled using the
// value_name argument.
inline bool parse_double(string_piece str, const char* value_name, double& value, string& error);

// Try to parse an double from given string. If the double cannot be parsed or does
// not fit doubleo double, an error is displayed and program exits.
inline double parse_double(string_piece str, const char* value_name);

//
// Definitions
//

bool parse_double(string_piece str, const char* value_name, double& value, string& error) {
  string_piece original = str;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Allow plus/minus
  bool negative = false;
  if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
    negative = str.str[0] == '-';
    str.str++, str.len--;
  }

  // Parse value, checking for overflow/underflow
  if (!str.len) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': empty string."), false;
  if (!(str.str[0] >= '0' || str.str[0] <= '9')) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': non-digit character found."), false;

  value = 0;
  while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
    value = 10 * value + (str.str[0] - '0');
    str.str++, str.len--;
  }

  // If there is a decimal point, parse the rest of the
  if (str.len && str.str[0] == '.') {
    double divider = 1;

    str.str++, str.len--;
    while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
      value = 10 * value + (str.str[0] - '0');
      divider *= 10.;
      str.str++, str.len--;
    }

    value /= divider;
  }
  if (!isfinite(value)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': overflow occured."), false;

  // Optionally parse an exponent
  if (str.len && (str.str[0] == 'e' || str.str[0] == 'E')) {
    str.str++, str.len--;

    double exponent = 0;
    bool exponent_negative = false;
    if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
      exponent_negative = str.str[0] == '-';
      str.str++, str.len--;
    }

    while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
      exponent = 10 * exponent + (str.str[0] - '0');
      str.str++, str.len--;
    }

    exponent = pow(10., exponent_negative ? -exponent : exponent);
    if (!isfinite(exponent)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': exponent overflow occured."), false;
    if (exponent == 0) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': exponent underflow occured."), false;

    if (value) {
      value *= exponent;
      if (!isfinite(value)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': overflow occured."), false;
      if (value == 0) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': underflow occured."), false;
    }
  }

  // Apply initial minus
  if (negative) value *= -1;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Check for remaining characters
  if (str.len) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': non-digit character found."), false;

  return true;
}

double parse_double(string_piece str, const char* value_name) {
  double result;
  string error;

  if (!parse_double(str, value_name, result, error))
    runtime_failure(error);

  return result;
}

} // namespace utils

/////////
// File: model/model_morphodita_parsito.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Versions:
// 1 - initial version
// 2 - add absolute lemmas (tagger_model::lemma == 2)
//   - use Arabic and space normalization

input_format* model_morphodita_parsito::new_tokenizer(const string& options) const {
  if (!tokenizer_factory)
    return nullptr;

  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  bool normalized_spaces = parsed_options.count("normalized_spaces");
  bool token_ranges = parsed_options.count("ranges");

  unique_ptr<input_format> result(new morphodita_tokenizer_wrapper(tokenizer_factory->new_tokenizer(), splitter.get(), normalized_spaces, token_ranges));

  // Presegmented
  if (parsed_options.count("presegmented") && result)
    result.reset(input_format::new_presegmented_tokenizer(result.release()));

  // Joint with parsing
  if (parsed_options.count("joint_with_parsing") && result) {
    int max_sentence_len = 20;
    if (parsed_options.count("joint_max_sentence_len") && !parse_int(parsed_options["joint_max_sentence_len"], "joint max sentence len", max_sentence_len, parse_error))
      return nullptr;

    double change_boundary_logprob = -0.5;
    if (parsed_options.count("joint_change_boundary_logprob") && !parse_double(parsed_options["joint_change_boundary_logprob"], "joint change boundary logprob", change_boundary_logprob, parse_error))
      return nullptr;

    double sentence_logprob = -0.5;
    if (parsed_options.count("joint_sentence_logprob") && !parse_double(parsed_options["joint_sentence_logprob"], "joint sentence logprob", sentence_logprob, parse_error))
      return nullptr;

    result.reset(new joint_with_parsing_tokenizer(result.release(), *this, max_sentence_len, change_boundary_logprob, sentence_logprob));
  }

  return result.release();
}

bool model_morphodita_parsito::tag(sentence& s, const string& /*options*/, string& error) const {
  error.clear();

  if (taggers.empty()) return error.assign("No tagger defined for the UDPipe model!"), false;
  if (s.empty()) return true;

  tagger_cache* c = tagger_caches.pop();
  if (!c) c = new tagger_cache();

  // Prepare input forms
  c->forms_normalized.resize(s.words.size() - 1);
  c->forms_string_pieces.resize(s.words.size() - 1);
  for (size_t i = 1; i < s.words.size(); i++)
    c->forms_string_pieces[i - 1] = normalize_form(s.words[i].form, c->forms_normalized[i - 1]);

  // Clear first
  for (size_t i = 1; i < s.words.size(); i++) {
    s.words[i].lemma.assign("_");
    s.words[i].upostag.clear();
    s.words[i].xpostag.clear();
    s.words[i].feats.clear();
  }

  // Fill information from the tagger models
  for (auto&& tagger : taggers) {
    if (!tagger.tagger) return error.assign("No tagger defined for the UDPipe model!"), false;

    tagger.tagger->tag(c->forms_string_pieces, c->lemmas);

    for (size_t i = 0; i < c->lemmas.size(); i++)
      fill_word_analysis(c->lemmas[i], tagger.upostag, tagger.lemma, tagger.xpostag, tagger.feats, s.words[i+1]);
  }

  tagger_caches.push(c);
  return true;
}

bool model_morphodita_parsito::parse(sentence& s, const string& options, string& error) const {
  return parse(s, options, error, nullptr);
}

bool model_morphodita_parsito::parse(sentence& s, const string& options, string& error, double* cost) const {
  error.clear();

  if (!parser) return error.assign("No parser defined for the UDPipe model!"), false;
  if (s.empty()) return true;

  parser_cache* c = parser_caches.pop();
  if (!c) c = new parser_cache();

  int beam_search = 5;
  if (!named_values::parse(options, c->options, error))
    return false;
  if (c->options.count("beam_search"))
    if (!parse_int(c->options["beam_search"], "beam_search", beam_search, error))
      return false;

  c->tree.clear();
  for (size_t i = 1; i < s.words.size(); i++) {
    c->tree.add_node(string());
    normalize_form(s.words[i].form, c->tree.nodes.back().form);
    normalize_lemma(s.words[i].lemma, c->tree.nodes.back().lemma);
    c->tree.nodes.back().upostag.assign(s.words[i].upostag);
    c->tree.nodes.back().xpostag.assign(s.words[i].xpostag);
    c->tree.nodes.back().feats.assign(s.words[i].feats);
    c->tree.nodes.back().deps.assign(s.words[i].deps);
    c->tree.nodes.back().misc.assign(s.words[i].misc);
  }

  parser->parse(c->tree, beam_search, cost);
  for (size_t i = 1; i < s.words.size(); i++)
    s.set_head(i, c->tree.nodes[i].head, c->tree.nodes[i].deprel);

  parser_caches.push(c);
  return true;
}

model* model_morphodita_parsito::load(istream& is) {
  char version;
  if (!is.get(version)) return nullptr;
  if (!(version >= 1 && version <= VERSION_LATEST)) return nullptr;

  // Because UDPipe 1.0 does not check the model version,
  // a specific sentinel was added since version 2 so that
  // loading of such model fail on UDPipe 1.0
  if (version >= 2) {
    char sentinel;
    if (!is.get(sentinel) || sentinel != 0x7F) return nullptr;
    if (!is.get(sentinel) || sentinel != 0x7F) return nullptr;
  }

  unique_ptr<model_morphodita_parsito> m(new model_morphodita_parsito((unsigned char)version));
  if (!m) return nullptr;

  char tokenizer;
  if (!is.get(tokenizer)) return nullptr;
  m->tokenizer_factory.reset(tokenizer ? morphodita::tokenizer_factory::load(is) : nullptr);
  if (tokenizer && !m->tokenizer_factory) return nullptr;
  m->splitter.reset(tokenizer ? multiword_splitter::load(is) : nullptr);
  if (tokenizer && !m->splitter) return nullptr;

  m->taggers.clear();
  char taggers; if (!is.get(taggers)) return nullptr;
  for (char i = 0; i < taggers; i++) {
    char lemma; if (!is.get(lemma)) return nullptr;
    char xpostag; if (!is.get(xpostag)) return nullptr;
    char feats; if (!is.get(feats)) return nullptr;
    morphodita::tagger* tagger = morphodita::tagger::load(is);
    if (!tagger) return nullptr;
    m->taggers.emplace_back(i == 0, int(lemma), bool(xpostag), bool(feats), tagger);
  }

  char parser;
  if (!is.get(parser)) return nullptr;
  m->parser.reset(parser ? parsito::parser::load(is) : nullptr);
  if (parser && !m->parser) return nullptr;

  return m.release();
}

model_morphodita_parsito::model_morphodita_parsito(unsigned version) : version(version) {}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::read_block(istream& is, string& block) const {
  block.clear();

  for (string line; getline(is, line); ) {
    block.append(line);
    block.push_back('\n');
  }

  if (is.eof() && !block.empty()) is.clear(istream::eofbit);
  return bool(is);
}

void model_morphodita_parsito::joint_with_parsing_tokenizer::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  sentence_id = 1;
  set_text("");
  sentences.clear();
  sentences_index = 0;
}

void model_morphodita_parsito::joint_with_parsing_tokenizer::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text.str = text_copy.c_str();
  }
  this->text = text;
}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::next_sentence(sentence& s, string& error) {
  error.clear();

  if (text.len) {
    sentences.clear();
    sentences_index = 0;

    tokenizer->set_text(text, false);

    sentence input;
    vector<sentence> paragraph;
    while (tokenizer->next_sentence(input, error)) {
      if (input.get_new_par() && !paragraph.empty()) {
        if (!parse_paragraph(paragraph, error)) return false;
        for (auto&& sentence : paragraph)
          sentences.push_back(sentence);
        paragraph.clear();
      }
      paragraph.push_back(input);
    }
    if (!error.empty()) return false;

    if (!paragraph.empty()) {
      if (!parse_paragraph(paragraph, error)) return false;
      for (auto&& sentence : paragraph)
        sentences.push_back(sentence);
    }

    text.len = 0;
  }

  if (sentences_index < sentences.size()) {
    s = sentences[sentences_index++];
    return true;
  }

  return false;
}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::parse_paragraph(vector<sentence>& paragraph, string& error) {
  sentence all_words;
  vector<bool> sentence_boundary(1, true);
  vector<bool> token_boundary(1, true);

  for (auto&& s : paragraph) {
    unsigned offset = all_words.words.size() - 1;
    for (unsigned i = 1; i < s.words.size(); i++) {
      all_words.words.push_back(s.words[i]);
      all_words.words.back().id += offset;
      sentence_boundary.push_back(i+1 == s.words.size());
      token_boundary.push_back(true);
    }

    for (auto&& mwt : s.multiword_tokens) {
      all_words.multiword_tokens.push_back(mwt);
      all_words.multiword_tokens.back().id_first += offset;
      all_words.multiword_tokens.back().id_last += offset;
      for (int i = all_words.multiword_tokens.back().id_first; i < all_words.multiword_tokens.back().id_last; i++)
        token_boundary[i] = false;
    }
  }

  vector<double> best_logprob(all_words.words.size(), -numeric_limits<double>::infinity()); best_logprob[0] = 0.;
  vector<unsigned> best_length(all_words.words.size(), 0);
  sentence s;

  for (unsigned start = 1; start < all_words.words.size(); start++) {
    if (!token_boundary[start - 1]) continue;
    s.clear();
    for (unsigned end = start + 1; end <= all_words.words.size() && (end - start) <= unsigned(max_sentence_len); end++) {
      s.words.push_back(all_words.words[end - 1]);
      s.words.back().id -= start - 1;
      if (!token_boundary[end - 1]) continue;

      for (unsigned i = 1; i < s.words.size(); i++) {
        s.words[i].head = -1;
        s.words[i].children.clear();
      }

      double cost;
      if (!model.tag(s, DEFAULT, error)) return false;
      if (!model.parse(s, DEFAULT, error, &cost)) return false;
      cost += sentence_logprob + change_boundary_logprob * (2 - int(sentence_boundary[start - 1]) - int(sentence_boundary[end - 1]));
      if (best_logprob[start - 1] + cost > best_logprob[end - 1]) {
        best_logprob[end - 1] = best_logprob[start - 1] + cost;
        best_length[end - 1] = end - start;
      }
    }
  }

  vector<unsigned> sentence_lengths;
  for (unsigned end = all_words.words.size(); end > 1; end -= best_length[end - 1])
    sentence_lengths.push_back(best_length[end - 1]);

  paragraph.clear();

  sentence_lengths.push_back(1);
  reverse(sentence_lengths.begin(), sentence_lengths.end());
  for (unsigned i = 1; i < sentence_lengths.size(); i++) {
    sentence_lengths[i] += sentence_lengths[i - 1];

    paragraph.emplace_back();
    while (!all_words.multiword_tokens.empty() && unsigned(all_words.multiword_tokens.front().id_first) < sentence_lengths[i]) {
      paragraph.back().multiword_tokens.push_back(all_words.multiword_tokens.front());
      paragraph.back().multiword_tokens.back().id_first -= sentence_lengths[i-1] - 1;
      paragraph.back().multiword_tokens.back().id_last -= sentence_lengths[i-1] - 1;
      all_words.multiword_tokens.erase(all_words.multiword_tokens.begin());
    }

    for (unsigned word = sentence_lengths[i - 1]; word < sentence_lengths[i]; word++) {
      paragraph.back().words.push_back(all_words.words[word]);
      paragraph.back().words.back().id -= sentence_lengths[i-1] - 1;
      paragraph.back().words.back().head = -1;
      paragraph.back().words.back().children.clear();
    }
  }

  if (!paragraph.empty()) {
    if (new_document) {
      paragraph.front().set_new_doc(true, document_id);
      new_document = false;
    }

    paragraph.front().set_new_par(true);
  }

  return true;
}

void model_morphodita_parsito::fill_word_analysis(const morphodita::tagged_lemma& analysis, bool upostag, int lemma, bool xpostag, bool feats, word& word) const {
  // Lemma
  if (lemma == 1) {
    word.lemma.assign(analysis.lemma);
  } else if (lemma == 2) {
    word.lemma.assign(analysis.lemma);

    // Lemma matching ~replacement~normalized_form is changed to replacement.
    if (analysis.lemma[0] == '~') {
      auto end = analysis.lemma.find('~', 1);
      if (end != string::npos) {
        normalize_form(word.form, word.lemma);
        if (analysis.lemma.compare(end + 1, string::npos, word.lemma) == 0)
          word.lemma.assign(analysis.lemma, 1, end - 1);
        else
          word.lemma.assign(analysis.lemma);
      }
    }
  }
  if (version == 2) {
    // Replace '\001' back to spaces
    for (auto && chr : word.lemma)
      if (chr == '\001')
        chr = ' ';
  } else if (version >= 3) {
    // Replace '0xC2 0xA0' back to spaces
    for (size_t i = 0; i + 1 < word.lemma.size(); i++)
      if (word.lemma[i] == char(0xC2) && word.lemma[i+1] == char(0xA0))
        word.lemma.replace(i, 2, 1, ' ');
  }

  if (!upostag && !xpostag && !feats) return;

  // UPOSTag
  char separator = analysis.tag[0];
  size_t start = min(size_t(1), analysis.tag.size()), end = min(analysis.tag.find(separator, 1), analysis.tag.size());
  if (upostag) word.upostag.assign(analysis.tag, start, end - start);

  if (!xpostag && !feats) return;

  // XPOSTag
  start = min(end + 1, analysis.tag.size());
  end = min(analysis.tag.find(separator, start), analysis.tag.size());
  if (xpostag) word.xpostag.assign(analysis.tag, start, end - start);

  if (!feats) return;

  // Features
  start = min(end + 1, analysis.tag.size());
  word.feats.assign(analysis.tag, start, analysis.tag.size() - start);
}

const string& model_morphodita_parsito::normalize_form(string_piece form, string& output) const {
  using unilib::utf8;

  // No normalization on version 1
  if (version <= 1) return output.assign(form.str, form.len);

  // If requested, replace space by \001 in version 2 and by &nbsp; (\u00a0) since version 3

  // Arabic normalization since version 2, implementation resulted from
  // discussion with Otakar Smrz and Nasrin Taghizadeh.
  // 1. Remove https://codepoints.net/U+0640 without any reasonable doubt :)
  // 2. Remove https://codepoints.net/U+0652
  // 3. Remove https://codepoints.net/U+0670
  // 4. Remove everything from https://codepoints.net/U+0653 to
  //    https://codepoints.net/U+0657 though they are probably very rare in date
  // 5. Remove everything from https://codepoints.net/U+064B to
  //    https://codepoints.net/U+0650
  // 6. Remove https://codepoints.net/U+0651
  // 7. Replace https://codepoints.net/U+0671 with https://codepoints.net/U+0627
  // 8. Replace https://codepoints.net/U+0622 with https://codepoints.net/U+0627
  // 9. Replace https://codepoints.net/U+0623 with https://codepoints.net/U+0627
  // 10. Replace https://codepoints.net/U+0625 with https://codepoints.net/U+0627
  // 11. Replace https://codepoints.net/U+0624 with https://codepoints.net/U+0648
  // 12. Replace https://codepoints.net/U+0626 with https://codepoints.net/U+064A
  // One might also consider replacing some Farsi characters that might be typed
  // unintentionally (by Iranians writing Arabic language texts):
  // 13. Replace https://codepoints.net/U+06CC with https://codepoints.net/U+064A
  // 14. Replace https://codepoints.net/U+06A9 with https://codepoints.net/U+0643
  // 15. Replace https://codepoints.net/U+06AA with https://codepoints.net/U+0643
  //
  // Not implemented:
  // There is additional challenge with data coming from Egypt (such as printed
  // or online newspapers), where the word-final https://codepoints.net/U+064A
  // may be switched for https://codepoints.net/U+0649 and visa versa. Also, the
  // word-final https://codepoints.net/U+0647 could actually represent https://
  // codepoints.net/U+0629. You can experiment with the following replacements,
  // but I would rather apply them only after classifying the whole document as
  // following such convention:
  // 1. Replace https://codepoints.net/U+0629 with https://codepoints.net/U+0647
  //    (frequent femine ending markers would appear like a third-person
  //    masculine pronoun clitic instead)
  // 2. Replace https://codepoints.net/U+0649 with https://codepoints.net/U+064A
  //    (some "weak" words would become even more ambiguous or appear as if
  //    with a first-person pronoun clitic)

  output.clear();
  for (auto&& chr : utf8::decoder(form.str, form.len)) {
    // Arabic normalization
    if (chr == 0x640 || (chr >= 0x64B && chr <= 0x657) || chr == 0x670) {}
    else if (chr == 0x622) utf8::append(output, 0x627);
    else if (chr == 0x623) utf8::append(output, 0x627);
    else if (chr == 0x624) utf8::append(output, 0x648);
    else if (chr == 0x625) utf8::append(output, 0x627);
    else if (chr == 0x626) utf8::append(output, 0x64A);
    else if (chr == 0x671) utf8::append(output, 0x627);
    else if (chr == 0x6A9) utf8::append(output, 0x643);
    else if (chr == 0x6AA) utf8::append(output, 0x643);
    else if (chr == 0x6CC) utf8::append(output, 0x64A);
    // Space normalization
    else if (chr == ' ' && version == 2) utf8::append(output, 0x01);
    else if (chr == ' ' && version >= 3) utf8::append(output, 0xA0);
    // Default
    else utf8::append(output, chr);
  }

  // Make sure we do not remove everything
  if (output.empty() && form.len)
    utf8::append(output, utf8::first(form.str, form.len));

  return output;
}

const string& model_morphodita_parsito::normalize_lemma(string_piece lemma, string& output) const {
  using unilib::utf8;

  // No normalization on version 1 and 2
  if (version <= 2) return output.assign(lemma.str, lemma.len);

  // Normalize spaces by &nbsp; since version 3
  output.clear();
  for (size_t i = 0; i < lemma.len; i++) {
    // Space normalization
    if (lemma.str[i] == ' ') utf8::append(output, 0xA0);
    // Default
    else output.push_back(lemma.str[i]);
  }

  return output;
}

/////////
// File: model/pipeline.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class pipeline {
 public:
  pipeline(const model* m, const string& input, const string& tagger, const string& parser, const string& output);

  void set_model(const model* m);
  void set_input(const string& input);
  void set_tagger(const string& tagger);
  void set_parser(const string& parser);
  void set_output(const string& output);

  void set_immediate(bool immediate);
  void set_document_id(const string& document_id);

  bool process(istream& is, ostream& os, string& error) const;

  static const string DEFAULT;
  static const string NONE;

 private:
  const model* m;
  string input, tokenizer, tagger, parser, output;
  string document_id;
  bool immediate;
};

/////////
// File: sentence/output_format.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class output_format {
 public:
  virtual ~output_format() {}

  virtual void write_sentence(const sentence& s, ostream& os) = 0;
  virtual void finish_document(ostream& /*os*/) {}

  // Static factory methods
  static output_format* new_output_format(const string& name);
  static output_format* new_conllu_output_format(const string& options = string());
  static output_format* new_epe_output_format(const string& options = string());
  static output_format* new_matxin_output_format(const string& options = string());
  static output_format* new_horizontal_output_format(const string& options = string());
  static output_format* new_plaintext_output_format(const string& options = string());
  static output_format* new_vertical_output_format(const string& options = string());

  static const string CONLLU_V1;
  static const string CONLLU_V2;
  static const string HORIZONTAL_PARAGRAPHS;
  static const string PLAINTEXT_NORMALIZED_SPACES;
  static const string VERTICAL_PARAGRAPHS;
};

/////////
// File: utils/getwhole.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Read whole content until EOF. All encountered \n are stored.
inline istream& getwhole(istream& is, string& whole);

//
// Definitions
//

istream& getwhole(istream& is, string& whole) {
  whole.clear();

  for (string line; getline(is, line); )
    whole.append(line).push_back('\n');

  if (is.eof() && !whole.empty()) is.clear(istream::eofbit);
  return is;
}

} // namespace utils

/////////
// File: model/pipeline.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string pipeline::DEFAULT;
const string pipeline::NONE = "none";

pipeline::pipeline(const model* m, const string& input, const string& tagger, const string& parser, const string& output) : immediate(false) {
  set_model(m);
  set_input(input);
  set_tagger(tagger);
  set_parser(parser);
  set_output(output);
}

void pipeline::set_model(const model* m) {
  this->m = m;
}

void pipeline::set_input(const string& input) {
  tokenizer.clear();

  if (input.empty()) {
    this->input = "conllu";
  } else if (input == "tokenize" || input == "tokenizer") {
    this->input = "tokenizer";
  } else if (input.compare(0, 10, "tokenizer=") == 0) {
    this->input = "tokenizer";
    tokenizer.assign(input, 10, string::npos);
  } else {
    this->input = input;
  }
}

void pipeline::set_tagger(const string& tagger) {
  this->tagger = tagger;
}

void pipeline::set_parser(const string& parser) {
  this->parser = parser;
}

void pipeline::set_output(const string& output) {
  this->output = output.empty() ? "conllu" : output;
}

void pipeline::set_immediate(bool immediate) {
  this->immediate = immediate;
}

void pipeline::set_document_id(const string& document_id) {
  this->document_id = document_id;
}

bool pipeline::process(istream& is, ostream& os, string& error) const {
  error.clear();

  sentence s;

  unique_ptr<input_format> reader;
  if (input == "tokenizer") {
    reader.reset(m->new_tokenizer(tokenizer));
    if (!reader) return error.assign("The model does not have a tokenizer!"), false;
  } else {
    reader.reset(input_format::new_input_format(input));
    if (!reader) return error.assign("The requested input format '").append(input).append("' does not exist!"), false;
  }
  reader->reset_document(document_id);

  unique_ptr<output_format> writer(output_format::new_output_format(output));
  if (!writer) return error.assign("The requested output format '").append(output).append("' does not exist!"), false;

  string block;
  while (immediate ? reader->read_block(is, block) : bool(getwhole(is, block))) {
    reader->set_text(block);
    while (reader->next_sentence(s, error)) {
      if (tagger != NONE)
        if (!m->tag(s, tagger, error))
          return false;

      if (parser != NONE)
        if (!m->parse(s, parser, error))
          return false;

      writer->write_sentence(s, os);
    }
    if (!error.empty()) return false;
  }
  writer->finish_document(os);

  return true;
}

/////////
// File: morphodita/derivator/derivation_formatter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class derivation_formatter {
 public:
  virtual ~derivation_formatter() {}

  // Perform the required derivation and store it directly in the lemma.
  virtual void format_derivation(string& lemma) const = 0;

  // Static factory methods.
  static derivation_formatter* new_none_derivation_formatter();
  static derivation_formatter* new_root_derivation_formatter(const derivator* derinet);
  static derivation_formatter* new_path_derivation_formatter(const derivator* derinet);
  static derivation_formatter* new_tree_derivation_formatter(const derivator* derinet);
  // String version of static factory method.
  static derivation_formatter* new_derivation_formatter(string_piece name, const derivator* derinet);
};

} // namespace morphodita

/////////
// File: morphodita/derivator/derivation_formatter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class none_derivation_formatter : public derivation_formatter {
  virtual void format_derivation(string& /*lemma*/) const override {}
};

derivation_formatter* derivation_formatter::new_none_derivation_formatter() {
  return new none_derivation_formatter();
}

class root_derivation_formatter : public derivation_formatter {
 public:
  root_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_derivation(string& lemma) const override {
    for (derivated_lemma parent; derinet->parent(lemma, parent); )
      lemma.assign(parent.lemma);
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_root_derivation_formatter(const derivator* derinet) {
  return derinet ? new root_derivation_formatter(derinet) : nullptr;
}

class path_derivation_formatter : public derivation_formatter {
 public:
  path_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_derivation(string& lemma) const override {
    string current(lemma);
    for (derivated_lemma parent; derinet->parent(current, parent); current.swap(parent.lemma))
      lemma.append(" ").append(parent.lemma);
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_path_derivation_formatter(const derivator* derinet) {
  return derinet ? new path_derivation_formatter(derinet) : nullptr;
}

class tree_derivation_formatter : public derivation_formatter {
 public:
  tree_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_derivation(string& lemma) const override {
    string root(lemma);
    for (derivated_lemma parent; derinet->parent(root, parent); root.swap(parent.lemma)) {}
    format_tree(root, lemma);
  }

  void format_tree(const string& root, string& tree) const {
    vector<derivated_lemma> children;

    tree.append(" ").append(root);
    if (derinet->children(root, children))
      for (auto&& child : children)
        format_tree(child.lemma, tree);
    tree.push_back(' ');
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_tree_derivation_formatter(const derivator* derinet) {
  return derinet ? new tree_derivation_formatter(derinet) : nullptr;
}

derivation_formatter* derivation_formatter::new_derivation_formatter(string_piece name, const derivator* derinet) {
  if (name == "none") return new_none_derivation_formatter();
  if (name == "root") return new_root_derivation_formatter(derinet);
  if (name == "path") return new_path_derivation_formatter(derinet);
  if (name == "tree") return new_tree_derivation_formatter(derinet);
  return nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/small_stringops.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
inline bool small_memeq(const void* a, const void* b, size_t len);
inline void small_memcpy(void* dest, const void* src, size_t len);

// Definitions
bool small_memeq(const void* a_void, const void* b_void, size_t len) {
  const char* a = (const char*)a_void;
  const char* b = (const char*)b_void;

  while (len--)
    if (*a++ != *b++)
      return false;
  return true;
}

void small_memcpy(void* dest_void, const void* src_void, size_t len) {
  char* dest = (char*)dest_void;
  const char* src = (const char*)src_void;

  while (len--)
    *dest++ = *src++;
}

} // namespace morphodita

/////////
// File: trainer/training_failure.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

class training_error : public runtime_error {
 public:
  training_error();

  static ostringstream message_collector;
};

#define training_failure(message) throw (training_error::message_collector << message, training_error())

} // namespace utils

/////////
// File: utils/binary_encoder.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class binary_encoder {
 public:
  inline binary_encoder();

  inline void add_1B(unsigned val);
  inline void add_2B(unsigned val);
  inline void add_4B(unsigned val);
  inline void add_float(double val);
  inline void add_double(double val);
  inline void add_str(string_piece str);
  inline void add_data(string_piece data);
  template <class T> inline void add_data(const vector<T>& data);
  template <class T> inline void add_data(const T* data, size_t elements);

  vector<unsigned char> data;
};

//
// Definitions
//

binary_encoder::binary_encoder() {
  data.reserve(16);
}

void binary_encoder::add_1B(unsigned val) {
  if (uint8_t(val) != val) training_failure("Should encode value " << val << " in one byte!");
  data.push_back(val);
}

void binary_encoder::add_2B(unsigned val) {
  if (uint16_t(val) != val) training_failure("Should encode value " << val << " in one byte!");
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(uint16_t));
}

void binary_encoder::add_4B(unsigned val) {
  if (uint32_t(val) != val) training_failure("Should encode value " << val << " in one byte!");
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(uint32_t));
}

void binary_encoder::add_float(double val) {
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(float));
}

void binary_encoder::add_double(double val) {
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(double));
}

void binary_encoder::add_str(string_piece str) {
  add_1B(str.len < 255 ? str.len : 255);
  if (!(str.len < 255)) add_4B(str.len);
  add_data(str);
}

void binary_encoder::add_data(string_piece data) {
  this->data.insert(this->data.end(), (const unsigned char*) data.str, (const unsigned char*) (data.str + data.len));
}

template <class T>
void binary_encoder::add_data(const vector<T>& data) {
  this->data.insert(this->data.end(), (const unsigned char*) data.data(), (const unsigned char*) (data.data() + data.size()));
}

template <class T>
void binary_encoder::add_data(const T* data, size_t elements) {
  this->data.insert(this->data.end(), (const unsigned char*) data, (const unsigned char*) (data + elements));
}

} // namespace utils

/////////
// File: utils/pointer_decoder.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class pointer_decoder {
 public:
  inline pointer_decoder(const unsigned char*& data);
  inline unsigned next_1B();
  inline unsigned next_2B();
  inline unsigned next_4B();
  inline void next_str(string& str);
  template <class T> inline const T* next(unsigned elements);

 private:
  const unsigned char*& data;
};

//
// Definitions
//

pointer_decoder::pointer_decoder(const unsigned char*& data) : data(data) {}

unsigned pointer_decoder::next_1B() {
  return *data++;
}

unsigned pointer_decoder::next_2B() {
  unsigned result = *(uint16_t*)data;
  data += sizeof(uint16_t);
  return result;
}

unsigned pointer_decoder::next_4B() {
  unsigned result = *(uint32_t*)data;
  data += sizeof(uint32_t);
  return result;
}

void pointer_decoder::next_str(string& str) {
  unsigned len = next_1B();
  if (len == 255) len = next_4B();
  str.assign(next<char>(len), len);
}

template <class T> const T* pointer_decoder::next(unsigned elements) {
  const T* result = (const T*) data;
  data += sizeof(T) * elements;
  return result;
}

} // namespace utils

/////////
// File: morphodita/morpho/persistent_unordered_map.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class persistent_unordered_map {
 public:
  // Accessing function
  template <class EntrySize>
  inline const unsigned char* at(const char* str, int len, EntrySize entry_size) const;

  template <class T>
  inline const T* at_typed(const char* str, int len) const;

  template <class EntryProcess>
  inline void iter(const char* str, int len, EntryProcess entry_process) const;

  template <class EntryProcess>
  inline void iter_all(EntryProcess entry_process) const;

  // Two helper functions accessing some internals
  inline int max_length() const;
  inline const unsigned char* data_start(int len) const;

  // Creation functions
  persistent_unordered_map() {}
  template <class Entry, class EntryEncode>
  persistent_unordered_map(const unordered_map<string, Entry>& map, double load_factor, EntryEncode entry_encode);
  template <class Entry, class EntryEncode>
  persistent_unordered_map(const unordered_map<string, Entry>& map, double load_factor, bool add_prefixes, bool add_suffixes, EntryEncode entry_encode);

  // Manual creation functions
  inline void resize(unsigned elems);
  inline void add(const char* str, int str_len, int data_len);
  inline void done_adding();
  inline unsigned char* fill(const char* str, int str_len, int data_len);
  inline void done_filling();

  // Serialization
  inline void load(binary_decoder& data);
  inline void save(binary_encoder& enc);

 private:
  struct fnv_hash;
  vector<fnv_hash> hashes;

  template <class Entry, class EntryEncode>
  void construct(const map<string, Entry>& map, double load_factor, EntryEncode entry_encode);
};

// Definitions
struct persistent_unordered_map::fnv_hash {
  fnv_hash(unsigned num) {
    mask = 1;
    while (mask < num)
      mask <<= 1;
    hash.resize(mask + 1);
    mask--;
  }
  fnv_hash(binary_decoder& data) {
    uint32_t size = data.next_4B();
    mask = size - 2;
    hash.resize(size);
    memcpy(hash.data(), data.next<uint32_t>(size), size * sizeof(uint32_t));

    size = data.next_4B();
    this->data.resize(size);
    memcpy(this->data.data(), data.next<char>(size), size);
  }

  inline uint32_t index(const char* data, int len) const {
    if (len <= 0) return 0;
    if (len == 1) return *(const uint8_t*)data;
    if (len == 2) return *(const uint16_t*)data;

    uint32_t hash = 2166136261U;
    while (len--)
      hash = (hash ^ unsigned(*data++)) * 16777619U;
    return hash & mask;
  }

  inline void save(binary_encoder& enc);

  unsigned mask;
  vector<uint32_t> hash;
  vector<unsigned char> data;
};

template <class EntrySize>
const unsigned char* persistent_unordered_map::at(const char* str, int len, EntrySize entry_size) const {
  if (unsigned(len) >= hashes.size()) return nullptr;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  if (len <= 2)
    return data != end ? data + len : nullptr;

  while (data < end) {
    if (small_memeq(str, data, len)) return data + len;
    data += len;
    pointer_decoder decoder(data);
    entry_size(decoder);
  }

  return nullptr;
}

template <class T>
const T* persistent_unordered_map::at_typed(const char* str, int len) const {
  if (unsigned(len) >= hashes.size()) return nullptr;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  if (len <= 2)
    return data != end ? (const T*)(data + len) : nullptr;

  while (data < end) {
    if (small_memeq(str, data, len)) return (const T*)(data + len);
    data += len + sizeof(T);
  }

  return nullptr;
}

template <class EntryProcess>
void persistent_unordered_map::iter(const char* str, int len, EntryProcess entry_process) const {
  if (unsigned(len) >= hashes.size()) return;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  while (data < end) {
    auto start = (const char*) data;
    data += len;
    pointer_decoder decoder(data);
    entry_process(start, decoder);
  }
}

template <class EntryProcess>
void persistent_unordered_map::iter_all(EntryProcess entry_process) const {
  for (unsigned len = 0; len < hashes.size(); len++) {
    const unsigned char* data = hashes[len].data.data();
    const unsigned char* end = data + hashes[len].data.size();

    while (data < end) {
      auto start = (const char*) data;
      data += len;
      pointer_decoder decoder(data);
      entry_process(start, len, decoder);
    }
  }
}

int persistent_unordered_map::max_length() const {
  return hashes.size();
}

const unsigned char* persistent_unordered_map::data_start(int len) const {
  return unsigned(len) < hashes.size() ? hashes[len].data.data() : nullptr;
}

void persistent_unordered_map::resize(unsigned elems) {
  if (hashes.size() == 0) hashes.emplace_back(1);
  else if (hashes.size() == 1) hashes.emplace_back(1<<8);
  else if (hashes.size() == 2) hashes.emplace_back(1<<16);
  else hashes.emplace_back(elems);
}

void persistent_unordered_map::add(const char* str, int str_len, int data_len) {
  if (unsigned(str_len) < hashes.size())
    hashes[str_len].hash[hashes[str_len].index(str, str_len)] += str_len + data_len;
}

void persistent_unordered_map::done_adding() {
  for (auto&& hash : hashes) {
    int total = 0;
    for (auto&& len : hash.hash) total += len, len = total - len;
    hash.data.resize(total);
  }
}

unsigned char* persistent_unordered_map::fill(const char* str, int str_len, int data_len) {
  if (unsigned(str_len) < hashes.size()) {
    unsigned index = hashes[str_len].index(str, str_len);
    unsigned offset = hashes[str_len].hash[index];
    small_memcpy(hashes[str_len].data.data() + offset, str, str_len);
    hashes[str_len].hash[index] += str_len + data_len;
    return hashes[str_len].data.data() + offset + str_len;
  }
  return nullptr;
}

void persistent_unordered_map::done_filling() {
  for (auto&& hash : hashes)
    for (int i = hash.hash.size() - 1; i >= 0; i--)
      hash.hash[i] = i > 0 ? hash.hash[i-1] : 0;
}

void persistent_unordered_map::load(binary_decoder& data) {
  unsigned sizes = data.next_1B();

  hashes.clear();
  for (unsigned i = 0; i < sizes; i++)
    hashes.emplace_back(data);
}

} // namespace morphodita

/////////
// File: morphodita/derivator/derivator_dictionary.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class derivator_dictionary : public derivator {
 public:
  virtual bool parent(string_piece lemma, derivated_lemma& parent) const override;
  virtual bool children(string_piece lemma, vector<derivated_lemma>& children) const override;

  bool load(istream& is);

 private:
  friend class morpho;
  const morpho* dictionary;
  persistent_unordered_map derinet;
};

} // namespace morphodita

/////////
// File: utils/compressor.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

class binary_decoder;
class binary_encoder;

class compressor {
 public:
  static bool load(istream& is, binary_decoder& data);
  static bool save(ostream& os, const binary_encoder& enc);
};

} // namespace utils

/////////
// File: morphodita/derivator/derivator_dictionary.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool derivator_dictionary::parent(string_piece lemma, derivated_lemma& parent) const {
  if (dictionary) lemma.len = dictionary->lemma_id_len(lemma);

  auto lemma_data = derinet.at(lemma.str, lemma.len, [](pointer_decoder& data) {
    data.next<char>(data.next_1B());
    data.next_4B();
    data.next<uint32_t>(data.next_2B());
  });
  if (lemma_data) {
    auto parent_encoded = *(uint32_t*)(lemma_data + 1 + *lemma_data);
    if (parent_encoded) {
      unsigned parent_len = parent_encoded & 0xFF;
      auto parent_data = derinet.data_start(parent_len) + (parent_encoded >> 8);
      parent.lemma.assign((const char*) parent_data, parent_len);
      if (parent_data[parent_len])
        parent.lemma.append((const char*) parent_data + parent_len + 1, parent_data[parent_len]);
      return true;
    }
  }
  parent.lemma.clear();
  return false;
}

bool derivator_dictionary::children(string_piece lemma, vector<derivated_lemma>& children) const {
  if (dictionary) lemma.len = dictionary->lemma_id_len(lemma);

  auto lemma_data = derinet.at(lemma.str, lemma.len, [](pointer_decoder& data) {
    data.next<char>(data.next_1B());
    data.next_4B();
    data.next<uint32_t>(data.next_2B());
  });
  if (lemma_data) {
    auto children_len = *(uint16_t*)(lemma_data + 1 + *lemma_data + 4);
    auto children_encoded = (uint32_t*)(lemma_data + 1 + *lemma_data + 4 + 2);
    if (children_len) {
      children.resize(children_len);
      for (unsigned i = 0; i < children_len; i++) {
        unsigned child_len = children_encoded[i] & 0xFF;
        auto child_data = derinet.data_start(child_len) + (children_encoded[i] >> 8);
        children[i].lemma.assign((const char*) child_data, child_len);
        if (child_data[child_len])
          children[i].lemma.append((const char*) child_data + child_len + 1, child_data[child_len]);
      }
      return true;
    }
  }
  children.clear();
  return false;
}

bool derivator_dictionary::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    for (int i = data.next_1B(); i > 0; i--)
      derinet.resize(data.next_4B());

    unsigned data_position = data.tell();
    vector<char> lemma, parent;
    for (int pass = 1; pass <= 3; pass++) {
      if (pass > 1) data.seek(data_position);

      lemma.clear();
      for (int i = data.next_4B(); i > 0; i--) {
        lemma.resize(lemma.size() - data.next_1B());
        for (int i = data.next_1B(); i > 0; i--)
          lemma.push_back(data.next_1B());

        unsigned char lemma_comment_len = data.next_1B();
        const char* lemma_comment = lemma_comment_len ? data.next<char>(lemma_comment_len) : nullptr;

        unsigned children = data.next_2B();

        if (pass == 3) parent.clear();
        enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
        int operations = data.next_1B();
        if (operations) {
          int remove_start = operations & REMOVE_START ? data.next_1B() : 0;
          int remove_end = operations & REMOVE_END ? data.next_1B() : 0;
          if (operations & ADD_START) {
            int add_start = data.next_1B();
            const char* str = data.next<char>(add_start);
            if (pass == 3) parent.assign(str, str + add_start);
          }
          if (pass == 3) parent.insert(parent.end(), lemma.begin() + remove_start, lemma.end() - remove_end);
          if (operations & ADD_END) {
            int add_end = data.next_1B();
            const char* str = data.next<char>(add_end);
            if (pass == 3) parent.insert(parent.end(), str, str + add_end);
          }
        }

        if (pass == 1) {
          derinet.add(lemma.data(), lemma.size(), 1 + lemma_comment_len + 4 + 2 + 4 * children);
        } else if (pass == 2) {
          unsigned char* lemma_data = derinet.fill(lemma.data(), lemma.size(), 1 + lemma_comment_len + 4 + 2 + 4 * children);
          *lemma_data++ = lemma_comment_len;
          while (lemma_comment_len--) *lemma_data++ = *lemma_comment++;
          *(uint32_t*)(lemma_data) = 0; lemma_data += sizeof(uint32_t);
          *(uint16_t*)(lemma_data) = children; lemma_data += sizeof(uint16_t);
          if (children) ((uint32_t*)lemma_data)[children - 1] = 0;
        } else if (pass == 3 && !parent.empty()) {
          auto lemma_data = derinet.at(lemma.data(), lemma.size(), [](pointer_decoder& data) {
            data.next<char>(data.next_1B());
            data.next_4B();
            data.next<uint32_t>(data.next_2B());
          });
          auto parent_data = derinet.at(parent.data(), parent.size(), [](pointer_decoder& data) {
            data.next<char>(data.next_1B());
            data.next_4B();
            data.next<uint32_t>(data.next_2B());
          });
          assert(lemma_data && parent_data);

          unsigned parent_offset = parent_data - parent.size() - derinet.data_start(parent.size());
          assert(parent.size() < (1<<8) && parent_offset < (1<<24));
          *(uint32_t*)(lemma_data + 1 + *lemma_data) = (parent_offset << 8) | parent.size();

          unsigned lemma_offset = lemma_data - lemma.size() - derinet.data_start(lemma.size());
          assert(lemma.size() < (1<<8) && lemma_offset < (1<<24));
          auto children_len = *(uint16_t*)(parent_data + 1 + *parent_data + 4);
          auto children = (uint32_t*)(parent_data + 1 + *parent_data + 4 + 2);
          auto child_index = children[children_len-1];
          children[child_index] = (lemma_offset << 8) | lemma.size();
          if (child_index+1 < children_len) children[children_len-1]++;
        }
      }

      if (pass == 1)
        derinet.done_adding();
      if (pass == 2)
        derinet.done_filling();
    }
  } catch (binary_decoder_error&) {
    return false;
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/casing_variants.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline void generate_casing_variants(string_piece form, string& form_uclc, string& form_lc) {
  using namespace unilib;

  // Detect uppercase+titlecase characters.
  bool first_Lut = false; // first character is uppercase or titlecase
  bool rest_has_Lut = false; // any character but first is uppercase or titlecase
  {
    string_piece form_tmp = form;
    first_Lut = unicode::category(utf8::decode(form_tmp.str, form_tmp.len)) & unicode::Lut;
    while (form_tmp.len && !rest_has_Lut)
      rest_has_Lut = unicode::category(utf8::decode(form_tmp.str, form_tmp.len)) & unicode::Lut;
  }

  // Generate all casing variants if needed (they are different than given form).
  // We only replace letters with their lowercase variants.
  // - form_uclc: first uppercase, rest lowercase
  // - form_lc: all lowercase

  if (first_Lut && !rest_has_Lut) { // common case allowing fast execution
    form_lc.reserve(form.len);
    string_piece form_tmp = form;
    utf8::append(form_lc, unicode::lowercase(utf8::decode(form_tmp.str, form_tmp.len)));
    form_lc.append(form_tmp.str, form_tmp.len);
  } else if (!first_Lut && rest_has_Lut) {
    form_lc.reserve(form.len);
    utf8::map(unicode::lowercase, form.str, form.len, form_lc);
  } else if (first_Lut && rest_has_Lut) {
    form_lc.reserve(form.len);
    form_uclc.reserve(form.len);
    string_piece form_tmp = form;
    char32_t first = utf8::decode(form_tmp.str, form_tmp.len);
    utf8::append(form_lc, unicode::lowercase(first));
    utf8::append(form_uclc, first);
    while (form_tmp.len) {
      char32_t lowercase = unicode::lowercase(utf8::decode(form_tmp.str, form_tmp.len));
      utf8::append(form_lc, lowercase);
      utf8::append(form_uclc, lowercase);
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct czech_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector<unsigned char> data;
};

// Definitions
int czech_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  // Lemma ends by a '-[0-9]', '`' or '_' on non-first position.
  for (unsigned len = 1; len < lemma.len; len++)
    if (lemma.str[len] == '`' || lemma.str[len] == '_' ||
        (lemma.str[len] == '-' && len+1 < lemma.len && lemma.str[len+1] >= '0' && lemma.str[len+1] <= '9'))
      return len;
  return lemma.len;
}

int czech_lemma_addinfo::lemma_id_len(string_piece lemma) {
  // Lemma ends by a '-[0-9]', '`' or '_' on non-first position.
  for (unsigned len = 1; len < lemma.len; len++) {
    if (lemma.str[len] == '`' || lemma.str[len] == '_')
      return len;
    if (lemma.str[len] == '-' && len+1 < lemma.len && lemma.str[len+1] >= '0' && lemma.str[len+1] <= '9') {
      len += 2;
      while (len < lemma.len && lemma.str[len] >= '0' && lemma.str[len] <= '9') len++;
      return len;
    }
  }
  return lemma.len;
}

string czech_lemma_addinfo::format(const unsigned char* addinfo, int addinfo_len) {
  string res;

  if (addinfo_len) {
    res.reserve(addinfo_len + 4);
    if (addinfo[0] != 255) {
      char num[5];
      sprintf(num, "-%u", addinfo[0]);
      res += num;
    }
    for (int i = 1; i < addinfo_len; i++)
      res += addinfo[i];
  }

  return res;
}

bool czech_lemma_addinfo::generatable(const unsigned char* addinfo, int addinfo_len) {
  for (int i = 1; i + 2 < addinfo_len; i++)
    if (addinfo[i] == '_' && addinfo[i+1] == ',' && addinfo[i+2] == 'x')
      return false;

  return true;
}

int czech_lemma_addinfo::parse(string_piece lemma, bool die_on_failure) {
  data.clear();

  const char* lemma_info = lemma.str + raw_lemma_len(lemma);
  if (lemma_info < lemma.str + lemma.len) {
    int lemma_num = 255;
    const char* lemma_additional_info = lemma_info;

    if (*lemma_info == '-') {
      lemma_num = strtol(lemma_info + 1, (char**) &lemma_additional_info, 10);

      if (lemma_additional_info == lemma_info + 1 || (*lemma_additional_info != '\0' && *lemma_additional_info != '`' && *lemma_additional_info != '_') || lemma_num < 0 || lemma_num >= 255) {
        if (die_on_failure)
          training_failure("Lemma number " << lemma_num << " in lemma " << lemma << " out of range!");
        else
          lemma_num = 255;
      }
    }
    data.emplace_back(lemma_num);
    while (lemma_additional_info < lemma.str + lemma.len)
      data.push_back(*(unsigned char*)lemma_additional_info++);

    if (data.size() > 255) {
      if (die_on_failure)
        training_failure("Too long lemma info " << lemma_info << " in lemma " << lemma << '!');
      else
        data.resize(255);
    }
  }

  return lemma_info - lemma.str;
}

bool czech_lemma_addinfo::match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len) {
  if (data.empty()) return true;
  if (data[0] != 255 && (!other_addinfo_len || other_addinfo[0] != data[0])) return false;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/tag_filter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class tag_filter {
 public:
  tag_filter(const char* filter = nullptr);

  inline bool matches(const char* tag) const;

 private:
  struct char_filter {
    char_filter(int pos, bool negate, const char* chars, int len) : pos(pos), negate(negate), chars(chars), len(len) {}

    int pos;
    bool negate;
    const char* chars;
    int len;
  };

  string wildcard;
  std::vector<char_filter> filters;
};

// Definitions
inline bool tag_filter::matches(const char* tag) const {
  if (filters.empty()) return true;

  int tag_pos = 0;
  for (auto&& filter : filters) {
    while (tag_pos < filter.pos)
      if (!tag[tag_pos++])
        return true;

    // We assume filter.len >= 1.
    bool matched = (*filter.chars == tag[tag_pos]) ^ filter.negate;
    for (int i = 1; i < filter.len && !matched; i++)
      matched = (filter.chars[i] == tag[tag_pos]) ^ filter.negate;
    if (!matched) return false;
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_dictionary.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class LemmaAddinfo>
class morpho_dictionary {
 public:
  void load(binary_decoder& data);
  void analyze(string_piece form, vector<tagged_lemma>& lemmas) const;
  bool generate(string_piece lemma, const tag_filter& filter, vector<tagged_lemma_forms>& lemmas_forms) const;
 private:
  persistent_unordered_map lemmas, roots, suffixes;

  vector<string> tags;
  vector<vector<pair<string, vector<uint16_t>>>> classes;
};

// Definitions
template <class LemmaAddinfo>
void morpho_dictionary<LemmaAddinfo>::load(binary_decoder& data) {
  // Prepare lemmas and roots hashes
  for (int i = data.next_1B(); i > 0; i--)
    lemmas.resize(data.next_4B());
  for (int i = data.next_1B(); i > 0; i--)
    roots.resize(data.next_4B());

  // Perform two pass over the lemmas and roots data, filling the hashes.

  vector<char> lemma(max(lemmas.max_length(), roots.max_length()));
  vector<char> root(max(lemmas.max_length(), roots.max_length()));
  unsigned data_position = data.tell();
  for (int pass = 1; pass <= 2; pass++) {
    if (pass > 1) data.seek(data_position);

    int lemma_len = 0;
    int root_len = 0;

    for (int i = data.next_4B(); i > 0; i--) {
      lemma_len -= data.next_1B();
      for (int i = data.next_1B(); i > 0; i--)
        lemma[lemma_len++] = data.next_1B();
      unsigned char lemma_info_len = data.next_1B();
      const char* lemma_info = lemma_info_len ? data.next<char>(lemma_info_len) : nullptr;
      unsigned lemma_roots = data.next_1B();

      unsigned char* lemma_data /* to keep compiler happy */ = nullptr;
      unsigned lemma_offset /* to keep compiler happy */ = 0;

      if (pass == 1) {
        lemmas.add(lemma.data(), lemma_len, 1 + lemma_info_len + 1 + lemma_roots * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));
      } else /*if (pass == 2)*/ {
        lemma_data = lemmas.fill(lemma.data(), lemma_len, 1 + lemma_info_len + 1 + lemma_roots * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));
        lemma_offset = lemma_data - lemma_len - lemmas.data_start(lemma_len);

        *lemma_data++ = lemma_info_len;
        if (lemma_info_len) small_memcpy(lemma_data, lemma_info, lemma_info_len), lemma_data += lemma_info_len;
        *lemma_data++ = lemma_roots;
      }

      small_memcpy(root.data(), lemma.data(), lemma_len); root_len = lemma_len;
      for (unsigned i = 0; i < lemma_roots; i++) {
        enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
        int operations = data.next_1B();
        if (operations & REMOVE_START) { int from = data.next_1B(), to = 0; while (from < root_len) root[to++] = root[from++]; root_len = to; }
        if (operations & REMOVE_END) root_len -= data.next_1B();
        if (operations & ADD_START) {
          int from = root_len, to = from + data.next_1B(); while (from > 0) root[--to] = root[--from]; root_len += to;
          for (int i = 0; i < to; i++) root[i] = data.next_1B();
        }
        if (operations & ADD_END)
          for (int len = data.next_1B(); len > 0; len--)
            root[root_len++] = data.next_1B();
        uint16_t clas = data.next_2B();

        if (pass == 1) { // for each root
          roots.add(root.data(), root_len, sizeof(uint16_t) + sizeof(uint32_t) + sizeof(uint8_t));
        } else /*if (pass == 2)*/ {
          unsigned char* root_data = roots.fill(root.data(), root_len, sizeof(uint16_t) + sizeof(uint32_t) + sizeof(uint8_t));
          unsigned root_offset = root_data - root_len - roots.data_start(root_len);

          *(uint16_t*)(root_data) = clas; root_data += sizeof(uint16_t);
          *(uint32_t*)(root_data) = lemma_offset; root_data += sizeof(uint32_t);
          *(uint8_t*)(root_data) = lemma_len; root_data += sizeof(uint8_t);
          assert(uint8_t(lemma_len) == lemma_len);

          *(uint32_t*)(lemma_data) = root_offset; lemma_data += sizeof(uint32_t);
          *(uint8_t*)(lemma_data) = root_len; lemma_data += sizeof(uint8_t);
          *(uint16_t*)(lemma_data) = clas; lemma_data += sizeof(uint16_t);
          assert(uint8_t(root_len) == root_len);
        }
      }
    }

    if (pass == 1) { // after the whole pass
      lemmas.done_adding();
      roots.done_adding();
    } else /*if (pass == 2)*/ {
      lemmas.done_filling();
      roots.done_filling();
    }
  }

  // Load tags
  tags.resize(data.next_2B());
  for (auto&& tag : tags) {
    tag.resize(data.next_1B());
    for (unsigned i = 0; i < tag.size(); i++)
      tag[i] = data.next_1B();
  }

  // Load suffixes
  suffixes.load(data);

  // Fill classes from suffixes
  suffixes.iter_all([this](const char* suffix, int len, pointer_decoder& data) mutable {
    unsigned classes_len = data.next_2B();
    const uint16_t* classes_ptr = data.next<uint16_t>(classes_len);
    // Following volatile is needed to overcome vectorizer bug in g++ 6.3.0 (among other versions).
    volatile const uint16_t* indices_ptr = data.next<uint16_t>(classes_len + 1);
    uint32_t tags_len = indices_ptr[0];
    for (unsigned i = 0; i < classes_len; i++)
      tags_len += uint16_t(indices_ptr[i + 1] - indices_ptr[i]);
    const uint16_t* tags_ptr = data.next<uint16_t>(tags_len);

    string suffix_str(suffix, len);
    uint32_t index = indices_ptr[0], prev_index = 0;
    for (unsigned i = 0; i < classes_len; i++) {
      if (classes_ptr[i] >= classes.size()) classes.resize(classes_ptr[i] + 1);
      prev_index = index;
      index += uint16_t(indices_ptr[i + 1] - indices_ptr[i]);
      classes[classes_ptr[i]].emplace_back(suffix_str, vector<uint16_t>(tags_ptr + prev_index, tags_ptr + index));
    }
  });
}

template <class LemmaAddinfo>
void morpho_dictionary<LemmaAddinfo>::analyze(string_piece form, vector<tagged_lemma>& lemmas) const {
  int max_suffix_len = suffixes.max_length();

  uint16_t* suff_stack[16]; vector<uint16_t*> suff_heap;
  uint16_t** suff = max_suffix_len <= 16 ? suff_stack : (suff_heap.resize(max_suffix_len), suff_heap.data());
  int suff_len = 0;
  for (int i = form.len; i >= 0 && suff_len < max_suffix_len; i--, suff_len++) {
    suff[suff_len] = (uint16_t*) suffixes.at(form.str + i, suff_len, [](pointer_decoder& data) {
      data.next<uint16_t>(2 * data.next_2B());
      data.next<uint16_t>(data.next_2B());
    });
    if (!suff[suff_len]) break;
  }

  for (int root_len = int(form.len) - --suff_len; suff_len >= 0 && root_len < int(roots.max_length()); suff_len--, root_len++)
    if (*suff[suff_len]) {
      unsigned suff_classes = *suff[suff_len];
      uint16_t* suff_data = suff[suff_len] + 1;

      roots.iter(form.str, root_len, [&](const char* root, pointer_decoder& root_data) {
        unsigned root_class = root_data.next_2B();
        unsigned lemma_offset = root_data.next_4B();
        unsigned lemma_len = root_data.next_1B();

        if (small_memeq(form.str, root, root_len)) {
          uint16_t* suffix_class_ptr = lower_bound(suff_data, suff_data + suff_classes, root_class);
          if (suffix_class_ptr < suff_data + suff_classes && *suffix_class_ptr == root_class) {
            const unsigned char* lemma_data = this->lemmas.data_start(lemma_len) + lemma_offset;
            string lemma((const char*)lemma_data, lemma_len);
            if (lemma_data[lemma_len]) lemma += LemmaAddinfo::format(lemma_data + lemma_len + 1, lemma_data[lemma_len]);

            uint16_t* suff_tag_indices = suff_data + suff_classes;
            uint16_t* suff_tags = suff_tag_indices + suff_classes + 1;
            for (unsigned i = suff_tag_indices[suffix_class_ptr - suff_data]; i < suff_tag_indices[suffix_class_ptr - suff_data + 1]; i++)
              lemmas.emplace_back(lemma, tags[suff_tags[i]]);
          }
        }
      });
    }
}

template <class LemmaAddinfo>
bool morpho_dictionary<LemmaAddinfo>::generate(string_piece lemma, const tag_filter& filter, vector<tagged_lemma_forms>& lemmas_forms) const {
  LemmaAddinfo addinfo;
  int raw_lemma_len = addinfo.parse(lemma);
  bool matched_lemma = false;

  lemmas.iter(lemma.str, raw_lemma_len, [&](const char* lemma_str, pointer_decoder& data) {
    unsigned lemma_info_len = data.next_1B();
    const auto* lemma_info = data.next<unsigned char>(lemma_info_len);
    unsigned lemma_roots_len = data.next_1B();
    auto* lemma_roots_ptr = data.next<unsigned char>(lemma_roots_len * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));

    if (small_memeq(lemma.str, lemma_str, raw_lemma_len) && addinfo.match_lemma_id(lemma_info, lemma_info_len) && LemmaAddinfo::generatable(lemma_info, lemma_info_len)) {
      matched_lemma = true;

      vector<tagged_form>* forms = nullptr;
      pointer_decoder lemma_roots(lemma_roots_ptr);
      for (unsigned i = 0; i < lemma_roots_len; i++) {
        unsigned root_offset = lemma_roots.next_4B();
        unsigned root_len = lemma_roots.next_1B();
        unsigned clas = lemma_roots.next_2B();

        const unsigned char* root_data = roots.data_start(root_len) + root_offset;
        for (auto&& suffix : classes[clas]) {
          string root_with_suffix;
          for (auto&& tag : suffix.second)
            if (filter.matches(tags[tag].c_str())) {
              if (!forms) {
                lemmas_forms.emplace_back(string(lemma.str, raw_lemma_len) + LemmaAddinfo::format(lemma_info, lemma_info_len));
                forms = &lemmas_forms.back().forms;
              }

              if (root_with_suffix.empty() && root_len + suffix.first.size()) {
                root_with_suffix.reserve(root_len + suffix.first.size());
                root_with_suffix.assign((const char*)root_data, root_len);
                root_with_suffix.append(suffix.first);
              }

              forms->emplace_back(root_with_suffix, tags[tag]);
            }
        }
      }
    }
  });

  return matched_lemma;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_prefix_guesser.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class MorphoDictionary>
class morpho_prefix_guesser {
 public:
  morpho_prefix_guesser(const MorphoDictionary& dictionary) : dictionary(dictionary) {}

  void load(binary_decoder& data);
  void analyze(string_piece form, vector<tagged_lemma>& lemmas);
  bool generate(string_piece lemma, const tag_filter& filter, vector<tagged_lemma_forms>& lemmas_forms);

 private:
  const MorphoDictionary& dictionary;
  vector<tag_filter> tag_filters;
  persistent_unordered_map prefixes_initial, prefixes_middle;
};

// Definitions
template <class MorphoDictionary>
void morpho_prefix_guesser<MorphoDictionary>::load(binary_decoder& data) {
  // Load and construct tag filters
  for (unsigned tag_filters_len = data.next_1B(); tag_filters_len; tag_filters_len--) {
    unsigned tag_filter_len = data.next_1B();
    string tag_filter(data.next<char>(tag_filter_len), tag_filter_len);

    tag_filters.emplace_back(tag_filter.c_str());
  }

  // Load prefixes
  prefixes_initial.load(data);
  prefixes_middle.load(data);
}

// Analyze can return non-unique lemma-tag pairs.
template <class MorphoDictionary>
void morpho_prefix_guesser<MorphoDictionary>::analyze(string_piece form, vector<tagged_lemma>& lemmas) {
  if (!form.len) return;

  vector<char> form_tmp;
  vector<unsigned> middle_masks;
  middle_masks.reserve(form.len);

  for (unsigned initial = 0; initial < form.len; initial++) {
    // Match the initial prefix.
    unsigned initial_mask = (1<<tag_filters.size()) - 1; // full mask for empty initial prefix
    if (initial) {
      auto found = prefixes_initial.at_typed<uint32_t>(form.str, initial);
      if (!found) break;
      initial_mask = *found;
    }

    // If we have found an initial prefix (including the empty one), match middle prefixes.
    if (initial_mask) {
      middle_masks.resize(initial);
      middle_masks.emplace_back(initial_mask);
      for (unsigned middle = initial; middle < middle_masks.size(); middle++) {
        if (!middle_masks[middle]) continue;
        // Try matching middle prefixes from current index.
        for (unsigned i = middle + 1; i < form.len; i++) {
          auto found = prefixes_middle.at_typed<uint32_t>(form.str + middle, i - middle);
          if (!found) break;
          if (*found) {
            if (i + 1 > middle_masks.size()) middle_masks.resize(i + 1);
            middle_masks[i] |= middle_masks[middle] & *found;
          }
        }

        // Try matching word forms if at least one middle prefix was found.
        if (middle > initial && middle < form.len ) {
          if (initial) {
            if (form_tmp.empty()) form_tmp.assign(form.str, form.str + form.len);
            small_memcpy(form_tmp.data() + middle - initial, form.str, initial);
          }
          unsigned lemmas_ori_size = lemmas.size();
          dictionary.analyze(string_piece((initial ? form_tmp.data() : form.str) + middle - initial, form.len - middle + initial), lemmas);
          unsigned lemmas_new_size = lemmas_ori_size;
          for (unsigned i = lemmas_ori_size; i < lemmas.size(); i++) {
            for (unsigned filter = 0; filter < tag_filters.size(); filter++)
              if ((middle_masks[middle] & (1<<filter)) && tag_filters[filter].matches(lemmas[i].tag.c_str())) {
                if (i == lemmas_new_size) {
                  lemmas[lemmas_new_size].lemma.insert(0, form.str + initial, middle - initial);
                } else {
                  lemmas[lemmas_new_size].lemma.reserve(lemmas[i].lemma.size() + middle - initial);
                  lemmas[lemmas_new_size].lemma.assign(form.str + initial, middle - initial);
                  lemmas[lemmas_new_size].lemma.append(lemmas[i].lemma);
                  lemmas[lemmas_new_size].tag = lemmas[i].tag;
                }
                lemmas_new_size++;
                break;
              }
          }
          if (lemmas_new_size < lemmas.size()) lemmas.erase(lemmas.begin() + lemmas_new_size, lemmas.end());
        }
      }
    }
  }
}

template <class MorphoDictionary>
bool morpho_prefix_guesser<MorphoDictionary>::generate(string_piece /*lemma*/, const tag_filter& /*filter*/, vector<tagged_lemma_forms>& /*lemmas_forms*/) {
  // Not implemented yet. Is it actually needed?
  return false;
}
} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser {
 public:
  void load(binary_decoder& data);
  typedef vector<string> used_rules;
  void analyze(string_piece form, vector<tagged_lemma>& lemmas, used_rules* used);

 private:
  vector<string> tags;
  unsigned default_tag;
  persistent_unordered_map rules;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/unicode_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class unicode_tokenizer : public tokenizer {
 public:
  enum { URL_EMAIL_LATEST = 2 };
  unicode_tokenizer(unsigned url_email_tokenizer);

  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(vector<string_piece>* forms, vector<token_range>* tokens) override;

  virtual bool next_sentence(vector<token_range>& tokens) = 0;

 protected:
  struct char_info {
    char32_t chr;
    unilib::unicode::category_t cat;
    const char* str;

    char_info(char32_t chr, const char* str) : chr(chr), cat(unilib::unicode::category(chr)), str(str) {}
  };
  vector<char_info> chars;
  size_t current;

  bool tokenize_url_email(vector<token_range>& tokens);
  bool emergency_sentence_split(const vector<token_range>& tokens);
  bool is_eos(const vector<token_range>& tokens, char32_t eos_chr, const unordered_set<string>* abbreviations);

 private:
  unsigned url_email_tokenizer;
  string text_buffer;
  vector<token_range> tokens_buffer;
  string eos_buffer;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/ragel_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class ragel_tokenizer : public unicode_tokenizer {
 public:
  ragel_tokenizer(unsigned url_email_tokenizer);

 protected:
  static inline uint8_t ragel_char(const char_info& chr);

 private:
  static void initialize_ragel_map();
  static vector<uint8_t> ragel_map;
  static atomic_flag ragel_map_flag;
  static void ragel_map_add(char32_t chr, uint8_t mapping);

  friend class unicode_tokenizer;
  static bool ragel_url_email(unsigned version, const vector<char_info>& chars, size_t& current_char, vector<token_range>& tokens);
};

uint8_t ragel_tokenizer::ragel_char(const char_info& chr) {
  return chr.chr < ragel_map.size() && ragel_map[chr.chr] != 128 ? ragel_map[chr.chr] : 128 + (uint32_t(chr.cat) * uint32_t(0x077CB531U) >> 27);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_tokenizer : public ragel_tokenizer {
 public:
  enum tokenizer_language { CZECH = 0, SLOVAK = 1 };
  enum { LATEST = 2 };
  czech_tokenizer(tokenizer_language language, unsigned version, const morpho* m = nullptr);

  virtual bool next_sentence(vector<token_range>& tokens) override;

 private:
  const morpho* m;
  const unordered_set<string>* abbreviations;
  vector<tagged_lemma> lemmas;

  void merge_hyphenated(vector<token_range>& tokens);

  static const unordered_set<string> abbreviations_czech;
  static const unordered_set<string> abbreviations_slovak;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_morpho.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_morpho : public morpho {
 public:
  using morpho_language = czech_tokenizer::tokenizer_language;

  czech_morpho(morpho_language language, unsigned version) : language(language), version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector<tagged_lemma>& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector<tagged_lemma_forms>& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const;

  morpho_language language;
  unsigned version;
  morpho_dictionary<czech_lemma_addinfo> dictionary;
  unique_ptr<morpho_prefix_guesser<decltype(dictionary)>> prefix_guesser;
  unique_ptr<morpho_statistical_guesser> statistical_guesser;

  string unknown_tag = "X@-------------";
  string number_tag = "C=-------------";
  string punctuation_tag = "Z:-------------";
};

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_morpho.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool czech_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load tag length
    unsigned tag_length = data.next_1B();
    if (tag_length < unknown_tag.size()) unknown_tag.erase(tag_length);
    if (tag_length < number_tag.size()) number_tag.erase(tag_length);
    if (tag_length < punctuation_tag.size()) punctuation_tag.erase(tag_length);

    // Load dictionary
    dictionary.load(data);

    // Optionally prefix guesser if present
    prefix_guesser.reset();
    if (data.next_1B()) {
      prefix_guesser.reset(new morpho_prefix_guesser<decltype(dictionary)>(dictionary));
      prefix_guesser->load(data);
    }

    // Optionally statistical guesser if present
    statistical_guesser.reset();
    if (data.next_1B()) {
      statistical_guesser.reset(new morpho_statistical_guesser());
      statistical_guesser->load(data);
    }
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int czech_morpho::analyze(string_piece form, guesser_mode guesser, vector<tagged_lemma>& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Then call analyze_special to handle numbers and punctuation.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // For the prefix guesser, use only form_lc.
    if (guesser == GUESSER && prefix_guesser)
      prefix_guesser->analyze(form_lc.empty() ? form : form_lc, lemmas);
    bool prefix_guesser_guesses = !lemmas.empty();

    // For the statistical guesser, use all casing variants.
    if (guesser == GUESSER && statistical_guesser) {
      if (form_uclc.empty() && form_lc.empty())
        statistical_guesser->analyze(form, lemmas, nullptr);
      else {
        morpho_statistical_guesser::used_rules used_rules; used_rules.reserve(3);
        statistical_guesser->analyze(form, lemmas, &used_rules);
        if (!form_uclc.empty()) statistical_guesser->analyze(form_uclc, lemmas, &used_rules);
        if (!form_lc.empty()) statistical_guesser->analyze(form_lc, lemmas, &used_rules);
      }
    }

    // Make sure results are unique lemma-tag pairs. Statistical guesser produces
    // unique lemma-tag pairs, but prefix guesser does not.
    if (prefix_guesser_guesses) {
      sort(lemmas.begin(), lemmas.end(), [](const tagged_lemma& a, const tagged_lemma& b) {
        int lemma_compare = a.lemma.compare(b.lemma);
        return lemma_compare < 0 || (lemma_compare == 0 && a.tag < b.tag);
      });
      auto lemmas_end = unique(lemmas.begin(), lemmas.end(), [](const tagged_lemma& a, const tagged_lemma& b) {
        return a.lemma == b.lemma && a.tag == b.tag;
      });
      if (lemmas_end != lemmas.end()) lemmas.erase(lemmas_end, lemmas.end());
    }

    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int czech_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode guesser, vector<tagged_lemma_forms>& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;

    if (guesser == GUESSER && prefix_guesser)
      if (prefix_guesser->generate(lemma, filter, forms))
        return GUESSER;
  }

  return -1;
}

int czech_morpho::raw_lemma_len(string_piece lemma) const {
  return czech_lemma_addinfo::raw_lemma_len(lemma);
}

int czech_morpho::lemma_id_len(string_piece lemma) const {
  return czech_lemma_addinfo::lemma_id_len(lemma);
}

int czech_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* czech_morpho::new_tokenizer() const {
  return new czech_tokenizer(language, version, this);
}

// What characters are considered punctuation except for the ones in unicode Punctuation category.
static bool punctuation_additional[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*$*/,
  0,0,0,0,0,0,1/*+*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*<*/,1/*=*/,1/*>*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,1/*^*/,0,1/*`*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*|*/,0,1/*~*/,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*caron*/};

// What characters of unicode Punctuation category are not considered punctuation.
static bool punctuation_exceptions[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,1/*paragraph*/};

void czech_morpho::analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers and punctuation.
  // Number is anything matching [+-]? is_Pn* ([.,] is_Pn*)? ([Ee] [+-]? is_Pn+)? for at least one is_Pn* nonempty.
  // Punctuation is any form beginning with either unicode punctuation or punctuation_exceptions character.
  // Beware that numbers takes precedence, so - is punctuation, -3 is number, -. is punctuation, -.3 is number.
  if (!form.len) return;

  string_piece form_ori = form;
  char32_t first = utf8::decode(form.str, form.len);

  // Try matching a number.
  char32_t codepoint = first;
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(form.str, form.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  if ((codepoint == '.' && form.len) || codepoint == ',') codepoint = utf8::decode(form.str, form.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(form.str, form.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(form.str, form.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  }

  if (any_digit && !form.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form_ori.str, form_ori.len - (codepoint == '.')), number_tag);
  } else if ((first < sizeof(punctuation_additional) && punctuation_additional[first]) ||
             ((unicode::category(first) & unicode::P) && (first >= sizeof(punctuation_exceptions) || !punctuation_exceptions[first])))
    lemmas.emplace_back(string(form_ori.str, form_ori.len), punctuation_tag);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct english_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector<unsigned char> data;
};

// Definitions
int english_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  // Lemma ends either by
  // - '^' on non-first position followed by nothing or [A-Za-z][-A-Za-z]*
  // - '+' on non-first position followed by nothing
  for (unsigned len = 1; len < lemma.len; len++) {
    if (len + 1 == lemma.len && (lemma.str[len] == '^' || lemma.str[len] == '+'))
      return len;
    if (len + 1 < lemma.len && lemma.str[len] == '^') {
      bool ok = true;
      for (unsigned i = len + 1; ok && i < lemma.len; i++)
        ok &= (lemma.str[i] >= 'A' && lemma.str[i] <= 'Z') ||
            (lemma.str[i] >= 'a' && lemma.str[i] <= 'z') ||
            (i > len + 1 && lemma.str[i] == '-');
      if (ok) return len;
    }
  }
  return lemma.len;
}

int english_lemma_addinfo::lemma_id_len(string_piece lemma) {
  // No lemma comments.
  return lemma.len;
}

string english_lemma_addinfo::format(const unsigned char* addinfo, int addinfo_len) {
  return string((const char*) addinfo, addinfo_len);
}

bool english_lemma_addinfo::generatable(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return true;
}

int english_lemma_addinfo::parse(string_piece lemma, bool /*die_on_failure*/) {
  data.clear();

  size_t len = raw_lemma_len(lemma);
  for (size_t i = len; i < lemma.len; i++)
    data.push_back(lemma.str[i]);

  return len;
}

bool english_lemma_addinfo::match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len) {
  if (data.empty()) return true;
  if (data.size() == 1 && data[0] == '^') return other_addinfo_len > 0 && other_addinfo[0] == '^';
  if (data.size() == 1 && data[0] == '+') return other_addinfo_len == 0;
  return data.size() == size_t(other_addinfo_len) && small_memeq(data.data(), other_addinfo, other_addinfo_len);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho_guesser.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_morpho_guesser {
 public:
  void load(binary_decoder& data);
  void analyze(string_piece form, string_piece form_lc, vector<tagged_lemma>& lemmas) const;
  bool analyze_proper_names(string_piece form, string_piece form_lc, vector<tagged_lemma>& lemmas) const;

 private:
  inline void add(const string& tag, const string& form, vector<tagged_lemma>& lemmas) const;
  inline void add(const string& tag, const string& tag2, const string& form, vector<tagged_lemma>& lemmas) const;
  inline void add(const string& tag, const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const;
  inline void add(const string& tag, const string& tag2, const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const;
  void add_NNS(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const;
  void add_NNPS(const string& form, vector<tagged_lemma>& lemmas) const;
  void add_VBG(const string& form, vector<tagged_lemma>& lemmas) const;
  void add_VBD_VBN(const string& form, vector<tagged_lemma>& lemmas) const;
  void add_VBZ(const string& form, vector<tagged_lemma>& lemmas) const;
  void add_JJR_RBR(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const;
  void add_JJS_RBS(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const;

  enum { NEGATION_LEN = 0, TO_FOLLOW = 1, TOTAL = 2 };
  vector<string> exceptions_tags;
  persistent_unordered_map exceptions;
  persistent_unordered_map negations;
  string CD = "CD", FW = "FW", JJ = "JJ", JJR = "JJR", JJS = "JJS",
         NN = "NN", NNP = "NNP", NNPS = "NNPS", NNS = "NNS", RB = "RB",
         RBR = "RBR", RBS = "RBS", SYM = "SYM", VB = "VB", VBD = "VBD",
         VBG = "VBG", VBN = "VBN", VBP = "VBP", VBZ = "VBZ";
};

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_morpho : public morpho {
 public:
  english_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector<tagged_lemma>& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector<tagged_lemma_forms>& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const;

  unsigned version;
  morpho_dictionary<english_lemma_addinfo> dictionary;
  english_morpho_guesser morpho_guesser;

  string unknown_tag = "UNK";
  string number_tag = "CD", nnp_tag = "NNP", ls_tag = "LS";
  string open_quotation_tag = "``", close_quotation_tag = "''";
  string open_parenthesis_tag = "(", close_parenthesis_tag = ")";
  string comma_tag = ",", dot_tag = ".", punctuation_tag = ":", hash_tag = "#", dollar_tag = "$";
  string sym_tag = "SYM", jj_tag = "JJ", nn_tag = "NN", nns_tag = "NNS", cc_tag = "CC", pos_tag = "POS", in_tag = "IN";
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/english_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_tokenizer : public ragel_tokenizer {
 public:
  enum { LATEST = 2 };
  english_tokenizer(unsigned version);

  virtual bool next_sentence(vector<token_range>& tokens) override;

 private:
  void split_token(vector<token_range>& tokens);

  static const unordered_set<string> abbreviations;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool english_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    dictionary.load(data);
    morpho_guesser.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int english_morpho::analyze(string_piece form, guesser_mode guesser, vector<tagged_lemma>& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty())
      return guesser == NO_GUESSER || !morpho_guesser.analyze_proper_names(form, form_lc.empty() ? form : form_lc, lemmas) ? NO_GUESSER : GUESSER;

    // Then call analyze_special to handle numbers, punctuation and symbols.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Use English guesser on form_lc if allowed.
    if (guesser == GUESSER)
      morpho_guesser.analyze(form, form_lc.empty() ? form : form_lc, lemmas);
    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int english_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector<tagged_lemma_forms>& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;
  }

  return -1;
}

int english_morpho::raw_lemma_len(string_piece lemma) const {
  return english_lemma_addinfo::raw_lemma_len(lemma);
}

int english_morpho::lemma_id_len(string_piece lemma) const {
  return english_lemma_addinfo::lemma_id_len(lemma);
}

int english_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* english_morpho::new_tokenizer() const {
  return new english_tokenizer(version <= 2 ? 1 : 2);
}

void english_morpho::analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers and punctuation.
  if (!form.len) return;

  // One-letter punctuation exceptions.
  if (form.len == 1)
    switch(*form.str) {
      case '.':
      case '!':
      case '?': lemmas.emplace_back(string(form.str, form.len), dot_tag); return;
      case ',': lemmas.emplace_back(string(form.str, form.len), comma_tag); return;
      case '#': lemmas.emplace_back(string(form.str, form.len), hash_tag); return;
      case '$': lemmas.emplace_back(string(form.str, form.len), dollar_tag); return;
      case '[': lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case ']': lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case '%': lemmas.emplace_back(string(form.str, form.len), jj_tag);
                lemmas.emplace_back(string(form.str, form.len), nn_tag); return;
      case '&': lemmas.emplace_back(string(form.str, form.len), cc_tag);
                lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case '*': lemmas.emplace_back(string(form.str, form.len), sym_tag);
                lemmas.emplace_back(string(form.str, form.len), nn_tag); return;
      case '@': lemmas.emplace_back(string(form.str, form.len), sym_tag);
                lemmas.emplace_back(string(form.str, form.len), in_tag); return;
      case '\'': lemmas.emplace_back(string(form.str, form.len), close_quotation_tag);
                 lemmas.emplace_back(string(form.str, form.len), pos_tag); return;
    }

  // Try matching a number: [+-]? is_Pn* (, is_Pn{3})? (. is_Pn*)? (s | [Ee] [+-]? is_Pn+)? with at least one digit
  string_piece number = form;
  char32_t codepoint = utf8::decode(number.str, number.len);
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  while (codepoint == ',') {
    string_piece group = number;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    any_digit = true;
    number = group;
    codepoint = utf8::decode(number.str, number.len);
  }
  if (codepoint == '.' && number.len) {
    codepoint = utf8::decode(number.str, number.len);
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }
  if (version >= 2 && any_digit && codepoint == 's' && !number.len) {
    lemmas.emplace_back(string(form.str, form.len), number_tag);
    lemmas.emplace_back(string(form.str, form.len - 1), nns_tag);
    return;
  }
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(number.str, number.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }
  if (any_digit && !number.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form.str, form.len - (codepoint == '.')), number_tag);
    lemmas.emplace_back(string(form.str, form.len - (codepoint == '.')), nnp_tag);
    if (form.len == 1 + (codepoint == '.') && *form.str >= '1' && *form.str <= '9')
      lemmas.emplace_back(string(form.str, form.len - (codepoint == '.')), ls_tag);
    return;
  }

  // Open quotation, end quotation, open parentheses, end parentheses, symbol, or other
  string_piece punctuation = form;
  bool open_quotation = true, close_quotation = true, open_parenthesis = true, close_parenthesis = true, any_punctuation = true, symbol = true;
  while ((symbol || any_punctuation) && punctuation.len) {
    codepoint = utf8::decode(punctuation.str, punctuation.len);
    if (open_quotation) open_quotation = codepoint == '`' || unicode::category(codepoint) & unicode::Pi;
    if (close_quotation) close_quotation = codepoint == '\'' || codepoint == '"' || unicode::category(codepoint) & unicode::Pf;
    if (open_parenthesis) open_parenthesis = unicode::category(codepoint) & unicode::Ps;
    if (close_parenthesis) close_parenthesis = unicode::category(codepoint) & unicode::Pe;
    if (any_punctuation) any_punctuation = unicode::category(codepoint) & unicode::P;
    if (symbol) symbol = codepoint == '*' || unicode::category(codepoint) & unicode::S;
  }
  if (!punctuation.len && open_quotation) { lemmas.emplace_back(string(form.str, form.len), open_quotation_tag); return; }
  if (!punctuation.len && close_quotation) { lemmas.emplace_back(string(form.str, form.len), close_quotation_tag); return; }
  if (!punctuation.len && open_parenthesis) { lemmas.emplace_back(string(form.str, form.len), open_parenthesis_tag); return; }
  if (!punctuation.len && close_parenthesis) { lemmas.emplace_back(string(form.str, form.len), close_parenthesis_tag); return; }
  if (!punctuation.len && symbol) { lemmas.emplace_back(string(form.str, form.len), sym_tag); return; }
  if (!punctuation.len && any_punctuation) { lemmas.emplace_back(string(form.str, form.len), punctuation_tag); return; }
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho_guesser.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// This code is a reimplementation of morphologic analyzer Morphium
// by Johanka Spoustova (Treex::Tool::EnglishMorpho::Analysis Perl module)
// and reimplementation of morphologic lemmatizer by Martin Popel
// (Treex::Tool::EnglishMorpho::Lemmatizer Perl module). The latter is based
// on morpha:
//   Minnen, G., J. Carroll and D. Pearce (2001). Applied morphological
//   processing of English, Natural Language Engineering, 7(3). 207-223.
// Morpha has been released under LGPL as a part of RASP system
//   http://ilexir.co.uk/applications/rasp/.

namespace morphodita {

void english_morpho_guesser::load(binary_decoder& data) {
  unsigned tags = data.next_2B();
  exceptions_tags.clear();
  exceptions_tags.reserve(tags);
  while (tags--) {
    unsigned len = data.next_1B();
    exceptions_tags.emplace_back(string(data.next<char>(len), len));
  }

  exceptions.load(data);
  negations.load(data);
}

static const char _tag_guesser_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 2, 2, 6, 2, 2, 7, 2,
	4, 6, 2, 4, 7, 2, 5, 6,
	2, 5, 7, 2, 6, 7, 3, 2,
	6, 7, 3, 4, 6, 7, 3, 5,
	6, 7
};

static const unsigned char _tag_guesser_key_offsets[] = {
	0, 19, 26, 34, 42, 50, 58, 66,
	74, 82, 90, 100, 108, 116, 124, 132,
	145, 153, 161, 168, 179, 195, 212, 220,
	228, 236
};

static const char _tag_guesser_trans_keys[] = {
	45, 46, 99, 100, 103, 105, 109, 110,
	114, 115, 116, 118, 120, 48, 57, 65,
	90, 97, 122, 45, 48, 57, 65, 90,
	97, 122, 45, 114, 48, 57, 65, 90,
	97, 122, 45, 111, 48, 57, 65, 90,
	97, 122, 45, 109, 48, 57, 65, 90,
	97, 122, 45, 101, 48, 57, 65, 90,
	97, 122, 45, 115, 48, 57, 65, 90,
	97, 122, 45, 101, 48, 57, 65, 90,
	97, 122, 45, 108, 48, 57, 65, 90,
	97, 122, 45, 115, 48, 57, 65, 90,
	97, 122, 45, 97, 101, 111, 48, 57,
	65, 90, 98, 122, 45, 101, 48, 57,
	65, 90, 97, 122, 45, 108, 48, 57,
	65, 90, 97, 122, 45, 109, 48, 57,
	65, 90, 97, 122, 45, 105, 48, 57,
	65, 90, 97, 122, 45, 97, 101, 105,
	111, 117, 121, 48, 57, 65, 90, 98,
	122, 45, 115, 48, 57, 65, 90, 97,
	122, 45, 101, 48, 57, 65, 90, 97,
	122, 45, 48, 57, 65, 90, 97, 122,
	45, 101, 114, 115, 116, 48, 57, 65,
	90, 97, 122, 45, 46, 105, 109, 118,
	120, 48, 57, 65, 90, 97, 98, 99,
	100, 101, 122, 45, 46, 101, 105, 109,
	118, 120, 48, 57, 65, 90, 97, 98,
	99, 100, 102, 122, 45, 110, 48, 57,
	65, 90, 97, 122, 45, 105, 48, 57,
	65, 90, 97, 122, 45, 101, 48, 57,
	65, 90, 97, 122, 45, 115, 48, 57,
	65, 90, 97, 122, 0
};

static const char _tag_guesser_single_lengths[] = {
	13, 1, 2, 2, 2, 2, 2, 2,
	2, 2, 4, 2, 2, 2, 2, 7,
	2, 2, 1, 5, 6, 7, 2, 2,
	2, 2
};

static const char _tag_guesser_range_lengths[] = {
	3, 3, 3, 3, 3, 3, 3, 3,
	3, 3, 3, 3, 3, 3, 3, 3,
	3, 3, 3, 3, 5, 5, 3, 3,
	3, 3
};

static const unsigned char _tag_guesser_index_offsets[] = {
	0, 17, 22, 28, 34, 40, 46, 52,
	58, 64, 70, 78, 84, 90, 96, 102,
	113, 119, 125, 130, 139, 151, 164, 170,
	176, 182
};

static const char _tag_guesser_indicies[] = {
	1, 2, 5, 6, 7, 5, 5, 8,
	9, 10, 11, 5, 5, 3, 4, 4,
	0, 13, 14, 15, 15, 12, 13, 16,
	14, 15, 15, 12, 13, 17, 14, 15,
	15, 12, 13, 18, 14, 15, 15, 12,
	13, 18, 14, 15, 15, 12, 13, 19,
	14, 15, 15, 12, 13, 20, 14, 15,
	15, 12, 13, 18, 14, 15, 15, 12,
	13, 21, 14, 15, 15, 12, 13, 22,
	23, 24, 14, 15, 15, 12, 13, 25,
	14, 15, 15, 12, 13, 23, 14, 15,
	15, 12, 13, 23, 14, 15, 15, 12,
	13, 26, 14, 15, 15, 12, 28, 15,
	15, 15, 15, 15, 15, 29, 26, 26,
	27, 31, 4, 32, 33, 33, 30, 13,
	23, 14, 15, 15, 12, 13, 14, 15,
	15, 12, 13, 34, 35, 36, 37, 14,
	15, 15, 12, 13, 38, 39, 39, 39,
	39, 14, 15, 15, 39, 15, 12, 13,
	38, 40, 39, 39, 39, 39, 14, 15,
	15, 39, 15, 12, 13, 41, 14, 15,
	15, 12, 13, 42, 14, 15, 15, 12,
	13, 18, 14, 15, 15, 12, 13, 43,
	14, 15, 15, 12, 0
};

static const char _tag_guesser_trans_targs[] = {
	18, 19, 20, 18, 18, 20, 21, 22,
	23, 24, 16, 25, 18, 19, 18, 1,
	3, 4, 18, 7, 8, 10, 11, 18,
	13, 12, 18, 18, 19, 18, 18, 19,
	18, 18, 2, 5, 6, 9, 20, 20,
	18, 14, 15, 17
};

static const char _tag_guesser_trans_actions[] = {
	29, 46, 29, 32, 11, 11, 11, 11,
	11, 11, 0, 11, 13, 35, 15, 0,
	0, 0, 1, 0, 0, 0, 0, 3,
	0, 0, 5, 17, 38, 20, 23, 42,
	26, 9, 0, 0, 0, 0, 13, 0,
	7, 0, 0, 0
};

static const char _tag_guesser_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 15, 15, 0, 0,
	0, 0
};

static const int tag_guesser_start = 0;

void english_morpho_guesser::analyze(string_piece form, string_piece form_lc, vector<tagged_lemma>& lemmas) const {
  // Try exceptions list
  auto* exception = exceptions.at(form_lc.str, form_lc.len, [](pointer_decoder& data){
    for (unsigned len = data.next_1B(); len; len--) {
      data.next<char>(data.next_1B());
      data.next<uint16_t>(data.next_1B());
    }
  });

  if (exception) {
    // Found in exceptions list
    pointer_decoder data(exception);
    for (unsigned len = data.next_1B(); len; len--) {
      unsigned lemma_len = data.next_1B();
      string lemma(data.next<char>(lemma_len), lemma_len);
      for (unsigned tags = data.next_1B(); tags; tags--)
        lemmas.emplace_back(lemma, exceptions_tags[data.next_2B()]);
    }
  } else {
    // Try stripping negative prefix and use rule guesser
    string lemma_lc(form_lc.str, form_lc.len);
    // Try finding negative prefix
    unsigned negation_len = 0;
    for (unsigned prefix = 1; prefix <= form_lc.len; prefix++) {
      auto found = negations.at(form_lc.str, prefix, [](pointer_decoder& data){ data.next<unsigned char>(TOTAL); });
      if (!found) break;
      if (found[NEGATION_LEN]) {
        if (form_lc.len - prefix >= found[TO_FOLLOW]) negation_len = found[NEGATION_LEN];
      }
    }

    // Add default tags
    add(FW, lemma_lc, lemmas);
    add(JJ, lemma_lc, negation_len, lemmas);
    add(RB, lemma_lc, negation_len, lemmas);
    add(NN, lemma_lc, negation_len, lemmas);
    add_NNS(lemma_lc, negation_len, lemmas);

    // Add specialized tags
    const char* p = form_lc.str; int cs;
    bool added_JJR_RBR = false, added_JJS_RBS = false, added_SYM = false, added_CD = false;

	{
	cs = tag_guesser_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form_lc.str + form_lc.len)) )
		goto _test_eof;
_resume:
	_keys = _tag_guesser_trans_keys + _tag_guesser_key_offsets[cs];
	_trans = _tag_guesser_index_offsets[cs];

	_klen = _tag_guesser_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _tag_guesser_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _tag_guesser_indicies[_trans];
	cs = _tag_guesser_trans_targs[_trans];

	if ( _tag_guesser_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _tag_guesser_actions + _tag_guesser_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (!added_JJR_RBR) added_JJR_RBR = true, add_JJR_RBR(lemma_lc, negation_len, lemmas); }
	break;
	case 1:
	{ if (!added_JJS_RBS) added_JJS_RBS = true, add_JJS_RBS(lemma_lc, negation_len, lemmas); }
	break;
	case 2:
	{ add_VBG(lemma_lc, lemmas); }
	break;
	case 3:
	{ add_VBD_VBN(lemma_lc, lemmas); }
	break;
	case 4:
	{ add_VBZ(lemma_lc, lemmas); }
	break;
	case 5:
	{ add(VB, lemma_lc, lemmas); add(VBP, lemma_lc, lemmas); }
	break;
	case 6:
	{ if (!added_SYM) added_SYM = true, add(SYM, lemma_lc, lemmas); }
	break;
	case 7:
	{ if (!added_CD) added_CD = true, add(CD, lemma_lc, lemmas); }
	break;
		}
	}

_again:
	if ( ++p != ( (form_lc.str + form_lc.len)) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form_lc.str + form_lc.len)) )
	{
	const char *__acts = _tag_guesser_actions + _tag_guesser_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 7:
	{ if (!added_CD) added_CD = true, add(CD, lemma_lc, lemmas); }
	break;
		}
	}
	}

	}

  }

  // Add proper names
  analyze_proper_names(form, form_lc, lemmas);
}

bool english_morpho_guesser::analyze_proper_names(string_piece form, string_piece form_lc, vector<tagged_lemma>& lemmas) const {
  // NNP if form_lc != form or form.str[0] =~ /[0-9']/, NNPS if form_lc != form
  bool is_NNP = form.str != form_lc.str || (form.len && (*form.str == '\'' || (*form.str >= '0' && *form.str <= '9')));
  bool is_NNPS = form.str != form_lc.str;
  if (!is_NNP && !is_NNPS) return false;

  bool was_NNP = false, was_NNPS = false;
  for (auto&& lemma : lemmas) {
    was_NNP |= lemma.tag == NNP;
    was_NNPS |= lemma.tag == NNPS;
  }
  if (!((is_NNP && !was_NNP) || (is_NNPS && !was_NNPS))) return false;

  string lemma(form.str, form.len);
  if (is_NNP && !was_NNP) add(NNP, lemma, lemmas);
  if (is_NNPS && !was_NNPS) add_NNPS(lemma, lemmas);
  return true;
}

inline void english_morpho_guesser::add(const string& tag, const string& form, vector<tagged_lemma>& lemmas) const {
  lemmas.emplace_back(form, tag);
}

inline void english_morpho_guesser::add(const string& tag, const string& tag2, const string& form, vector<tagged_lemma>& lemmas) const {
  add(tag, form, lemmas);
  add(tag2, form, lemmas);
}

inline void english_morpho_guesser::add(const string& tag, const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const {
  lemmas.emplace_back(negation_len ? form.substr(negation_len) + "^" + form.substr(0, negation_len) : form, tag);
}

inline void english_morpho_guesser::add(const string& tag, const string& tag2, const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const {
  add(tag, form, negation_len, lemmas);
  add(tag2, form, negation_len, lemmas);
}

// Common definitions (written backwards)
#define REM(str, len) (str.substr(0, str.size() - len))
#define REM_ADD(str, len, add) (str.substr(0, str.size() - len).append(add))

static const char _NNS_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 1, 8, 1, 9, 1, 10, 1,
	11, 1, 12, 1, 13
};

static const char _NNS_key_offsets[] = {
	0, 0, 2, 3, 4, 5, 7, 17,
	17, 29, 30, 35, 35, 36, 37, 37,
	37, 44, 45, 53, 63, 72
};

static const char _NNS_trans_keys[] = {
	110, 115, 101, 109, 101, 99, 115, 98,
	100, 102, 104, 106, 110, 112, 116, 118,
	122, 104, 122, 98, 100, 102, 103, 106,
	110, 112, 116, 118, 120, 111, 97, 101,
	105, 111, 117, 105, 119, 104, 105, 111,
	115, 118, 120, 122, 115, 97, 101, 105,
	110, 111, 114, 115, 117, 98, 100, 102,
	104, 106, 110, 112, 116, 118, 122, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _NNS_single_lengths[] = {
	0, 2, 1, 1, 1, 2, 0, 0,
	2, 1, 5, 0, 1, 1, 0, 0,
	7, 1, 8, 0, 7, 0
};

static const char _NNS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 5, 0,
	5, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 5, 1, 0
};

static const char _NNS_index_offsets[] = {
	0, 0, 3, 5, 7, 9, 12, 18,
	19, 27, 29, 35, 36, 38, 40, 41,
	42, 50, 52, 61, 67, 76
};

static const char _NNS_indicies[] = {
	0, 2, 1, 3, 1, 4, 1, 6,
	5, 7, 7, 1, 8, 8, 8, 8,
	8, 1, 9, 11, 10, 10, 10, 10,
	10, 10, 1, 12, 1, 13, 13, 13,
	13, 13, 1, 14, 15, 1, 16, 1,
	17, 1, 18, 19, 20, 21, 22, 7,
	23, 1, 24, 1, 25, 25, 25, 26,
	25, 27, 28, 29, 1, 30, 30, 30,
	30, 30, 1, 31, 31, 31, 31, 31,
	31, 33, 32, 1, 17, 0
};

static const char _NNS_trans_targs[] = {
	2, 0, 4, 3, 15, 15, 16, 15,
	7, 15, 15, 17, 15, 11, 15, 13,
	15, 15, 5, 6, 8, 18, 12, 20,
	15, 15, 9, 10, 15, 19, 15, 15,
	14, 21
};

static const char _NNS_trans_actions[] = {
	0, 0, 0, 0, 1, 27, 27, 21,
	0, 23, 25, 25, 19, 0, 17, 0,
	5, 11, 0, 0, 0, 21, 0, 21,
	3, 9, 0, 0, 15, 9, 7, 13,
	0, 15
};

static const int NNS_start = 1;

void english_morpho_guesser::add_NNS(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = NNS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _NNS_trans_keys + _NNS_key_offsets[cs];
	_trans = _NNS_index_offsets[cs];

	_klen = _NNS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _NNS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _NNS_indicies[_trans];
	cs = _NNS_trans_targs[_trans];

	if ( _NNS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _NNS_actions + _NNS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = "an";    }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 1, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = "fe";    }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 2, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 3, append = "y";     }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(NNS, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

static const char _NNPS_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 1, 6, 1, 7, 1, 8, 1,
	9, 1, 10, 1, 11, 1, 12, 1,
	14, 1, 15, 1, 16, 2, 0, 1,
	2, 3, 4, 2, 13, 14
};

static const unsigned char _NNPS_key_offsets[] = {
	0, 0, 4, 6, 8, 10, 12, 16,
	36, 36, 60, 62, 72, 72, 74, 76,
	78, 78, 98, 98, 100, 102, 104, 104,
	118, 120, 136, 156, 174, 174
};

static const char _NNPS_trans_keys[] = {
	78, 83, 110, 115, 69, 101, 77, 109,
	77, 109, 69, 101, 67, 83, 99, 115,
	66, 68, 70, 72, 74, 78, 80, 84,
	86, 90, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 72, 90, 104, 122,
	66, 68, 70, 71, 74, 78, 80, 84,
	86, 88, 98, 100, 102, 103, 106, 110,
	112, 116, 118, 120, 79, 111, 65, 69,
	73, 79, 85, 97, 101, 105, 111, 117,
	73, 105, 87, 119, 87, 119, 66, 68,
	70, 72, 74, 78, 80, 84, 86, 90,
	98, 100, 102, 104, 106, 110, 112, 116,
	118, 122, 73, 105, 69, 101, 69, 101,
	72, 73, 79, 83, 86, 88, 90, 104,
	105, 111, 115, 118, 120, 122, 83, 115,
	65, 69, 73, 78, 79, 82, 83, 85,
	97, 101, 105, 110, 111, 114, 115, 117,
	66, 68, 70, 72, 74, 78, 80, 84,
	86, 90, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 65, 69, 73, 79,
	85, 89, 90, 97, 101, 105, 111, 117,
	121, 122, 66, 88, 98, 120, 72, 73,
	79, 83, 86, 88, 90, 104, 105, 111,
	115, 118, 120, 122, 0
};

static const char _NNPS_single_lengths[] = {
	0, 4, 2, 2, 2, 2, 4, 0,
	0, 4, 2, 10, 0, 2, 2, 2,
	0, 0, 0, 2, 2, 2, 0, 14,
	2, 16, 0, 14, 0, 14
};

static const char _NNPS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 10,
	0, 10, 0, 0, 0, 0, 0, 0,
	0, 10, 0, 0, 0, 0, 0, 0,
	0, 0, 10, 2, 0, 0
};

static const unsigned char _NNPS_index_offsets[] = {
	0, 0, 5, 8, 11, 14, 17, 22,
	33, 34, 49, 52, 63, 64, 67, 70,
	73, 74, 85, 86, 89, 92, 95, 96,
	111, 114, 131, 142, 159, 160
};

static const char _NNPS_indicies[] = {
	0, 2, 3, 4, 1, 5, 6, 1,
	7, 8, 1, 8, 8, 1, 10, 11,
	9, 12, 12, 12, 12, 1, 13, 13,
	13, 13, 13, 13, 13, 13, 13, 13,
	1, 14, 16, 15, 16, 15, 15, 15,
	15, 15, 15, 15, 15, 15, 15, 15,
	1, 17, 17, 1, 18, 18, 18, 18,
	18, 18, 18, 18, 18, 18, 1, 19,
	20, 21, 1, 22, 23, 1, 23, 23,
	1, 24, 25, 25, 25, 25, 25, 25,
	25, 25, 25, 25, 1, 26, 21, 21,
	1, 6, 6, 1, 11, 11, 9, 1,
	27, 28, 29, 30, 31, 12, 32, 27,
	33, 29, 30, 34, 12, 32, 1, 35,
	35, 1, 36, 36, 36, 37, 36, 38,
	39, 40, 36, 36, 36, 37, 36, 38,
	39, 40, 1, 41, 41, 41, 41, 41,
	41, 41, 41, 41, 41, 1, 42, 42,
	42, 42, 42, 42, 44, 42, 42, 42,
	42, 42, 42, 44, 43, 43, 1, 24,
	27, 33, 29, 30, 34, 12, 32, 27,
	33, 29, 30, 34, 12, 32, 1, 0
};

static const char _NNPS_trans_targs[] = {
	2, 0, 5, 20, 21, 3, 4, 22,
	22, 22, 23, 29, 22, 8, 22, 22,
	24, 22, 12, 22, 14, 15, 22, 22,
	22, 18, 22, 6, 7, 9, 25, 13,
	27, 17, 19, 22, 22, 10, 11, 22,
	26, 22, 22, 16, 28
};

static const char _NNPS_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 29,
	1, 27, 27, 27, 21, 0, 35, 25,
	25, 19, 0, 17, 0, 0, 32, 5,
	11, 0, 23, 0, 0, 0, 21, 0,
	21, 0, 0, 3, 9, 0, 0, 15,
	9, 7, 13, 0, 15
};

static const int NNPS_start = 1;

void english_morpho_guesser::add_NNPS(const string& form, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = NNPS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _NNPS_trans_keys + _NNPS_key_offsets[cs];
	_trans = _NNPS_index_offsets[cs];

	_klen = _NNPS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _NNPS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _NNPS_indicies[_trans];
	cs = _NNPS_trans_targs[_trans];

	if ( _NNPS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _NNPS_actions + _NNPS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = "AN";    }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = "an";    }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = "FE";    }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 3, append = "fe";    }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 2, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 1, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 1, append = nullptr; }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 3, append = "Y";     }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 3, append = "y";     }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 2, append = nullptr; }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(NNPS, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBG_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 1, 6, 1, 7, 1, 9, 1,
	10, 1, 11, 1, 12, 1, 13, 1,
	14, 1, 15, 1, 16, 1, 17, 2,
	0, 12, 2, 3, 4, 2, 5, 9,
	2, 5, 10, 2, 8, 9, 2, 9,
	10, 2, 11, 12, 3, 0, 2, 12,
	3, 2, 11, 12
};

static const short _VBG_key_offsets[] = {
	0, 0, 1, 2, 3, 9, 14, 24,
	29, 34, 44, 46, 47, 48, 49, 50,
	51, 52, 59, 66, 68, 70, 71, 72,
	73, 74, 75, 76, 81, 89, 90, 91,
	92, 93, 94, 96, 97, 98, 99, 100,
	101, 102, 127, 127, 136, 137, 142, 153,
	162, 171, 181, 186, 191, 197, 207, 207,
	216, 228, 229, 240, 240, 249, 258, 267,
	276, 285, 290, 302, 313, 318, 324, 334,
	344, 355, 362, 373, 382, 391, 391, 402,
	413, 415, 416, 417, 417, 418, 426, 437,
	442, 448, 458, 468, 479, 486, 497, 504,
	510, 519, 528, 537, 543
};

static const char _VBG_trans_keys[] = {
	103, 110, 105, 97, 101, 105, 111, 117,
	121, 97, 101, 105, 111, 117, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 122,
	97, 101, 105, 111, 117, 97, 101, 105,
	111, 117, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 98, 122, 97, 101, 105, 111, 117,
	98, 122, 97, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 97, 101, 105, 110, 111, 115, 117,
	120, 105, 112, 105, 109, 101, 98, 114,
	105, 114, 112, 105, 109, 101, 98, 99,
	100, 102, 103, 104, 106, 107, 108, 109,
	110, 111, 112, 113, 114, 115, 116, 117,
	118, 119, 120, 121, 122, 97, 105, 97,
	98, 101, 105, 111, 117, 122, 99, 120,
	113, 97, 101, 105, 111, 117, 98, 99,
	100, 105, 111, 117, 122, 97, 101, 102,
	120, 97, 100, 101, 105, 111, 117, 122,
	98, 120, 97, 101, 102, 105, 111, 117,
	122, 98, 120, 97, 101, 103, 105, 110,
	111, 117, 122, 98, 120, 97, 101, 105,
	111, 117, 101, 110, 111, 115, 120, 101,
	110, 111, 112, 115, 120, 97, 101, 104,
	105, 111, 116, 117, 122, 98, 120, 97,
	101, 105, 106, 111, 117, 122, 98, 120,
	98, 99, 100, 105, 107, 111, 117, 122,
	97, 101, 102, 120, 105, 97, 101, 105,
	108, 111, 114, 117, 119, 122, 98, 120,
	97, 101, 105, 109, 111, 117, 122, 98,
	120, 97, 101, 105, 110, 111, 117, 122,
	98, 120, 97, 101, 105, 111, 112, 117,
	122, 98, 120, 97, 101, 105, 111, 113,
	117, 122, 98, 120, 97, 101, 105, 111,
	114, 117, 122, 98, 120, 97, 101, 105,
	111, 117, 98, 99, 100, 105, 108, 111,
	116, 117, 97, 101, 102, 122, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 101, 110, 111, 115, 120, 101, 110,
	111, 112, 115, 120, 101, 105, 110, 111,
	115, 120, 98, 116, 118, 122, 101, 105,
	110, 111, 115, 120, 98, 116, 118, 122,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 98, 101, 110, 111, 114,
	115, 120, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 97, 101, 105,
	111, 115, 117, 122, 98, 120, 97, 101,
	105, 111, 116, 117, 122, 98, 120, 122,
	98, 100, 102, 104, 106, 110, 112, 116,
	118, 120, 122, 98, 100, 102, 104, 106,
	110, 112, 116, 118, 120, 98, 114, 112,
	114, 113, 97, 101, 105, 108, 111, 117,
	98, 122, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 101, 110, 111,
	115, 120, 101, 110, 111, 112, 115, 120,
	101, 105, 110, 111, 115, 120, 98, 116,
	118, 122, 101, 105, 110, 111, 115, 120,
	98, 116, 118, 122, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 98,
	101, 110, 111, 114, 115, 120, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 97, 101, 105, 111, 117, 98, 122,
	97, 101, 105, 111, 117, 121, 97, 101,
	105, 111, 117, 118, 122, 98, 120, 97,
	101, 105, 111, 117, 119, 122, 98, 120,
	97, 101, 105, 111, 117, 120, 122, 98,
	119, 97, 101, 105, 111, 117, 121, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _VBG_single_lengths[] = {
	0, 1, 1, 1, 6, 5, 0, 5,
	5, 0, 2, 1, 1, 1, 1, 1,
	1, 5, 5, 0, 2, 1, 1, 1,
	1, 1, 1, 5, 8, 1, 1, 1,
	1, 1, 2, 1, 1, 1, 1, 1,
	1, 23, 0, 7, 1, 5, 7, 7,
	7, 8, 5, 5, 6, 8, 0, 7,
	8, 1, 9, 0, 7, 7, 7, 7,
	7, 5, 8, 5, 5, 6, 6, 6,
	5, 7, 5, 7, 7, 0, 1, 1,
	2, 1, 1, 0, 1, 6, 5, 5,
	6, 6, 6, 5, 7, 5, 5, 6,
	7, 7, 7, 6, 7
};

static const char _VBG_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 5, 0,
	0, 5, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 0, 1, 0, 0, 2, 1,
	1, 1, 0, 0, 0, 1, 0, 1,
	2, 0, 1, 0, 1, 1, 1, 1,
	1, 0, 2, 3, 0, 0, 2, 2,
	3, 0, 3, 1, 1, 0, 5, 5,
	0, 0, 0, 0, 0, 1, 3, 0,
	0, 2, 2, 3, 0, 3, 1, 0,
	1, 1, 1, 0, 1
};

static const short _VBG_index_offsets[] = {
	0, 0, 2, 4, 6, 13, 19, 25,
	31, 37, 43, 46, 48, 50, 52, 54,
	56, 58, 65, 72, 74, 77, 79, 81,
	83, 85, 87, 89, 95, 104, 106, 108,
	110, 112, 114, 117, 119, 121, 123, 125,
	127, 129, 154, 155, 164, 166, 172, 182,
	191, 200, 210, 216, 222, 229, 239, 240,
	249, 260, 262, 273, 274, 283, 292, 301,
	310, 319, 325, 336, 345, 351, 358, 367,
	376, 385, 393, 402, 411, 420, 421, 428,
	435, 438, 440, 442, 443, 445, 453, 462,
	468, 475, 484, 493, 502, 510, 519, 526,
	533, 542, 551, 560, 567
};

static const unsigned char _VBG_indicies[] = {
	0, 1, 2, 1, 3, 1, 4, 4,
	4, 4, 4, 4, 1, 5, 5, 5,
	5, 6, 1, 7, 7, 7, 7, 7,
	1, 8, 8, 8, 8, 9, 1, 5,
	5, 5, 5, 10, 1, 11, 11, 11,
	11, 11, 1, 11, 12, 1, 11, 1,
	13, 1, 11, 1, 14, 1, 11, 1,
	11, 1, 5, 5, 5, 5, 6, 15,
	1, 5, 5, 5, 5, 6, 16, 1,
	4, 1, 17, 18, 1, 17, 1, 19,
	1, 17, 1, 20, 1, 17, 1, 17,
	1, 21, 22, 21, 23, 24, 1, 25,
	26, 25, 27, 28, 29, 25, 30, 1,
	31, 1, 31, 1, 32, 1, 31, 1,
	31, 1, 33, 34, 1, 33, 1, 35,
	1, 33, 1, 36, 1, 33, 1, 33,
	1, 38, 39, 40, 41, 42, 43, 44,
	45, 46, 47, 48, 49, 50, 51, 52,
	53, 54, 55, 56, 57, 58, 59, 60,
	37, 1, 1, 61, 62, 61, 61, 61,
	61, 63, 63, 1, 64, 1, 65, 65,
	65, 65, 65, 1, 67, 68, 67, 66,
	66, 66, 67, 66, 67, 1, 69, 62,
	69, 69, 69, 69, 63, 63, 1, 61,
	61, 62, 61, 61, 61, 63, 63, 1,
	66, 66, 68, 66, 70, 66, 66, 67,
	67, 1, 71, 71, 71, 71, 71, 1,
	72, 73, 74, 75, 76, 1, 72, 73,
	74, 11, 75, 76, 1, 61, 61, 62,
	61, 61, 77, 61, 63, 63, 1, 78,
	61, 61, 61, 62, 61, 61, 63, 63,
	1, 63, 79, 63, 61, 62, 61, 61,
	63, 61, 63, 1, 7, 1, 61, 61,
	61, 68, 61, 80, 61, 80, 67, 67,
	1, 5, 61, 61, 61, 62, 61, 61,
	63, 63, 1, 81, 81, 82, 62, 81,
	81, 63, 63, 1, 81, 81, 81, 81,
	62, 81, 63, 63, 1, 61, 61, 61,
	61, 62, 61, 63, 63, 1, 61, 83,
	61, 84, 62, 61, 63, 63, 1, 5,
	5, 5, 5, 6, 1, 85, 86, 85,
	5, 86, 5, 86, 6, 5, 85, 1,
	87, 88, 89, 90, 91, 85, 85, 85,
	1, 87, 92, 89, 93, 94, 1, 87,
	92, 89, 17, 93, 94, 1, 87, 17,
	88, 89, 90, 91, 85, 85, 1, 87,
	20, 88, 89, 90, 91, 85, 85, 1,
	95, 88, 89, 90, 91, 85, 85, 85,
	1, 17, 87, 92, 89, 18, 93, 94,
	1, 87, 97, 89, 98, 99, 96, 96,
	96, 1, 66, 66, 66, 66, 100, 66,
	67, 67, 1, 101, 102, 103, 61, 62,
	61, 63, 63, 1, 104, 106, 106, 106,
	106, 106, 106, 105, 107, 107, 107, 107,
	107, 107, 1, 31, 108, 1, 31, 1,
	109, 1, 105, 110, 104, 5, 5, 5,
	112, 5, 6, 111, 1, 113, 114, 115,
	116, 117, 111, 111, 111, 1, 113, 118,
	115, 119, 120, 1, 113, 118, 115, 33,
	119, 120, 1, 113, 33, 114, 115, 116,
	117, 111, 111, 1, 113, 36, 114, 115,
	116, 117, 111, 111, 1, 121, 114, 115,
	116, 117, 111, 111, 111, 1, 33, 113,
	118, 115, 34, 119, 120, 1, 113, 123,
	115, 124, 125, 122, 122, 122, 1, 5,
	5, 5, 5, 6, 111, 1, 4, 4,
	4, 4, 4, 4, 1, 66, 66, 66,
	66, 66, 68, 67, 67, 1, 81, 81,
	81, 81, 81, 62, 63, 63, 1, 81,
	81, 81, 81, 81, 62, 63, 63, 1,
	126, 126, 126, 126, 126, 4, 1, 127,
	127, 127, 127, 127, 129, 130, 128, 1,
	0
};

static const char _VBG_trans_targs[] = {
	2, 0, 3, 41, 42, 42, 44, 42,
	42, 44, 44, 51, 52, 13, 15, 42,
	42, 68, 69, 23, 25, 77, 78, 83,
	84, 42, 80, 29, 82, 31, 33, 42,
	32, 87, 88, 37, 39, 4, 43, 46,
	47, 48, 49, 53, 55, 56, 58, 60,
	61, 19, 62, 63, 64, 75, 76, 95,
	96, 97, 98, 99, 100, 5, 45, 42,
	42, 6, 7, 42, 45, 8, 50, 9,
	10, 11, 12, 14, 16, 54, 42, 57,
	59, 17, 18, 65, 66, 67, 74, 20,
	70, 22, 71, 72, 21, 24, 26, 73,
	67, 70, 71, 72, 45, 27, 85, 94,
	42, 42, 79, 28, 81, 30, 42, 86,
	93, 34, 89, 36, 90, 91, 35, 38,
	40, 92, 86, 89, 90, 91, 65, 65,
	42, 42, 45
};

static const char _VBG_trans_actions[] = {
	0, 0, 0, 29, 23, 15, 15, 3,
	46, 46, 40, 0, 0, 0, 0, 5,
	34, 0, 0, 0, 0, 15, 15, 15,
	15, 11, 11, 0, 11, 0, 0, 9,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 21,
	0, 0, 0, 23, 0, 0, 19, 19,
	7, 0, 0, 49, 49, 0, 49, 0,
	0, 0, 0, 0, 0, 19, 17, 19,
	49, 0, 0, 27, 27, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	25, 25, 25, 25, 56, 0, 9, 9,
	13, 43, 43, 0, 9, 0, 37, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 7, 7, 7, 7, 23, 1,
	31, 1, 52
};

static const char _VBG_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 3, 0, 0, 3, 3,
	3, 3, 0, 3, 3, 3, 0, 3,
	3, 0, 3, 0, 3, 3, 3, 3,
	3, 0, 0, 25, 25, 25, 25, 25,
	25, 25, 25, 3, 3, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 7, 7,
	7, 7, 7, 7, 7, 7, 0, 0,
	3, 3, 3, 0, 3
};

static const int VBG_start = 1;

void english_morpho_guesser::add_VBG(const string& form, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBG_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBG_trans_keys + _VBG_key_offsets[cs];
	_trans = _VBG_index_offsets[cs];

	_klen = _VBG_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBG_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBG_indicies[_trans];
	cs = _VBG_trans_targs[_trans];

	if ( _VBG_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBG_actions + _VBG_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 3, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 3, append = "e";     }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = "e";     }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 3, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = "e";     }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 3, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 3, append = "e";     }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 3, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 3, append = "e";     }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 3, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 3, append = "e";     }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 3, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 3, append = "e";     }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 3, append = nullptr; }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 3, append = "e";     }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 3, append = nullptr; }
	break;
	case 17:
	{ if (best > 'r') best = 'r', remove = 3, append = "e";     }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form.c_str() + form.size())) )
	{
	const char *__acts = _VBG_actions + _VBG_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = "e";     }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 3, append = "e";     }
	break;
		}
	}
	}

	_out: {}
	}

  add(VBG, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBD_VBN_actions[] = {
	0, 1, 0, 1, 2, 1, 3, 1,
	4, 1, 5, 1, 6, 1, 7, 1,
	8, 1, 9, 1, 10, 1, 11, 1,
	13, 1, 14, 1, 15, 1, 16, 1,
	17, 2, 1, 16, 2, 4, 5, 2,
	8, 16, 2, 9, 13, 2, 9, 14,
	2, 12, 13, 2, 13, 14, 2, 15,
	16, 3, 1, 3, 16, 3, 3, 15,
	16
};

static const short _VBD_VBN_key_offsets[] = {
	0, 0, 2, 3, 9, 14, 24, 29,
	34, 44, 46, 47, 48, 49, 50, 51,
	52, 60, 67, 74, 76, 77, 78, 79,
	80, 81, 82, 87, 95, 96, 97, 98,
	99, 100, 102, 103, 104, 105, 106, 107,
	108, 114, 115, 140, 140, 149, 150, 155,
	166, 175, 184, 194, 199, 204, 210, 220,
	220, 229, 241, 242, 253, 253, 262, 271,
	280, 289, 298, 303, 316, 327, 332, 338,
	348, 358, 369, 376, 387, 396, 405, 405,
	416, 427, 429, 430, 431, 431, 432, 440,
	451, 456, 462, 472, 482, 493, 500, 511,
	518, 524, 533, 542, 551
};

static const char _VBD_VBN_trans_keys[] = {
	100, 110, 101, 97, 101, 105, 111, 117,
	121, 97, 101, 105, 111, 117, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 122,
	97, 101, 105, 111, 117, 97, 101, 105,
	111, 117, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 121, 98, 122, 97, 101, 105, 111,
	117, 98, 122, 97, 101, 105, 111, 117,
	98, 122, 98, 114, 105, 114, 112, 105,
	109, 101, 97, 101, 105, 111, 117, 97,
	101, 105, 110, 111, 115, 117, 120, 105,
	112, 105, 109, 101, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 121, 101, 98, 99, 100, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 117, 118, 119, 120,
	121, 122, 97, 111, 97, 98, 101, 105,
	111, 117, 122, 99, 120, 113, 97, 101,
	105, 111, 117, 98, 99, 100, 105, 111,
	117, 122, 97, 101, 102, 120, 97, 100,
	101, 105, 111, 117, 122, 98, 120, 97,
	101, 102, 105, 111, 117, 122, 98, 120,
	97, 101, 103, 105, 110, 111, 117, 122,
	98, 120, 97, 101, 105, 111, 117, 101,
	110, 111, 115, 120, 101, 110, 111, 112,
	115, 120, 97, 101, 104, 105, 111, 116,
	117, 122, 98, 120, 97, 101, 105, 106,
	111, 117, 122, 98, 120, 98, 99, 100,
	105, 107, 111, 117, 122, 97, 101, 102,
	120, 105, 97, 101, 105, 108, 111, 114,
	117, 119, 122, 98, 120, 97, 101, 105,
	109, 111, 117, 122, 98, 120, 97, 101,
	105, 110, 111, 117, 122, 98, 120, 97,
	101, 105, 111, 112, 117, 122, 98, 120,
	97, 101, 105, 111, 113, 117, 122, 98,
	120, 97, 101, 105, 111, 114, 117, 122,
	98, 120, 97, 101, 105, 111, 117, 98,
	99, 100, 105, 108, 110, 111, 116, 117,
	97, 101, 102, 122, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 101,
	110, 111, 115, 120, 101, 110, 111, 112,
	115, 120, 101, 105, 110, 111, 115, 120,
	98, 116, 118, 122, 101, 105, 110, 111,
	115, 120, 98, 116, 118, 122, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 98, 101, 110, 111, 114, 115, 120,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 97, 101, 105, 111, 115,
	117, 122, 98, 120, 97, 101, 105, 111,
	116, 117, 122, 98, 120, 122, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 120,
	122, 98, 100, 102, 104, 106, 110, 112,
	116, 118, 120, 98, 114, 112, 114, 113,
	97, 101, 105, 108, 111, 117, 98, 122,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 101, 110, 111, 115, 120,
	101, 110, 111, 112, 115, 120, 101, 105,
	110, 111, 115, 120, 98, 116, 118, 122,
	101, 105, 110, 111, 115, 120, 98, 116,
	118, 122, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 98, 101, 110,
	111, 114, 115, 120, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 97,
	101, 105, 111, 117, 98, 122, 97, 101,
	105, 111, 117, 121, 97, 101, 105, 111,
	117, 118, 122, 98, 120, 97, 101, 105,
	111, 117, 119, 122, 98, 120, 97, 101,
	105, 111, 117, 120, 122, 98, 119, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _VBD_VBN_single_lengths[] = {
	0, 2, 1, 6, 5, 0, 5, 5,
	0, 2, 1, 1, 1, 1, 1, 1,
	6, 5, 5, 2, 1, 1, 1, 1,
	1, 1, 5, 8, 1, 1, 1, 1,
	1, 2, 1, 1, 1, 1, 1, 1,
	6, 1, 23, 0, 7, 1, 5, 7,
	7, 7, 8, 5, 5, 6, 8, 0,
	7, 8, 1, 9, 0, 7, 7, 7,
	7, 7, 5, 9, 5, 5, 6, 6,
	6, 5, 7, 5, 7, 7, 0, 1,
	1, 2, 1, 1, 0, 1, 6, 5,
	5, 6, 6, 6, 5, 7, 5, 5,
	6, 7, 7, 7, 7
};

static const char _VBD_VBN_range_lengths[] = {
	0, 0, 0, 0, 0, 5, 0, 0,
	5, 0, 0, 0, 0, 0, 0, 0,
	1, 1, 1, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 1, 0, 1, 0, 0, 2,
	1, 1, 1, 0, 0, 0, 1, 0,
	1, 2, 0, 1, 0, 1, 1, 1,
	1, 1, 0, 2, 3, 0, 0, 2,
	2, 3, 0, 3, 1, 1, 0, 5,
	5, 0, 0, 0, 0, 0, 1, 3,
	0, 0, 2, 2, 3, 0, 3, 1,
	0, 1, 1, 1, 1
};

static const short _VBD_VBN_index_offsets[] = {
	0, 0, 3, 5, 12, 18, 24, 30,
	36, 42, 45, 47, 49, 51, 53, 55,
	57, 65, 72, 79, 82, 84, 86, 88,
	90, 92, 94, 100, 109, 111, 113, 115,
	117, 119, 122, 124, 126, 128, 130, 132,
	134, 141, 143, 168, 169, 178, 180, 186,
	196, 205, 214, 224, 230, 236, 243, 253,
	254, 263, 274, 276, 287, 288, 297, 306,
	315, 324, 333, 339, 351, 360, 366, 373,
	382, 391, 400, 408, 417, 426, 435, 436,
	443, 450, 453, 455, 457, 458, 460, 468,
	477, 483, 490, 499, 508, 517, 525, 534,
	541, 548, 557, 566, 575
};

static const unsigned char _VBD_VBN_indicies[] = {
	0, 2, 1, 3, 1, 4, 4, 4,
	4, 4, 4, 1, 5, 5, 5, 5,
	6, 1, 7, 7, 7, 7, 7, 1,
	8, 8, 8, 8, 9, 1, 5, 5,
	5, 5, 10, 1, 11, 11, 11, 11,
	11, 1, 11, 12, 1, 11, 1, 13,
	1, 11, 1, 14, 1, 11, 1, 11,
	1, 4, 4, 4, 4, 4, 16, 15,
	1, 5, 5, 5, 5, 6, 17, 1,
	5, 5, 5, 5, 6, 18, 1, 19,
	20, 1, 19, 1, 21, 1, 19, 1,
	22, 1, 19, 1, 19, 1, 23, 24,
	23, 25, 26, 1, 27, 28, 27, 29,
	30, 31, 27, 32, 1, 33, 1, 33,
	1, 34, 1, 33, 1, 33, 1, 35,
	36, 1, 35, 1, 37, 1, 35, 1,
	38, 1, 35, 1, 35, 1, 39, 39,
	39, 39, 39, 4, 1, 40, 1, 42,
	43, 44, 45, 46, 47, 48, 49, 50,
	51, 52, 53, 54, 55, 56, 57, 58,
	59, 60, 61, 62, 63, 64, 41, 1,
	1, 65, 66, 65, 65, 65, 65, 4,
	4, 1, 67, 1, 68, 68, 68, 68,
	68, 1, 70, 71, 70, 69, 69, 69,
	70, 69, 70, 1, 72, 66, 72, 72,
	72, 72, 4, 4, 1, 65, 65, 66,
	65, 65, 65, 4, 4, 1, 69, 69,
	71, 69, 73, 69, 69, 70, 70, 1,
	74, 74, 74, 74, 74, 1, 75, 76,
	77, 78, 79, 1, 75, 76, 77, 11,
	78, 79, 1, 65, 65, 66, 65, 65,
	80, 65, 4, 4, 1, 81, 65, 65,
	65, 66, 65, 65, 4, 4, 1, 4,
	82, 4, 65, 66, 65, 65, 4, 65,
	4, 1, 7, 1, 65, 65, 65, 71,
	65, 83, 65, 83, 70, 70, 1, 5,
	65, 65, 65, 66, 65, 65, 4, 4,
	1, 84, 84, 85, 66, 84, 84, 4,
	4, 1, 84, 84, 84, 84, 66, 84,
	4, 4, 1, 65, 65, 65, 65, 66,
	65, 4, 4, 1, 65, 86, 65, 87,
	66, 65, 4, 4, 1, 5, 5, 5,
	5, 6, 1, 88, 89, 88, 5, 89,
	89, 5, 89, 6, 5, 88, 1, 90,
	91, 92, 93, 94, 88, 88, 88, 1,
	90, 95, 92, 96, 97, 1, 90, 95,
	92, 19, 96, 97, 1, 90, 19, 91,
	92, 93, 94, 88, 88, 1, 90, 22,
	91, 92, 93, 94, 88, 88, 1, 98,
	91, 92, 93, 94, 88, 88, 88, 1,
	19, 90, 95, 92, 20, 96, 97, 1,
	90, 100, 92, 101, 102, 99, 99, 99,
	1, 69, 69, 69, 69, 103, 69, 70,
	70, 1, 104, 105, 106, 65, 66, 65,
	4, 4, 1, 107, 109, 109, 109, 109,
	109, 109, 108, 110, 110, 110, 110, 110,
	110, 1, 33, 111, 1, 33, 1, 112,
	1, 108, 113, 107, 5, 5, 5, 115,
	5, 6, 114, 1, 116, 117, 118, 119,
	120, 114, 114, 114, 1, 116, 121, 118,
	122, 123, 1, 116, 121, 118, 35, 122,
	123, 1, 116, 35, 117, 118, 119, 120,
	114, 114, 1, 116, 38, 117, 118, 119,
	120, 114, 114, 1, 124, 117, 118, 119,
	120, 114, 114, 114, 1, 35, 116, 121,
	118, 36, 122, 123, 1, 116, 126, 118,
	127, 128, 125, 125, 125, 1, 5, 5,
	5, 5, 6, 114, 1, 4, 4, 4,
	4, 4, 4, 1, 69, 69, 69, 69,
	69, 71, 70, 70, 1, 84, 84, 84,
	84, 84, 66, 4, 4, 1, 84, 84,
	84, 84, 84, 66, 4, 4, 1, 129,
	129, 129, 129, 129, 131, 132, 130, 1,
	0
};

static const char _VBD_VBN_trans_targs[] = {
	2, 0, 41, 42, 43, 43, 45, 43,
	43, 45, 45, 52, 53, 12, 14, 43,
	43, 43, 43, 69, 70, 22, 24, 78,
	79, 84, 85, 43, 81, 28, 83, 30,
	32, 43, 31, 88, 89, 36, 38, 66,
	43, 3, 44, 47, 48, 49, 50, 54,
	16, 56, 57, 59, 61, 62, 63, 64,
	65, 76, 77, 96, 97, 98, 99, 40,
	100, 4, 46, 43, 5, 6, 43, 46,
	7, 51, 8, 9, 10, 11, 13, 15,
	55, 43, 58, 60, 17, 18, 66, 67,
	68, 75, 19, 71, 21, 72, 73, 20,
	23, 25, 74, 68, 71, 72, 73, 46,
	26, 86, 95, 43, 43, 80, 27, 82,
	29, 43, 87, 94, 33, 90, 35, 91,
	92, 34, 37, 39, 93, 87, 90, 91,
	92, 66, 43, 43, 46
};

static const char _VBD_VBN_trans_actions[] = {
	0, 0, 0, 31, 29, 25, 25, 5,
	51, 51, 45, 0, 0, 0, 0, 15,
	39, 9, 36, 0, 0, 0, 0, 25,
	25, 25, 25, 21, 21, 0, 21, 0,
	0, 19, 0, 0, 0, 0, 0, 29,
	1, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 27, 0, 0, 0, 0,
	0, 0, 29, 17, 0, 0, 54, 54,
	0, 54, 0, 0, 0, 0, 0, 0,
	29, 27, 29, 54, 0, 0, 13, 13,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 7, 7, 7, 7, 61,
	0, 19, 19, 23, 48, 48, 0, 19,
	0, 42, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 17, 17, 17,
	17, 3, 33, 3, 57
};

static const char _VBD_VBN_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 5, 0, 0, 5,
	5, 5, 5, 0, 5, 5, 5, 0,
	5, 5, 0, 5, 0, 5, 5, 5,
	5, 5, 0, 0, 11, 11, 11, 11,
	11, 11, 11, 11, 5, 5, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 17,
	17, 17, 17, 17, 17, 17, 17, 0,
	0, 5, 5, 5, 5
};

static const int VBD_VBN_start = 1;

void english_morpho_guesser::add_VBD_VBN(const string& form, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBD_VBN_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBD_VBN_trans_keys + _VBD_VBN_key_offsets[cs];
	_trans = _VBD_VBN_index_offsets[cs];

	_klen = _VBD_VBN_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBD_VBN_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBD_VBN_indicies[_trans];
	cs = _VBD_VBN_trans_targs[_trans];

	if ( _VBD_VBN_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBD_VBN_actions + _VBD_VBN_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 1, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 3, append = "y";     }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 2, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 1, append = nullptr; }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 1, append = nullptr; }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 2, append = nullptr; }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 1, append = nullptr; }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 2, append = nullptr; }
	break;
	case 17:
	{ if (best > 'r') best = 'r', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form.c_str() + form.size())) )
	{
	const char *__acts = _VBD_VBN_actions + _VBD_VBN_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
		}
	}
	}

	_out: {}
	}

  add(VBD, VBN, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBZ_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 1, 8
};

static const char _VBZ_key_offsets[] = {
	0, 0, 1, 2, 4, 14, 14, 25,
	26, 31, 31, 31, 31, 37, 45, 54
};

static const char _VBZ_trans_keys[] = {
	115, 101, 99, 115, 98, 100, 102, 104,
	106, 110, 112, 116, 118, 122, 122, 98,
	100, 102, 104, 106, 110, 112, 116, 118,
	120, 111, 97, 101, 105, 111, 117, 104,
	105, 111, 115, 120, 122, 97, 101, 105,
	110, 111, 114, 115, 117, 97, 101, 105,
	111, 117, 121, 122, 98, 120, 0
};

static const char _VBZ_single_lengths[] = {
	0, 1, 1, 2, 0, 0, 1, 1,
	5, 0, 0, 0, 6, 8, 7, 0
};

static const char _VBZ_range_lengths[] = {
	0, 0, 0, 0, 5, 0, 5, 0,
	0, 0, 0, 0, 0, 0, 1, 0
};

static const char _VBZ_index_offsets[] = {
	0, 0, 2, 4, 7, 13, 14, 21,
	23, 29, 30, 31, 32, 39, 48, 57
};

static const char _VBZ_indicies[] = {
	0, 1, 3, 2, 4, 4, 1, 5,
	5, 5, 5, 5, 1, 6, 7, 7,
	7, 7, 7, 7, 1, 8, 1, 9,
	9, 9, 9, 9, 1, 8, 10, 1,
	11, 12, 13, 14, 4, 15, 1, 16,
	16, 16, 17, 16, 18, 19, 16, 1,
	20, 20, 20, 20, 20, 20, 22, 21,
	1, 10, 0
};

static const char _VBZ_trans_targs[] = {
	2, 0, 11, 12, 11, 5, 11, 11,
	11, 9, 11, 3, 4, 6, 13, 14,
	11, 7, 8, 11, 11, 10, 15
};

static const char _VBZ_trans_actions[] = {
	0, 0, 17, 17, 11, 0, 13, 15,
	9, 0, 3, 0, 0, 0, 11, 11,
	1, 0, 0, 7, 5, 0, 7
};

static const int VBZ_start = 1;

void english_morpho_guesser::add_VBZ(const string& form, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBZ_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBZ_trans_keys + _VBZ_key_offsets[cs];
	_trans = _VBZ_index_offsets[cs];

	_klen = _VBZ_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBZ_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBZ_indicies[_trans];
	cs = _VBZ_trans_targs[_trans];

	if ( _VBZ_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBZ_actions + _VBZ_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 1, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 3, append = "y";     }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(VBZ, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _JJR_RBR_actions[] = {
	0, 1, 0, 1, 1, 1, 3, 1,
	4, 1, 5, 2, 1, 4, 2, 2,
	5, 2, 4, 5
};

static const unsigned char _JJR_RBR_key_offsets[] = {
	0, 0, 1, 2, 26, 26, 32, 37,
	50, 56, 62, 73, 79, 85, 91, 102,
	103, 109, 115, 117, 123, 129, 135, 146,
	152, 163, 169, 175, 181
};

static const char _JJR_RBR_trans_keys[] = {
	114, 101, 98, 99, 100, 101, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 117, 118, 119, 120,
	121, 122, 97, 98, 101, 105, 111, 117,
	97, 101, 105, 111, 117, 98, 99, 100,
	105, 111, 117, 122, 97, 101, 102, 109,
	112, 120, 97, 100, 101, 105, 111, 117,
	97, 101, 102, 105, 111, 117, 97, 101,
	103, 105, 111, 117, 122, 98, 109, 112,
	120, 97, 101, 104, 105, 111, 117, 97,
	101, 105, 106, 111, 117, 97, 101, 105,
	107, 111, 117, 97, 101, 105, 108, 111,
	117, 122, 98, 109, 112, 120, 101, 97,
	101, 105, 109, 111, 117, 97, 101, 105,
	110, 111, 117, 97, 122, 97, 101, 105,
	111, 112, 117, 97, 101, 105, 111, 113,
	117, 97, 101, 105, 111, 114, 117, 97,
	101, 105, 111, 115, 117, 122, 98, 109,
	112, 120, 97, 101, 105, 111, 116, 117,
	97, 101, 105, 111, 117, 118, 122, 98,
	109, 112, 120, 97, 101, 105, 111, 117,
	119, 97, 101, 105, 111, 117, 120, 97,
	101, 105, 111, 117, 121, 97, 101, 105,
	111, 117, 122, 0
};

static const char _JJR_RBR_single_lengths[] = {
	0, 1, 1, 24, 0, 6, 5, 7,
	6, 6, 7, 6, 6, 6, 7, 1,
	6, 6, 0, 6, 6, 6, 7, 6,
	7, 6, 6, 6, 6
};

static const char _JJR_RBR_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 3,
	0, 0, 2, 0, 0, 0, 2, 0,
	0, 0, 1, 0, 0, 0, 2, 0,
	2, 0, 0, 0, 0
};

static const unsigned char _JJR_RBR_index_offsets[] = {
	0, 0, 2, 4, 29, 30, 37, 43,
	54, 61, 68, 78, 85, 92, 99, 109,
	111, 118, 125, 127, 134, 141, 148, 158,
	165, 175, 182, 189, 196
};

static const char _JJR_RBR_indicies[] = {
	0, 1, 2, 1, 4, 5, 6, 7,
	8, 9, 10, 11, 12, 13, 14, 15,
	16, 17, 18, 19, 20, 21, 7, 22,
	23, 24, 25, 26, 3, 1, 27, 28,
	27, 27, 27, 27, 1, 29, 29, 29,
	29, 29, 1, 30, 31, 30, 27, 27,
	27, 30, 27, 30, 30, 1, 27, 28,
	27, 27, 27, 27, 1, 27, 27, 28,
	27, 27, 27, 1, 27, 27, 31, 27,
	27, 27, 30, 30, 30, 1, 27, 27,
	28, 27, 27, 27, 1, 27, 27, 27,
	28, 27, 27, 1, 27, 27, 27, 28,
	27, 27, 1, 27, 27, 27, 32, 27,
	27, 30, 30, 30, 1, 1, 33, 27,
	27, 27, 28, 27, 27, 1, 34, 34,
	34, 28, 34, 34, 1, 29, 1, 34,
	34, 34, 34, 28, 34, 1, 27, 27,
	27, 27, 28, 27, 1, 27, 27, 27,
	27, 28, 27, 1, 27, 27, 27, 27,
	31, 27, 30, 30, 30, 1, 27, 27,
	27, 27, 28, 27, 1, 27, 27, 27,
	27, 27, 31, 30, 30, 30, 1, 34,
	34, 34, 34, 34, 28, 1, 34, 34,
	34, 34, 34, 28, 1, 27, 27, 27,
	27, 27, 28, 1, 27, 27, 27, 27,
	27, 28, 1, 0
};

static const char _JJR_RBR_trans_targs[] = {
	2, 0, 3, 4, 5, 7, 8, 4,
	9, 10, 11, 4, 12, 13, 14, 16,
	17, 19, 20, 21, 22, 23, 24, 25,
	26, 27, 28, 6, 4, 4, 4, 4,
	15, 4, 18
};

static const char _JJR_RBR_trans_actions[] = {
	0, 0, 0, 9, 9, 9, 9, 17,
	9, 9, 9, 14, 9, 9, 9, 9,
	9, 9, 9, 9, 9, 9, 9, 9,
	9, 9, 9, 7, 3, 5, 7, 11,
	11, 1, 7
};

static const int JJR_RBR_start = 1;

void english_morpho_guesser::add_JJR_RBR(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = JJR_RBR_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _JJR_RBR_trans_keys + _JJR_RBR_key_offsets[cs];
	_trans = _JJR_RBR_index_offsets[cs];

	_klen = _JJR_RBR_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _JJR_RBR_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _JJR_RBR_indicies[_trans];
	cs = _JJR_RBR_trans_targs[_trans];

	if ( _JJR_RBR_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _JJR_RBR_actions + _JJR_RBR_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 3, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = "y";     }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(JJR, RBR, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

static const char _JJS_RBS_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 2, 0, 5, 2, 1, 4, 2,
	3, 5
};

static const unsigned char _JJS_RBS_key_offsets[] = {
	0, 0, 1, 2, 3, 25, 25, 25,
	31, 44, 50, 56, 67, 73, 79, 85,
	96, 102, 108, 114, 120, 126, 137, 143,
	154, 160, 166, 172, 178, 178, 183, 183,
	183, 184
};

static const char _JJS_RBS_trans_keys[] = {
	116, 115, 101, 98, 99, 100, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 118, 119, 120, 121,
	122, 97, 98, 101, 105, 111, 117, 98,
	99, 100, 105, 111, 117, 122, 97, 101,
	102, 109, 112, 120, 97, 100, 101, 105,
	111, 117, 97, 101, 102, 105, 111, 117,
	97, 101, 103, 105, 111, 117, 122, 98,
	109, 112, 120, 97, 101, 104, 105, 111,
	117, 97, 101, 105, 106, 111, 117, 97,
	101, 105, 107, 111, 117, 97, 101, 105,
	108, 111, 117, 122, 98, 109, 112, 120,
	97, 101, 105, 109, 111, 117, 97, 101,
	105, 110, 111, 117, 97, 101, 105, 111,
	112, 117, 97, 101, 105, 111, 113, 117,
	97, 101, 105, 111, 114, 117, 97, 101,
	105, 111, 115, 117, 122, 98, 109, 112,
	120, 97, 101, 105, 111, 116, 117, 97,
	101, 105, 111, 117, 118, 122, 98, 109,
	112, 120, 97, 101, 105, 111, 117, 119,
	97, 101, 105, 111, 117, 120, 97, 101,
	105, 111, 117, 121, 97, 101, 105, 111,
	117, 122, 97, 101, 105, 111, 117, 101,
	97, 122, 0
};

static const char _JJS_RBS_single_lengths[] = {
	0, 1, 1, 1, 22, 0, 0, 6,
	7, 6, 6, 7, 6, 6, 6, 7,
	6, 6, 6, 6, 6, 7, 6, 7,
	6, 6, 6, 6, 0, 5, 0, 0,
	1, 0
};

static const char _JJS_RBS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	3, 0, 0, 2, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 2, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1
};

static const unsigned char _JJS_RBS_index_offsets[] = {
	0, 0, 2, 4, 6, 29, 30, 31,
	38, 49, 56, 63, 73, 80, 87, 94,
	104, 111, 118, 125, 132, 139, 149, 156,
	166, 173, 180, 187, 194, 195, 201, 202,
	203, 205
};

static const char _JJS_RBS_indicies[] = {
	0, 1, 2, 1, 3, 1, 5, 6,
	7, 8, 9, 10, 11, 12, 13, 14,
	15, 16, 17, 18, 19, 20, 21, 22,
	23, 24, 25, 26, 4, 27, 28, 29,
	30, 29, 29, 29, 29, 27, 31, 32,
	31, 29, 29, 29, 31, 29, 31, 31,
	27, 29, 30, 29, 29, 29, 29, 27,
	29, 29, 30, 29, 29, 29, 27, 29,
	29, 32, 29, 29, 29, 31, 31, 31,
	27, 29, 29, 30, 29, 29, 29, 27,
	29, 29, 29, 30, 29, 29, 27, 29,
	29, 29, 30, 29, 29, 27, 29, 29,
	29, 33, 29, 29, 31, 31, 31, 27,
	29, 29, 29, 30, 29, 29, 27, 34,
	34, 34, 30, 34, 34, 27, 34, 34,
	34, 34, 30, 34, 27, 29, 29, 29,
	29, 30, 29, 27, 29, 29, 29, 29,
	30, 29, 27, 29, 29, 29, 29, 32,
	29, 31, 31, 31, 27, 29, 29, 29,
	29, 30, 29, 27, 29, 29, 29, 29,
	29, 32, 31, 31, 31, 27, 34, 34,
	34, 34, 34, 30, 27, 34, 34, 34,
	34, 34, 30, 27, 29, 29, 29, 29,
	29, 30, 27, 29, 29, 29, 29, 29,
	30, 27, 1, 35, 35, 35, 35, 35,
	28, 28, 27, 28, 36, 35, 28, 0
};

static const char _JJS_RBS_trans_targs[] = {
	2, 0, 3, 4, 5, 7, 8, 9,
	10, 11, 12, 31, 13, 14, 15, 16,
	17, 18, 19, 20, 21, 22, 23, 24,
	25, 26, 27, 6, 28, 29, 30, 30,
	30, 32, 33, 28, 28
};

static const char _JJS_RBS_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 3, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 7, 5, 1, 5,
	12, 12, 5, 15, 9
};

static const int JJS_RBS_start = 1;

void english_morpho_guesser::add_JJS_RBS(const string& form, unsigned negation_len, vector<tagged_lemma>& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = JJS_RBS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _JJS_RBS_trans_keys + _JJS_RBS_key_offsets[cs];
	_trans = _JJS_RBS_index_offsets[cs];

	_klen = _JJS_RBS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _JJS_RBS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _JJS_RBS_indicies[_trans];
	cs = _JJS_RBS_trans_targs[_trans];

	if ( _JJS_RBS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _JJS_RBS_actions + _JJS_RBS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 3, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 4, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 4, append = "y";     }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 2, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(JJS, RBS, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/external_morpho.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class external_morpho : public morpho {
 public:
  external_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector<tagged_lemma>& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector<tagged_lemma_forms>& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);

 private:
  unsigned version;

  string unknown_tag;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer : public ragel_tokenizer {
 public:
  enum { LATEST = 2 };
  generic_tokenizer(unsigned version);

  virtual bool next_sentence(vector<token_range>& tokens) override;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/external_morpho.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool external_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load unknown_tag
    unsigned length = data.next_1B();
    unknown_tag.assign(data.next<char>(length), length);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int external_morpho::analyze(string_piece form, guesser_mode /*guesser*/, vector<tagged_lemma>& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Start by skipping the first form
    string_piece lemmatags = form;
    while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
    if (lemmatags.len) lemmatags.len--, lemmatags.str++;

    // Split lemmatags using ' ' into lemma-tag pairs.
    while (lemmatags.len) {
      auto lemma_start = lemmatags.str;
      while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
      if (!lemmatags.len) break;
      auto lemma_len = lemmatags.str - lemma_start;
      lemmatags.len--, lemmatags.str++;

      auto tag_start = lemmatags.str;
      while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
      auto tag_len = lemmatags.str - tag_start;
      if (lemmatags.len) lemmatags.len--, lemmatags.str++;

      lemmas.emplace_back(string(lemma_start, lemma_len), string(tag_start, tag_len));
    }

    if (!lemmas.empty()) return NO_GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int external_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector<tagged_lemma_forms>& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    // Start by locating the lemma
    string_piece formtags = lemma;
    while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
    string_piece real_lemma(lemma.str, lemma.len - formtags.len);
    if (formtags.len) formtags.len--, formtags.str++;

    // Split formtags using ' ' into form-tag pairs.
    bool any_result = false;
    while (formtags.len) {
      auto form_start = formtags.str;
      while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
      if (!formtags.len) break;
      auto form_len = formtags.str - form_start;
      formtags.len--, formtags.str++;

      auto tag_start = formtags.str;
      while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
      auto tag_len = formtags.str - tag_start;
      if (formtags.len) formtags.len--, formtags.str++;

      any_result = true;
      string tag(tag_start, tag_len);
      if (filter.matches(tag.c_str())) {
        if (forms.empty()) forms.emplace_back(string(real_lemma.str, real_lemma.len));
        forms.back().forms.emplace_back(string(form_start, form_len), tag);
      }
    }

    if (any_result) return NO_GUESSER;
  }

  return -1;
}

int external_morpho::raw_lemma_len(string_piece lemma) const {
  unsigned lemma_len = 0;
  while (lemma_len < lemma.len && lemma.str[lemma_len] != ' ') lemma_len++;
  return lemma_len;
}

int external_morpho::lemma_id_len(string_piece lemma) const {
  unsigned lemma_len = 0;
  while (lemma_len < lemma.len && lemma.str[lemma_len] != ' ') lemma_len++;
  return lemma_len;
}

int external_morpho::raw_form_len(string_piece form) const {
  unsigned form_len = 0;
  while (form_len < form.len && form.str[form_len] != ' ') form_len++;
  return form_len;
}

tokenizer* external_morpho::new_tokenizer() const {
  return new generic_tokenizer(version);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct generic_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector<unsigned char> data;
};

// Definitions
int generic_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  return lemma.len;
}

int generic_lemma_addinfo::lemma_id_len(string_piece lemma) {
  return lemma.len;
}

string generic_lemma_addinfo::format(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return string();
}

bool generic_lemma_addinfo::generatable(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return true;
}

int generic_lemma_addinfo::parse(string_piece lemma, bool /*die_on_failure*/) {
  return lemma.len;
}

bool generic_lemma_addinfo::match_lemma_id(const unsigned char* /*other_addinfo*/, int /*other_addinfo_len*/) {
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_morpho : public morpho {
 public:
  generic_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector<tagged_lemma>& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector<tagged_lemma_forms>& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const;

  unsigned version;
  morpho_dictionary<generic_lemma_addinfo> dictionary;
  unique_ptr<morpho_statistical_guesser> statistical_guesser;

  string unknown_tag, number_tag, punctuation_tag, symbol_tag;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool generic_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load tags
    unsigned length = data.next_1B();
    unknown_tag.assign(data.next<char>(length), length);
    length = data.next_1B();
    number_tag.assign(data.next<char>(length), length);
    length = data.next_1B();
    punctuation_tag.assign(data.next<char>(length), length);
    length = data.next_1B();
    symbol_tag.assign(data.next<char>(length), length);

    // Load dictionary
    dictionary.load(data);

    // Optionally statistical guesser if present
    statistical_guesser.reset();
    if (data.next_1B()) {
      statistical_guesser.reset(new morpho_statistical_guesser());
      statistical_guesser->load(data);
    }
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int generic_morpho::analyze(string_piece form, guesser_mode guesser, vector<tagged_lemma>& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Then call analyze_special to handle numbers, punctuation and symbols.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // For the statistical guesser, use all casing variants.
    if (guesser == GUESSER && statistical_guesser) {
      if (form_uclc.empty() && form_lc.empty())
        statistical_guesser->analyze(form, lemmas, nullptr);
      else {
        morpho_statistical_guesser::used_rules used_rules; used_rules.reserve(3);
        statistical_guesser->analyze(form, lemmas, &used_rules);
        if (!form_uclc.empty()) statistical_guesser->analyze(form_uclc, lemmas, &used_rules);
        if (!form_lc.empty()) statistical_guesser->analyze(form_lc, lemmas, &used_rules);
      }
    }
    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int generic_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector<tagged_lemma_forms>& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;
  }

  return -1;
}

int generic_morpho::raw_lemma_len(string_piece lemma) const {
  return generic_lemma_addinfo::raw_lemma_len(lemma);
}

int generic_morpho::lemma_id_len(string_piece lemma) const {
  return generic_lemma_addinfo::lemma_id_len(lemma);
}

int generic_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* generic_morpho::new_tokenizer() const {
  return new generic_tokenizer(version);
}

void generic_morpho::analyze_special(string_piece form, vector<tagged_lemma>& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers, punctuation and symbols.
  // Number is anything matching [+-]? is_Pn* ([.,] is_Pn*)? ([Ee] [+-]? is_Pn+)? for at least one is_Pn* nonempty.
  // Punctuation is any form beginning with either unicode punctuation or punctuation_exceptions character.
  // Beware that numbers takes precedence, so - is punctuation, -3 is number, -. is punctuation, -.3 is number.
  if (!form.len) return;

  string_piece number = form;
  char32_t first = utf8::decode(number.str, number.len);

  // Try matching a number.
  char32_t codepoint = first;
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  if ((codepoint == '.' && number.len) || codepoint == ',') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(number.str, number.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }

  if (any_digit && !number.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form.str, form.len - (codepoint == '.')), number_tag);
    return;
  }

  // Try matching punctuation or symbol.
  bool punctuation = true, symbol = true;
  string_piece form_ori = form;
  while (form.len) {
    codepoint = utf8::decode(form.str, form.len);
    punctuation = punctuation && unicode::category(codepoint) & unicode::P;
    symbol = symbol && unicode::category(codepoint) & unicode::S;
  }
  if (punctuation)
    lemmas.emplace_back(string(form_ori.str, form_ori.len), punctuation_tag);
  else if (symbol)
    lemmas.emplace_back(string(form_ori.str, form_ori.len), symbol_tag);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_morpho_encoder {
 public:
  struct tags {
    string unknown_tag, number_tag, punctuation_tag, symbol_tag;
  };
  static void encode(istream& in_dictionary, int max_suffix_len, const tags& tags, istream& in_statistical_guesser, ostream& out_morpho);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/persistent_unordered_map_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template <class Entry, class EntryEncode>
persistent_unordered_map::persistent_unordered_map(const unordered_map<string, Entry>& map, double load_factor, EntryEncode entry_encode) {
  construct(std::map<string, Entry>(map.begin(), map.end()), load_factor, entry_encode);
}

template <class Entry, class EntryEncode>
persistent_unordered_map::persistent_unordered_map(const unordered_map<string, Entry>& map, double load_factor, bool add_prefixes, bool add_suffixes, EntryEncode entry_encode) {
  // Copy data, possibly including prefixes and suffixes
  std::map<string, Entry> enlarged_map(map.begin(), map.end());

  for (auto&& entry : map) {
    const string& key = entry.first;

    if (!key.empty() && add_prefixes)
      for (unsigned i = key.size() - 1; i; i--)
        enlarged_map[key.substr(0, i)];

    if (!key.empty() && add_suffixes)
      for (unsigned i = 1; i < key.size(); i++)
        enlarged_map[key.substr(i)];
  }

  construct(enlarged_map, load_factor, entry_encode);
}

// We could (and used to) use unordered_map as input parameter.
// Nevertheless, as order is unspecified, the resulting persistent_unordered_map
// has different collision chains when generated on 32-bit and 64-bit machines.
// To guarantee uniform binary representation, we use map instead.
template <class Entry, class EntryEncode>
void persistent_unordered_map::construct(const map<string, Entry>& map, double load_factor, EntryEncode entry_encode) {
  // 1) Count number of elements for each size
  vector<int> sizes;
  for (auto&& elem : map) {
    unsigned len = elem.first.size();
    if (len >= sizes.size()) sizes.resize(len + 1);
    sizes[len]++;
  }
  for (auto&& size : sizes)
    resize(unsigned(load_factor * size));

  // 2) Add sizes of element data
  for (auto&& elem : map) {
    binary_encoder enc;
    entry_encode(enc, elem.second);
    add(elem.first.c_str(), elem.first.size(), enc.data.size());
  }
  done_adding();

  // 3) Fill in element data
  for (auto&& elem : map) {
    binary_encoder enc;
    entry_encode(enc, elem.second);
    small_memcpy(fill(elem.first.c_str(), elem.first.size(), enc.data.size()), enc.data.data(), enc.data.size());
  }
  done_filling();
}

void persistent_unordered_map::save(binary_encoder& enc) {
  enc.add_1B(hashes.size());

  for (auto&& hash : hashes)
    hash.save(enc);
}

void persistent_unordered_map::fnv_hash::save(binary_encoder& enc) {
  enc.add_4B(hash.size());
  enc.add_data(hash);

  enc.add_4B(data.size());
  enc.add_data(data);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/raw_morpho_dictionary_reader.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class raw_morpho_dictionary_reader {
 public:
  raw_morpho_dictionary_reader(istream& in) : in(in) {}
  bool next_lemma(string& lemma, vector<pair<string, string>>& tagged_forms);
 private:
  istream& in;
  string line;
  vector<string> tokens;
  unordered_set<string> seen_lemmas;
};

} // namespace morphodita

/////////
// File: utils/new_unique_ptr.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

template<typename T, typename... Args>
unique_ptr<T> new_unique_ptr(Args&&... args) {
  return unique_ptr<T>(new T(std::forward<Args>(args)...));
}

} // namespace utils

/////////
// File: morphodita/morpho/morpho_dictionary_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class LemmaAddinfo>
class morpho_dictionary_encoder {
 public:
  static void encode(istream& is, int max_suffix_len, binary_encoder& enc);
};

// Definitions
template <class LemmaAddinfo>
class dictionary {
 public:
  void load(istream& is, int max_suffix_len);
  void encode(binary_encoder& enc);

 private:
  class trie {
   public:
    trie() : depth(0) {}

    void add(const char* str) {
      if (!*str) return;

      for (auto&& child : children)
        if (child.first == *str) {
          child.second->add(str + 1);
          depth = max(depth, 1 + child.second->depth);
          return;
        }
      children.emplace_back(*str, new_unique_ptr<trie>());
      children.back().second->add(str + 1);
      depth = max(depth, 1 + children.back().second->depth);
    }

    string find_candidate_prefix(int max_suffix_len) {
      string current, best;
      int best_length = 0;
      find_candidate_prefix(max_suffix_len, current, best, best_length, 0);
      return best;
    }
    void find_candidate_prefix(int max_suffix_len, string& current, string& best, int& best_length, int length) {
      if (depth < max_suffix_len && length > best_length) {
        best = current;
        best_length = length;
      }
      for (auto&& child : children) {
        current.push_back(child.first);
        child.second->find_candidate_prefix(max_suffix_len, current, best, best_length, children.size() == 1 ? length + 1 : 1);
        current.resize(current.size() - 1);
      }
    }

    vector<pair<char, unique_ptr<trie>>> children;
    int depth;
  };

  class histogram {
   public:
    void add(const string& str) {
      if (str.size() >= lengths.size()) lengths.resize(str.size() + 1);
      lengths[str.size()].insert(str);
    }

    void encode(binary_encoder& enc) {
      enc.add_1B(lengths.size());
      for (auto&& set : lengths)
        enc.add_4B(set.size());
    }

    vector<unordered_set<string>> lengths;
  };

  struct lemma_info {
    lemma_info(string lemma) {
      this->lemma = lemma.substr(0, addinfo.parse(lemma, true));
    }

    string lemma;
    LemmaAddinfo addinfo;
    struct lemma_form_info {
      lemma_form_info(string form, int clas) : form(form), clas(clas) {}

      string form;
      int clas;

      bool operator<(const lemma_form_info& other) const { return form < other.form || (form == other.form && clas < other.clas); }
    };
    vector<lemma_form_info> forms;

    bool operator<(const lemma_info& other) const { return lemma < other.lemma || (lemma == other.lemma && addinfo.data < other.addinfo.data); }
  };

  unordered_map<string, int> classes;
  unordered_map<string, map<int, vector<int>>> suffixes;

  vector<string> tags;
  unordered_map<string, int> tags_map;

  histogram lemmas_hist, forms_hist;

  vector<lemma_info> lemmas;
};

template <class LemmaAddinfo>
void morpho_dictionary_encoder<LemmaAddinfo>::encode(istream& is, int max_suffix_len, binary_encoder& enc) {
  dictionary<LemmaAddinfo> dict;

  // Load the dictionary and create classes
  dict.load(is, max_suffix_len);

  // Encode the dictionary
  dict.encode(enc);
}

template <class LemmaAddinfo>
void dictionary<LemmaAddinfo>::load(istream& is, int max_suffix_len) {
  // Load lemmas and create classes
  raw_morpho_dictionary_reader raw(is);
  string lemma;
  vector<pair<string, string>> forms;
  while(raw.next_lemma(lemma, forms)) {
    // Make sure forms are unique
    sort(forms.begin(), forms.end());
    auto forms_end = unique(forms.begin(), forms.end());
    if (forms_end != forms.end()) {
//      cerr << "Warning: repeated form-tag in lemma " << lemma << '.' << endl;
      forms.erase(forms_end, forms.end());
    }

    // Create lemma_info
    lemmas.emplace_back(lemma);
    auto& lemma_info = lemmas.back();
    lemmas_hist.add(lemma_info.lemma);

    // Create classes
    while (!forms.empty()) {
      trie t;
      for (auto&& form : forms)
        t.add(form.first.c_str());

      // Find prefix of forms in class being added.
      string prefix = t.find_candidate_prefix(max_suffix_len);

      // Find forms of the class being added.
      auto start = forms.begin();
      while (start != forms.end() && start->first.compare(0, prefix.size(), prefix) != 0) start++;
      if (start == forms.end()) training_failure("Internal error when generating classes, cannot find prefix '" << prefix << "'!");
      auto end = start;
      while (end != forms.end() && end->first.compare(0, prefix.size(), prefix) == 0) end++;

      // Find common prefix of class forms -- may be larger than prefix.
      int common_prefix = prefix.size();
      while (common_prefix < int(start->first.size()) && start->first[common_prefix] == (end-1)->first[common_prefix]) common_prefix++;

      string clas;
      for (auto form = start; form != end; form++) {
        if (!clas.empty()) clas.push_back('\t');
        clas.append(form->first, common_prefix, string::npos);
        clas.push_back('\t');
        clas.append(form->second);
      }

      auto class_it = classes.emplace(clas, classes.size());
      int class_id = class_it.first->second;
      if (class_it.second) {
        // New class, add it, together with its tags.
        for (auto form = start; form != end; form++) {
          int tag = tags_map.emplace(form->second, tags.size()).first->second;
          if (tag >= int(tags.size())) tags.emplace_back(form->second);
          suffixes[form->first.substr(common_prefix)][class_id].emplace_back(tag);
        }
      }

      // Move forms in the class being added to lemma and remove them from unprocessed forms.
      lemma_info.forms.emplace_back(start->first.substr(0, common_prefix), class_id);
      forms_hist.add(lemma_info.forms.back().form);
      forms.erase(start, end);
    }
    stable_sort(lemma_info.forms.begin(), lemma_info.forms.end());
  }
  stable_sort(lemmas.begin(), lemmas.end());
}

template <class LemmaAddinfo>
void dictionary<LemmaAddinfo>::encode(binary_encoder& enc) {
  // Encode lemmas and forms
  lemmas_hist.encode(enc);
  forms_hist.encode(enc);

  string prev = "";
  enc.add_4B(lemmas.size());
  for (auto&& lemma : lemmas) {
    int cpl = 0;
    while (prev[cpl] && prev[cpl] == lemma.lemma[cpl]) cpl++;

    enc.add_1B(prev.length() - cpl);
    enc.add_1B(lemma.lemma.size() - cpl);
    enc.add_data(lemma.lemma.substr(cpl));
    enc.add_1B(lemma.addinfo.data.size());
    enc.add_data(lemma.addinfo.data);
    enc.add_1B(lemma.forms.size());

    string prev_form = lemma.lemma;
    for (auto&& lemma_form : lemma.forms) {
      unsigned best_prev_from = 0, best_form_from = 0, best_len = 0;
      for (unsigned prev_from = 0; prev_from < prev_form.size(); prev_from++)
        for (unsigned form_from = 0; form_from < lemma_form.form.size(); form_from++) {
          unsigned len = 0;
          while (prev_from + len < prev_form.size() && form_from + len < lemma_form.form.size() && prev_form[prev_from+len] == lemma_form.form[form_from+len]) len++;
          if (len > best_len) best_prev_from = prev_from, best_form_from = form_from, best_len = len;
        }

      enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
      enc.add_1B(REMOVE_START * (best_prev_from>0) + REMOVE_END * (best_prev_from+best_len<prev_form.size()) +
             ADD_START * (best_form_from>0) + ADD_END * (best_form_from+best_len<lemma_form.form.size()));
      if (best_prev_from > 0) enc.add_1B(best_prev_from);
      if (best_prev_from + best_len < prev_form.size()) enc.add_1B(prev_form.size() - best_prev_from - best_len);
      if (best_form_from > 0) {
        enc.add_1B(best_form_from);
        enc.add_data(lemma_form.form.substr(0, best_form_from));
      }
      if (best_form_from + best_len < lemma_form.form.size()) {
        enc.add_1B(lemma_form.form.size() - best_form_from - best_len);
        enc.add_data(lemma_form.form.substr(best_form_from + best_len));
      }
      enc.add_2B(lemma_form.clas);

      prev_form = lemma_form.form;
    }

    prev = lemma.lemma;
  }

  // Encode tags
  enc.add_2B(tags.size());
  for (auto&& tag : tags) {
    enc.add_1B(tag.size());
    enc.add_data(tag);
  }

  // Encode classes
  persistent_unordered_map(suffixes, 5, false, true, [](binary_encoder& enc, const map<int, vector<int>>& suffix) {
    enc.add_2B(suffix.size());
    for (auto&& clas : suffix)
      enc.add_2B(clas.first);
    uint32_t tags = 0, prev_tags = 0;
    for (auto&& clas : suffix) {
      enc.add_2B(tags - prev_tags < (1<<16) ? uint16_t(tags) : tags);
      prev_tags = tags;
      tags += clas.second.size();
    }
    enc.add_2B(tags - prev_tags < (1<<16) ? uint16_t(tags) : tags);
    for (auto&& clas : suffix)
      for (auto&& tag : clas.second)
        enc.add_2B(tag);
  }).save(enc);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_prefix_guesser_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_prefix_guesser_encoder {
 public:
  static void encode(istream& is, binary_encoder& enc);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser_encoder {
 public:
  static void encode(istream& is, binary_encoder& enc);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho_encoder.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void generic_morpho_encoder::encode(istream& in_dictionary, int max_suffix_len, const tags& tags, istream& in_statistical_guesser, ostream& out_morpho) {
  binary_encoder enc;

  enc.add_1B(tags.unknown_tag.size());
  enc.add_data(tags.unknown_tag);
  enc.add_1B(tags.number_tag.size());
  enc.add_data(tags.number_tag);
  enc.add_1B(tags.punctuation_tag.size());
  enc.add_data(tags.punctuation_tag);
  enc.add_1B(tags.symbol_tag.size());
  enc.add_data(tags.symbol_tag);

//  cerr << "Encoding dictionary." << endl;
  morpho_dictionary_encoder<generic_lemma_addinfo>::encode(in_dictionary, max_suffix_len, enc);

  // Load and encode statistical guesser if requested
  enc.add_1B(bool(in_statistical_guesser));
  if (in_statistical_guesser) {
//    cerr << "Encoding statistical guesser." << endl;
    morpho_statistical_guesser_encoder::encode(in_statistical_guesser, enc);
  }

  // done, save the dictionary
//  cerr << "Compressing dictionary." << endl;
  if (!compressor::save(out_morpho, enc)) training_failure("Cannot compress and write dictionary to file!");
//  cerr << "Dictionary saved." << endl;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_ids.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_ids {
 public:
  enum morpho_id {
    CZECH = 0,
    ENGLISH_V1 = 1,
    GENERIC = 2,
    EXTERNAL = 3,
    ENGLISH_V2 = 4,
    ENGLISH_V3 = 5, ENGLISH = ENGLISH_V3,
    SLOVAK_PDT = 6,
    DERIVATOR_DICTIONARY = 7,
  };

  static bool parse(const string& str, morpho_id& id) {
    if (str == "czech") return id = CZECH, true;
    if (str == "english") return id = ENGLISH, true;
    if (str == "external") return id = EXTERNAL, true;
    if (str == "generic") return id = GENERIC, true;
    if (str == "slovak_pdt") return id = SLOVAK_PDT, true;
    return false;
  }
};

typedef morpho_ids::morpho_id morpho_id;

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

morpho* morpho::load(istream& is) {
  morpho_id id = morpho_id(is.get());
  switch (id) {
    case morpho_ids::CZECH:
      {
        auto res = new_unique_ptr<czech_morpho>(czech_morpho::morpho_language::CZECH, 1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::ENGLISH_V1:
    case morpho_ids::ENGLISH_V2:
    case morpho_ids::ENGLISH_V3:
      {
        auto res = new_unique_ptr<english_morpho>(id == morpho_ids::ENGLISH_V1 ? 1 :
                                                  id == morpho_ids::ENGLISH_V2 ? 2 :
                                                  3);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::EXTERNAL:
      {
        auto res = new_unique_ptr<external_morpho>(1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::GENERIC:
      {
        auto res = new_unique_ptr<generic_morpho>(1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::SLOVAK_PDT:
      {
        auto res = new_unique_ptr<czech_morpho>(czech_morpho::morpho_language::SLOVAK, 3);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::DERIVATOR_DICTIONARY:
      {
        auto derinet = new_unique_ptr<derivator_dictionary>();
        if (!derinet->load(is)) return nullptr;

        unique_ptr<morpho> dictionary(load(is));
        if (!dictionary) return nullptr;
        derinet->dictionary = dictionary.get();
        dictionary->derinet.reset(derinet.release());
        return dictionary.release();
      }
  }

  return nullptr;
}

morpho* morpho::load(const char* fname) {
  ifstream f(fname, ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

const derivator* morpho::get_derivator() const {
  return derinet.get();
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser::load(binary_decoder& data) {
  // Load tags and default tag
  tags.resize(data.next_2B());
  for (auto&& tag : tags) {
    tag.resize(data.next_1B());
    for (unsigned i = 0; i < tag.size(); i++)
      tag[i] = data.next_1B();
  }
  default_tag = data.next_2B();

  // Load rules
  rules.load(data);
}

// Helper method for analyze.
static bool contains(morpho_statistical_guesser::used_rules* used, const string& rule) {
  if (!used) return false;

  for (auto&& used_rule : *used)
    if (used_rule == rule)
      return true;

  return false;
}

// Produces unique lemma-tag pairs.
void morpho_statistical_guesser::analyze(string_piece form, vector<tagged_lemma>& lemmas, morpho_statistical_guesser::used_rules* used) {
  unsigned lemmas_initial_size = lemmas.size();

  // We have rules in format "suffix prefix" in rules.
  // Find the matching rule with longest suffix and of those with longest prefix.
  string rule_label; rule_label.reserve(12);
  unsigned suffix_len = 0;
  for (; suffix_len < form.len; suffix_len++) {
    rule_label.push_back(form.str[form.len - (suffix_len + 1)]);
    if (!rules.at(rule_label.c_str(), rule_label.size(), [](pointer_decoder& data){ data.next<char>(data.next_2B()); }))
      break;
  }

  for (suffix_len++; suffix_len--; ) {
    rule_label.resize(suffix_len);
    rule_label.push_back(' ');

    const unsigned char* rule = nullptr;
    unsigned rule_prefix_len = 0;
    for (unsigned prefix_len = 0; prefix_len + suffix_len <= form.len; prefix_len++) {
      if (prefix_len) rule_label.push_back(form.str[prefix_len - 1]);
      const unsigned char* found = rules.at(rule_label.c_str(), rule_label.size(), [](pointer_decoder& data){ data.next<char>(data.next_2B()); });
      if (!found) break;
      if (*(found += sizeof(uint16_t))) {
        rule = found;
        rule_prefix_len = prefix_len;
      }
    }

    if (rule) {
      rule_label.resize(suffix_len + 1 + rule_prefix_len);
      if (rule_label.size() > 1 && !contains(used, rule_label)) { // ignore rule ' '
        if (used) used->push_back(rule_label);
        for (int rules_len = *rule++; rules_len; rules_len--) {
          unsigned pref_del_len = *rule++; const char* pref_del = (const char*)rule; rule += pref_del_len;
          unsigned pref_add_len = *rule++; const char* pref_add = (const char*)rule; rule += pref_add_len;
          unsigned suff_del_len = *rule++; const char* suff_del = (const char*)rule; rule += suff_del_len;
          unsigned suff_add_len = *rule++; const char* suff_add = (const char*)rule; rule += suff_add_len;
          unsigned tags_len = *rule++; const uint16_t* tags = (const uint16_t*)rule; rule += tags_len * sizeof(uint16_t);

          if (pref_del_len + suff_del_len > form.len ||
              (pref_del_len && !small_memeq(pref_del, form.str, pref_del_len)) ||
              (suff_del_len && !small_memeq(suff_del, form.str + form.len - suff_del_len, suff_del_len)) ||
              (form.len + pref_add_len - pref_del_len + suff_add_len - suff_del_len == 0))
            continue;

          string lemma;
          lemma.reserve(form.len + pref_add_len - pref_del_len + suff_add_len - suff_del_len);
          if (pref_add_len) lemma.append(pref_add, pref_add_len);
          if (pref_del_len + suff_del_len < form.len) lemma.append(form.str + pref_del_len, form.len - pref_del_len - suff_del_len);
          if (suff_add_len) lemma.append(suff_add, suff_add_len);
          while (tags_len--)
            lemmas.emplace_back(lemma, this->tags[*tags++]);
        }
      }
      break;
    }
  }

  // If nothing was found, use default tag.
  if (lemmas.size() == lemmas_initial_size)
    if (!contains(used, string())) {
      if (used) used->push_back(string());
      lemmas.emplace_back(string(form.str, form.len), tags[default_tag]);
    }
}

} // namespace morphodita

/////////
// File: utils/split.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Split given text on the separator character.
inline void split(const string& text, char sep, vector<string>& tokens);
inline void split(string_piece text, char sep, vector<string_piece>& tokens);

//
// Definitions
//

void split(const string& text, char sep, vector<string>& tokens) {
  tokens.clear();
  if (text.empty()) return;

  string::size_type index = 0;
  for (string::size_type next; (next = text.find(sep, index)) != string::npos; index = next + 1)
    tokens.emplace_back(text, index, next - index);

  tokens.emplace_back(text, index);
}

void split(string_piece text, char sep, vector<string_piece>& tokens) {
  tokens.clear();
  if (!text.len) return;

  const char* str = text.str;
  for (const char* next; (next = (const char*) memchr(str, sep, text.str + text.len - str)); str = next + 1)
    tokens.emplace_back(str, next - str);

  tokens.emplace_back(str, text.str + text.len - str);
}

} // namespace utils

/////////
// File: morphodita/morpho/morpho_statistical_guesser_encoder.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser_encoder::encode(istream& is, binary_encoder& enc) {
  unordered_map<string, vector<pair<vector<string>, vector<int>>>> statistical_guesser;
  vector<string> tags;
  unordered_map<string, int> tags_map;

  // Load statistical guesser
  string line;
  vector<string> tokens;
  if (!getline(is, line)) training_failure("Missing first line with default tag in statistical guesser file");
  int statistical_guesser_default = tags_map.emplace(line.data(), tags.size()).first->second;
  if (unsigned(statistical_guesser_default) >= tags.size()) tags.emplace_back(line.data());

  while (getline(is, line)) {
    split(line, '\t', tokens);
    if (tokens.size() < 3 || (tokens.size() % 2) != 1) training_failure("Cannot parse line " << line << " in statistical guesser file!");

    vector<string> affixes;
    split(tokens[0], ' ', affixes);
    if (affixes.size() != 2) training_failure("Cannot parse prefix_suffix '" << tokens[0] << "' in statistical guesser file!");
    reverse(affixes[1].begin(), affixes[1].end());

    auto& rules = statistical_guesser[affixes[1] + ' ' + affixes[0]];
    for (unsigned i = 1; i < tokens.size(); i+= 2) {
      vector<string> replacements;
      split(tokens[i], ' ', replacements);
      if (replacements.size() != 4) training_failure("Cannot parse replacement rule '" << tokens[i] << "' in statistical guesser file!");

      vector<string> rule_tags;
      split(tokens[i+1], ' ', rule_tags);
      vector<int> decoded_tags;
      for (auto&& rule_tag : rule_tags) {
        int tag = tags_map.emplace(rule_tag, tags.size()).first->second;
        if (unsigned(tag) >= tags.size()) tags.emplace_back(rule_tag);
        decoded_tags.emplace_back(tag);
      }

      rules.emplace_back(replacements, decoded_tags);
    }
  }

  // Encode statistical guesser
  enc.add_2B(tags.size());
  for (auto&& tag : tags) {
    enc.add_1B(tag.size());
    enc.add_data(tag);
  }
  enc.add_2B(statistical_guesser_default);

  persistent_unordered_map(statistical_guesser, 5, true, false, [](binary_encoder& enc, vector<pair<vector<string>, vector<int>>> rules) {
    binary_encoder e;
    e.add_1B(rules.size());
    for (auto&& rule : rules) {
      if (rule.first.size() != 4) training_failure("Replacement rule not of size 4 in statistical guesser!");
      for (auto&& affix : rule.first) {
        e.add_1B(affix.size());
        e.add_data(affix);
      }
      e.add_1B(rule.second.size());
      for (auto&& tag : rule.second)
        e.add_2B(tag);
    }
    enc.add_2B(e.data.size());
    enc.add_data(e.data);
  }).save(enc);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_trainer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser_trainer {
 public:
  static void train(istream& is, unsigned suffix_len, unsigned rules_per_suffix, unsigned max_prefixes, unsigned min_prefix_count, ostream& os);

 private:
  struct instance {
    string form, lemma, tag;
    string lemma_rule, form_prefix;

    instance(const string& form, const string& lemma, const string& tag);
  };

  enum casing { CASE_LC, CASE_UCLC, CASE_UC, CASE_OTHER };
  static casing get_casing(const string& word, bool allow_nonletters);
  static void set_casing(const string& original, casing c, string& word);
  static bool suffix(const string& word, unsigned& length);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_trainer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser_trainer::train(istream& is, unsigned suffix_len, unsigned rules_per_suffix, unsigned max_prefixes, unsigned min_prefix_count, ostream& os) {
  vector<instance> data;

  // Load training data
  string form;
  vector<string> tokens;
  for (string line; getline(is, line);) {
    if (line.empty()) continue;

    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("The guesser training line '" << line << "' does not contain three columns!");
    if (tokens[0].empty() || tokens[1].empty() || tokens[2].empty()) training_failure("The guesser training line '" << line << "' contains an empty column!");

    // Normalize case
    casing form_case = get_casing(tokens[0], false);
    casing lemma_case = get_casing(tokens[1], true);
    if ((lemma_case == CASE_LC && (form_case == CASE_UCLC || form_case == CASE_UC)) ||
        (lemma_case == CASE_UCLC && form_case == CASE_UC)) {
      set_casing(tokens[0], lemma_case, form);
    } else {
      form.swap(tokens[0]);
    }

    data.emplace_back(form, tokens[1], tokens[2]);
  }

  // Generate at most max_prefixes prefixes with min_prefix_count
  unordered_map<string, unordered_set<string>> prefixes_with_forms;
  for (auto&& instance : data)
    if (!instance.form_prefix.empty())
      prefixes_with_forms[instance.form_prefix].insert(instance.form);

  vector<pair<unsigned, string>> prefixes_with_counts;
  for (auto&& prefix : prefixes_with_forms)
    if (prefix.second.size() >= min_prefix_count)
      prefixes_with_counts.emplace_back(prefix.second.size(), prefix.first);

  if (prefixes_with_counts.size() > max_prefixes) {
    sort(prefixes_with_counts.begin(), prefixes_with_counts.end(), greater<pair<unsigned, string>>());
    prefixes_with_counts.resize(max_prefixes);
  }

  unordered_set<string> prefixes;
  prefixes.emplace();
  for (auto&& prefix : prefixes_with_counts)
    prefixes.insert(prefix.second);

  // Generate the guesser rules
  unordered_map<string, unordered_set<string>> tags;
  unordered_map<string, unordered_map<string, unordered_set<string>>> rules;
  unordered_set<string> suffixes;
  string prefix_suffix, tag_lemma_rule;
  for (auto&& instance : data) {
    // Add tag
    tags[instance.tag].insert(instance.form);

    // Find longest matching prefix
    unsigned prefix_length = 0;
    for (auto&& prefix : prefixes)
      if (prefix.size() > prefix_length && instance.form.compare(0, prefix.size(), prefix) == 0)
        prefix_length = prefix.size();

    tag_lemma_rule.assign(instance.lemma_rule).append("\t").append(instance.tag);

    // Add prefix + all suffixes of length 1..suffix_len to rules
    for (unsigned length = 0, utf8_length = 0; length < suffix_len && suffix(instance.form, utf8_length); length++) {
      prefix_suffix.assign(instance.form, 0, prefix_length).append(" ").append(instance.form, instance.form.size() - utf8_length, utf8_length);
      rules[prefix_suffix][tag_lemma_rule].insert(instance.form);
      suffixes.emplace(instance.form, instance.form.size() - utf8_length, utf8_length);
    }
  }

  // Start generating the guesser description by writing the most "frequent" tag
  string most_frequent_tag; unsigned most_frequent_tag_count = 0;
  for (auto&& tag : tags)
    if (tag.second.size() > most_frequent_tag_count)
      most_frequent_tag.assign(tag.first), most_frequent_tag_count = tag.second.size();

  os << most_frequent_tag << endl;

  // For every prefix-suffix, write at most rules_per_suffix most "frequent" rules
  string rule_key, output;
  unordered_set<string> rules_set;
  vector<pair<unsigned, string>> rules_counts;
  for (auto&& suffix : suffixes) {
    for (auto&& prefix : prefixes) {
      rules_counts.clear();
      rules_set.clear();

      // Gather at most rules_per_suffix rules
      for (int prefix_len = int(prefix.size()); prefix_len >= 0; prefix_len -= prefix.empty() ? 1 : prefix.size()) {
        for (int suffix_len = int(suffix.size()); rules_counts.size() < rules_per_suffix && suffix_len > 0; suffix_len--) {
          rule_key.assign(prefix, 0, prefix_len).append(" ").append(suffix, suffix.size() - suffix_len, suffix_len);
          if (!rules.count(rule_key)) continue;

          unsigned rules_counts_original = rules_counts.size();
          for (auto&& entry : rules[rule_key])
            if (!rules_set.count(entry.first)) {
              rules_counts.emplace_back(entry.second.size(), entry.first);
              rules_set.insert(entry.first);
            }

          sort(rules_counts.begin() + rules_counts_original, rules_counts.end(), greater<pair<unsigned, string>>());

          if (rules_counts.size() >= rules_per_suffix) {
            rules_counts.resize(rules_per_suffix);
            break;
          }
        }
        // Stop if there are no rules for given prefix
        if (rules_set.empty()) break;
      }
      if (!rules_set.empty()) {
        // Write the chosen rules
        output.assign(prefix).append(" ").append(suffix);
        for (unsigned i = 0; i < rules_counts.size(); i++) {
          unsigned tab = rules_counts[i].second.find('\t');

          output.append("\t").append(rules_counts[i].second, 0, tab).append("\t").append(rules_counts[i].second, tab + 1, string::npos);

          // Join rules with same lemma_rule
          for (unsigned start = i; i+1 < rules_counts.size() && rules_counts[i+1].second.compare(0, tab + 1, rules_counts[start].second, 0, tab + 1) == 0; i++)
            output.append(" ").append(rules_counts[i+1].second, tab + 1, string::npos);
        }
        os << output << endl;
      }
    }
  }
}

morpho_statistical_guesser_trainer::instance::instance(const string& form, const string& lemma, const string& tag)
  : form(form), lemma(lemma), tag(tag)
{
  using namespace unilib;

  unsigned length_best = 0;
  int form_best = 0, lemma_best = 0;
  for (int offset = -int(lemma.size() - 1); offset < int(form.size()) - 1; offset++) {
    unsigned form_offset = max(0, offset);
    unsigned lemma_offset = max(0, -offset);
    for (unsigned length = 0; form_offset < form.size() && lemma_offset < lemma.size(); form_offset++, lemma_offset++)
      if (form[form_offset] == lemma[lemma_offset]) {
        if (++length > length_best && utf8::valid(form.c_str() + form_offset + 1 - length, length))
          length_best = length, form_best = form_offset + 1 - length, lemma_best = lemma_offset + 1 - length;
      } else {
        length = 0;
      }
  }

  form_prefix.assign(form, 0, lemma_best == 0 ? form_best : 0);
  lemma_rule.assign(form, 0, form_best).append(" ").append(lemma, 0, lemma_best).append(" ")
      .append(form, form_best + length_best, string::npos).append(" ").append(lemma, lemma_best + length_best, string::npos);
}

morpho_statistical_guesser_trainer::casing morpho_statistical_guesser_trainer::get_casing(const string& word, bool allow_nonletters) {
  using namespace unilib;

  casing c = CASE_OTHER;
  int index = 0;
  for (auto&& chr : utf8::decoder(word)) {
    auto cat = unicode::category(chr);

    // Return OTHER for non-letters
    if (allow_nonletters && index >= 2 && cat & ~unicode::L) continue;
    if (cat & ~unicode::L) return CASE_OTHER;

    if (index == 0) {
      c = cat & unicode::Ll ? CASE_LC : CASE_UC;
    } else if (c == CASE_UC && index == 1) {
      c = cat & unicode::Ll ? CASE_UCLC : CASE_UC;
    } else if (c == CASE_UC) {
      if (cat & ~unicode::Lut) return CASE_OTHER;
    } else /*CASE_LC or CASE_UCLC*/ {
      if (cat & ~unicode::Ll) return CASE_OTHER;
    }
    index++;
  }
  return c;
}

void morpho_statistical_guesser_trainer::set_casing(const string& original, casing c, string& word) {
  using namespace unilib;

  word.clear();
  bool first = true;
  for (auto&& chr : utf8::decoder(original)) {
    utf8::append(word, (c == CASE_UC || (c == CASE_UCLC && first)) ? unicode::uppercase(chr) : unicode::lowercase(chr));
    first = false;
  }
}

bool morpho_statistical_guesser_trainer::suffix(const string& word, unsigned& length) {
  using namespace unilib;

  unsigned additional = 1;
  while (additional + length <= word.size() && !utf8::valid(word.c_str() + word.size() - length - additional, additional))
    additional++;

  if (additional + length > word.size()) return false;

  length += additional;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/raw_morpho_dictionary_reader.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool raw_morpho_dictionary_reader::next_lemma(string& lemma, vector<pair<string, string>>& tagged_forms) {
  if (line.empty()) {
    if (!getline(in, line))
      return false;
    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("Line " << line << " does not have three columns!");
  }

  lemma = tokens[0];
  if (seen_lemmas.count(lemma))
    training_failure("Raw morphological dictionary contains lemma '" << lemma << "' multiple times - all forms of one lemma must be in continuous region!");
  seen_lemmas.insert(lemma);

  tagged_forms.clear();
  tagged_forms.emplace_back(tokens[2], tokens[1]);
  while (getline(in, line)) {
    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("Line " << line << " does not have three columns!");

    if (lemma != tokens[0]) break;
    tagged_forms.emplace_back(tokens[2], tokens[1]);
  }

  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/tag_filter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tag_filter::tag_filter(const char* filter) {
  if (!filter) return;

  wildcard.assign(filter);
  filter = wildcard.c_str();

  for (int tag_pos = 0; *filter; tag_pos++, filter++) {
    if (*filter == '?') continue;
    if (*filter == '[') {
      filter++;

      bool negate = false;
      if (*filter == '^') negate = true, filter++;

      const char* chars = filter;
      for (bool first = true; *filter && (first || *filter != ']'); first = false) filter++;

      filters.emplace_back(tag_pos, negate, chars, filter - chars);
      if (!*filter) break;
    } else {
      filters.emplace_back(tag_pos, false, filter, 1);
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/elementary_features.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
enum elementary_feature_type { PER_FORM, PER_TAG, DYNAMIC };
enum elementary_feature_range { ONLY_CURRENT, ANY_OFFSET };

typedef uint32_t elementary_feature_value;
enum :elementary_feature_value { elementary_feature_unknown = 0, elementary_feature_empty = 1 };

struct elementary_feature_description {
  string name;
  elementary_feature_type type;
  elementary_feature_range range;
  int index;
  int map_index;
};

template<class Map>
class elementary_features {
 public:
  bool load(istream& is);
  bool save(ostream& out);

  vector<Map> maps;
};

class persistent_elementary_feature_map : public persistent_unordered_map {
 public:
  persistent_elementary_feature_map() : persistent_unordered_map() {}
  persistent_elementary_feature_map(const persistent_unordered_map&& map) : persistent_unordered_map(map) {}

  elementary_feature_value value(const char* feature, int len) const {
    auto* it = at_typed<elementary_feature_value>(feature, len);
    return it ? *it : elementary_feature_unknown;
  }
};

// Definitions
template <class Map>
inline bool elementary_features<Map>::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    maps.resize(data.next_1B());
    for (auto&& map : maps)
      map.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

} // namespace morphodita

/////////
// File: morphodita/tagger/vli.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class T>
class vli {
 public:
  static int max_length();
  static void encode(T value, char*& where);
  static T decode(const char*& from);
};

// Definitions
template <>
inline int vli<uint32_t>::max_length() {
  return 5;
}

template <>
inline void vli<uint32_t>::encode(uint32_t value, char*& where) {
  if (value < 0x80) *where++ = value;
  else if (value < 0x4000) *where++ = (value >> 7) | 0x80u, *where++ = value & 0x7Fu;
  else if (value < 0x200000) *where++ = (value >> 14) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
  else if (value < 0x10000000) *where++ = (value >> 21) | 0x80u, *where++ = ((value >> 14) & 0x7Fu) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
  else *where++ = (value >> 28) | 0x80u, *where++ = ((value >> 21) & 0x7Fu) | 0x80u, *where++ = ((value >> 14) & 0x7Fu) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
}

template <>
inline uint32_t vli<uint32_t>::decode(const char*& from) {
  uint32_t value = 0;
  while (((unsigned char)(*from)) & 0x80u) value = (value << 7) | (((unsigned char)(*from++)) ^ 0x80u);
  value = (value << 7) | ((unsigned char)(*from++));
  return value;
}

} // namespace morphodita

/////////
// File: morphodita/tagger/feature_sequences.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
typedef int32_t feature_sequence_score;
typedef int64_t feature_sequences_score;

struct feature_sequence_element {
  elementary_feature_type type;
  int elementary_index;
  int sequence_index;

  feature_sequence_element() {}
  feature_sequence_element(elementary_feature_type type, int elementary_index, int sequence_index) : type(type), elementary_index(elementary_index), sequence_index(sequence_index) {}
};

struct feature_sequence {
  vector<feature_sequence_element> elements;
  int dependant_range = 1;
};

template <class ElementaryFeatures, class Map>
class feature_sequences {
 public:
  typedef typename ElementaryFeatures::per_form_features per_form_features;
  typedef typename ElementaryFeatures::per_tag_features per_tag_features;
  typedef typename ElementaryFeatures::dynamic_features dynamic_features;

  void parse(int window_size, istream& is);
  bool load(istream& is);
  bool save(ostream& os);

  struct cache;

  inline void initialize_sentence(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, cache& c) const;
  inline void compute_dynamic_features(int form_index, int tag_index, const dynamic_features* prev_dynamic, dynamic_features& dynamic, cache& c) const;
  inline feature_sequences_score score(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, cache& c) const;
  void feature_keys(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, vector<string>& keys, cache& c) const;

  ElementaryFeatures elementary;
  vector<Map> scores;
  vector<feature_sequence> sequences;
};

class persistent_feature_sequence_map : public persistent_unordered_map {
 public:
  persistent_feature_sequence_map() : persistent_unordered_map() {}
  persistent_feature_sequence_map(const persistent_unordered_map&& map) : persistent_unordered_map(map) {}

  feature_sequence_score score(const char* feature, int len) const {
    auto* it = at_typed<feature_sequence_score>(feature, len);
    return it ? *it : 0;
  }
};

template <class ElementaryFeatures> using persistent_feature_sequences = feature_sequences<ElementaryFeatures, persistent_feature_sequence_map>;

// Definitions
template <class ElementaryFeatures, class Map>
inline bool feature_sequences<ElementaryFeatures, Map>::load(istream& is) {
  if (!elementary.load(is)) return false;

  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    sequences.resize(data.next_1B());
    for (auto&& sequence : sequences) {
      sequence.dependant_range = data.next_4B();
      sequence.elements.resize(data.next_1B());
      for (auto&& element : sequence.elements) {
        element.type = elementary_feature_type(data.next_4B());
        element.elementary_index = data.next_4B();
        element.sequence_index = data.next_4B();
      }
    }

    scores.resize(data.next_1B());
    for (auto&& score : scores)
      score.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

template <class ElementaryFeatures, class Map>
struct feature_sequences<ElementaryFeatures, Map>::cache {
  const vector<string_piece>* forms;
  const vector<vector<tagged_lemma>>* analyses;
  vector<per_form_features> elementary_per_form;
  vector<vector<per_tag_features>> elementary_per_tag;

  struct cache_element {
    vector<char> key;
    int key_size;
    feature_sequence_score score;

    cache_element(int elements) : key(vli<elementary_feature_value>::max_length() * elements), key_size(0), score(0) {}
  };
  vector<cache_element> caches;
  vector<const per_tag_features*> window;
  vector<char> key;
  feature_sequences_score score;

  cache(const feature_sequences<ElementaryFeatures, Map>& self) : score(0) {
    caches.reserve(self.sequences.size());
    int max_sequence_elements = 0, max_window_size = 1;
    for (auto&& sequence : self.sequences) {
      caches.emplace_back(sequence.elements.size());
      if (int(sequence.elements.size()) > max_sequence_elements) max_sequence_elements = sequence.elements.size();
      for (auto&& element : sequence.elements)
        if (element.type == PER_TAG && 1 - element.sequence_index > max_window_size)
          max_window_size = 1 - element.sequence_index;
    }
    key.resize(max_sequence_elements * vli<elementary_feature_value>::max_length());
    window.resize(max_window_size);
  }
};

template <class ElementaryFeatures, class Map>
void feature_sequences<ElementaryFeatures, Map>::initialize_sentence(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, cache& c) const {
  // Store forms and forms_size
  c.forms = &forms;
  c.analyses = &analyses;

  // Enlarge elementary features vectors if needed
  if (forms.size() > c.elementary_per_form.size()) c.elementary_per_form.resize(forms.size() * 2);
  if (forms.size() > c.elementary_per_tag.size()) c.elementary_per_tag.resize(forms.size() * 2);
  for (unsigned i = 0; i < forms.size(); i++)
    if (analyses[i].size() > c.elementary_per_tag[i].size())
      c.elementary_per_tag[i].resize(analyses[i].size() * 2);

  // Compute elementary features
  elementary.compute_features(forms, analyses, c.elementary_per_form, c.elementary_per_tag);

  // Clear score cache, because scores may have been modified
  c.score = 0;
  for (auto&& cache : c.caches)
    cache.key_size = cache.score = 0;
}

template <class ElementaryFeatures, class Map>
void feature_sequences<ElementaryFeatures, Map>::compute_dynamic_features(int form_index, int tag_index, const dynamic_features* prev_dynamic, dynamic_features& dynamic, cache& c) const {
  elementary.compute_dynamic_features((*c.analyses)[form_index][tag_index], c.elementary_per_form[form_index], c.elementary_per_tag[form_index][tag_index], form_index > 0 ? prev_dynamic : nullptr, dynamic);
}

template <class ElementaryFeatures, class Map>
feature_sequences_score feature_sequences<ElementaryFeatures, Map>::score(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, cache& c) const {
  // Start by creating a window of per_tag_features*
  for (int i = 0; i < int(c.window.size()) && form_index - i >= 0; i++)
    c.window[i] = &c.elementary_per_tag[form_index - i][tags_window[i]];

  // Compute the score
  feature_sequences_score result = c.score;
  for (unsigned i = 0; i < sequences.size(); i++) {
    if (tags_unchanged >= sequences[i].dependant_range)
      break;

    char* key = c.key.data();
    for (unsigned j = 0; j < sequences[i].elements.size(); j++) {
      auto& element = sequences[i].elements[j];
      elementary_feature_value value;

      switch (element.type) {
        case PER_FORM:
          value = form_index + element.sequence_index < 0 || unsigned(form_index + element.sequence_index) >= c.forms->size() ? elementary_feature_empty : c.elementary_per_form[form_index + element.sequence_index].values[element.elementary_index];
          break;
        case PER_TAG:
          value = form_index + element.sequence_index < 0 ? elementary_feature_empty : c.window[-element.sequence_index]->values[element.elementary_index];
          break;
        case DYNAMIC:
        default:
          value = dynamic.values[element.elementary_index];
      }

      if (value == elementary_feature_unknown) {
        key = c.key.data();
        break;
      }
      vli<elementary_feature_value>::encode(value, key);
    }

    result -= c.caches[i].score;
    int key_size = key - c.key.data();
    if (!key_size) {
      c.caches[i].score = 0;
      c.caches[i].key_size = 0;
    } else if (key_size != c.caches[i].key_size || !small_memeq(c.key.data(), c.caches[i].key.data(), key_size)) {
      c.caches[i].score = scores[i].score(c.key.data(), key_size);
      c.caches[i].key_size = key_size;
      small_memcpy(c.caches[i].key.data(), c.key.data(), key_size);
    }
    result += c.caches[i].score;
  }

  c.score = result;
  return result;
}

template <class ElementaryFeatures, class Map>
void feature_sequences<ElementaryFeatures, Map>::feature_keys(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, vector<string>& keys, cache& c) const {
  score(form_index, tags_window, tags_unchanged, dynamic, c);

  keys.resize(c.caches.size());
  for (unsigned i = 0; i < c.caches.size(); i++)
    keys[i].assign(c.caches[i].key.data(), c.caches[i].key_size);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/viterbi.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class FeatureSequences>
class viterbi {
 public:
  viterbi(const FeatureSequences& features, int decoding_order, int window_size)
      : features(features), decoding_order(decoding_order), window_size(window_size) {}

  struct cache;
  void tag(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, cache& c, vector<int>& tags) const;

 private:
  struct node;

  const FeatureSequences& features;
  int decoding_order, window_size;
};

// Definitions
template <class FeatureSequences>
struct viterbi<FeatureSequences>::cache {
  vector<node> nodes;
  typename FeatureSequences::cache features_cache;

  cache(const viterbi<FeatureSequences>& self) : features_cache(self.features) {}
};

template <class FeatureSequences>
struct viterbi<FeatureSequences>::node {
  int tag;
  int prev;
  feature_sequences_score score;
  typename FeatureSequences::dynamic_features dynamic;
};

template <class FeatureSequences>
void viterbi<FeatureSequences>::tag(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, cache& c, vector<int>& tags) const {
  if (!forms.size()) return;

  // Count number of nodes and allocate
  unsigned nodes = 0;
  for (unsigned i = 0, states = 1; i < forms.size(); i++) {
    if (analyses[i].empty()) return;
    states = (i+1 >= unsigned(decoding_order) ? states / analyses[i-decoding_order+1].size() : states) * analyses[i].size();
    nodes += states;
  }
  if (nodes > c.nodes.size()) c.nodes.resize(nodes);

  // Init feature sequences
  features.initialize_sentence(forms, analyses, c.features_cache);

  int window_stack[16]; vector<int> window_heap;
  int* window = window_size <= 16 ? window_stack : (window_heap.resize(window_size), window_heap.data());
  typename FeatureSequences::dynamic_features dynamic;
  feature_sequences_score score;

  // Compute all nodes score
  int nodes_prev = -1, nodes_now = 0;
  for (unsigned i = 0; i < forms.size(); i++) {
    int nodes_next = nodes_now;

    for (int j = 0; j < window_size; j++) window[j] = -1;
    for (int tag = 0; tag < int(analyses[i].size()); tag++)
      for (int prev = nodes_prev; prev < nodes_now; prev++) {
        // Compute predecessors and number of unchanges
        int same_tags = window[0] == tag;
        window[0] = tag;
        for (int p = prev, n = 1; p >= 0 && n < window_size; p = c.nodes[p].prev, n++) {
          same_tags += same_tags == n && window[n] == c.nodes[p].tag;
          window[n] = c.nodes[p].tag;
        }

        // Compute dynamic elementary features and score
        features.compute_dynamic_features(i, tag, prev >= 0 ? &c.nodes[prev].dynamic : nullptr, dynamic, c.features_cache);
        score = (nodes_prev + 1 == nodes_now && analyses[i].size() == 1 ? 0 : features.score(i, window, same_tags, dynamic, c.features_cache)) +
            (prev >= 0 ? c.nodes[prev].score : 0);

        // Update existing node or create a new one
        if (same_tags >= decoding_order-1) {
          if (score <= c.nodes[nodes_next-1].score) continue;
          nodes_next--;
        }
        c.nodes[nodes_next].tag = tag;
        c.nodes[nodes_next].prev = prev;
        c.nodes[nodes_next].score = score;
        c.nodes[nodes_next++].dynamic = dynamic;
      }

    nodes_prev = nodes_now;
    nodes_now = nodes_next;
  }

  // Choose the best ending node
  int best = nodes_prev;
  for (int node = nodes_prev + 1; node < nodes_now; node++)
    if (c.nodes[node].score > c.nodes[best].score)
      best = node;

  for (int i = forms.size() - 1; i >= 0; i--, best = c.nodes[best].prev)
    tags[i] = c.nodes[best].tag;
}

} // namespace morphodita

/////////
// File: morphodita/tagger/conllu_elementary_features.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class Map>
class conllu_elementary_features : public elementary_features<Map> {
 public:
  conllu_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_FORM, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, PREFIX5, PREFIX6, PREFIX7, PREFIX8, PREFIX9, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, SUFFIX5, SUFFIX6, SUFFIX7, SUFFIX8, SUFFIX9, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG_UPOS, TAG_CASE, TAG_GENDER, TAG_NUMBER, TAG_NEGATIVE, TAG_PERSON, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_FORM, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_FORM, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_PREFIX5, MAP_PREFIX6, MAP_PREFIX7, MAP_PREFIX8, MAP_PREFIX9, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_SUFFIX5, MAP_SUFFIX6, MAP_SUFFIX7, MAP_SUFFIX8, MAP_SUFFIX9, MAP_TAG, MAP_TAG_UPOS, MAP_TAG_CASE, MAP_TAG_GENDER, MAP_TAG_NUMBER, MAP_TAG_NEGATIVE, MAP_TAG_PERSON, MAP_LEMMA, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector<elementary_feature_description> descriptions;

  void compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features<Map>::maps;
};

typedef conllu_elementary_features<persistent_elementary_feature_map> persistent_conllu_elementary_features;

// Definitions
template <class Map>
conllu_elementary_features<Map>::conllu_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template <class Map>
vector<elementary_feature_description> conllu_elementary_features<Map>::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbForm", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_FORM, MAP_FORM},
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Prefix5", PER_FORM, ONLY_CURRENT, PREFIX5, MAP_PREFIX5},
  {"Prefix6", PER_FORM, ONLY_CURRENT, PREFIX6, MAP_PREFIX6},
  {"Prefix7", PER_FORM, ONLY_CURRENT, PREFIX7, MAP_PREFIX7},
  {"Prefix8", PER_FORM, ONLY_CURRENT, PREFIX8, MAP_PREFIX8},
  {"Prefix9", PER_FORM, ONLY_CURRENT, PREFIX9, MAP_PREFIX9},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},
  {"Suffix5", PER_FORM, ONLY_CURRENT, SUFFIX5, MAP_SUFFIX5},
  {"Suffix6", PER_FORM, ONLY_CURRENT, SUFFIX6, MAP_SUFFIX6},
  {"Suffix7", PER_FORM, ONLY_CURRENT, SUFFIX7, MAP_SUFFIX7},
  {"Suffix8", PER_FORM, ONLY_CURRENT, SUFFIX8, MAP_SUFFIX8},
  {"Suffix9", PER_FORM, ONLY_CURRENT, SUFFIX9, MAP_SUFFIX9},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"TagUPos", PER_TAG, ANY_OFFSET, TAG_UPOS, MAP_TAG_UPOS},
  {"TagCase", PER_TAG, ANY_OFFSET, TAG_CASE, MAP_TAG_CASE},
  {"TagGender", PER_TAG, ANY_OFFSET, TAG_GENDER, MAP_TAG_GENDER},
  {"TagNumber", PER_TAG, ANY_OFFSET, TAG_NUMBER, MAP_TAG_NUMBER},
  {"TagNegative", PER_TAG, ANY_OFFSET, TAG_NEGATIVE, MAP_TAG_NEGATIVE},
  {"TagPerson", PER_TAG, ANY_OFFSET, TAG_PERSON, MAP_TAG_PERSON},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbForm", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_FORM, MAP_FORM},
};

template <class Map>
void conllu_elementary_features<Map>::compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_form = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      const string& tag = analyses[i][j].tag;
      const string& lemma = analyses[i][j].lemma;

      // Tag consists of three parts separated by tag[0] character
      // - first is TAG_UPOS,
      // - second is TAG_LPOS,
      // - then there is any number of | separated named fields in format Name=Value
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(tag.c_str(), tag.size());
      per_tag[i][j].values[TAG_UPOS] = per_tag[i][j].values[TAG_CASE] = per_tag[i][j].values[TAG_GENDER] = elementary_feature_empty;
      per_tag[i][j].values[TAG_NUMBER] = per_tag[i][j].values[TAG_NEGATIVE] = per_tag[i][j].values[TAG_PERSON] = elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(lemma.c_str(), lemma.size());

      char separator = tag[0];
      size_t index = tag.find(separator, 1);
      if (index == string::npos) index = tag.size();
      per_tag[i][j].values[TAG_UPOS] = maps[MAP_TAG_UPOS].value(tag.c_str() + (index ? 1 : 0), index - (index ? 1 : 0));

      if (index < tag.size()) index++;
      if (index < tag.size()) index = tag.find(separator, index);
      if (index < tag.size()) index++;
      for (size_t length; index < tag.size(); index += length + 1) {
        length = tag.find('|', index);
        length = (length == string::npos ? tag.size() : length) - index;

        for (size_t equal_sign = 0; equal_sign + 1 < length; equal_sign++)
          if (tag[index + equal_sign] == '=') {
            int value = -1, map;
            switch (equal_sign) {
              case 4:
                if (tag.compare(index, equal_sign, "Case") == 0) value = TAG_CASE, map = MAP_TAG_CASE;
                break;
              case 6:
                if (tag.compare(index, equal_sign, "Gender") == 0) value = TAG_GENDER, map = MAP_TAG_GENDER;
                if (tag.compare(index, equal_sign, "Number") == 0) value = TAG_NUMBER, map = MAP_TAG_NUMBER;
                if (tag.compare(index, equal_sign, "Person") == 0) value = TAG_PERSON, map = MAP_TAG_PERSON;
                break;
              case 8:
                if (tag.compare(index, equal_sign, "Negative") == 0) value = TAG_NEGATIVE, map = MAP_TAG_NEGATIVE;
                break;
            }

            if (value >= 0)
              per_tag[i][j].values[value] = maps[map].value(tag.c_str() + index + equal_sign + 1, length - equal_sign - 1);
            break;
          }
      }

      if (tag.size() >= 2 && tag[1] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_FORM] = following_verb_form;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_form = per_form[i].values[FORM];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_unknown;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_unknown;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_unknown;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_unknown;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_empty;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_empty;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_empty;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_empty;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[18] = {0, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, 0, 0, 0, 0, 0, 0, 0, 0}; // careful here regarding forms shorter than 9 characters
      int index = 0;
      while (form.len) {
        indices[(index++) % 18] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 10 || (!form.len && index < 10)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
          per_form[i].values[PREFIX5] = maps[MAP_PREFIX5].value(form_start, indices[5]);
          per_form[i].values[PREFIX6] = maps[MAP_PREFIX6].value(form_start, indices[6]);
          per_form[i].values[PREFIX7] = maps[MAP_PREFIX7].value(form_start, indices[7]);
          per_form[i].values[PREFIX8] = maps[MAP_PREFIX8].value(form_start, indices[8]);
          per_form[i].values[PREFIX9] = maps[MAP_PREFIX9].value(form_start, indices[9]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index+18-1) % 18], form.str - form_start - indices[(index+18-1) % 18]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index+18-2) % 18], form.str - form_start - indices[(index+18-2) % 18]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index+18-3) % 18], form.str - form_start - indices[(index+18-3) % 18]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index+18-4) % 18], form.str - form_start - indices[(index+18-4) % 18]);
      per_form[i].values[SUFFIX5] = maps[MAP_SUFFIX5].value(form_start + indices[(index+18-5) % 18], form.str - form_start - indices[(index+18-5) % 18]);
      per_form[i].values[SUFFIX6] = maps[MAP_SUFFIX6].value(form_start + indices[(index+18-6) % 18], form.str - form_start - indices[(index+18-6) % 18]);
      per_form[i].values[SUFFIX7] = maps[MAP_SUFFIX7].value(form_start + indices[(index+18-7) % 18], form.str - form_start - indices[(index+18-7) % 18]);
      per_form[i].values[SUFFIX8] = maps[MAP_SUFFIX8].value(form_start + indices[(index+18-8) % 18], form.str - form_start - indices[(index+18-8) % 18]);
      per_form[i].values[SUFFIX9] = maps[MAP_SUFFIX9].value(form_start + indices[(index+18-9) % 18], form.str - form_start - indices[(index+18-9) % 18]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template <class Map>
void conllu_elementary_features<Map>::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_FORM] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_FORM];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_FORM] = elementary_feature_empty;
  }

  if (tag.tag.size() >= 2 && tag.tag[1] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_FORM] = per_form.values[FORM];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_FORM] = dynamic.values[PREVIOUS_VERB_FORM];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/czech_elementary_features.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class Map>
class czech_elementary_features : public elementary_features<Map> {
 public:
  czech_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_LEMMA, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG3, TAG5, TAG25, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_LEMMA, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_LEMMA, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_LEMMA, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_TAG, MAP_TAG3, MAP_TAG5, MAP_TAG25, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector<elementary_feature_description> descriptions;

  void compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features<Map>::maps;
};

typedef czech_elementary_features<persistent_elementary_feature_map> persistent_czech_elementary_features;

// Definitions
template <class Map>
czech_elementary_features<Map>::czech_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template <class Map>
vector<elementary_feature_description> czech_elementary_features<Map>::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbLemma", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_LEMMA, MAP_LEMMA },
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"Tag3", PER_TAG, ANY_OFFSET, TAG3, MAP_TAG3},
  {"Tag5", PER_TAG, ANY_OFFSET, TAG5, MAP_TAG5},
  {"Tag25", PER_TAG, ANY_OFFSET, TAG25, MAP_TAG25},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbLemma", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_LEMMA, MAP_LEMMA}
};

template <class Map>
void czech_elementary_features<Map>::compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_lemma = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      char tag25[2];
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(analyses[i][j].tag.c_str(), analyses[i][j].tag.size());
      per_tag[i][j].values[TAG3] = analyses[i][j].tag.size() >= 3 ? maps[MAP_TAG3].value(analyses[i][j].tag.c_str() + 2, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG5] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG5].value(analyses[i][j].tag.c_str() + 4, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG25] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG25].value((tag25[0] = analyses[i][j].tag[1], tag25[1] = analyses[i][j].tag[4], tag25), 2) : elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == analyses[i][j].lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(analyses[i][j].lemma.c_str(), analyses[i][j].lemma.size());

      if (analyses[i][j].tag[0] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = analyses[i][j].tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && analyses[i][j].lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_LEMMA] = following_verb_lemma;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_lemma = per_tag[i][verb_candidate].values[LEMMA];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = per_form[i].values[PREFIX4] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = per_form[i].values[SUFFIX4] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = per_form[i].values[PREFIX4] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = per_form[i].values[SUFFIX4] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[8] = {0, form.len, form.len, form.len, form.len, 0, 0, 0}; // careful here regarding forms shorter than 4 characters
      int index = 0;
      while (form.len) {
        indices[(index++)&7] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 5 || (!form.len && index < 5)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index-1)&7], form.str - form_start - indices[(index-1)&7]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index-2)&7], form.str - form_start - indices[(index-2)&7]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index-3)&7], form.str - form_start - indices[(index-3)&7]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index-4)&7], form.str - form_start - indices[(index-4)&7]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template <class Map>
void czech_elementary_features<Map>::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& /*per_form*/, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_LEMMA] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_LEMMA];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_LEMMA] = elementary_feature_empty;
  }

  if (tag.tag[0] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = per_tag.values[LEMMA];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = dynamic.values[PREVIOUS_VERB_LEMMA];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/generic_elementary_features.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class Map>
class generic_elementary_features : public elementary_features<Map> {
 public:
  generic_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_LEMMA, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, PREFIX5, PREFIX6, PREFIX7, PREFIX8, PREFIX9, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, SUFFIX5, SUFFIX6, SUFFIX7, SUFFIX8, SUFFIX9, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG1, TAG2, TAG3, TAG4, TAG5, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_LEMMA, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_LEMMA, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_PREFIX5, MAP_PREFIX6, MAP_PREFIX7, MAP_PREFIX8, MAP_PREFIX9, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_SUFFIX5, MAP_SUFFIX6, MAP_SUFFIX7, MAP_SUFFIX8, MAP_SUFFIX9, MAP_TAG, MAP_TAG1, MAP_TAG2, MAP_TAG3, MAP_TAG4, MAP_TAG5, MAP_LEMMA, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector<elementary_feature_description> descriptions;

  void compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features<Map>::maps;
};

typedef generic_elementary_features<persistent_elementary_feature_map> persistent_generic_elementary_features;

// Definitions
template <class Map>
generic_elementary_features<Map>::generic_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template <class Map>
vector<elementary_feature_description> generic_elementary_features<Map>::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbLemma", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_LEMMA, MAP_LEMMA },
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Prefix5", PER_FORM, ONLY_CURRENT, PREFIX5, MAP_PREFIX5},
  {"Prefix6", PER_FORM, ONLY_CURRENT, PREFIX6, MAP_PREFIX6},
  {"Prefix7", PER_FORM, ONLY_CURRENT, PREFIX7, MAP_PREFIX7},
  {"Prefix8", PER_FORM, ONLY_CURRENT, PREFIX8, MAP_PREFIX8},
  {"Prefix9", PER_FORM, ONLY_CURRENT, PREFIX9, MAP_PREFIX9},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},
  {"Suffix5", PER_FORM, ONLY_CURRENT, SUFFIX5, MAP_SUFFIX5},
  {"Suffix6", PER_FORM, ONLY_CURRENT, SUFFIX6, MAP_SUFFIX6},
  {"Suffix7", PER_FORM, ONLY_CURRENT, SUFFIX7, MAP_SUFFIX7},
  {"Suffix8", PER_FORM, ONLY_CURRENT, SUFFIX8, MAP_SUFFIX8},
  {"Suffix9", PER_FORM, ONLY_CURRENT, SUFFIX9, MAP_SUFFIX9},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"Tag1", PER_TAG, ANY_OFFSET, TAG1, MAP_TAG1},
  {"Tag2", PER_TAG, ANY_OFFSET, TAG2, MAP_TAG2},
  {"Tag3", PER_TAG, ANY_OFFSET, TAG3, MAP_TAG3},
  {"Tag4", PER_TAG, ANY_OFFSET, TAG4, MAP_TAG4},
  {"Tag5", PER_TAG, ANY_OFFSET, TAG5, MAP_TAG5},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbLemma", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_LEMMA, MAP_LEMMA}
};

template <class Map>
void generic_elementary_features<Map>::compute_features(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<per_form_features>& per_form, vector<vector<per_tag_features>>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_lemma = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(analyses[i][j].tag.c_str(), analyses[i][j].tag.size());
      per_tag[i][j].values[TAG1] = analyses[i][j].tag.size() >= 1 ? maps[MAP_TAG1].value(analyses[i][j].tag.c_str() + 0, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG2] = analyses[i][j].tag.size() >= 2 ? maps[MAP_TAG2].value(analyses[i][j].tag.c_str() + 1, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG3] = analyses[i][j].tag.size() >= 3 ? maps[MAP_TAG3].value(analyses[i][j].tag.c_str() + 2, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG4] = analyses[i][j].tag.size() >= 4 ? maps[MAP_TAG4].value(analyses[i][j].tag.c_str() + 3, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG5] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG5].value(analyses[i][j].tag.c_str() + 4, 1) : elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == analyses[i][j].lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(analyses[i][j].lemma.c_str(), analyses[i][j].lemma.size());

      if (analyses[i][j].tag[0] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = analyses[i][j].tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && analyses[i][j].lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_LEMMA] = following_verb_lemma;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_lemma = per_tag[i][verb_candidate].values[LEMMA];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_unknown;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_unknown;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_unknown;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_unknown;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_empty;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_empty;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_empty;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_empty;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[18] = {0, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, 0, 0, 0, 0, 0, 0, 0, 0}; // careful here regarding forms shorter than 9 characters
      int index = 0;
      while (form.len) {
        indices[(index++) % 18] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 10 || (!form.len && index < 10)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
          per_form[i].values[PREFIX5] = maps[MAP_PREFIX5].value(form_start, indices[5]);
          per_form[i].values[PREFIX6] = maps[MAP_PREFIX6].value(form_start, indices[6]);
          per_form[i].values[PREFIX7] = maps[MAP_PREFIX7].value(form_start, indices[7]);
          per_form[i].values[PREFIX8] = maps[MAP_PREFIX8].value(form_start, indices[8]);
          per_form[i].values[PREFIX9] = maps[MAP_PREFIX9].value(form_start, indices[9]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index+18-1) % 18], form.str - form_start - indices[(index+18-1) % 18]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index+18-2) % 18], form.str - form_start - indices[(index+18-2) % 18]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index+18-3) % 18], form.str - form_start - indices[(index+18-3) % 18]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index+18-4) % 18], form.str - form_start - indices[(index+18-4) % 18]);
      per_form[i].values[SUFFIX5] = maps[MAP_SUFFIX5].value(form_start + indices[(index+18-5) % 18], form.str - form_start - indices[(index+18-5) % 18]);
      per_form[i].values[SUFFIX6] = maps[MAP_SUFFIX6].value(form_start + indices[(index+18-6) % 18], form.str - form_start - indices[(index+18-6) % 18]);
      per_form[i].values[SUFFIX7] = maps[MAP_SUFFIX7].value(form_start + indices[(index+18-7) % 18], form.str - form_start - indices[(index+18-7) % 18]);
      per_form[i].values[SUFFIX8] = maps[MAP_SUFFIX8].value(form_start + indices[(index+18-8) % 18], form.str - form_start - indices[(index+18-8) % 18]);
      per_form[i].values[SUFFIX9] = maps[MAP_SUFFIX9].value(form_start + indices[(index+18-9) % 18], form.str - form_start - indices[(index+18-9) % 18]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template <class Map>
void generic_elementary_features<Map>::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& /*per_form*/, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_LEMMA] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_LEMMA];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_LEMMA] = elementary_feature_empty;
  }

  if (tag.tag[0] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = per_tag.values[LEMMA];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = dynamic.values[PREVIOUS_VERB_LEMMA];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/perceptron_tagger.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template<class FeatureSequences>
class perceptron_tagger : public tagger {
 public:
  perceptron_tagger(int decoding_order, int window_size);

  bool load(istream& is);
  virtual const morpho* get_morpho() const override;
  virtual void tag(const vector<string_piece>& forms, vector<tagged_lemma>& tags, morpho::guesser_mode guesser = morpho::guesser_mode(-1)) const override;
  virtual void tag_analyzed(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<int>& tags) const override;

 private:
  int decoding_order, window_size;

  unique_ptr<morpho> dict;
  bool use_guesser;
  FeatureSequences features;
  typedef viterbi<FeatureSequences> viterbi_decoder;
  viterbi_decoder decoder;
  struct cache {
    vector<string_piece> forms;
    vector<vector<tagged_lemma>> analyses;
    vector<int> tags;
    typename viterbi_decoder::cache decoder_cache;

    cache(const perceptron_tagger<FeatureSequences>& self) : decoder_cache(self.decoder) {}
  };

  mutable threadsafe_stack<cache> caches;
};

// Definitions

template<class FeatureSequences>
perceptron_tagger<FeatureSequences>::perceptron_tagger(int decoding_order, int window_size)
  : decoding_order(decoding_order), window_size(window_size), decoder(features, decoding_order, window_size) {}

template<class FeatureSequences>
bool perceptron_tagger<FeatureSequences>::load(istream& is) {
  if (dict.reset(morpho::load(is)), !dict) return false;
  use_guesser = is.get();
  if (!features.load(is)) return false;
  return true;
}

template<class FeatureSequences>
const morpho* perceptron_tagger<FeatureSequences>::get_morpho() const {
  return dict.get();
}

template<class FeatureSequences>
void perceptron_tagger<FeatureSequences>::tag(const vector<string_piece>& forms, vector<tagged_lemma>& tags, morpho::guesser_mode guesser) const {
  tags.clear();
  if (!dict) return;

  cache* c = caches.pop();
  if (!c) c = new cache(*this);

  c->forms.resize(forms.size());
  if (c->analyses.size() < forms.size()) c->analyses.resize(forms.size());
  for (unsigned i = 0; i < forms.size(); i++) {
    c->forms[i] = forms[i];
    c->forms[i].len = dict->raw_form_len(forms[i]);
    dict->analyze(forms[i], guesser >= 0 ? guesser : use_guesser ? morpho::GUESSER : morpho::NO_GUESSER, c->analyses[i]);
  }

  if (c->tags.size() < forms.size()) c->tags.resize(forms.size() * 2);
  decoder.tag(c->forms, c->analyses, c->decoder_cache, c->tags);

  for (unsigned i = 0; i < forms.size(); i++)
    tags.emplace_back(c->analyses[i][c->tags[i]]);

  caches.push(c);
}

template<class FeatureSequences>
void perceptron_tagger<FeatureSequences>::tag_analyzed(const vector<string_piece>& forms, const vector<vector<tagged_lemma>>& analyses, vector<int>& tags) const {
  tags.clear();

  cache* c = caches.pop();
  if (!c) c = new cache(*this);

  tags.resize(forms.size());
  decoder.tag(forms, analyses, c->decoder_cache, tags);

  caches.push(c);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger_ids.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagger_ids {
 public:
  enum tagger_id {
    CZECH2 = 0, CZECH3 = 1, CZECH2_3 = 6,
    /* 2 was used internally for ENGLISH3, but never released publicly */
    GENERIC2 = 3, GENERIC3 = 4, GENERIC4 = 5, GENERIC2_3 = 7,
    CONLLU2 = 8, CONLLU2_3 = 9, CONLLU3 = 10,
  };

  static bool parse(const string& str, tagger_id& id) {
    if (str == "czech2") return id = CZECH2, true;
    if (str == "czech2_3") return id = CZECH2_3, true;
    if (str == "czech3") return id = CZECH3, true;
    if (str == "generic2") return id = GENERIC2, true;
    if (str == "generic2_3") return id = GENERIC2_3, true;
    if (str == "generic3") return id = GENERIC3, true;
    if (str == "generic4") return id = GENERIC4, true;
    if (str == "conllu2") return id = CONLLU2, true;
    if (str == "conllu2_3") return id = CONLLU2_3, true;
    if (str == "conllu3") return id = CONLLU3, true;
    return false;
  }

  static int decoding_order(tagger_id id) {
    switch (id) {
      case CZECH2: return 2;
      case CZECH2_3: return 2;
      case CZECH3: return 3;
      case GENERIC2: return 2;
      case GENERIC2_3: return 2;
      case GENERIC3: return 3;
      case GENERIC4: return 4;
      case CONLLU2: return 2;
      case CONLLU2_3: return 2;
      case CONLLU3: return 3;
    }
    return 0;
  }

  static int window_size(tagger_id id) {
    switch (id) {
      case CZECH2_3: return 3;
      case GENERIC2_3: return 3;
      case CONLLU2_3: return 3;
      default: break;
    }
    return decoding_order(id);
  }
};

typedef tagger_ids::tagger_id tagger_id;

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tagger* tagger::load(istream& is) {
  tagger_id id = tagger_id(is.get());
  switch (id) {
    case tagger_ids::CZECH2:
    case tagger_ids::CZECH2_3:
    case tagger_ids::CZECH3:
      {
        auto res = new_unique_ptr<perceptron_tagger<persistent_feature_sequences<persistent_czech_elementary_features>>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
    case tagger_ids::GENERIC2:
    case tagger_ids::GENERIC2_3:
    case tagger_ids::GENERIC3:
    case tagger_ids::GENERIC4:
      {
        auto res = new_unique_ptr<perceptron_tagger<persistent_feature_sequences<persistent_generic_elementary_features>>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
    case tagger_ids::CONLLU2:
    case tagger_ids::CONLLU2_3:
    case tagger_ids::CONLLU3:
      {
        auto res = new_unique_ptr<perceptron_tagger<persistent_feature_sequences<persistent_conllu_elementary_features>>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
  }

  return nullptr;
}

tagger* tagger::load(const char* fname) {
  ifstream f(fname, ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

tokenizer* tagger::new_tokenizer() const {
  auto morpho = get_morpho();
  return morpho ? morpho->new_tokenizer() : nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/tagset_converter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagset_converter {
 public:
  virtual ~tagset_converter() {}

  // Convert a tag-lemma pair to a different tag set.
  virtual void convert(tagged_lemma& tagged_lemma) const = 0;
  // Convert a result of analysis to a different tag set. Apart from calling
  // convert, any repeated entry is removed.
  virtual void convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const = 0;
  // Convert a result of generation to a different tag set. Apart from calling
  // convert, any repeated entry is removed.
  virtual void convert_generated(vector<tagged_lemma_forms>& forms) const = 0;

  // Static factory methods
  static tagset_converter* new_identity_converter();

  static tagset_converter* new_pdt_to_conll2009_converter();
  static tagset_converter* new_strip_lemma_comment_converter(const morpho& dictionary);
  static tagset_converter* new_strip_lemma_id_converter(const morpho& dictionary);
};

// Helper method for creating tagset_converter from instance name.
tagset_converter* new_tagset_converter(const string& name, const morpho& dictionary);

// Helper methods making sure remapped results are unique.
void tagset_converter_unique_analyzed(vector<tagged_lemma>& tagged_lemmas);
void tagset_converter_unique_generated(vector<tagged_lemma_forms>& forms);

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/identity_tagset_converter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class identity_tagset_converter : public tagset_converter {
 public:
  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const override;
  virtual void convert_generated(vector<tagged_lemma_forms>& forms) const override;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/identity_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void identity_tagset_converter::convert(tagged_lemma& /*tagged_lemma*/) const {}

void identity_tagset_converter::convert_analyzed(vector<tagged_lemma>& /*tagged_lemmas*/) const {}

void identity_tagset_converter::convert_generated(vector<tagged_lemma_forms>& /*forms*/) const {}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/pdt_to_conll2009_tagset_converter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class pdt_to_conll2009_tagset_converter : public tagset_converter {
 public:
  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const override;
  virtual void convert_generated(vector<tagged_lemma_forms>& forms) const override;

 private:
  inline void convert_tag(const string& lemma, string& tag) const;
  inline bool convert_lemma(string& lemma) const;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/pdt_to_conll2009_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char* names[15] = {"POS", "SubPOS", "Gen", "Num", "Cas", "PGe", "PNu", "Per", "Ten", "Gra", "Neg", "Voi", "", "", "Var"};

inline void pdt_to_conll2009_tagset_converter::convert_tag(const string& lemma, string& tag) const {
  char pdt_tag[15];
  strncpy(pdt_tag, tag.c_str(), 15);

  // Clear the tag
  tag.clear();

  // Fill FEAT of filled tag characters
  for (int i = 0; i < 15 && pdt_tag[i]; i++)
    if (pdt_tag[i] != '-') {
      if (!tag.empty()) tag.push_back('|');
      tag.append(names[i]);
      tag.push_back('=');
      tag.push_back(pdt_tag[i]);
    }

  // Try adding Sem FEAT
  for (unsigned i = 0; i + 2 < lemma.size(); i++)
    if (lemma[i] == '_' && lemma[i + 1] == ';') {
      if (!tag.empty()) tag.push_back('|');
      tag.append("Sem=");
      tag.push_back(lemma[i + 2]);
      break;
    }
}

inline bool pdt_to_conll2009_tagset_converter::convert_lemma(string& lemma) const {
  unsigned raw_lemma = czech_lemma_addinfo::raw_lemma_len(lemma);
  return raw_lemma < lemma.size() ? (lemma.resize(raw_lemma), true) : false;
}

void pdt_to_conll2009_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_tag(tagged_lemma.lemma, tagged_lemma.tag);
  convert_lemma(tagged_lemma.lemma);
}

void pdt_to_conll2009_tagset_converter::convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas) {
    convert_tag(tagged_lemma.lemma, tagged_lemma.tag);
    lemma_changed |= convert_lemma(tagged_lemma.lemma);
  }

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void pdt_to_conll2009_tagset_converter::convert_generated(vector<tagged_lemma_forms>& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms) {
    for (auto&& tagged_form : tagged_lemma_forms.forms)
      convert_tag(tagged_lemma_forms.lemma, tagged_form.tag);
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);
  }

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_comment_tagset_converter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class strip_lemma_comment_tagset_converter : public tagset_converter {
 public:
  strip_lemma_comment_tagset_converter(const morpho& dictionary) : dictionary(dictionary) {}

  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const override;
  virtual void convert_generated(vector<tagged_lemma_forms>& forms) const override;

 private:
  inline bool convert_lemma(string& lemma) const;
  const morpho& dictionary;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_comment_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline bool strip_lemma_comment_tagset_converter::convert_lemma(string& lemma) const {
  unsigned lemma_id_len = dictionary.lemma_id_len(lemma);
  return lemma_id_len < lemma.size() ? (lemma.resize(lemma_id_len), true) : false;
}

void strip_lemma_comment_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_lemma(tagged_lemma.lemma);
}

void strip_lemma_comment_tagset_converter::convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas)
    lemma_changed |= convert_lemma(tagged_lemma.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void strip_lemma_comment_tagset_converter::convert_generated(vector<tagged_lemma_forms>& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms)
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_id_tagset_converter.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class strip_lemma_id_tagset_converter : public tagset_converter {
 public:
  strip_lemma_id_tagset_converter(const morpho& dictionary) : dictionary(dictionary) {}

  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const override;
  virtual void convert_generated(vector<tagged_lemma_forms>& forms) const override;

 private:
  inline bool convert_lemma(string& lemma) const;
  const morpho& dictionary;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_id_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline bool strip_lemma_id_tagset_converter::convert_lemma(string& lemma) const {
  unsigned raw_lemma_len = dictionary.raw_lemma_len(lemma);
  return raw_lemma_len < lemma.size() ? (lemma.resize(raw_lemma_len), true) : false;
}

void strip_lemma_id_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_lemma(tagged_lemma.lemma);
}

void strip_lemma_id_tagset_converter::convert_analyzed(vector<tagged_lemma>& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas)
    lemma_changed |= convert_lemma(tagged_lemma.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void strip_lemma_id_tagset_converter::convert_generated(vector<tagged_lemma_forms>& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms)
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/tagset_converter.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tagset_converter* tagset_converter::new_identity_converter() {
  return new identity_tagset_converter();
}

tagset_converter* tagset_converter::new_pdt_to_conll2009_converter() {
  return new pdt_to_conll2009_tagset_converter();
}

tagset_converter* tagset_converter::new_strip_lemma_comment_converter(const morpho& dictionary) {
  return new strip_lemma_comment_tagset_converter(dictionary);
}

tagset_converter* tagset_converter::new_strip_lemma_id_converter(const morpho& dictionary) {
  return new strip_lemma_id_tagset_converter(dictionary);
}

tagset_converter* new_tagset_converter(const string& name, const morpho& dictionary) {
  if (name == "pdt_to_conll2009") return tagset_converter::new_pdt_to_conll2009_converter();
  if (name == "strip_lemma_comment") return tagset_converter::new_strip_lemma_comment_converter(dictionary);
  if (name == "strip_lemma_id") return tagset_converter::new_strip_lemma_id_converter(dictionary);
  return nullptr;
}

void tagset_converter_unique_analyzed(vector<tagged_lemma>& tagged_lemmas) {
  // Remove possible lemma-tag pair duplicates
  struct tagged_lemma_comparator {
    inline static bool eq(const tagged_lemma& a, const tagged_lemma& b) { return a.lemma == b.lemma && a.tag == b.tag; }
    inline static bool lt(const tagged_lemma& a, const tagged_lemma& b) { int lemma_compare = a.lemma.compare(b.lemma); return lemma_compare < 0 || (lemma_compare == 0 && a.tag < b.tag); }
  };

  sort(tagged_lemmas.begin(), tagged_lemmas.end(), tagged_lemma_comparator::lt);
  tagged_lemmas.resize(unique(tagged_lemmas.begin(), tagged_lemmas.end(), tagged_lemma_comparator::eq) - tagged_lemmas.begin());
}

void tagset_converter_unique_generated(vector<tagged_lemma_forms>& forms) {
  // Regroup and if needed remove duplicate form-tag pairs for each lemma
  for (unsigned i = 0; i < forms.size(); i++) {
    bool any_merged = false;
    for (unsigned j = forms.size() - 1; j > i; j--)
      if (forms[j].lemma == forms[i].lemma) {
        // Same lemma was found. Merge form-tag pairs
        for (auto&& tagged_form : forms[j].forms)
          forms[i].forms.emplace_back(move(tagged_form));

        // Remove lemma j by moving it to end and deleting
        if (j < forms.size() - 1) {
          forms[j].lemma.swap(forms[forms.size() - 1].lemma);
          forms[j].forms.swap(forms[forms.size() - 1].forms);
        }
        forms.pop_back();
        any_merged = true;
      }

    if (any_merged && forms[i].forms.size() > 1) {
      // Remove duplicate form-tag pairs
      struct tagged_form_comparator {
        inline static bool eq(const tagged_form& a, const tagged_form& b) { return a.tag == b.tag && a.form == b.form; }
        inline static bool lt(const tagged_form& a, const tagged_form& b) { int tag_compare = a.tag.compare(b.tag); return tag_compare < 0 || (tag_compare == 0 && a.form < b.form); }
      };

      sort(forms[i].forms.begin(), forms[i].forms.end(), tagged_form_comparator::lt);
      forms[i].forms.resize(unique(forms[i].forms.begin(), forms[i].forms.end(), tagged_form_comparator::eq) - forms[i].forms.begin());
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _czech_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2
};

static const char _czech_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const short _czech_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _czech_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _czech_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 51, 54,
	89, 94, 98, 101, 105, 110, 111, 116,
	117, 122, 136, 143, 148, 151, 163
};

static const short _czech_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 43u, 45u,
	159u, 48u, 57u, 159u, 48u, 57u, 9u, 10u,
	13u, 32u, 33u, 44u, 46u, 47u, 63u, 129u,
	131u, 135u, 142u, 147u, 157u, 159u, 160u, 301u,
	557u, 811u, 1067u, 0u, 42u, 48u, 57u, 58u,
	64u, 65u, 90u, 91u, 96u, 97u, 122u, 123u,
	255u, 9u, 10u, 13u, 32u, 147u, 9u, 13u,
	32u, 147u, 9u, 32u, 147u, 9u, 10u, 32u,
	147u, 9u, 10u, 13u, 32u, 147u, 13u, 9u,
	10u, 13u, 32u, 147u, 10u, 9u, 10u, 13u,
	32u, 147u, 13u, 32u, 34u, 39u, 41u, 59u,
	93u, 125u, 139u, 141u, 147u, 161u, 9u, 10u,
	44u, 46u, 69u, 101u, 159u, 48u, 57u, 69u,
	101u, 159u, 48u, 57u, 159u, 48u, 57u, 129u,
	131u, 135u, 151u, 155u, 157u, 65u, 90u, 97u,
	122u, 142u, 143u, 159u, 48u, 57u, 0
};

static const char _czech_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 3, 1, 21,
	5, 4, 3, 4, 5, 1, 5, 1,
	5, 12, 5, 3, 1, 6, 1
};

static const char _czech_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 7,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 1, 3, 1
};

static const unsigned char _czech_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 50, 53,
	82, 88, 93, 97, 102, 108, 110, 116,
	118, 124, 138, 145, 150, 153, 163
};

static const char _czech_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 8,
	8, 0, 8, 8, 0, 10, 11, 12,
	10, 13, 9, 13, 9, 13, 16, 16,
	16, 16, 10, 16, 15, 13, 9, 17,
	9, 17, 9, 15, 9, 16, 9, 16,
	9, 14, 10, 19, 20, 10, 10, 18,
	10, 21, 10, 10, 18, 10, 10, 10,
	18, 10, 21, 10, 10, 18, 10, 22,
	23, 10, 10, 18, 25, 24, 10, 22,
	26, 10, 10, 18, 25, 24, 10, 23,
	26, 10, 10, 18, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 27, 28, 28, 29, 29, 15, 15,
	27, 29, 29, 6, 6, 27, 8, 8,
	27, 16, 16, 16, 16, 16, 16, 16,
	16, 16, 27, 15, 15, 27, 0
};

static const char _czech_tokenizer_trans_targs[] = {
	7, 1, 2, 7, 1, 3, 19, 6,
	20, 7, 8, 12, 16, 17, 0, 18,
	21, 22, 7, 9, 11, 10, 13, 14,
	7, 7, 15, 7, 4, 5
};

static const char _czech_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 4, 0,
	0, 7, 0, 0, 0, 4, 0, 4,
	0, 0, 8, 0, 0, 0, 0, 0,
	9, 10, 0, 11, 0, 0
};

static const char _czech_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 5,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const char _czech_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 6,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const unsigned char _czech_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 0,
	19, 19, 19, 19, 19, 25, 19, 25,
	19, 28, 28, 28, 28, 28, 28
};

static const int czech_tokenizer_start = 7;

// The list of lower cased words that when preceding eos do not end sentence.
// Note: because of VS, we cannot list the abbreviations directly in UTF-8,
// because the compilation of utf-8 encoded sources fail on some locales
// (e.g., Japanese).
// perl -CSD -ple 'use Encode;s/([^[:ascii:]])/join("", map {sprintf "\\%o", ord($_)} split(m@@, encode("utf-8", $1)))/ge'
// perl -CSD -ple 'use Encode;s/\\([0-7]{3})\\([0-7]{3})/decode("utf-8", chr(oct($1)).chr(oct($2)))/ge'
const unordered_set<string> czech_tokenizer::abbreviations_czech = {
  // Titles
  "prof", "csc", "drsc", "doc", "phd", "ph", "d",
  "judr", "mddr", "mudr", "mvdr", "paeddr", "paedr", "phdr", "rndr", "rsdr", "dr",
  "ing", "arch", "mgr", "bc", "mag", "mba", "bca", "mga",
  "gen", "plk", "pplk", "npor", "por", "ppor", "kpt", "mjr", "sgt", "pls", "p", "s",
  "p", "p\303\255", "fa", "fy", "mr", "mrs", "ms", "miss", "tr", "sv",
  // Geographic names
  "angl", "fr", "\304\215es", "ces", "\304\215s", "cs", "slov", "n\304\233m", "nem", "it", "pol", "ma\304\217", "mad", "rus",
  "sev", "v\303\275ch", "vych", "ji\305\276", "jiz", "z\303\241p", "zap",
  // Common abbrevs
  "adr", "\304\215", "c", "eg", "ev", "g", "hod", "j", "kr", "m", "max", "min", "mj", "nap\305\231", "napr",
  "okr", "pop\305\231", "popr", "pozn", "r", "\305\231", "red", "rep", "resp", "srov", "st", "st\305\231", "str",
  "sv", "tel", "tj", "tzv", "\303\272", "u", "uh", "ul", "um", "zl", "zn",
};

const unordered_set<string> czech_tokenizer::abbreviations_slovak = {
  // Titles
  "prof", "csc", "drsc", "doc", "phd", "ph", "d",
  "judr", "mddr", "mudr", "mvdr", "paeddr", "paedr", "phdr", "rndr", "rsdr", "dr",
  "ing", "arch", "mgr", "bc", "mag", "mba", "bca", "mga",
  "gen", "plk", "pplk", "npor", "por", "ppor", "kpt", "mjr", "sgt", "pls", "p", "s",
  "p", "p\303\255", "fa", "fy", "mr", "mrs", "ms", "miss", "tr", "sv",
  // Geographic names
  "angl", "fr", "\304\215es", "ces", "\304\215s", "cs", "slov", "nem", "it", "po\304\276", "pol", "ma\304\217", "mad",
  "rus", "sev", "v\303\275ch", "vych", "ju\305\276", "juz", "z\303\241p", "zap",
  // Common abbrevs
  "adr", "\304\215", "c", "eg", "ev", "g", "hod", "j", "kr", "m", "max", "min", "mj", "napr",
  "okr", "popr", "pozn", "r", "red", "rep", "resp", "srov", "st", "str",
  "sv", "tel", "tj", "tzv", "\303\272", "u", "uh", "ul", "um", "zl", "zn",
};

czech_tokenizer::czech_tokenizer(tokenizer_language language, unsigned version, const morpho* m)
  : ragel_tokenizer(version <= 1 ? 1 : 2), m(m) {
  switch (language) {
    case CZECH:
      abbreviations = &abbreviations_czech;
      break;
    case SLOVAK:
      abbreviations = &abbreviations_slovak;
      break;
  }
}

void czech_tokenizer::merge_hyphenated(vector<token_range>& tokens) {
  using namespace unilib;

  if (!m) return;
  if (tokens.empty() || chars[tokens.back().start].cat & ~unicode::L) return;

  unsigned matched_hyphens = 0;
  for (unsigned hyphens = 1; hyphens <= 2; hyphens++) {
    // Are the tokens a sequence of 'hyphens' hyphenated tokens?
    if (tokens.size() < 2*hyphens + 1) break;
    unsigned first_hyphen = tokens.size() - 2*hyphens;
    if (tokens[first_hyphen].length != 1 || chars[tokens[first_hyphen].start].cat & ~unicode::P ||
        tokens[first_hyphen].start + tokens[first_hyphen].length != tokens[first_hyphen + 1].start ||
        tokens[first_hyphen-1].start + tokens[first_hyphen-1].length != tokens[first_hyphen].start ||
        chars[tokens[first_hyphen-1].start].cat & ~unicode::L)
      break;

    if (m->analyze(string_piece(chars[tokens[first_hyphen-1].start].str, chars[tokens.back().start + tokens.back().length].str - chars[tokens[first_hyphen-1].start].str), morpho::NO_GUESSER, lemmas) >= 0)
      matched_hyphens = hyphens;
  }

  if (matched_hyphens) {
    unsigned first = tokens.size() - 2*matched_hyphens - 1;
    tokens[first].length = tokens.back().start + tokens.back().length - tokens[first].start;
    tokens.resize(first + 1);
  }
}

bool czech_tokenizer::next_sentence(vector<token_range>& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = czech_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _czech_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _czech_tokenizer_cond_lengths[cs];
	_keys = _czech_tokenizer_cond_keys + (_czech_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _czech_tokenizer_cond_spaces[_czech_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _czech_tokenizer_trans_keys + _czech_tokenizer_key_offsets[cs];
	_trans = _czech_tokenizer_index_offsets[cs];

	_klen = _czech_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _czech_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _czech_tokenizer_indicies[_trans];
_eof_trans:
	cs = _czech_tokenizer_trans_targs[_trans];

	if ( _czech_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _czech_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, abbreviations);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _czech_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _czech_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _czech_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/english_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// The list of lowercased words that when preceding eos do not end sentence.
const unordered_set<string> english_tokenizer::abbreviations = {
  // Titles
  "adj", "adm", "adv", "assoc", "asst", "bart", "bldg", "brig", "bros", "capt",
  "cmdr", "col", "comdr", "con", "corp", "cpl", "d", "dr", "dr", "drs", "ens",
  "gen", "gov", "hon", "hosp", "hr", "insp", "lt", "mm", "mr", "mrs", "ms",
  "maj", "messrs", "mlle", "mme", "mr", "mrs", "ms", "msgr", "op", "ord",
  "pfc", "ph", "phd", "prof", "pvt", "rep", "reps", "res", "rev", "rt", "sen",
  "sens", "sfc", "sgt", "sr", "st", "supt", "surg", "univ",
  // Common abbrevs
  "addr", "approx", "apr", "aug", "calif", "co", "corp", "dec", "def", "e",
  "e.g", "eg", "feb", "fla", "ft", "gen", "gov", "hrs", "i.", "i.e", "ie",
  "inc", "jan", "jr", "ltd", "mar", "max", "min", "mph", "mt", "n", "nov",
  "oct", "ont", "pa", "pres", "rep", "rev", "s", "sec", "sen", "sep", "sept",
  "sgt", "sr", "tel", "un", "univ", "v", "va", "vs", "w", "yrs",
};

static const char _english_tokenizer_split_token_key_offsets[] = {
	0, 0, 16, 20, 22, 26, 28, 30,
	32, 34, 36, 44, 46, 50, 52, 54,
	56, 58, 60, 62, 64, 66, 68, 72,
	74, 76, 78, 80, 82, 82
};

static const unsigned char _english_tokenizer_split_token_trans_keys[] = {
	65u, 68u, 69u, 76u, 77u, 78u, 83u, 84u,
	97u, 100u, 101u, 108u, 109u, 110u, 115u, 116u,
	78u, 84u, 110u, 116u, 78u, 110u, 65u, 79u,
	97u, 111u, 87u, 119u, 71u, 103u, 84u, 116u,
	79u, 111u, 39u, 161u, 77u, 82u, 86u, 89u,
	109u, 114u, 118u, 121u, 77u, 109u, 69u, 73u,
	101u, 105u, 76u, 108u, 39u, 161u, 68u, 100u,
	76u, 108u, 39u, 161u, 69u, 101u, 82u, 114u,
	79u, 111u, 77u, 109u, 39u, 79u, 111u, 161u,
	78u, 110u, 78u, 110u, 78u, 110u, 65u, 97u,
	67u, 99u, 0
};

static const char _english_tokenizer_split_token_single_lengths[] = {
	0, 16, 4, 2, 4, 2, 2, 2,
	2, 2, 8, 2, 4, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 4, 2,
	2, 2, 2, 2, 0, 0
};

static const char _english_tokenizer_split_token_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0
};

static const unsigned char _english_tokenizer_split_token_index_offsets[] = {
	0, 0, 17, 22, 25, 30, 33, 36,
	39, 42, 45, 54, 57, 62, 65, 68,
	71, 74, 77, 80, 83, 86, 89, 94,
	97, 100, 103, 106, 109, 110
};

static const char _english_tokenizer_split_token_indicies[] = {
	0, 2, 3, 4, 2, 5, 2, 6,
	0, 2, 3, 4, 2, 5, 2, 6,
	1, 7, 8, 7, 8, 1, 9, 9,
	1, 10, 11, 10, 11, 1, 12, 12,
	1, 12, 12, 1, 13, 13, 1, 11,
	11, 1, 14, 14, 1, 15, 2, 2,
	16, 15, 2, 2, 16, 1, 17, 17,
	1, 18, 11, 18, 11, 1, 12, 12,
	1, 19, 19, 1, 12, 12, 1, 2,
	2, 1, 20, 20, 1, 21, 21, 1,
	22, 22, 1, 23, 23, 1, 12, 12,
	1, 24, 25, 25, 24, 1, 14, 14,
	1, 26, 26, 1, 27, 27, 1, 28,
	28, 1, 12, 12, 1, 1, 1, 0
};

static const char _english_tokenizer_split_token_trans_targs[] = {
	2, 0, 9, 10, 16, 17, 22, 3,
	7, 4, 5, 6, 28, 8, 29, 11,
	14, 12, 13, 15, 18, 19, 20, 21,
	23, 24, 25, 26, 27
};

static const char _english_tokenizer_split_token_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 1,
	1, 0, 0, 0, 0, 0, 2, 1,
	1, 0, 0, 0, 1, 0, 0, 0,
	0, 0, 1, 0, 0
};

static const char _english_tokenizer_split_token_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 3, 0
};

static const int english_tokenizer_split_token_start = 1;

void english_tokenizer::split_token(vector<token_range>& tokens) {
  if (tokens.empty() || chars[tokens.back().start].cat & ~unilib::unicode::L) return;

  size_t index = tokens.back().start, end = index + tokens.back().length;
  int cs;
  size_t split_mark = 0, split_len = 0;

	{
	cs = english_tokenizer_split_token_start;
	}

	{
	int _klen;
	const unsigned char *_keys;
	int _trans;

	if ( ( index) == ( end) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _english_tokenizer_split_token_trans_keys + _english_tokenizer_split_token_key_offsets[cs];
	_trans = _english_tokenizer_split_token_index_offsets[cs];

	_klen = _english_tokenizer_split_token_single_lengths[cs];
	if ( _klen > 0 ) {
		const unsigned char *_lower = _keys;
		const unsigned char *_mid;
		const unsigned char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) < *_mid )
				_upper = _mid - 1;
			else if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _english_tokenizer_split_token_range_lengths[cs];
	if ( _klen > 0 ) {
		const unsigned char *_lower = _keys;
		const unsigned char *_mid;
		const unsigned char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _english_tokenizer_split_token_indicies[_trans];
	cs = _english_tokenizer_split_token_trans_targs[_trans];

	if ( _english_tokenizer_split_token_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _english_tokenizer_split_token_trans_actions[_trans] ) {
	case 1:
	{ split_mark = index - tokens.back().start + 1; }
	break;
	case 2:
	{ split_mark = index - tokens.back().start + 1; }
	{ split_len = split_mark; {( index)++; goto _out; } }
	break;
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++( index) != ( end) )
		goto _resume;
	_test_eof: {}
	if ( ( index) == ( end) )
	{
	switch ( _english_tokenizer_split_token_eof_actions[cs] ) {
	case 3:
	{ split_len = split_mark; {( index)++; goto _out; } }
	break;
	}
	}

	_out: {}
	}

  if (split_len && split_len < end) {
    tokens.back().length -= split_len;
    tokens.emplace_back(end - split_len, split_len);
  }
}

static const char _english_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2
};

static const char _english_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 2, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const short _english_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _english_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _english_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 49, 52,
	55, 60, 63, 98, 103, 107, 110, 114,
	119, 120, 125, 126, 131, 145, 152, 156,
	161, 164, 179, 192, 206
};

static const short _english_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 159u, 48u,
	57u, 159u, 48u, 57u, 159u, 48u, 57u, 43u,
	45u, 159u, 48u, 57u, 159u, 48u, 57u, 9u,
	10u, 13u, 32u, 33u, 44u, 46u, 47u, 63u,
	129u, 131u, 135u, 142u, 147u, 157u, 159u, 160u,
	301u, 557u, 811u, 1067u, 0u, 42u, 48u, 57u,
	58u, 64u, 65u, 90u, 91u, 96u, 97u, 122u,
	123u, 255u, 9u, 10u, 13u, 32u, 147u, 9u,
	13u, 32u, 147u, 9u, 32u, 147u, 9u, 10u,
	32u, 147u, 9u, 10u, 13u, 32u, 147u, 13u,
	9u, 10u, 13u, 32u, 147u, 10u, 9u, 10u,
	13u, 32u, 147u, 13u, 32u, 34u, 39u, 41u,
	59u, 93u, 125u, 139u, 141u, 147u, 161u, 9u,
	10u, 44u, 46u, 69u, 101u, 159u, 48u, 57u,
	44u, 46u, 69u, 101u, 69u, 101u, 159u, 48u,
	57u, 159u, 48u, 57u, 39u, 45u, 129u, 131u,
	135u, 151u, 155u, 157u, 161u, 65u, 90u, 97u,
	122u, 142u, 143u, 45u, 129u, 131u, 135u, 151u,
	155u, 157u, 65u, 90u, 97u, 122u, 142u, 143u,
	39u, 129u, 131u, 135u, 151u, 155u, 157u, 161u,
	65u, 90u, 97u, 122u, 142u, 143u, 159u, 48u,
	57u, 0
};

static const char _english_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 1, 1, 1,
	3, 1, 21, 5, 4, 3, 4, 5,
	1, 5, 1, 5, 12, 5, 4, 3,
	1, 9, 7, 8, 1
};

static const char _english_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 1,
	1, 1, 7, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 1, 0, 1,
	1, 3, 3, 3, 1
};

static const unsigned char _english_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 48, 51,
	54, 59, 62, 91, 97, 102, 106, 111,
	117, 119, 125, 127, 133, 147, 154, 159,
	164, 167, 180, 191, 203
};

static const char _english_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 0,
	8, 8, 0, 9, 9, 0, 10, 10,
	11, 11, 0, 11, 11, 0, 13, 14,
	15, 13, 16, 12, 16, 12, 16, 19,
	19, 19, 19, 13, 19, 18, 16, 12,
	20, 12, 20, 12, 18, 12, 19, 12,
	19, 12, 17, 13, 22, 23, 13, 13,
	21, 13, 24, 13, 13, 21, 13, 13,
	13, 21, 13, 24, 13, 13, 21, 13,
	25, 26, 13, 13, 21, 28, 27, 13,
	25, 29, 13, 13, 21, 28, 27, 13,
	26, 29, 13, 13, 21, 4, 4, 5,
	5, 5, 5, 5, 5, 5, 5, 4,
	5, 4, 30, 31, 32, 33, 33, 18,
	18, 30, 31, 32, 33, 33, 30, 33,
	33, 9, 9, 30, 11, 11, 30, 34,
	35, 19, 19, 19, 19, 19, 19, 34,
	19, 19, 19, 30, 35, 19, 19, 19,
	19, 19, 19, 19, 19, 19, 30, 34,
	19, 19, 19, 19, 19, 19, 34, 19,
	19, 19, 30, 18, 18, 30, 0
};

static const char _english_tokenizer_trans_targs[] = {
	10, 1, 2, 10, 1, 3, 5, 6,
	22, 23, 9, 24, 10, 11, 15, 19,
	20, 0, 21, 25, 28, 10, 12, 14,
	13, 16, 17, 10, 10, 18, 10, 4,
	7, 8, 26, 27
};

static const char _english_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 0, 0,
	4, 4, 0, 0, 7, 0, 0, 0,
	4, 0, 4, 0, 0, 8, 0, 0,
	0, 0, 0, 9, 10, 0, 11, 0,
	0, 0, 0, 0
};

static const char _english_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 5, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const char _english_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 6, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const unsigned char _english_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 0, 22, 22, 22, 22, 22,
	28, 22, 28, 22, 31, 31, 31, 31,
	31, 31, 31, 31, 31
};

static const int english_tokenizer_start = 10;

english_tokenizer::english_tokenizer(unsigned version) : ragel_tokenizer(version <= 1 ? 1 : 2) {}

bool english_tokenizer::next_sentence(vector<token_range>& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = english_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _english_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _english_tokenizer_cond_lengths[cs];
	_keys = _english_tokenizer_cond_keys + (_english_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _english_tokenizer_cond_spaces[_english_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _english_tokenizer_trans_keys + _english_tokenizer_key_offsets[cs];
	_trans = _english_tokenizer_index_offsets[cs];

	_klen = _english_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _english_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _english_tokenizer_indicies[_trans];
_eof_trans:
	cs = _english_tokenizer_trans_targs[_trans];

	if ( _english_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _english_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, &abbreviations);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _english_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _english_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _english_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _generic_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2
};

static const char _generic_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const short _generic_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _generic_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _generic_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 51, 54,
	89, 94, 98, 101, 105, 110, 111, 116,
	117, 122, 136, 142, 147, 150, 162
};

static const short _generic_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 43u, 45u,
	159u, 48u, 57u, 159u, 48u, 57u, 9u, 10u,
	13u, 32u, 33u, 44u, 46u, 47u, 63u, 129u,
	131u, 135u, 142u, 147u, 157u, 159u, 160u, 301u,
	557u, 811u, 1067u, 0u, 42u, 48u, 57u, 58u,
	64u, 65u, 90u, 91u, 96u, 97u, 122u, 123u,
	255u, 9u, 10u, 13u, 32u, 147u, 9u, 13u,
	32u, 147u, 9u, 32u, 147u, 9u, 10u, 32u,
	147u, 9u, 10u, 13u, 32u, 147u, 13u, 9u,
	10u, 13u, 32u, 147u, 10u, 9u, 10u, 13u,
	32u, 147u, 13u, 32u, 34u, 39u, 41u, 59u,
	93u, 125u, 139u, 141u, 147u, 161u, 9u, 10u,
	46u, 69u, 101u, 159u, 48u, 57u, 69u, 101u,
	159u, 48u, 57u, 159u, 48u, 57u, 129u, 131u,
	135u, 151u, 155u, 157u, 65u, 90u, 97u, 122u,
	142u, 143u, 159u, 48u, 57u, 0
};

static const char _generic_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 3, 1, 21,
	5, 4, 3, 4, 5, 1, 5, 1,
	5, 12, 4, 3, 1, 6, 1
};

static const char _generic_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 7,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 1, 3, 1
};

static const unsigned char _generic_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 50, 53,
	82, 88, 93, 97, 102, 108, 110, 116,
	118, 124, 138, 144, 149, 152, 162
};

static const char _generic_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 8,
	8, 0, 8, 8, 0, 10, 11, 12,
	10, 13, 9, 13, 9, 13, 16, 16,
	16, 16, 10, 16, 15, 13, 9, 17,
	9, 17, 9, 15, 9, 16, 9, 16,
	9, 14, 10, 19, 20, 10, 10, 18,
	10, 21, 10, 10, 18, 10, 10, 10,
	18, 10, 21, 10, 10, 18, 10, 22,
	23, 10, 10, 18, 25, 24, 10, 22,
	26, 10, 10, 18, 25, 24, 10, 23,
	26, 10, 10, 18, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 27, 28, 29, 29, 15, 15, 27,
	29, 29, 6, 6, 27, 8, 8, 27,
	16, 16, 16, 16, 16, 16, 16, 16,
	16, 27, 15, 15, 27, 0
};

static const char _generic_tokenizer_trans_targs[] = {
	7, 1, 2, 7, 1, 3, 19, 6,
	20, 7, 8, 12, 16, 17, 0, 18,
	21, 22, 7, 9, 11, 10, 13, 14,
	7, 7, 15, 7, 4, 5
};

static const char _generic_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 4, 0,
	0, 7, 0, 0, 0, 4, 0, 4,
	0, 0, 8, 0, 0, 0, 0, 0,
	9, 10, 0, 11, 0, 0
};

static const char _generic_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 5,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const char _generic_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 6,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const unsigned char _generic_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 0,
	19, 19, 19, 19, 19, 25, 19, 25,
	19, 28, 28, 28, 28, 28, 28
};

static const int generic_tokenizer_start = 7;

generic_tokenizer::generic_tokenizer(unsigned version) : ragel_tokenizer(version <= 1 ? 1 : 2) {}

bool generic_tokenizer::next_sentence(vector<token_range>& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = generic_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _generic_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _generic_tokenizer_cond_lengths[cs];
	_keys = _generic_tokenizer_cond_keys + (_generic_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _generic_tokenizer_cond_spaces[_generic_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _generic_tokenizer_trans_keys + _generic_tokenizer_key_offsets[cs];
	_trans = _generic_tokenizer_index_offsets[cs];

	_klen = _generic_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _generic_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _generic_tokenizer_indicies[_trans];
_eof_trans:
	cs = _generic_tokenizer_trans_targs[_trans];

	if ( _generic_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _generic_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, nullptr);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _generic_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _generic_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _generic_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer_factory : public tokenizer_factory {
 public:
  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  unsigned version;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* generic_tokenizer_factory::new_tokenizer() const {
  return new generic_tokenizer(version);
}

bool generic_tokenizer_factory::load(istream& is) {
  version = is.get();

  return bool(is);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer_factory_encoder {
 public:
  static void encode(unsigned version, ostream& os);
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory_encoder.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void generic_tokenizer_factory_encoder::encode(unsigned version, ostream& os) {
  os.put(version);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_network.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations

class gru_tokenizer_network {
 public:
  virtual ~gru_tokenizer_network() {}

  template <int R, int C> struct matrix {
    float w[R][C];
    float b[R];

    void clear();
    void load(binary_decoder& data);
  };

  enum { NO_SPLIT, END_OF_TOKEN, END_OF_SENTENCE, OUTCOMES };
  struct outcome_t {
    int outcome;
    float w[3];
    const float* embedding;
  };
  struct char_info {
    char32_t chr;
    unilib::unicode::category_t cat;

    char_info() {}
    char_info(char32_t chr, unilib::unicode::category_t cat) : chr(chr), cat(cat) {}
  };

  virtual void classify(const vector<char_info>& chars, vector<outcome_t>& outcomes) const = 0;

  static gru_tokenizer_network* load(binary_decoder& data);
};

template <int D>
class gru_tokenizer_network_implementation : public gru_tokenizer_network {
 public:
  virtual void classify(const vector<char_info>& chars, vector<outcome_t>& outcomes) const override;

  static gru_tokenizer_network_implementation<D>* load(binary_decoder& data);

 protected:
  void cache_embeddings();

  struct cached_embedding {
    matrix<1, D> e;
    matrix<6, D> cache;
  };

  struct gru {
    matrix<D,D> X, X_r, X_z;
    matrix<D,D> H, H_r, H_z;

    void load(binary_decoder& data);
  };

  unordered_map<char32_t, cached_embedding> embeddings;
  cached_embedding empty_embedding;
  gru gru_fwd, gru_bwd;
  matrix<3, D> projection_fwd, projection_bwd;
  unordered_map<unilib::unicode::category_t, char32_t> unknown_chars;
};

// Definitions

template <int R, int C>
void gru_tokenizer_network::matrix<R, C>::clear() {
  for (int i = 0; i < R; i++) fill_n(w[i], C, 0.f);
  fill_n(b, R, 0.f);
}

template <int R, int C>
void gru_tokenizer_network::matrix<R, C>::load(binary_decoder& data) {
  for (int i = 0; i < R; i++) copy_n(data.next<float>(C), C, w[i]);
  copy_n(data.next<float>(R), R, b);
}

template <int D>
void gru_tokenizer_network_implementation<D>::gru::load(binary_decoder& data) {
  X.load(data);
  X_r.load(data);
  X_z.load(data);
  H.load(data);
  H_r.load(data);
  H_z.load(data);
}

template <int D>
void gru_tokenizer_network_implementation<D>::classify(const vector<char_info>& chars, vector<outcome_t>& outcomes) const {
  if (chars.empty()) return;

  // Resolve embeddings, possibly with unknown_chars or empty_embedding
  for (size_t i = 0; i < chars.size(); i++) {
    auto embedding = embeddings.find(chars[i].chr);
    if (embedding != embeddings.end()) {
      outcomes[i].embedding = embedding->second.cache.w[0];
    } else {
      auto unknown_char = unknown_chars.find(chars[i].cat);
      if (unknown_char != unknown_chars.end()) embedding = embeddings.find(unknown_char->second);
      outcomes[i].embedding = embedding != embeddings.end() ? embedding->second.cache.w[0] : empty_embedding.cache.w[0];
    }
  }

  // Clear outcome probabilities
  for (auto&& outcome : outcomes)
    for (int i = 0; i < 3; i++)
      outcome.w[i] = projection_fwd.b[i];

  // Perform forward & backward GRU
  matrix<1, D> state, update, reset, candidate;
  for (int dir = 0; dir < 2; dir++) {
    auto& gru = dir == 0 ? gru_fwd : gru_bwd;
    auto& projection = dir == 0 ? projection_fwd : projection_bwd;

    state.clear();
    for (size_t i = 0; i < outcomes.size(); i++) {
      auto& outcome = outcomes[dir == 0 ? i : outcomes.size() - 1 - i];
      auto* embedding_cache = outcome.embedding + (dir == 1) * 3 * D;

      for (int j = 0; j < D; j++) {
        update.w[0][j] = gru.X_z.b[j] + embedding_cache[2*D + j];
        reset.w[0][j] = gru.X_r.b[j] + embedding_cache[D + j];
        for (int k = 0; k < D; k++) {
          update.w[0][j] += state.w[0][k] * gru.H_z.w[j][k];
          reset.w[0][j] += state.w[0][k] * gru.H_r.w[j][k];
        }
        update.w[0][j] = 1.f / (1.f + exp(-update.w[0][j]));
        reset.w[0][j] = 1.f / (1.f + exp(-reset.w[0][j]));
        reset.w[0][j] *= state.w[0][j];
      }
      for (int j = 0; j < D; j++) {
        candidate.w[0][j] = gru.X.b[j] + embedding_cache[j];
        for (int k = 0; k < D; k++)
          candidate.w[0][j] += reset.w[0][k] * gru.H.w[j][k];
        candidate.w[0][j] = tanh(candidate.w[0][j]);
        state.w[0][j] = update.w[0][j] * state.w[0][j] + (1.f - update.w[0][j]) * candidate.w[0][j];
      }

      for (int j = 0; j < 3; j++)
        for (int k = 0; k < D; k++)
          outcome.w[j] += projection.w[j][k] * state.w[0][k];
    }
  }

  // Choose the outcome with the highest weight
  for (auto&& outcome : outcomes) {
    outcome.outcome = outcome.w[1] > outcome.w[0];
    if (outcome.w[2] > outcome.w[outcome.outcome]) outcome.outcome = 2;
  }
}

template <int D>
gru_tokenizer_network_implementation<D>* gru_tokenizer_network_implementation<D>::load(binary_decoder& data) {
  unique_ptr<gru_tokenizer_network_implementation<D>> network(new gru_tokenizer_network_implementation<D>());

  for (unsigned chars = data.next_4B(); chars; chars--) {
    auto& embedding = network->embeddings[data.next_4B()];
    copy_n(data.next<float>(D), D, embedding.e.w[0]);
  }
  fill_n(network->empty_embedding.e.w[0], D, 0.f);

  network->gru_fwd.load(data);
  network->gru_bwd.load(data);
  network->projection_fwd.load(data);
  network->projection_bwd.load(data);

  network->unknown_chars.clear();
  for (unsigned unknown_chars_len = data.next_1B(); unknown_chars_len; unknown_chars_len--) {
    unilib::unicode::category_t cat = data.next_4B();
    network->unknown_chars[cat] = data.next_4B();
  }

  network->cache_embeddings();

  return network.release();
}

template <int D>
void gru_tokenizer_network_implementation<D>::cache_embeddings() {
  for (auto&& embedding : embeddings) {
    auto& e = embedding.second.e;
    auto& cache = embedding.second.cache;

    fill_n(cache.w[0], 6*D, 0.f);
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[0][i] += e.w[0][j] * gru_fwd.X.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[1][i] += e.w[0][j] * gru_fwd.X_r.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[2][i] += e.w[0][j] * gru_fwd.X_z.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[3][i] += e.w[0][j] * gru_bwd.X.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[4][i] += e.w[0][j] * gru_bwd.X_r.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[5][i] += e.w[0][j] * gru_bwd.X_z.w[i][j];
  }
  fill_n(empty_embedding.cache.w[0], 6*D, 0.f);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class gru_tokenizer : public unicode_tokenizer {
 public:
  gru_tokenizer(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const gru_tokenizer_network& network)
      : unicode_tokenizer(url_email_tokenizer), segment(segment), allow_spaces(allow_spaces), network_index(0), network_length(0), network(network) {}

  virtual bool next_sentence(vector<token_range>& tokens) override;

 private:
  inline bool is_space(size_t index);
  int next_outcome();

  unsigned segment;
  bool allow_spaces;
  unsigned network_index, network_length;
  vector<gru_tokenizer_network::char_info> network_chars;
  vector<gru_tokenizer_network::outcome_t> network_outcomes;
  vector<size_t> network_offsets;
  const gru_tokenizer_network& network;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool gru_tokenizer::is_space(size_t index) {
  return (chars[index].cat & unilib::unicode::Zs) || chars[index].chr == '\r' || chars[index].chr == '\n' || chars[index].chr == '\t';
}

bool gru_tokenizer::next_sentence(vector<token_range>& tokens) {
  tokens.clear();

  // Reset tokenizer on new text
  if (current == 0) network_index = network_length = 0;

  // Tokenize until EOS
  for (bool eos = false; !eos && !emergency_sentence_split(tokens); ) {
    while (current < chars.size() - 1 && is_space(current))
      next_outcome();
    if (current >= chars.size() - 1) break;

    // We have a beginning of a token. Try if it is an URL.
    if (tokenize_url_email(tokens)) {
      while (network_index < network_length && network_offsets[network_index] < current) network_index++;
      continue;
    }

    // Slurp current token
    size_t token_start = current;
    do {
      int outcome = next_outcome();
      eos = outcome == gru_tokenizer_network::END_OF_SENTENCE;
      if (outcome != gru_tokenizer_network::NO_SPLIT) break;
    } while (current < chars.size() - 1);
    tokens.emplace_back(token_start, current - token_start);
  }

  return !tokens.empty();
}

int gru_tokenizer::next_outcome() {
  if (network_index >= network_length) {
    // Compute required window
    network_index = 0;
    network_length = 0;
    network_chars.clear();
    network_outcomes.clear();
    network_offsets.clear();

    // Prepare data for the classification
    for (size_t offset = current;
         network_offsets.push_back(offset), offset < chars.size() - 1 && network_length < segment;
         network_length++, offset++) {
      if (is_space(offset)) {
        network_chars.emplace_back(' ', unilib::unicode::Zs);
        while (offset + 1 < chars.size() - 1 && is_space(offset + 1)) offset++;
      } else {
        network_chars.emplace_back(chars[offset].chr, chars[offset].cat);
      }
    }
    // Add a space to the end on the EOD
    if (network_length < segment && network_chars.back().chr != ' ')
      network_chars.emplace_back(' ', unilib::unicode::Zs);
    network_outcomes.resize(network_chars.size());

    // Perform the classification
    network.classify(network_chars, network_outcomes);

    // Add spacing token/sentence breaks
    for (size_t i = 0; i < network_length - 1; i++)
      if (is_space(network_offsets[i+1])) {
        // Detect EOS on the following space or \n\n or \r\n\r\n, or if there is end of text
        bool eos = network_outcomes[i+1].outcome == gru_tokenizer_network::END_OF_SENTENCE;
        if (i + 2 == network_length) eos = true;
        for (size_t j = network_offsets[i+1]; j + 1 < network_offsets[i+2] && !eos; j++)
          eos = (chars[j].chr == '\n' && chars[j+1].chr == '\n') ||
                (j + 3 < network_offsets[i+2] && chars[j].chr == '\r' && chars[j+1].chr == '\n' && chars[j+2].chr == '\r' && chars[j+3].chr == '\n');
        if (eos) network_outcomes[i].outcome = gru_tokenizer_network::END_OF_SENTENCE;

        if (network_outcomes[i].outcome == gru_tokenizer_network::NO_SPLIT)
          // Force EOT if not allowing spaces, and also detect EOT on the following space
          if (!allow_spaces || network_outcomes[i+1].outcome == gru_tokenizer_network::END_OF_TOKEN)
            network_outcomes[i].outcome = gru_tokenizer_network::END_OF_TOKEN;
      }

    // Adjust network_length to suitable break
    if (network_length == segment && network_length >= 10) {
      network_length -= 5;
      while (network_length > segment / 2)
        if (network_outcomes[--network_length].outcome != gru_tokenizer_network::NO_SPLIT)
          break;
    }
  }
  return current = network_offsets[network_index + 1], network_outcomes[network_index++].outcome;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_factory.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class gru_tokenizer_factory : public tokenizer_factory {
 public:
  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);

 private:
  unsigned url_email_tokenizer;
  unsigned segment;
  bool allow_spaces;

  unique_ptr<gru_tokenizer_network> network;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* gru_tokenizer_factory::new_tokenizer() const {
  return new gru_tokenizer(url_email_tokenizer, segment, allow_spaces, *network);
}

bool gru_tokenizer_factory::load(istream& is) {
  char version;
  if (!is.get(version)) return false;
  if (!(version >= 1 && version <= 2)) return false;

  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    url_email_tokenizer = data.next_1B();
    segment = data.next_2B();
    allow_spaces = version >= 2 ? data.next_1B() : false /*false was default for version 1*/;

    network.reset(gru_tokenizer_network::load(data));
    if (!network) return false;
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_network.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

gru_tokenizer_network* gru_tokenizer_network::load(binary_decoder& data) {
  if (data.next_1B() != 1) return nullptr;
  switch (data.next_1B()) {
    case 16: return gru_tokenizer_network_implementation<16>::load(data);
    case 24: return gru_tokenizer_network_implementation<24>::load(data);
    case 64: return gru_tokenizer_network_implementation<64>::load(data);
  }
  return nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_trainer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct tokenized_sentence {
  u32string sentence;
  vector<token_range> tokens;
};

class gru_tokenizer_trainer {
 public:
  enum { URL_EMAIL_LATEST = unicode_tokenizer::URL_EMAIL_LATEST };

  static bool train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned dimension, unsigned epochs,
                    unsigned batch_size, float learning_rate, float learning_rate_final, float dropout,
                    float initialization_range, bool early_stopping, const vector<tokenized_sentence>& data,
                    const vector<tokenized_sentence>& heldout, ostream& os, string& error);
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_network_trainer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

//
// Declarations
//

template <int D>
class gru_tokenizer_network_trainer : public gru_tokenizer_network_implementation<D> {
 public:
  bool train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned epochs, unsigned batch_size,
             float learning_rate, float learning_rate_final, float dropout, float initialization_range,
             bool early_stopping, const vector<tokenized_sentence>& data, const vector<tokenized_sentence>& heldout,
             binary_encoder& enc, string& error);

 private:
  template <int R, int C> using matrix = gru_tokenizer_network::matrix<R, C>;
  using typename gru_tokenizer_network_implementation<D>::cached_embedding;
  using typename gru_tokenizer_network_implementation<D>::gru;

  template <int R, int C> struct matrix_trainer {
    matrix<R, C>& original;
    float w_g[R][C], b_g[R];
    float w_m[R][C], b_m[R];
    float w_v[R][C], b_v[R];

    matrix_trainer(matrix<R, C>& original) : original(original), w_g(), b_g(), w_m(), b_m(), w_v(), b_v() {}
    void update_weights(float learning_rate);
  };
  struct gru_trainer {
    matrix_trainer<D,D> X, X_r, X_z;
    matrix_trainer<D,D> H, H_r, H_z;
    vector<matrix<1, D>> states, updates, resets, resetstates, candidates, dropouts;

    gru_trainer(gru& g, unsigned segment)
        : X(g.X), X_r(g.X_r), X_z(g.X_z), H(g.H), H_r(g.H_r), H_z(g.H_z), states(segment + 1),
        updates(segment), resets(segment), resetstates(segment), candidates(segment), dropouts(segment) {}
    void update_weights(float learning_rate);
  };

  struct f1_info { double precision, recall, f1; };
  void evaluate(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const vector<tokenized_sentence>& heldout,
                f1_info& tokens_f1, f1_info& sentences_f1);
  void evaluate_f1(const vector<token_range>& system, const vector<token_range>& gold, f1_info& f1);

  template <int R, int C> void random_matrix(matrix<R,C>& m, mt19937& generator, float range, float bias);
  void random_gru(gru& g,  mt19937& generator, float range);

  template <int R, int C> void save_matrix(const matrix<R,C>& m, binary_encoder& enc);
  void save_gru(const gru& g, binary_encoder& enc);
};

//
// Definitions
//

template <int D>
bool gru_tokenizer_network_trainer<D>::train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned epochs, unsigned batch_size,
                                             float learning_rate_initial, float learning_rate_final, float dropout,
                                             float initialization_range, bool early_stopping, const vector<tokenized_sentence>& data,
                                             const vector<tokenized_sentence>& heldout, binary_encoder& enc, string& error) {
  if (segment < 10) return error.assign("Segment size must be at least 10!"), false;

  unsigned characters = 0;
  for (auto&& sentence : data)
    characters += sentence.sentence.size();
  if (characters < segment) return error.assign("Not enought training data for the gru_tokenizer!"), false;

  mt19937 generator;

  float dropout_multiplier = 1.f / (1.f - dropout);
  bernoulli_distribution dropout_distribution(dropout);

  // Generate embeddings
  for (auto&& sentence : data)
    for (auto&& chr : sentence.sentence)
      if (!this->embeddings.count(chr)) {
        cached_embedding embedding;
        random_matrix(embedding.e, generator, initialization_range, 0.f);
        this->embeddings.emplace(chr, embedding);
      }
  this->empty_embedding.e.clear();

  // Initialize weights
  random_gru(this->gru_fwd, generator, initialization_range);
  random_gru(this->gru_bwd, generator, initialization_range);
  random_matrix(this->projection_fwd, generator, initialization_range, 0.f); this->projection_fwd.b[this->NO_SPLIT] = 1.f;
  random_matrix(this->projection_bwd, generator, initialization_range, 0.f); this->projection_bwd.b[this->NO_SPLIT] = 1.f;

  // Train the network
  unordered_map<char32_t, matrix_trainer<1, D>> embeddings;
  for (auto&& embedding : this->embeddings)
    embeddings.emplace(embedding.first, embedding.second.e);
  vector<matrix_trainer<1, D>*> chosen_embeddings(segment);
  vector<matrix<1, D>> embedding_dropouts(segment);
  gru_trainer gru_fwd(this->gru_fwd, segment), gru_bwd(this->gru_bwd, segment);
  matrix_trainer<3, D> projection_fwd(this->projection_fwd), projection_bwd(this->projection_bwd);
  float learning_rate = learning_rate_initial, b1t = 1.f, b2t = 1.f;

  float best_combined_f1 = 0.f; unsigned best_combined_f1_epoch = 0;
  gru_tokenizer_network_trainer<D> best_combined_f1_network;

  size_t training_offset = 0, training_shift;
  vector<gru_tokenizer_network::char_info> training_input, instance_input(segment);
  vector<gru_tokenizer_network::outcome_t> training_output, instance_output(segment);
  vector<int> permutation; for (size_t i = 0; i < data.size(); i++) permutation.push_back(permutation.size());
  for (unsigned epoch = 0; epoch < epochs; epoch++) {
    double logprob = 0;
    int total = 0, correct = 0;

    for (int instance = 0, instances = 10000; instance < instances; instance++) {
      // Prepare input instance
      if (training_offset + segment >= training_input.size()) {
        shuffle(permutation.begin(), permutation.end(), generator);
        training_input.clear(); training_output.clear();
        for (auto&& index : permutation) {
          auto& sentence = data[index];
          if (sentence.tokens.empty()) continue;

          training_offset = training_input.size();
          training_input.resize(training_offset + sentence.sentence.size());
          training_output.resize(training_offset + sentence.sentence.size());
          for (size_t i = 0; i < sentence.sentence.size(); i++) {
            training_input[training_offset + i].chr = sentence.sentence[i];
            training_output[training_offset + i].outcome = gru_tokenizer_network::NO_SPLIT;
          }
          for (size_t i = 0; i < sentence.tokens.size(); i++)
            training_output[training_offset + sentence.tokens[i].start + sentence.tokens[i].length - 1].outcome =
                i+1 < sentence.tokens.size() ? gru_tokenizer_network::END_OF_TOKEN : gru_tokenizer_network::END_OF_SENTENCE;
        }
        training_offset = 0;
      }
      copy_n(training_input.begin() + training_offset, segment, instance_input.begin());
      copy_n(training_output.begin() + training_offset, segment, instance_output.begin());

      // Shift training_offset
      for (training_shift = segment - 5; training_shift > segment / 2; training_shift--)
        if (instance_output[training_shift-1].outcome != gru_tokenizer_network::NO_SPLIT || instance_input[training_shift-1].chr == ' ')
          break;
      training_offset += training_shift;

      // Forward pass
      for (unsigned i = 0; i < segment; i++) {
        chosen_embeddings[i] = &embeddings.at(instance_input[i].chr);
        for (unsigned k = 0; k < D; k++)
          embedding_dropouts[i].w[0][k] = dropout && dropout_distribution(generator) ? 0.f : dropout_multiplier;
        for (int j = 0; j < 3; j++)
          instance_output[i].w[j] = projection_fwd.original.b[j];
      }

      for (int dir = 0; dir < 2; dir++) {
        auto& gru = dir == 0 ? gru_fwd : gru_bwd;
        auto& projection = dir == 0 ? projection_fwd : projection_bwd;

        gru.states[0].clear();
        for (size_t i = 0; i < segment; i++) {
          auto& embedding = chosen_embeddings[dir == 0 ? i : segment - 1 - i];
          auto& embedding_dropout = embedding_dropouts[dir == 0 ? i : segment - 1 - i];
          auto& output = instance_output[dir == 0 ? i : segment - 1 - i];

          for (int j = 0; j < D; j++) {
            gru.updates[i].w[0][j] = gru.X_z.original.b[j];
            gru.resets[i].w[0][j] = gru.X_r.original.b[j];
            for (int k = 0; k < D; k++) {
              gru.updates[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X_z.original.w[j][k] + gru.states[i].w[0][k] * gru.H_z.original.w[j][k];
              gru.resets[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X_r.original.w[j][k] + gru.states[i].w[0][k] * gru.H_r.original.w[j][k];
            }
            gru.updates[i].w[0][j] = 1.f / (1.f + exp(-gru.updates[i].w[0][j]));
            gru.resets[i].w[0][j] = 1.f / (1.f + exp(-gru.resets[i].w[0][j]));
            gru.resetstates[i].w[0][j] = gru.resets[i].w[0][j] * gru.states[i].w[0][j];
          }
          for (int j = 0; j < D; j++) {
            gru.candidates[i].w[0][j] = gru.X.original.b[j];
            for (int k = 0; k < D; k++)
              gru.candidates[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X.original.w[j][k] + gru.resetstates[i].w[0][k] * gru.H.original.w[j][k];
            gru.candidates[i].w[0][j] = tanh(gru.candidates[i].w[0][j]);
            gru.states[i+1].w[0][j] = gru.updates[i].w[0][j] * gru.states[i].w[0][j] + (1.f - gru.updates[i].w[0][j]) * gru.candidates[i].w[0][j];
          }

          for (int j = 0; j < D; j++)
            gru.dropouts[i].w[0][j] = dropout && dropout_distribution(generator) ? 0.f : dropout_multiplier * gru.states[i+1].w[0][j];

          for (int j = 0; j < 3; j++)
            for (int k = 0; k < D; k++)
              output.w[j] += projection.original.w[j][k] * gru.dropouts[i].w[0][k];
        }
      }

      for (auto&& output : instance_output) {
        int best = output.w[1] > output.w[0];
        if (output.w[2] > output.w[best]) best = 2;
        float maximum = output.w[best], sum = 0;
        for (int j = 0; j < 3; j++) sum += (output.w[j] = exp(output.w[j] - maximum));
        sum = 1.f / sum;
        for (int j = 0; j < 3; j++) output.w[j] *= sum;

        total++;
        correct += best == output.outcome;
        logprob += log(output.w[output.outcome]);
      }

      // Backward pass
      for (auto&& output : instance_output)
        for (int j = 0; j < 3; j++)
          output.w[j] = (output.outcome == j) - output.w[j];

      for (int dir = 0; dir < 2; dir++) {
        auto& gru = dir == 0 ? gru_fwd : gru_bwd;
        auto& projection = dir == 0 ? projection_fwd : projection_bwd;

        matrix<1, D> state_g, update_g, candidate_g, reset_g, resetstate_g;
        state_g.clear();
        for (size_t i = segment; i--; ) {
          auto& embedding = chosen_embeddings[dir == 0 ? i : segment - 1 - i];
          auto& embedding_dropout = embedding_dropouts[dir == 0 ? i : segment - 1 - i];
          auto& output = instance_output[dir == 0 ? i : segment - 1 - i];

          for (int j = 0; j < D; j++) // These for cycles are swapped because
            for (int k = 0; k < 3; k++) // g++-4.8 generates wrong code otherwise.
              projection.w_g[k][j] += gru.dropouts[i].w[0][j] * output.w[k];

          for (int j = 0; j < D; j++)
            if (gru.dropouts[i].w[0][j])
              for (int k = 0; k < 3; k++)
                state_g.w[0][j] += projection.original.w[k][j] * output.w[k];

          resetstate_g.clear();
          for (int j = 0; j < D; j++) {
            update_g.w[0][j] = state_g.w[0][j] * (gru.states[i].w[0][j] - gru.candidates[i].w[0][j]);
            candidate_g.w[0][j] = state_g.w[0][j] * (1.f - gru.updates[i].w[0][j]);
            state_g.w[0][j] = state_g.w[0][j] * gru.updates[i].w[0][j];

            candidate_g.w[0][j] *= 1 - gru.candidates[i].w[0][j] * gru.candidates[i].w[0][j];
            gru.X.b_g[j] += candidate_g.w[0][j];
            for (int k = 0; k < D; k++) {
              gru.X.w_g[j][k] += candidate_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H.w_g[j][k] += candidate_g.w[0][j] * gru.resetstates[i].w[0][k];
              embedding->w_g[0][k] += embedding_dropout.w[0][k] * candidate_g.w[0][j] * gru.X.original.w[j][k];
              resetstate_g.w[0][k] += candidate_g.w[0][j] * gru.H.original.w[j][k];
            }
          }
          for (int j = 0; j < D; j++) {
            state_g.w[0][j] += resetstate_g.w[0][j] * gru.resets[i].w[0][j];
            reset_g.w[0][j] = resetstate_g.w[0][j] * gru.states[i].w[0][j];

            update_g.w[0][j] *= gru.updates[i].w[0][j] * (1 - gru.updates[i].w[0][j]);
            reset_g.w[0][j] *= gru.resets[i].w[0][j] * (1 - gru.resets[i].w[0][j]);

            gru.X_z.b_g[j] += update_g.w[0][j];
            gru.X_r.b_g[j] += reset_g.w[0][j];
            for (int k = 0; k < D; k++) {
              gru.X_z.w_g[j][k] += update_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H_z.w_g[j][k] += update_g.w[0][j] * gru.states[i].w[0][k];
              gru.X_r.w_g[j][k] += reset_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H_r.w_g[j][k] += reset_g.w[0][j] * gru.states[i].w[0][k];
              embedding->w_g[0][k] += embedding_dropout.w[0][k] * (update_g.w[0][j] * gru.X_z.original.w[j][k] +
                                                                   reset_g.w[0][j] * gru.X_r.original.w[j][k]);
              state_g.w[0][k] += update_g.w[0][j] * gru.H_z.original.w[j][k] + reset_g.w[0][j] * gru.H_r.original.w[j][k];
            }
          }
        }
      }

      // Update the weights
      if (batch_size == 1 ||
          instance+1 == instances ||
          (instance+1) % batch_size == 0) {
        b1t *= 0.9f;
        b2t *= 0.999f;
        float learning_rate_biased = learning_rate * sqrt(1-b2t) / (1-b1t);

        if (batch_size == 1)
          for (auto&& chosen_embedding : chosen_embeddings)
            chosen_embedding->update_weights(learning_rate_biased);
        else
          for (auto&& embedding : embeddings)
            embedding.second.update_weights(learning_rate_biased);
        gru_fwd.update_weights(learning_rate_biased);
        gru_bwd.update_weights(learning_rate_biased);
        projection_fwd.update_weights(learning_rate_biased);
        projection_bwd.update_weights(learning_rate_biased);
      }
    }
    if (learning_rate_final && learning_rate_final != learning_rate_initial)
      learning_rate = exp(((epochs - epoch - 2) * log(learning_rate_initial) + (epoch + 1) * log(learning_rate_final)) / (epochs - 1));

    // Evaluate
    cerr << "Epoch " << epoch+1 << ", logprob: " << scientific << setprecision(4) << logprob
         << ", training acc: " << fixed << setprecision(2) << 100. * correct / double(total) << "%";
    if (!heldout.empty()) {
      f1_info tokens, sentences;
      evaluate(url_email_tokenizer, segment, allow_spaces, heldout, tokens, sentences);
      cerr << ", heldout tokens: " << 100. * tokens.precision << "%P/" << 100. * tokens.recall << "%R/"
           << 100. * tokens.f1 << "%, sentences: " << 100. * sentences.precision << "%P/"
           << 100. * sentences.recall << "%R/" << 100. * sentences.f1 << "%";

      if (early_stopping && sentences.f1 + tokens.f1 > best_combined_f1) {
        best_combined_f1 = sentences.f1 + tokens.f1;
        best_combined_f1_epoch = epoch;
        best_combined_f1_network = *this;
      }
      if (early_stopping && best_combined_f1 && epoch - best_combined_f1_epoch > 30) {
        cerr << endl << "Stopping after 30 iterations of not improving sum of sentence and token f1." << endl;
        break;
      }
    }
    cerr << endl;
  }

  // Choose best network if desired
  if (early_stopping && best_combined_f1) {
    cerr << "Choosing parameters from epoch " << best_combined_f1_epoch+1 << "." << endl;
    this->embeddings = best_combined_f1_network.embeddings;
    this->gru_fwd = best_combined_f1_network.gru_fwd;
    this->gru_bwd = best_combined_f1_network.gru_bwd;
    this->projection_fwd = best_combined_f1_network.projection_fwd;
    this->projection_bwd = best_combined_f1_network.projection_bwd;
  }

  // Encode the network
  enc.add_1B(1);
  enc.add_1B(D);

  enc.add_4B(this->embeddings.size());
  for (auto&& embedding : this->embeddings) {
    enc.add_4B(embedding.first);
    enc.add_data(embedding.second.e.w[0], D);
  }
  save_gru(this->gru_fwd, enc);
  save_gru(this->gru_bwd, enc);
  save_matrix(this->projection_fwd, enc);
  save_matrix(this->projection_bwd, enc);

  return true;
}

template <int D> template <int R, int C>
void gru_tokenizer_network_trainer<D>::matrix_trainer<R, C>::update_weights(float learning_rate) {
  for (int i = 0; i < R; i++) {
    for (int j = 0; j < C; j++) {
      w_m[i][j] = 0.9 * w_m[i][j] + (1-0.9) * w_g[i][j];
      w_v[i][j] = 0.999 * w_v[i][j] + (1-0.999) * w_g[i][j] * w_g[i][j];
      original.w[i][j] += learning_rate * w_m[i][j] / (sqrt(w_v[i][j]) + 1e-8);
    }
    b_m[i] = 0.9 * b_m[i] + (1-0.9) * b_g[i];
    b_v[i] = 0.999 * b_v[i] + (1-0.999) * b_g[i] * b_g[i];
    original.b[i] += learning_rate * b_m[i] / (sqrt(b_v[i]) + 1e-8);
  }

  for (int i = 0; i < R; i++) {
    for (int j = 0; j < C; j++)
      w_g[i][j] = 0.f;
    b_g[i] = 0.f;
  }
}

template <int D>
void gru_tokenizer_network_trainer<D>::gru_trainer::update_weights(float learning_rate) {
  X.update_weights(learning_rate);
  X_r.update_weights(learning_rate);
  X_z.update_weights(learning_rate);
  H.update_weights(learning_rate);
  H_r.update_weights(learning_rate);
  H_z.update_weights(learning_rate);
}

template <int D>
void gru_tokenizer_network_trainer<D>::evaluate(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const vector<tokenized_sentence>& heldout,
                                                f1_info& tokens_f1, f1_info& sentences_f1) {
  // Generate gold data
  vector<token_range> gold_sentences, gold_tokens;
  u32string text;
  for (auto&& sentence : heldout) {
    if (sentence.tokens.empty()) continue;

    gold_sentences.emplace_back(text.size() + sentence.tokens.front().start, sentence.tokens.back().start + sentence.tokens.back().length - sentence.tokens.front().start);
    for (auto&& token : sentence.tokens)
      gold_tokens.emplace_back(text.size() + token.start, token.length);
    text.append(sentence.sentence);
  }

  // Generate system data
  vector<token_range> system_sentences, system_tokens, tokens;
  string text_utf8;

  this->cache_embeddings();
  gru_tokenizer tokenizer(url_email_tokenizer, segment, allow_spaces, *this);
  unilib::utf8::encode(text, text_utf8);
  tokenizer.set_text(text_utf8);

  while (tokenizer.next_sentence(tokens))
    if (!tokens.empty()) {
      system_sentences.emplace_back(tokens.front().start, tokens.back().start + tokens.back().length - tokens.front().start);
      system_tokens.insert(system_tokens.end(), tokens.begin(), tokens.end());
    }

  evaluate_f1(system_tokens, gold_tokens, tokens_f1);
  evaluate_f1(system_sentences, gold_sentences, sentences_f1);
}

template <int D>
void gru_tokenizer_network_trainer<D>::evaluate_f1(const vector<token_range>& system, const vector<token_range>& gold, f1_info& f1) {
  size_t both = 0;
  for (size_t si = 0, gi = 0; si < system.size() || gi < gold.size(); )
    if (si < system.size() && (gi == gold.size() || system[si].start < gold[gi].start))
      si++;
    else if (gi < gold.size() && (si == system.size() || gold[gi].start < system[si].start))
      gi++;
    else
      both += system[si++].length == gold[gi++].length;

  f1.precision = system.size() ? both / double(system.size()) : 0.;
  f1.recall = gold.size() ? both / double(gold.size()) : 0.;
  f1.f1 = system.size()+gold.size() ? 2 * both / double(system.size() + gold.size()) : 0.;
}

template <int D> template <int R, int C>
void gru_tokenizer_network_trainer<D>::random_matrix(matrix<R,C>& m, mt19937& generator, float range, float bias) {
  uniform_real_distribution<float> uniform(-range, range);
  for (int i = 0; i < R; i++) {
    m.b[i] = bias;
    for (int j = 0; j < C; j++)
      m.w[i][j] = uniform(generator);
  }
}

template <int D>
void gru_tokenizer_network_trainer<D>::random_gru(gru& g, mt19937& generator, float range) {
  random_matrix(g.X, generator, range, 0.f);
  random_matrix(g.X_r, generator, range, 1.f);
  random_matrix(g.X_z, generator, range, 1.f);
  random_matrix(g.H, generator, range, 0.f);
  random_matrix(g.H_r, generator, range, 1.f);
  random_matrix(g.H_z, generator, range, 1.f);
}

template <int D> template <int R, int C>
void gru_tokenizer_network_trainer<D>::save_matrix(const matrix<R,C>& m, binary_encoder& enc) {
  for (int i = 0; i < R; i++)
    enc.add_data(m.w[i], C);
  enc.add_data(m.b, R);
}

template <int D>
void gru_tokenizer_network_trainer<D>::save_gru(const gru& g, binary_encoder& enc) {
  save_matrix(g.X, enc);
  save_matrix(g.X_r, enc);
  save_matrix(g.X_z, enc);
  save_matrix(g.H, enc);
  save_matrix(g.H_r, enc);
  save_matrix(g.H_z, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_trainer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool gru_tokenizer_trainer::train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned dimension, unsigned epochs,
                                  unsigned batch_size, float learning_rate, float learning_rate_final, float dropout,
                                  float initialization_range, bool early_stopping, const vector<tokenized_sentence>& data,
                                  const vector<tokenized_sentence>& heldout, ostream& os, string& error) {
  using namespace unilib;

  error.clear();

  // Start encoding the tokenizer
  os.put(2);

  binary_encoder enc;
  enc.add_1B(url_email_tokenizer);
  enc.add_2B(segment);
  enc.add_1B(allow_spaces);

  // Train the GRU network
  if (dimension == 16) {
    gru_tokenizer_network_trainer<16> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else if (dimension == 24) {
    gru_tokenizer_network_trainer<24> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else if (dimension == 64) {
    gru_tokenizer_network_trainer<64> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else {
    return error.assign("Gru tokenizer dimension '").append(to_string(dimension)).append("' is not supported!"), false;
  }

  // Compute best substitutions for every category
  unordered_map<unicode::category_t, unordered_map<char32_t, unsigned>> counts;
  for (auto&& sentence : data)
    for (auto&& chr : sentence.sentence)
      counts[unicode::category(chr)][chr]++;

  unordered_map<unicode::category_t, char32_t> unknown_chars;
  for (auto&& count : counts) {
    char32_t best_chr = 0;
    unsigned best = 0;
    for (auto&& chr : count.second)
      if (chr.second > best)
        best = chr.second, best_chr = chr.first;
    if (best_chr)
      unknown_chars.emplace(count.first, best_chr);
  }
  enc.add_1B(unknown_chars.size());
  for (auto&& unknown_char : unknown_chars) {
    enc.add_4B(unknown_char.first);
    enc.add_4B(unknown_char.second);
  }

  if (!compressor::save(os, enc)) return error.assign("Cannot save gru_tokenizer_factory!"), false;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/ragel_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _ragel_url_email_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 2, 3, 3, 4, 5,
	6, 7, 8, 9, 10, 11, 12, 13,
	14, 15, 16
};

static const char _ragel_url_email_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 1, 1, 0, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1
};

static const short _ragel_url_email_cond_keys[] = {
	41u, 41u, 47u, 47u, 47u, 47u, 41u, 41u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 0
};

static const char _ragel_url_email_cond_spaces[] = {
	1, 0, 0, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0
};

static const short _ragel_url_email_key_offsets[] = {
	0, 0, 15, 29, 41, 54, 63, 71,
	78, 86, 92, 100, 117, 145, 154, 162,
	171, 179, 188, 196, 204, 215, 225, 233,
	241, 252, 262, 270, 278, 289, 299, 315,
	330, 346, 360, 376, 393, 409, 426, 442,
	459, 475, 491, 510, 528, 544, 560, 579,
	597, 613, 629, 648, 666, 682, 698, 714,
	725, 726, 741, 752, 756, 773, 801, 812,
	823, 834, 848, 861, 879, 893, 908, 926,
	944, 962, 983
};

static const short _ragel_url_email_trans_keys[] = {
	33u, 48u, 49u, 50u, 95u, 36u, 37u, 39u,
	46u, 51u, 57u, 65u, 90u, 97u, 122u, 33u,
	58u, 64u, 95u, 36u, 37u, 39u, 46u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 95u, 36u,
	37u, 39u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 33u, 64u, 95u, 36u, 37u, 39u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 48u, 49u,
	50u, 51u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 48u, 57u, 65u, 90u, 97u, 122u, 45u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 33u, 39u, 41u, 61u,
	95u, 36u, 47u, 48u, 57u, 58u, 59u, 63u,
	64u, 65u, 90u, 97u, 122u, 33u, 39u, 40u,
	44u, 46u, 61u, 63u, 95u, 129u, 131u, 135u,
	151u, 809u, 1065u, 36u, 38u, 42u, 57u, 58u,
	59u, 64u, 90u, 97u, 122u, 142u, 143u, 155u,
	159u, 48u, 49u, 50u, 51u, 57u, 65u, 90u,
	97u, 122u, 45u, 46u, 48u, 57u, 65u, 90u,
	97u, 122u, 48u, 49u, 50u, 51u, 57u, 65u,
	90u, 97u, 122u, 45u, 46u, 48u, 57u, 65u,
	90u, 97u, 122u, 48u, 49u, 50u, 51u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 53u, 48u,
	52u, 54u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 48u, 53u, 54u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 53u, 48u, 52u, 54u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 53u,
	54u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	53u, 48u, 52u, 54u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 53u, 54u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 58u, 64u, 95u,
	36u, 37u, 39u, 46u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 58u, 64u, 95u, 36u, 37u,
	39u, 46u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 48u, 49u, 50u, 58u, 64u, 95u, 36u,
	37u, 39u, 46u, 51u, 57u, 65u, 90u, 97u,
	122u, 33u, 45u, 46u, 58u, 64u, 95u, 36u,
	37u, 39u, 44u, 48u, 57u, 65u, 90u, 97u,
	122u, 33u, 48u, 49u, 50u, 58u, 64u, 95u,
	36u, 37u, 39u, 46u, 51u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 48u, 49u, 50u, 58u, 64u,
	95u, 36u, 37u, 39u, 46u, 51u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 53u, 58u,
	64u, 95u, 36u, 37u, 39u, 44u, 48u, 52u,
	54u, 57u, 65u, 90u, 97u, 122u, 33u, 45u,
	46u, 58u, 64u, 95u, 36u, 37u, 39u, 44u,
	48u, 53u, 54u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 53u, 58u, 64u, 95u, 36u,
	37u, 39u, 44u, 48u, 52u, 54u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 53u, 54u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 36u, 37u, 39u, 44u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 36u, 37u, 39u, 44u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	53u, 58u, 64u, 95u, 36u, 37u, 39u, 44u,
	48u, 52u, 54u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 53u, 54u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 47u, 95u, 36u, 37u, 39u,
	57u, 65u, 90u, 97u, 122u, 47u, 33u, 48u,
	49u, 50u, 95u, 36u, 37u, 39u, 46u, 51u,
	57u, 65u, 90u, 97u, 122u, 45u, 46u, 58u,
	303u, 559u, 48u, 57u, 65u, 90u, 97u, 122u,
	303u, 559u, 48u, 57u, 33u, 39u, 41u, 61u,
	95u, 36u, 47u, 48u, 57u, 58u, 59u, 63u,
	64u, 65u, 90u, 97u, 122u, 33u, 39u, 40u,
	44u, 46u, 61u, 63u, 95u, 129u, 131u, 135u,
	151u, 809u, 1065u, 36u, 38u, 42u, 57u, 58u,
	59u, 64u, 90u, 97u, 122u, 142u, 143u, 155u,
	159u, 45u, 46u, 58u, 303u, 559u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 58u, 303u,
	559u, 48u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 58u, 303u, 559u, 48u, 57u, 65u, 90u,
	97u, 122u, 45u, 46u, 53u, 58u, 303u, 559u,
	48u, 52u, 54u, 57u, 65u, 90u, 97u, 122u,
	45u, 46u, 58u, 303u, 559u, 48u, 53u, 54u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 303u, 559u, 36u, 37u, 39u,
	44u, 48u, 57u, 65u, 90u, 97u, 122u, 33u,
	95u, 303u, 559u, 36u, 37u, 39u, 46u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 64u, 95u,
	303u, 559u, 36u, 37u, 39u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 33u, 45u, 46u, 58u,
	64u, 95u, 303u, 559u, 36u, 37u, 39u, 44u,
	48u, 57u, 65u, 90u, 97u, 122u, 33u, 45u,
	46u, 58u, 64u, 95u, 303u, 559u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 303u, 559u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 53u, 58u, 64u,
	95u, 303u, 559u, 36u, 37u, 39u, 44u, 48u,
	52u, 54u, 57u, 65u, 90u, 97u, 122u, 33u,
	45u, 46u, 58u, 64u, 95u, 303u, 559u, 36u,
	37u, 39u, 44u, 48u, 53u, 54u, 57u, 65u,
	90u, 97u, 122u, 0
};

static const char _ragel_url_email_single_lengths[] = {
	0, 5, 4, 2, 3, 3, 2, 1,
	2, 0, 2, 5, 14, 3, 2, 3,
	2, 3, 2, 2, 3, 2, 2, 2,
	3, 2, 2, 2, 3, 2, 6, 5,
	6, 4, 6, 7, 6, 7, 6, 7,
	6, 6, 7, 6, 6, 6, 7, 6,
	6, 6, 7, 6, 6, 6, 6, 3,
	1, 5, 5, 2, 5, 14, 5, 5,
	5, 6, 5, 8, 4, 5, 8, 8,
	8, 9, 8
};

static const char _ragel_url_email_range_lengths[] = {
	0, 5, 5, 5, 5, 3, 3, 3,
	3, 3, 3, 6, 7, 3, 3, 3,
	3, 3, 3, 3, 4, 4, 3, 3,
	4, 4, 3, 3, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 5, 5,
	5, 5, 6, 6, 5, 5, 6, 6,
	5, 5, 6, 6, 5, 5, 5, 4,
	0, 5, 3, 1, 6, 7, 3, 3,
	3, 4, 4, 5, 5, 5, 5, 5,
	5, 6, 6
};

static const short _ragel_url_email_index_offsets[] = {
	0, 0, 11, 21, 29, 38, 45, 51,
	56, 62, 66, 72, 84, 106, 113, 119,
	126, 132, 139, 145, 151, 159, 166, 172,
	178, 186, 193, 199, 205, 213, 220, 232,
	243, 255, 265, 277, 290, 302, 315, 327,
	340, 352, 364, 378, 391, 403, 415, 429,
	442, 454, 466, 480, 493, 505, 517, 529,
	537, 539, 550, 559, 563, 575, 597, 606,
	615, 624, 635, 645, 659, 669, 680, 694,
	708, 722, 738
};

static const char _ragel_url_email_indicies[] = {
	0, 2, 3, 4, 0, 0, 0, 5,
	6, 6, 1, 0, 7, 8, 0, 0,
	0, 0, 0, 0, 1, 9, 9, 9,
	9, 9, 9, 9, 1, 9, 8, 9,
	9, 9, 9, 9, 9, 1, 10, 11,
	12, 13, 14, 14, 1, 15, 16, 14,
	14, 14, 1, 15, 14, 14, 14, 1,
	15, 17, 14, 14, 14, 1, 14, 18,
	18, 1, 15, 17, 14, 19, 19, 1,
	20, 21, 21, 20, 20, 20, 21, 20,
	20, 21, 21, 1, 22, 22, 24, 22,
	22, 23, 22, 23, 23, 23, 23, 23,
	25, 26, 23, 23, 22, 23, 23, 23,
	23, 1, 27, 28, 29, 30, 18, 18,
	1, 15, 31, 14, 14, 14, 1, 32,
	33, 34, 35, 18, 18, 1, 15, 36,
	14, 14, 14, 1, 37, 38, 39, 40,
	18, 18, 1, 15, 36, 35, 14, 14,
	1, 15, 36, 32, 14, 14, 1, 15,
	36, 41, 35, 32, 14, 14, 1, 15,
	36, 32, 14, 14, 14, 1, 15, 31,
	30, 14, 14, 1, 15, 31, 27, 14,
	14, 1, 15, 31, 42, 30, 27, 14,
	14, 1, 15, 31, 27, 14, 14, 14,
	1, 15, 16, 13, 14, 14, 1, 15,
	16, 10, 14, 14, 1, 15, 16, 43,
	13, 10, 14, 14, 1, 15, 16, 10,
	14, 14, 14, 1, 0, 44, 45, 7,
	8, 0, 0, 0, 46, 46, 46, 1,
	0, 44, 7, 8, 0, 0, 0, 46,
	46, 46, 1, 0, 44, 47, 7, 8,
	0, 0, 0, 46, 46, 46, 1, 0,
	7, 8, 0, 0, 0, 46, 48, 48,
	1, 0, 44, 47, 7, 8, 0, 0,
	0, 46, 49, 49, 1, 0, 50, 51,
	52, 7, 8, 0, 0, 0, 53, 48,
	48, 1, 0, 44, 54, 7, 8, 0,
	0, 0, 46, 46, 46, 1, 0, 55,
	56, 57, 7, 8, 0, 0, 0, 58,
	48, 48, 1, 0, 44, 59, 7, 8,
	0, 0, 0, 46, 46, 46, 1, 0,
	60, 61, 62, 7, 8, 0, 0, 0,
	63, 48, 48, 1, 0, 44, 59, 7,
	8, 0, 0, 0, 58, 46, 46, 1,
	0, 44, 59, 7, 8, 0, 0, 0,
	55, 46, 46, 1, 0, 44, 59, 64,
	7, 8, 0, 0, 0, 58, 55, 46,
	46, 1, 0, 44, 59, 7, 8, 0,
	0, 0, 55, 46, 46, 46, 1, 0,
	44, 54, 7, 8, 0, 0, 0, 53,
	46, 46, 1, 0, 44, 54, 7, 8,
	0, 0, 0, 50, 46, 46, 1, 0,
	44, 54, 65, 7, 8, 0, 0, 0,
	53, 50, 46, 46, 1, 0, 44, 54,
	7, 8, 0, 0, 0, 50, 46, 46,
	46, 1, 0, 44, 45, 7, 8, 0,
	0, 0, 5, 46, 46, 1, 0, 44,
	45, 7, 8, 0, 0, 0, 2, 46,
	46, 1, 0, 44, 45, 66, 7, 8,
	0, 0, 0, 5, 2, 46, 46, 1,
	0, 44, 45, 7, 8, 0, 0, 0,
	2, 46, 46, 46, 1, 0, 44, 47,
	7, 8, 0, 0, 0, 46, 67, 67,
	1, 0, 44, 47, 7, 8, 0, 0,
	0, 46, 68, 68, 1, 0, 44, 47,
	69, 8, 0, 0, 0, 46, 68, 68,
	1, 9, 70, 9, 9, 9, 9, 9,
	1, 71, 1, 0, 2, 3, 4, 0,
	0, 0, 5, 46, 46, 1, 15, 17,
	72, 21, 23, 14, 19, 19, 1, 21,
	23, 72, 1, 20, 21, 21, 20, 20,
	20, 21, 20, 20, 21, 21, 1, 22,
	22, 24, 22, 22, 23, 22, 23, 23,
	23, 23, 23, 25, 26, 23, 23, 22,
	23, 23, 23, 23, 1, 15, 17, 72,
	21, 23, 14, 14, 14, 1, 15, 17,
	72, 21, 23, 40, 14, 14, 1, 15,
	17, 72, 21, 23, 37, 14, 14, 1,
	15, 17, 73, 72, 21, 23, 40, 37,
	14, 14, 1, 15, 17, 72, 21, 23,
	37, 14, 14, 14, 1, 0, 44, 47,
	74, 8, 0, 21, 23, 0, 0, 46,
	49, 49, 1, 9, 9, 21, 23, 9,
	9, 75, 9, 9, 1, 9, 8, 9,
	21, 23, 9, 9, 75, 9, 9, 1,
	0, 44, 47, 74, 8, 0, 21, 23,
	0, 0, 46, 46, 46, 1, 0, 44,
	47, 74, 8, 0, 21, 23, 0, 0,
	63, 46, 46, 1, 0, 44, 47, 74,
	8, 0, 21, 23, 0, 0, 60, 46,
	46, 1, 0, 44, 47, 76, 74, 8,
	0, 21, 23, 0, 0, 63, 60, 46,
	46, 1, 0, 44, 47, 74, 8, 0,
	21, 23, 0, 0, 60, 46, 46, 46,
	1, 0
};

static const char _ragel_url_email_trans_targs[] = {
	2, 0, 30, 48, 50, 49, 52, 3,
	5, 4, 6, 26, 28, 27, 8, 7,
	13, 9, 10, 58, 11, 60, 12, 61,
	61, 12, 61, 14, 22, 24, 23, 15,
	16, 18, 20, 19, 17, 62, 63, 65,
	64, 21, 25, 29, 31, 35, 32, 33,
	34, 67, 36, 44, 46, 45, 37, 38,
	40, 42, 41, 39, 70, 71, 73, 72,
	43, 47, 51, 53, 54, 55, 56, 57,
	59, 66, 68, 69, 74
};

static const char _ragel_url_email_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 1, 0, 1, 0, 1,
	2, 3, 4, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 1, 1,
	1, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 1, 1, 1,
	0, 0, 0, 0, 0, 0, 0, 0,
	1, 1, 1, 1, 1
};

static const int ragel_url_email_start = 1;

vector<uint8_t> ragel_tokenizer::ragel_map;
atomic_flag ragel_tokenizer::ragel_map_flag = ATOMIC_FLAG_INIT;

ragel_tokenizer::ragel_tokenizer(unsigned url_email_tokenizer) : unicode_tokenizer(url_email_tokenizer) {
  initialize_ragel_map();
}

void ragel_tokenizer::initialize_ragel_map() {
  while (ragel_map_flag.test_and_set()) {}
  if (ragel_map.empty()) {
    for (uint8_t ascii = 0; ascii < 128; ascii++)
      ragel_map.push_back(ascii);

    ragel_map_add(U'\u2026', 160); // horizontal ellipsis (TRIPLE DOT)
    ragel_map_add(U'\u2019', 161); // right single quotation mark
    ragel_map_add(U'\u2018', 162); // left single quotation mark
    ragel_map_add(U'\u2010', 163); // hyphen
  }
  ragel_map_flag.clear();
}

void ragel_tokenizer::ragel_map_add(char32_t chr, uint8_t mapping) {
  if (chr >= ragel_map.size())
    ragel_map.resize(chr + 1, 128);
  ragel_map[chr] = mapping;
}

bool ragel_tokenizer::ragel_url_email(unsigned version, const vector<char_info>& chars, size_t& current, vector<token_range>& tokens) {
  int cs;

  size_t start = current, end = current, parens = 0;

	{
	cs = ragel_url_email_start;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_widec = ( ragel_char(chars[current]));
	_klen = _ragel_url_email_cond_lengths[cs];
	_keys = _ragel_url_email_cond_keys + (_ragel_url_email_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _ragel_url_email_cond_spaces[_ragel_url_email_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 version >= 2  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
parens ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _ragel_url_email_trans_keys + _ragel_url_email_key_offsets[cs];
	_trans = _ragel_url_email_index_offsets[cs];

	_klen = _ragel_url_email_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _ragel_url_email_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _ragel_url_email_indicies[_trans];
	cs = _ragel_url_email_trans_targs[_trans];

	if ( _ragel_url_email_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _ragel_url_email_trans_actions[_trans] ) {
	case 3:
	{parens-=!!parens;}
	break;
	case 1:
	{ end = current + 1; }
	break;
	case 2:
	{parens++;}
	{ end = current + 1; }
	break;
	case 4:
	{parens-=!!parens;}
	{ end = current + 1; }
	break;
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  if (end > start) {
    tokens.emplace_back(start, end - start);
    current = end;
    return true;
  } else {
    current = start;
    return false;
  }
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/vertical_tokenizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class vertical_tokenizer : public unicode_tokenizer {
 public:
  vertical_tokenizer() : unicode_tokenizer(0) {}

  virtual bool next_sentence(vector<token_range>& tokens) override;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* tokenizer::new_vertical_tokenizer() {
  return new vertical_tokenizer();
}

tokenizer* tokenizer::new_czech_tokenizer() {
  return new czech_tokenizer(czech_tokenizer::CZECH, czech_tokenizer::LATEST);
}

tokenizer* tokenizer::new_english_tokenizer() {
  return new english_tokenizer(english_tokenizer::LATEST);
}

tokenizer* tokenizer::new_generic_tokenizer() {
  return new generic_tokenizer(generic_tokenizer::LATEST);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_ids.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tokenizer_ids {
 public:
  enum tokenizer_id {
    CZECH = 0,
    ENGLISH = 1,
    GENERIC = 2,
    GRU = 3,
  };

  static bool parse(const string& str, tokenizer_id& id) {
    if (str == "czech") return id = CZECH, true;
    if (str == "english") return id = ENGLISH, true;
    if (str == "generic") return id = GENERIC, true;
    if (str == "gru") return id = GRU, true;
    return false;
  }
};

typedef tokenizer_ids::tokenizer_id tokenizer_id;

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer_factory* tokenizer_factory::load(istream& is) {
  tokenizer_id id = tokenizer_id(is.get());
  switch (id) {
    case tokenizer_ids::GENERIC:
      {
        auto res = new_unique_ptr<generic_tokenizer_factory>();
        if (res->load(is)) return res.release();
        break;
      }
    case tokenizer_ids::GRU:
      {
        auto res = new_unique_ptr<gru_tokenizer_factory>();
        if (res->load(is)) return res.release();
        break;
      }
    case tokenizer_ids::CZECH:
      break;
    case tokenizer_ids::ENGLISH:
      break;
  }

  return nullptr;
}

tokenizer_factory* tokenizer_factory::load(const char* fname) {
  ifstream f(fname, ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/unicode_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

unicode_tokenizer::unicode_tokenizer(unsigned url_email_tokenizer) : url_email_tokenizer(url_email_tokenizer) {
  ragel_tokenizer::initialize_ragel_map();

  set_text(string_piece(nullptr, 0));
}

void unicode_tokenizer::set_text(string_piece text, bool make_copy /*= false*/) {
  using namespace unilib;

  if (make_copy && text.str) {
    text_buffer.assign(text.str, text.len);
    text.str = text_buffer.c_str();
  }
  current = 0;

  chars.clear();
  for (const char* curr_str = text.str; text.len; curr_str = text.str)
    chars.emplace_back(utf8::decode(text.str, text.len), curr_str);
  chars.emplace_back(0, text.str);
}

bool unicode_tokenizer::next_sentence(vector<string_piece>* forms, vector<token_range>* tokens_ptr) {
  vector<token_range>& tokens = tokens_ptr ? *tokens_ptr : tokens_buffer;
  tokens.clear();
  if (forms) forms->clear();
  if (current >= chars.size() - 1) return false;

  bool result = next_sentence(tokens);
  if (forms)
    for (auto&& token : tokens)
      forms->emplace_back(chars[token.start].str, chars[token.start + token.length].str - chars[token.start].str);

  return result;
}

bool unicode_tokenizer::tokenize_url_email(vector<token_range>& tokens) {
  if (current >= chars.size() - 1) return false;

  return url_email_tokenizer ? ragel_tokenizer::ragel_url_email(url_email_tokenizer, chars, current, tokens) : false;
}

bool unicode_tokenizer::emergency_sentence_split(const vector<token_range>& tokens) {
  using namespace unilib;

  // Implement emergency splitting for large sentences
  return tokens.size() >= 500 ||
         (tokens.size() >= 450 && chars[tokens.back().start].cat & unicode::P) ||
         (tokens.size() >= 400 && chars[tokens.back().start].cat & unicode::Po);
}

bool unicode_tokenizer::is_eos(const vector<token_range>& tokens, char32_t eos_chr, const unordered_set<string>* abbreviations) {
  using namespace unilib;

  if (eos_chr == '.' && !tokens.empty()) {
    // Ignore one-letter capitals before dot
    if (tokens.back().length == 1 && chars[tokens.back().start].cat & unicode::Lut)
      return false;

    // Ignore specified abbreviations
    if (abbreviations) {
      eos_buffer.clear();
      for (size_t i = 0; i < tokens.back().length; i++)
        utf8::append(eos_buffer, unicode::lowercase(chars[tokens.back().start + i].chr));
      if (abbreviations->count(eos_buffer))
        return false;
    }
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/vertical_tokenizer.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool vertical_tokenizer::next_sentence(vector<token_range>& tokens) {
  if (current >= chars.size() - 1) return false;

  while (true) {
    size_t line_start = current;
    while (current < chars.size() - 1 && chars[current].chr != '\r' && chars[current].chr != '\n') current++;

    size_t line_end = current;
    if (current < chars.size() - 1) {
      current++;
      if (current < chars.size() - 1 &&
          ((chars[current-1].chr == '\r' && chars[current].chr == '\n') ||
           (chars[current-1].chr == '\n' && chars[current].chr == '\r')))
        current++;
    }

    if (line_start < line_end)
      tokens.emplace_back(line_start, line_end - line_start);
    else
      break;
  }

  return true;
}

} // namespace morphodita

/////////
// File: unilib/version.h
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

struct version {
  unsigned major;
  unsigned minor;
  unsigned patch;
  std::string prerelease;

  // Returns current version.
  static version current();
};

} // namespace unilib

/////////
// File: morphodita/version/version.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class version {
 public:
  unsigned major;
  unsigned minor;
  unsigned patch;
  string prerelease;

  // Returns current MorphoDiTa version.
  static version current();

  // Returns multi-line formated version and copyright string.
  static string version_and_copyright(const string& other_libraries = string());
};

} // namespace morphodita

/////////
// File: morphodita/version/version.cpp
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

version version::current() {
  return {1, 9, 3, "devel"};
}

// Returns multi-line formated version and copyright string.
string version::version_and_copyright(const string& other_libraries) {
  ostringstream info;

  auto morphodita = version::current();
  auto unilib = unilib::version::current();

  info << "MorphoDiTa version " << morphodita.major << '.' << morphodita.minor << '.' << morphodita.patch
       << (morphodita.prerelease.empty() ? "" : "-") << morphodita.prerelease
       << " (using UniLib " << unilib.major << '.' << unilib.minor << '.' << unilib.patch
       << (other_libraries.empty() ? "" : " and ") << other_libraries << ")\n"
          "Copyright 2015 by Institute of Formal and Applied Linguistics, Faculty of\n"
          "Mathematics and Physics, Charles University in Prague, Czech Republic.";

  return info.str();
}

} // namespace morphodita

/////////
// File: parsito/configuration/configuration.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void configuration::init(tree* t) {
  assert(t);

  t->unlink_all_nodes();
  this->t = t;

  stack.clear();
  if (!t->nodes.empty()) stack.push_back(0);

  buffer.clear();
  buffer.reserve(t->nodes.size());
  for (size_t i = t->nodes.size(); i > 1; i--)
    buffer.push_back(i - 1);
}

bool configuration::final() {
  return buffer.empty() && stack.size() <= 1;
}

} // namespace parsito

/////////
// File: parsito/configuration/node_extractor.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class node_extractor {
 public:
  unsigned node_count() const;
  void extract(const configuration& conf, vector<int>& nodes) const;

  bool create(string_piece description, string& error);

 private:
  enum start_t { STACK = 0, BUFFER = 1 };
  enum direction_t { PARENT = 0, CHILD = 1 };
  struct node_selector {
    pair<start_t, int> start;
    vector<pair<direction_t, int>> directions;

    node_selector(start_t start, int start_index) : start(start, start_index) {}
  };

  vector<node_selector> selectors;
};

} // namespace parsito

/////////
// File: parsito/configuration/node_extractor.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

unsigned node_extractor::node_count() const {
  return selectors.size();
}

void node_extractor::extract(const configuration& conf, vector<int>& nodes) const {
  nodes.clear();
  for (auto&& selector : selectors) {
    // Start by locating starting node
    int current = -1;
    switch (selector.start.first) {
      case STACK:
        if (selector.start.second < int(conf.stack.size()))
          current = conf.stack[conf.stack.size() - 1 - selector.start.second];
        break;
      case BUFFER:
        if (selector.start.second < int(conf.buffer.size()))
          current = conf.buffer[conf.buffer.size() - 1 - selector.start.second];
        break;
    }

    // Follow directions to the final node
    if (current >= 0)
      for (auto&& direction : selector.directions) {
        const node& node = conf.t->nodes[current];
        switch (direction.first) {
          case PARENT:
            current = node.head ? node.head : -1;
            break;
          case CHILD:
            current = direction.second >= 0 && direction.second < int(node.children.size()) ?
                        node.children[direction.second] :
                      direction.second < 0 && -direction.second <= int(node.children.size()) ?
                        node.children[node.children.size() + direction.second] :
                        -1;
            break;
        }
        if (current <= 0) break;
      }

    // Add the selected node
    nodes.push_back(current);
  }
}

bool node_extractor::create(string_piece description, string& error) {
  selectors.clear();
  error.clear();

  vector<string_piece> lines, parts, words;
  split(description, '\n', lines);
  for (auto&& line : lines) {
    if (!line.len || line.str[0] == '#') continue;

    // Separate start and directions
    split(line, ',', parts);

    // Parse start
    split(parts[0], ' ', words);
    if (words.size() != 2)
      return error.assign("The node selector '").append(parts[0].str, parts[0].len).append("' on line '").append(line.str, line.len).append("' does not contain two space separated values!"), false;

    start_t start;
    if (words[0] == "stack")
      start = STACK;
    else if (words[0] == "buffer")
      start = BUFFER;
    else
      return error.assign("Cannot parse starting location '").append(words[0].str, words[0].len).append("' on line '").append(line.str, line.len).append(".!"), false;

    int start_index;
    if (!parse_int(words[1], "starting index", start_index, error)) return false;

    selectors.emplace_back(start, start_index);

    // Parse directions
    for (size_t i = 1; i < parts.size(); i++) {
      split(parts[i], ' ', words);
      if (words.empty())
        return error.assign("Empty node selector on line '").append(line.str, line.len).append(".!"), false;

      if (words[0] == "parent") {
        if (words.size() != 1)
          return error.assign("The node selector '").append(parts[i].str, parts[i].len).append("' on line '").append(line.str, line.len).append("' does not contain one space separated value!"), false;
        selectors.back().directions.emplace_back(PARENT, 0);
      } else if (words[0] == "child") {
        if (words.size() != 2)
          return error.assign("The node selector '").append(parts[i].str, parts[i].len).append("' on line '").append(line.str, line.len).append("' does not contain two space separated values!"), false;
        int child_index;
        if (!parse_int(words[1], "child index", child_index, error)) return false;
        selectors.back().directions.emplace_back(CHILD, child_index);
      } else {
        return error.assign("Cannot parse direction location '").append(words[0].str, words[0].len).append("' on line '").append(line.str, line.len).append(".!"), false;
      }
    }
  }

  return true;
}

} // namespace parsito

/////////
// File: parsito/configuration/value_extractor.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class value_extractor {
 public:
  void extract(const node& n, string& value) const;

  bool create(string_piece description, string& error);

 private:
  enum value_t { FORM = 0, LEMMA = 1, LEMMA_ID = 2, TAG = 3, UNIVERSAL_TAG = 4,
    FEATS = 5, UNIVERSAL_TAG_FEATS = 6, DEPREL = 7 };
  value_t selector;
};

} // namespace parsito

/////////
// File: parsito/configuration/value_extractor.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void value_extractor::extract(const node& n, string& value) const {
  switch (selector) {
    case FORM:
      value.assign(n.form);
      break;
    case LEMMA:
      value.assign(n.lemma);
      break;
    case LEMMA_ID:
      if (!n.misc.empty()) {
        // Try finding LId= in misc column
        auto lid = n.misc.find("LId=");
        if (lid != string::npos) {
          lid += 4;

          // Find optional | ending the lemma_id
          auto lid_end = n.misc.find('|', lid);
          if (lid_end == string::npos) lid_end = n.misc.size();

          // Store the lemma_id
          value.assign(n.misc, lid, lid_end - lid);
          break;
        }
      }
      value.assign(n.lemma);
      break;
    case TAG:
      value.assign(n.xpostag);
      break;
    case UNIVERSAL_TAG:
      value.assign(n.upostag);
      break;
    case FEATS:
      value.assign(n.feats);
      break;
    case UNIVERSAL_TAG_FEATS:
      value.assign(n.upostag).append(n.feats);
      break;
    case DEPREL:
      value.assign(n.deprel);
      break;
  }
}

bool value_extractor::create(string_piece description, string& error) {
  error.clear();

  if (description == "form")
    selector = FORM;
  else if (description == "lemma")
    selector = LEMMA;
  else if (description == "lemma_id")
    selector = LEMMA_ID;
  else if (description == "tag")
    selector = TAG;
  else if (description == "universal_tag")
    selector = UNIVERSAL_TAG;
  else if (description == "feats")
    selector = FEATS;
  else if (description == "universal_tag_feats")
    selector = UNIVERSAL_TAG_FEATS;
  else if (description == "deprel")
    selector = DEPREL;
  else
    return error.assign("Cannot parse value selector '").append(description.str, description.len).append("'!"), false;

  return true;
}

} // namespace parsito

/////////
// File: parsito/embedding/embedding.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class embedding {
 public:
  unsigned dimension;

  int lookup_word(const string& word, string& buffer) const;
  int unknown_word() const;
  float* weight(int id); // nullptr for wrong id
  const float* weight(int id) const; // nullpt for wrong id

  bool can_update_weights(int id) const;

  void load(binary_decoder& data);
  void save(binary_encoder& enc) const;

  void create(unsigned dimension, int updatable_index, const vector<pair<string, vector<float>>>& words, const vector<float>& unknown_weights);
  void export_embeddings(vector<pair<string, vector<float>>>& words, vector<float>& unknown_weights) const;
 private:
  int updatable_index, unknown_index;

  unordered_map<string, int> dictionary;
  vector<float> weights;
};

} // namespace parsito

/////////
// File: parsito/embedding/embedding.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

int embedding::lookup_word(const string& word, string& buffer) const {
  using namespace unilib;

  auto it = dictionary.find(word);
  if (it != dictionary.end()) return it->second;

  // We now apply several heuristics to find a match

  // Try locating uppercase/titlecase characters which we could lowercase
  bool first = true;
  unicode::category_t first_category = 0, other_categories = 0;
  for (auto&& chr : utf8::decoder(word)) {
    (first ? first_category : other_categories) |= unicode::category(chr);
    first = false;
  }

  if ((first_category & unicode::Lut) && (other_categories & unicode::Lut)) {
    // Lowercase all characters but the first
    buffer.clear();
    first = true;
    for (auto&& chr : utf8::decoder(word)) {
      utf8::append(buffer, first ? chr : unicode::lowercase(chr));
      first = false;
    }

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  if ((first_category & unicode::Lut) || (other_categories & unicode::Lut)) {
    utf8::map(unicode::lowercase, word, buffer);

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  // If the word starts with digit and contain only digits and non-letter characters
  // i.e. large number, date, time, try replacing it with first digit only.
  if ((first_category & unicode::N) && !(other_categories & unicode::L)) {
    buffer.clear();
    utf8::append(buffer, utf8::first(word));

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  return unknown_index;
}

int embedding::unknown_word() const {
  return unknown_index;
}

float* embedding::weight(int id) {
  if (id < 0 || id * dimension >= weights.size()) return nullptr;
  return weights.data() + id * dimension;
}

const float* embedding::weight(int id) const {
  if (id < 0 || id * dimension >= weights.size()) return nullptr;
  return weights.data() + id * dimension;
}

void embedding::load(binary_decoder& data) {
  // Load dimemsion
  dimension = data.next_4B();

  updatable_index = numeric_limits<decltype(updatable_index)>::max();

  // Load dictionary
  dictionary.clear();
  string word;
  for (unsigned size = data.next_4B(); size; size--) {
    data.next_str(word);
    dictionary.emplace(word, dictionary.size());
  }

  unknown_index = data.next_1B() ? dictionary.size() : -1;

  // Load weights
  const float* weights_ptr = data.next<float>(dimension * (dictionary.size() + (unknown_index >= 0)));
  weights.assign(weights_ptr, weights_ptr + dimension * (dictionary.size() + (unknown_index >= 0)));
}

} // namespace parsito

/////////
// File: parsito/embedding/embedding_encode.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void embedding::save(binary_encoder& enc) const {
  // Save dimension and update_weight
  enc.add_4B(dimension);

  // Save the dictionary
  vector<string_piece> words(dictionary.size());
  for (auto&& entry : dictionary) {
    assert(entry.second >= 0 && entry.second < int(dictionary.size()));
    words[entry.second] = entry.first;
  }
  enc.add_4B(dictionary.size());
  for (auto&& word : words)
    enc.add_str(word);

  enc.add_1B(unknown_index >= 0);

  // Save the weights
  enc.add_data(weights);
}

bool embedding::can_update_weights(int id) const {
  return id >= int(updatable_index);
}

void embedding::create(unsigned dimension, int updatable_index, const vector<pair<string, vector<float>>>& words, const vector<float>& unknown_weights) {
  this->dimension = dimension;
  this->updatable_index = updatable_index;

  dictionary.clear();
  weights.clear();
  for (auto&& word : words) {
    assert(word.second.size() == dimension);
    dictionary.emplace(word.first, dictionary.size());
    weights.insert(weights.end(), word.second.begin(), word.second.end());
  }

  if (unknown_weights.empty()) {
    this->unknown_index = -1;
  } else {
    this->unknown_index = dictionary.size();
    weights.insert(weights.end(), unknown_weights.begin(), unknown_weights.end());
  }
}

void embedding::export_embeddings(vector<pair<string, vector<float>>>& words, vector<float>& unknown_weights) const {
  words.clear();
  unknown_weights.clear();

  if (dictionary.empty()) return;

  assert(unknown_index < 0 || unknown_index == int(dictionary.size()));

  words.resize(dictionary.size());
  for (auto&& entry : dictionary) {
    words[entry.second].first = entry.first;
    words[entry.second].second.assign(weights.data() + entry.second * dimension, weights.data() + entry.second * dimension + dimension);
  }
  if (unknown_index >= 0)
    unknown_weights.assign(weights.data() + unknown_index * dimension, weights.data() + unknown_index * dimension + dimension);
}

} // namespace parsito

/////////
// File: parsito/network/activation_function.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct activation_function {
  enum type { TANH = 0, CUBIC = 1, RELU = 2 };

  static bool create(string_piece name, type& activation) {
    if (name == "tanh") return activation = TANH, true;
    if (name == "cubic") return activation = CUBIC, true;
    if (name == "relu") return activation = RELU, true;
    return false;
  }
};

} // namespace parsito

/////////
// File: parsito/network/neural_network.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class neural_network {
 public:
  typedef vector<vector<vector<float>>> embeddings_cache;

  void propagate(const vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences,
                 vector<float>& hidden_layer, vector<float>& outcomes, const embeddings_cache* cache = nullptr, bool softmax = true) const;

  void load(binary_decoder& data);
  void generate_tanh_cache();
  void generate_embeddings_cache(const vector<embedding>& embeddings, embeddings_cache& cache, unsigned max_words) const;

 private:
  friend class neural_network_trainer;

  void load_matrix(binary_decoder& data, vector<vector<float>>& m);

  activation_function::type hidden_layer_activation;
  vector<vector<float>> weights[2];

  vector<float> tanh_cache;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void neural_network::load_matrix(binary_decoder& data, vector<vector<float>>& m) {
  unsigned rows = data.next_4B();
  unsigned columns = data.next_4B();

  m.resize(rows);
  for (auto&& row : m) {
    const float* row_ptr = data.next<float>(columns);
    row.assign(row_ptr, row_ptr + columns);
  }
}

void neural_network::load(binary_decoder& data) {
  hidden_layer_activation = activation_function::type(data.next_1B());
  load_matrix(data, weights[0]);
  load_matrix(data, weights[1]);
}

void neural_network::propagate(const vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences,
                               vector<float>& hidden_layer, vector<float>& outcomes, const embeddings_cache* cache, bool softmax) const {
  assert(!weights[0].empty());
  assert(!weights[1].empty());
  for (auto&& embedding_ids : embedding_ids_sequences) if (embedding_ids) assert(embeddings.size() == embedding_ids->size());

  unsigned hidden_layer_size = weights[0].front().size();
  unsigned outcomes_size = weights[1].front().size();

  outcomes.assign(outcomes_size, 0);

  // Hidden layer
  hidden_layer.assign(hidden_layer_size, 0);

  unsigned index = 0;
  for (unsigned sequence = 0; sequence < embedding_ids_sequences.size(); sequence++)
    for (unsigned i = 0; i < embeddings.size(); index += embeddings[i].dimension, i++)
      if (embedding_ids_sequences[sequence] && embedding_ids_sequences[sequence]->at(i) >= 0) {
        unsigned word = embedding_ids_sequences[sequence]->at(i);
        if (cache && i < cache->size() && word < cache->at(i).size()) {
          // Use cache
          const float* precomputed = cache->at(i)[word].data() + sequence * hidden_layer_size;
          for (unsigned j = 0; j < hidden_layer_size; j++)
            hidden_layer[j] += precomputed[j];
        } else {
          // Compute directly
          const float* embedding = embeddings[i].weight(word);
          for (unsigned j = 0; j < embeddings[i].dimension; j++)
            for (unsigned k = 0; k < hidden_layer_size; k++)
              hidden_layer[k] += embedding[j] * weights[0][index + j][k];
        }
      }
  for (unsigned i = 0; i < hidden_layer_size; i++) // Bias
    hidden_layer[i] += weights[0][index][i];

  // Activation function
  switch (hidden_layer_activation) {
    case activation_function::TANH:
      if (!tanh_cache.empty())
        for (auto&& weight : hidden_layer)
          weight = weight <= -10 ? -1 : weight >= 10 ? 1 : tanh_cache[int(weight * 32768 + 10 * 32768)];
      else
        for (auto&& weight : hidden_layer)
          weight = tanh(weight);
      break;
    case activation_function::CUBIC:
      for (auto&& weight : hidden_layer)
        weight = weight * weight * weight;
      break;
    case activation_function::RELU:
      for (auto&& weight : hidden_layer)
        if (weight < 0) weight = 0;
      break;
  }

  for (unsigned i = 0; i < hidden_layer_size; i++)
    for (unsigned j = 0; j < outcomes_size; j++)
      outcomes[j] += hidden_layer[i] * weights[1][i][j];
  for (unsigned i = 0; i < outcomes_size; i++) // Bias
    outcomes[i] += weights[1][hidden_layer_size][i];

  // Softmax if requested
  if (softmax) {
    float max = outcomes[0];
    for (unsigned i = 1; i < outcomes_size; i++) if (outcomes[i] > max) max = outcomes[i];

    float sum = 0;
    for (unsigned i = 0; i < outcomes_size; i++) sum += (outcomes[i] = exp(outcomes[i] - max));
    sum = 1 / sum;

    for (unsigned i = 0; i < outcomes_size; i++) outcomes[i] *= sum;
  }
}

void neural_network::generate_tanh_cache() {
  tanh_cache.resize(2 * 10 * 32768);
  for (unsigned i = 0; i < tanh_cache.size(); i++)
    tanh_cache[i] = tanh(i / 32768.0 - 10);
}

void neural_network::generate_embeddings_cache(const vector<embedding>& embeddings, embeddings_cache& cache, unsigned max_words) const {
  unsigned embeddings_dim = 0;
  for (auto&& embedding : embeddings) embeddings_dim += embedding.dimension;

  unsigned sequences = weights[0].size() / embeddings_dim;
  assert(sequences * embeddings_dim + 1 == weights[0].size());

  unsigned hidden_layer_size = weights[0].front().size();

  cache.resize(embeddings.size());
  for (unsigned i = 0, weight_index = 0; i < embeddings.size(); weight_index += embeddings[i].dimension, i++) {
    unsigned words = 0;
    while (words < max_words && embeddings[i].weight(words)) words++;

    cache[i].resize(words);
    for (unsigned word = 0; word < words; word++) {
      const float* embedding = embeddings[i].weight(word);

      cache[i][word].assign(sequences * hidden_layer_size, 0);
      for (unsigned sequence = 0, index = weight_index; sequence < sequences; index += embeddings_dim, sequence++)
        for (unsigned j = 0; j < embeddings[i].dimension; j++)
          for (unsigned k = 0; k < hidden_layer_size; k++)
            cache[i][word][sequence * hidden_layer_size + k] += embedding[j] * weights[0][index + j][k];
    }
  }
}

} // namespace parsito

/////////
// File: parsito/network/network_parameters.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct network_trainer {
  enum network_trainer_algorithm {
    SGD,
    SGD_MOMENTUM,
    ADAGRAD,
    ADADELTA,
    ADAM,
  };

  network_trainer_algorithm algorithm;
  float learning_rate, learning_rate_final;
  float momentum, momentum2;
  float epsilon;
};

struct network_parameters {
  unsigned iterations;
  int structured_interval;
  unsigned hidden_layer;
  activation_function::type hidden_layer_type;
  network_trainer trainer;
  unsigned batch_size;
  float initialization_range;
  float l1_regularization;
  float l2_regularization;
  float maxnorm_regularization;
  float dropout_hidden, dropout_input;
  bool early_stopping;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network_trainer.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class neural_network_trainer {
 public:
  neural_network_trainer(neural_network& network, unsigned input_size, unsigned output_size,
                         const network_parameters& parameters, mt19937& generator);

  bool next_iteration();

  struct workspace {
    unsigned batch = 0;
    vector<float> outcomes;
    vector<float> hidden_layer;
    vector<float> error_outcomes;
    vector<float> error_hidden;

    // Delta accumulators
    vector<vector<float>> weights_batch[2];
    vector<vector<vector<float>>> error_embedding;
    vector<vector<unsigned>> error_embedding_nonempty;

    // Trainer data
    struct trainer_data {
      float delta = 0;
      float gradient = 0;
    };
    vector<vector<trainer_data>> weights_trainer[2];
    vector<vector<vector<trainer_data>>> embedding_trainer;

    // Dropout vectors
    vector<bool> input_dropout;
    vector<bool> hidden_dropout;
    vector<unsigned> hidden_kept;
  };
  void propagate(const vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, workspace& w) const;
  void backpropagate(vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, unsigned required_outcome, workspace& w);

  void finalize_sentence();

  void save_network(binary_encoder& enc) const;

 private:
  struct trainer_sgd {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_sgd_momentum {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adagrad {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adadelta {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adam {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  template <class TRAINER> void backpropagate_template(vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, unsigned required_outcome, workspace& w);

  void l1_regularize();
  void maxnorm_regularize();

  void save_matrix(const vector<vector<float>>& m, binary_encoder& enc) const;

  neural_network& network;
  mt19937& generator;
  unsigned iteration, iterations, steps;
  network_trainer trainer;
  unsigned batch_size;
  float l1_regularization, l2_regularization, maxnorm_regularization;
  float dropout_hidden, dropout_input;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network_trainer.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

neural_network_trainer::neural_network_trainer(neural_network& network, unsigned input_size, unsigned output_size,
                                               const network_parameters& parameters, mt19937& generator) : network(network), generator(generator) {
  // Initialize hidden layer
  network.hidden_layer_activation = parameters.hidden_layer_type;
  if (parameters.hidden_layer) {
    float uniform_pre_hidden_range = parameters.initialization_range > 0 ? parameters.initialization_range :
        -parameters.initialization_range * sqrt(6.0 / float(input_size + parameters.hidden_layer));
    uniform_real_distribution<float> uniform_pre_hidden(-uniform_pre_hidden_range, uniform_pre_hidden_range);

    network.weights[0].resize(input_size + 1/*bias*/);
    for (auto&& row : network.weights[0]) {
      row.resize(parameters.hidden_layer);
      for (auto&& weight : row)
        weight = uniform_pre_hidden(generator);
    }

    float uniform_post_hidden_range = parameters.initialization_range > 0 ? parameters.initialization_range :
        -parameters.initialization_range * sqrt(6.0 / float(output_size + parameters.hidden_layer));
    uniform_real_distribution<float> uniform_post_hidden(-uniform_post_hidden_range, uniform_post_hidden_range);

    network.weights[1].resize(parameters.hidden_layer + 1/*bias*/);
    for (auto&& row : network.weights[1]) {
      row.resize(output_size);
      for (auto&& weight : row)
        weight = uniform_post_hidden(generator);
    }
  }

  // Store the network_parameters
  iteration = steps = 0;
  iterations = parameters.iterations;
  trainer = parameters.trainer;
  batch_size = parameters.batch_size;
  l1_regularization = parameters.l1_regularization;
  l2_regularization = parameters.l2_regularization;
  maxnorm_regularization = parameters.maxnorm_regularization;
  dropout_hidden = parameters.dropout_hidden;
  dropout_input = parameters.dropout_input;

  // Maxnorm regularize the created weights
  if (maxnorm_regularization) maxnorm_regularize();
}

bool neural_network_trainer::next_iteration() {
  if (iteration++ >= iterations) return false;

  if (trainer.algorithm != network_trainer::ADADELTA)
    if (trainer.learning_rate != trainer.learning_rate_final && iteration > 1)
      trainer.learning_rate =
          exp(((iterations - iteration) * log(trainer.learning_rate) + log(trainer.learning_rate_final)) / (iterations - iteration + 1));

  return true;
}

void neural_network_trainer::propagate(const vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, workspace& w) const {
  // Initialize dropout if requested
  if (dropout_input) {
    w.input_dropout.resize(network.weights[0].size());
    bernoulli_distribution dropout(dropout_input);
    for (auto&& flag : w.input_dropout)
      flag = dropout(generator);
  }

  if (dropout_hidden) {
    w.hidden_dropout.resize(network.weights[1].size());
    bernoulli_distribution dropout(dropout_hidden);
    for (auto&& flag : w.hidden_dropout)
      flag = dropout(generator);
  }
  w.hidden_kept.clear();
  for (unsigned i = 0; i < network.weights[0].front().size(); i++)
    if (w.hidden_dropout.empty() || !w.hidden_dropout[i])
      w.hidden_kept.push_back(i);

  // Propagate
  unsigned hidden_layer_size = network.weights[0].front().size();
  unsigned outcomes_size = network.weights[1].front().size();

  w.outcomes.assign(outcomes_size, 0);

  // Hidden layer
  w.hidden_layer.assign(hidden_layer_size, 0);

  unsigned index = 0;
  for (auto&& embedding_ids : embedding_ids_sequences)
    // Note: The unnecessary brackets on the following for cycle are needed
    // to compile on VS 2015 Update 3, which otherwise fail to compile it.
    for (unsigned i = 0; i < embeddings.size(); i++) {
      if (embedding_ids && (*embedding_ids)[i] >= 0) {
        const float* embedding = embeddings[i].weight((*embedding_ids)[i]);
        for (unsigned dimension = embeddings[i].dimension; dimension; dimension--, embedding++, index++)
          if (w.input_dropout.empty() || !w.input_dropout[index])
            for (auto&& j : w.hidden_kept)
              w.hidden_layer[j] += *embedding * network.weights[0][index][j];
      } else {
        index += embeddings[i].dimension;
      }
    }
  if (dropout_input) { // Dropout normalization
    float dropout_factor = 1. / (1. - dropout_input);
    for (auto&& i : w.hidden_kept)
      w.hidden_layer[i] *= dropout_factor;
  }
  for (auto&& i : w.hidden_kept) // Bias
    w.hidden_layer[i] += network.weights[0][index][i];

  // Activation function
  switch (network.hidden_layer_activation) {
    case activation_function::TANH:
      for (auto&& weight : w.hidden_layer)
        weight = tanh(weight);
      break;
    case activation_function::CUBIC:
      for (auto&& weight : w.hidden_layer)
        weight = weight * weight * weight;
      break;
    case activation_function::RELU:
      for (auto&& weight : w.hidden_layer)
        if (weight < 0) weight = 0;
      break;
  }
  if (dropout_hidden) { // Dropout normalization
    float dropout_factor = 1. / (1. - dropout_hidden);
    for (auto&& i : w.hidden_kept)
      w.hidden_layer[i] *= dropout_factor;
  }

  for (auto&& i : w.hidden_kept)
    for (unsigned j = 0; j < outcomes_size; j++)
      w.outcomes[j] += w.hidden_layer[i] * network.weights[1][i][j];
  for (unsigned i = 0; i < outcomes_size; i++) // Bias
    w.outcomes[i] += network.weights[1][hidden_layer_size][i];

  // Softmax
  float max = w.outcomes[0];
  for (unsigned i = 1; i < outcomes_size; i++) if (w.outcomes[i] > max) max = w.outcomes[i];

  float sum = 0;
  for (unsigned i = 0; i < outcomes_size; i++) sum += (w.outcomes[i] = exp(w.outcomes[i] - max));
  sum = 1 / sum;

  for (unsigned i = 0; i < outcomes_size; i++) w.outcomes[i] *= sum;
}

// SGD
bool neural_network_trainer::trainer_sgd::need_trainer_data = false;
float neural_network_trainer::trainer_sgd::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& /*data*/) {
  return trainer.learning_rate * gradient;
}

// SGD with momentum
bool neural_network_trainer::trainer_sgd_momentum::need_trainer_data = true;
float neural_network_trainer::trainer_sgd_momentum::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.delta = trainer.momentum * data.delta + trainer.learning_rate * gradient;
  return data.delta;
}

// AdaGrad
bool neural_network_trainer::trainer_adagrad::need_trainer_data = true;
float neural_network_trainer::trainer_adagrad::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient += gradient * gradient;
  return trainer.learning_rate / sqrt(data.gradient + trainer.epsilon) * gradient;
}

// AdaDelta
bool neural_network_trainer::trainer_adadelta::need_trainer_data = true;
float neural_network_trainer::trainer_adadelta::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient = trainer.momentum * data.gradient + (1 - trainer.momentum) * gradient * gradient;
  float delta = sqrt(data.delta + trainer.epsilon) / sqrt(data.gradient + trainer.epsilon) * gradient;
  data.delta = trainer.momentum * data.delta + (1 - trainer.momentum) * delta * delta;
  return delta;
}

// Adam
bool neural_network_trainer::trainer_adam::need_trainer_data = true;
float neural_network_trainer::trainer_adam::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient = trainer.momentum * data.gradient + (1 - trainer.momentum) * gradient;
  data.delta = trainer.momentum2 * data.delta + (1 - trainer.momentum2) * gradient * gradient;
  return trainer.learning_rate * data.gradient / sqrt(data.delta + trainer.epsilon);
}

// Backpropagation
template <class TRAINER>
void neural_network_trainer::backpropagate_template(vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, unsigned required_outcome, workspace& w) {
  size_t hidden_layer_size = network.weights[0].front().size();
  size_t outcomes_size = network.weights[1].front().size();

  // Allocate space for delta accumulators
  if (network.weights[0].size() > w.weights_batch[0].size()) w.weights_batch[0].resize(network.weights[0].size());
  if (network.weights[1].size() > w.weights_batch[1].size()) w.weights_batch[1].resize(network.weights[1].size());
  if (embeddings.size() > w.error_embedding.size()) w.error_embedding.resize(embeddings.size());
  if (embeddings.size() > w.error_embedding_nonempty.size()) w.error_embedding_nonempty.resize(embeddings.size());

  // Allocate space for trainer_data if required)
  workspace::trainer_data none_trainer_data;
  if (TRAINER::need_trainer_data) {
    while (network.weights[0].size() > w.weights_trainer[0].size()) w.weights_trainer[0].emplace_back(network.weights[0].front().size());
    while (network.weights[1].size() > w.weights_trainer[1].size()) w.weights_trainer[1].emplace_back(outcomes_size);
  }

  // Compute error vector
  w.error_outcomes.resize(outcomes_size);
  for (unsigned i = 0; i < outcomes_size; i++)
    w.error_outcomes[i] = (i == required_outcome) - w.outcomes[i];

  // Backpropagate error_outcomes to error_hidden
  w.error_hidden.assign(hidden_layer_size, 0);
  for (auto&& i : w.hidden_kept)
    for (unsigned j = 0; j < outcomes_size; j++)
      w.error_hidden[i] += network.weights[1][i][j] * w.error_outcomes[j];
  // Dropout normalization
  if (dropout_hidden) {
    float dropout_factor = 1. / (1. - dropout_hidden);
    for (auto&& i : w.hidden_kept)
      w.error_hidden[i] *= dropout_factor;
  }

  // Perform activation function derivation
  switch (network.hidden_layer_activation) {
    case activation_function::TANH:
      for (auto&& i : w.hidden_kept)
        w.error_hidden[i] *= 1 - w.hidden_layer[i] * w.hidden_layer[i];
      break;
    case activation_function::CUBIC:
      for (auto&& i : w.hidden_kept) {
        float hidden_layer = cbrt(w.hidden_layer[i]);
        w.error_hidden[i] *= 3 * hidden_layer * hidden_layer;
      }
      break;
    case activation_function::RELU:
      for (auto&& i : w.hidden_kept)
        if (w.hidden_layer[i] <= 0)
          w.error_hidden[i] = 0;
      break;
  }

  // Update weights[1]
  for (auto&& i : w.hidden_kept) {
    if (w.weights_batch[1][i].empty()) w.weights_batch[1][i].resize(outcomes_size);
    for (unsigned j = 0; j < outcomes_size; j++)
      w.weights_batch[1][i][j] += w.hidden_layer[i] * w.error_outcomes[j];
  }
  // Bias
  if (w.weights_batch[1][hidden_layer_size].empty()) w.weights_batch[1][hidden_layer_size].resize(outcomes_size);
  for (unsigned i = 0; i < outcomes_size; i++)
    w.weights_batch[1][hidden_layer_size][i] += w.error_outcomes[i];

  // Dropout normalization
  if (dropout_input) {
    float dropout_factor = 1. / (1. - dropout_input);
    for (auto&& i : w.hidden_kept)
      w.error_hidden[i] *= dropout_factor;
  }
  // Update weights[0] and backpropagate to error_embedding
  unsigned index = 0;
  for (auto&& embedding_ids : embedding_ids_sequences)
    // Note: The unnecessary brackets on the following for cycle are needed
    // to compile on VS 2015 Update 3, which otherwise fail to compile it.
    for (unsigned i = 0; i < embeddings.size(); i++) {
      if (embedding_ids && (*embedding_ids)[i] >= 0) {
        int embedding_id = (*embedding_ids)[i];

        float* error_embedding = nullptr; // Accumulate embedding error if required
        if (embeddings[i].can_update_weights(embedding_id)) {
          if (w.error_embedding[i].size() <= unsigned(embedding_id)) w.error_embedding[i].resize(embedding_id + 1);
          if (w.error_embedding[i][embedding_id].empty()) {
            w.error_embedding[i][embedding_id].assign(embeddings[i].dimension, 0);
            w.error_embedding_nonempty[i].emplace_back(embedding_id);
          }
          error_embedding = w.error_embedding[i][embedding_id].data();
        }

        const float* embedding = embeddings[i].weight(embedding_id);
        for (unsigned dimension = embeddings[i].dimension; dimension; dimension--, index++, embedding++, error_embedding += !!error_embedding)
          if (w.input_dropout.empty() || !w.input_dropout[index]) {
            if (error_embedding)
              for (auto&& j : w.hidden_kept)
                *error_embedding += network.weights[0][index][j] * w.error_hidden[j];
            if (w.weights_batch[0][index].empty()) w.weights_batch[0][index].resize(hidden_layer_size);
            for (auto&& j : w.hidden_kept)
              w.weights_batch[0][index][j] += *embedding * w.error_hidden[j];
          }
      } else {
        index += embeddings[i].dimension;
      }
    }
  // Bias
  {
    float negate_input_dropout = 1. - dropout_hidden;
    if (w.weights_batch[0][index].empty()) w.weights_batch[0][index].resize(hidden_layer_size);
    for (auto&& i : w.hidden_kept)
      w.weights_batch[0][index][i] += w.error_hidden[i] * negate_input_dropout;
  }

  // End if not at the end of the batch
  if (++w.batch < batch_size) return;
  w.batch = 0;

  // Update hidden weights
  if (!network.weights[0].empty())
    for (int i = 0; i < 2; i++) {
      for (unsigned j = 0; j < w.weights_batch[i].size(); j++)
        if (!w.weights_batch[i][j].empty()) {
          for (unsigned k = 0; k < w.weights_batch[i][j].size(); k++)
            network.weights[i][j][k] += TRAINER::delta(w.weights_batch[i][j][k], trainer, TRAINER::need_trainer_data ? w.weights_trainer[i][j][k] : none_trainer_data) - (j+1 == w.weights_batch[i].size() ? /*bias*/ 0. : l2_regularization) * network.weights[i][j][k];
          w.weights_batch[i][j].clear();
        }
    }

  // Update embedding weights using error_embedding
  for (unsigned i = 0; i < embeddings.size(); i++) {
    for (auto&& id : w.error_embedding_nonempty[i]) {
      if (TRAINER::need_trainer_data) {
        if (w.embedding_trainer.size() <= i) w.embedding_trainer.resize(i + 1);
        if (w.embedding_trainer[i].size() <= id) w.embedding_trainer[i].resize(id + 1);
        if (w.embedding_trainer[i][id].size() < embeddings[i].dimension) w.embedding_trainer[i][id].resize(embeddings[i].dimension);
      }
      float* embedding = embeddings[i].weight(id);
      for (unsigned j = 0; j < embeddings[i].dimension; j++)
        embedding[j] += TRAINER::delta(w.error_embedding[i][id][j], trainer, TRAINER::need_trainer_data ? w.embedding_trainer[i][id][j] : none_trainer_data) - l2_regularization * embedding[j];
      w.error_embedding[i][id].clear();
    }
    w.error_embedding_nonempty[i].clear();
  }

  // Maxnorm regularize the updated weights
  if (maxnorm_regularization) maxnorm_regularize();
}

void neural_network_trainer::backpropagate(vector<embedding>& embeddings, const vector<const vector<int>*>& embedding_ids_sequences, unsigned required_outcome, workspace& w) {
  steps++;

  switch (trainer.algorithm) {
    case network_trainer::SGD:
      backpropagate_template<trainer_sgd>(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::SGD_MOMENTUM:
      backpropagate_template<trainer_sgd_momentum>(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADAGRAD:
      backpropagate_template<trainer_adagrad>(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADADELTA:
      backpropagate_template<trainer_adadelta>(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADAM:
      float original_learning_rate = trainer.learning_rate;
      trainer.learning_rate *= sqrt(1-pow(trainer.momentum2, steps)) / (1-pow(trainer.momentum, steps));
      backpropagate_template<trainer_adam>(embeddings, embedding_ids_sequences, required_outcome, w);
      trainer.learning_rate = original_learning_rate;
      return;
  }

  training_failure("Internal error, unsupported trainer!");
}

void neural_network_trainer::l1_regularize() {
  if (!l1_regularization) return;

  for (auto&& weights : network.weights)
    for (unsigned i = 0; i + 1 /*ignore biases*/ < weights.size(); i++) {
      auto& row = weights[i];
      for (auto&& weight : row)
        if (weight < l1_regularization) weight += l1_regularization;
        else if (weight > l1_regularization) weight -= l1_regularization;
        else weight = 0;
    }
}

void neural_network_trainer::maxnorm_regularize() {
  if (!maxnorm_regularization) return;

  for (unsigned i = 0; i < 2; i++)
    for (unsigned j = 0; j < network.weights[i].front().size(); j++) {
      float length = 0;
      for (auto&& row : network.weights[i])
        length += row[j] * row[j];

      if (length > 0 && length > maxnorm_regularization * maxnorm_regularization) {
        float factor = 1 / sqrt(length / (maxnorm_regularization * maxnorm_regularization));
        for (auto&& row : network.weights[i])
          row[j] *= factor;
      }
    }
}

void neural_network_trainer::finalize_sentence() {
  if (l1_regularization) l1_regularize();
}

void neural_network_trainer::save_matrix(const vector<vector<float>>& m, binary_encoder& enc) const {
  enc.add_4B(m.size());
  enc.add_4B(m.empty() ? 0 : m.front().size());

  for (auto&& row : m) {
    assert(row.size() == m.front().size());
    enc.add_data(row);
  }
}

void neural_network_trainer::save_network(binary_encoder& enc) const {
  enc.add_1B(network.hidden_layer_activation);
  save_matrix(network.weights[0], enc);
  save_matrix(network.weights[1], enc);
}

} // namespace parsito

/////////
// File: parsito/transition/transition.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Abstract transition class
class transition {
 public:
  virtual ~transition() {}

  virtual bool applicable(const configuration& conf) const = 0;
  virtual int perform(configuration& conf) const = 0;
};

// Specific transition classes
class transition_left_arc : public transition {
 public:
  transition_left_arc(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_right_arc : public transition {
 public:
  transition_right_arc(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_shift : public transition {
 public:
  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
};

class transition_swap : public transition {
 public:
  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
};

class transition_left_arc_2 : public transition {
 public:
  transition_left_arc_2(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_right_arc_2 : public transition {
 public:
  transition_right_arc_2(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_oracle.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_oracle {
 public:
  virtual ~transition_oracle() {}

  struct predicted_transition {
    unsigned best;
    unsigned to_follow;

    predicted_transition(unsigned best, unsigned to_follow) : best(best), to_follow(to_follow) {}
  };

  class tree_oracle {
   public:
    virtual ~tree_oracle() {}

    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const = 0;
    virtual void interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const = 0;
  };

  virtual unique_ptr<tree_oracle> create_tree_oracle(const tree& gold) const = 0;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system {
 public:
  virtual ~transition_system() {}

  virtual unsigned transition_count() const;
  virtual bool applicable(const configuration& conf, unsigned transition) const;
  virtual int perform(configuration& conf, unsigned transition) const;
  virtual transition_oracle* oracle(const string& name) const = 0;

  static transition_system* create(const string& name, const vector<string>& labels);

 protected:
  transition_system(const vector<string>& labels) : labels(labels) {}

  const vector<string>& labels;
  vector<unique_ptr<transition>> transitions;
};

} // namespace parsito

/////////
// File: parsito/parser/parser_nn.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class parser_nn : public parser {
 public:
  parser_nn(bool versioned);

  virtual void parse(tree& t, unsigned beam_size = 0, double* cost = nullptr) const override;

 protected:
  virtual void load(binary_decoder& data, unsigned cache) override;

 private:
  friend class parser_nn_trainer;
  void parse_greedy(tree& t, double* cost) const;
  void parse_beam_search(tree& t, unsigned beam_size, double* cost) const;

  bool versioned;
  unsigned version;
  bool single_root;
  enum { VERSION_LATEST = 2 };

  vector<string> labels;
  unique_ptr<transition_system> system;

  node_extractor nodes;

  vector<value_extractor> values;
  vector<embedding> embeddings;

  neural_network network;
  neural_network::embeddings_cache embeddings_cache;

  struct workspace {
    workspace(bool single_root) : conf(single_root) {}

    configuration conf;

    string word, word_buffer;
    vector<vector<int>> embeddings;
    vector<vector<string>> embeddings_values;

    vector<int> extracted_nodes;
    vector<const vector<int>*> extracted_embeddings;

    vector<float> outcomes, network_buffer;

    // Beam-size structures
    struct beam_size_configuration {
      beam_size_configuration(bool single_root) : conf(single_root) {}

      configuration conf;
      vector<int> heads;
      vector<string> deprels;
      double cost;

      void refresh_tree();
      void save_tree();
    };
    struct beam_size_alternative {
      const beam_size_configuration* bs_conf;
      int transition;
      double cost;
      bool operator<(const beam_size_alternative& other) const { return cost > other.cost; }

      beam_size_alternative(const beam_size_configuration* bs_conf, int transition, double cost)
          : bs_conf(bs_conf), transition(transition), cost(cost) {}
    };
    vector<beam_size_configuration> bs_confs[2]; size_t bs_confs_size[2];
    vector<beam_size_alternative> bs_alternatives;
  };
  mutable threadsafe_stack<workspace> workspaces;
};

} // namespace parsito

/////////
// File: parsito/parser/parser.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

parser* parser::load(const char* file, unsigned cache) {
  ifstream in(file, ifstream::in | ifstream::binary);
  if (!in.is_open()) return nullptr;
  return load(in, cache);
}

parser* parser::load(istream& in, unsigned cache) {
  unique_ptr<parser> result;

  binary_decoder data;
  if (!compressor::load(in, data)) return nullptr;

  try {
    string name;
    data.next_str(name);

    result.reset(create(name));
    if (!result) return nullptr;

    result->load(data, cache);
  } catch (binary_decoder_error&) {
    return nullptr;
  }

  return result && data.is_end() ? result.release() : nullptr;
}

parser* parser::create(const string& name) {
  if (name == "nn") return new parser_nn(false);
  if (name == "nn_versioned") return new parser_nn(true);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/parser/parser_nn.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Versions:
// 1: initial version
// 2: add ReLU activation function

parser_nn::parser_nn(bool versioned) : versioned(versioned) {}

void parser_nn::parse(tree& t, unsigned beam_size, double* cost) const {
  if (beam_size > 1)
    parse_beam_search(t, beam_size, cost);
  else
    parse_greedy(t, cost);
}

void parser_nn::parse_greedy(tree& t, double* cost) const {
  assert(system);
  if (cost) *cost = 0.;

  // Retrieve or create workspace
  workspace* w = workspaces.pop();
  if (!w) w = new workspace(single_root);

  // Create configuration
  w->conf.init(&t);

  // Compute embeddings of all nodes
  if (w->embeddings.size() < t.nodes.size()) w->embeddings.resize(t.nodes.size());
  for (size_t i = 0; i < t.nodes.size(); i++) {
    if (w->embeddings[i].size() < embeddings.size()) w->embeddings[i].resize(embeddings.size());
    for (size_t j = 0; j < embeddings.size(); j++) {
      values[j].extract(t.nodes[i], w->word);
      w->embeddings[i][j] = embeddings[j].lookup_word(w->word, w->word_buffer);
    }
  }

  // Compute which transitions to perform and perform them
  int transitions = 0;
  for (; !w->conf.final(); transitions++) {
    // Extract nodes from the configuration
    nodes.extract(w->conf, w->extracted_nodes);
    w->extracted_embeddings.resize(w->extracted_nodes.size());
    for (size_t i = 0; i < w->extracted_nodes.size(); i++)
      w->extracted_embeddings[i] = w->extracted_nodes[i] >= 0 ? &w->embeddings[w->extracted_nodes[i]] : nullptr;

    // Classify using neural network
    network.propagate(embeddings, w->extracted_embeddings, w->network_buffer, w->outcomes, &embeddings_cache, cost ? true : false);

    // Find most probable applicable transition
    int best = -1;
    for (unsigned i = 0; i < w->outcomes.size(); i++)
      if (system->applicable(w->conf, i) && (best < 0 || w->outcomes[i] > w->outcomes[best]))
        best = i;

    // Perform the best transition
    int child = system->perform(w->conf, best);
    if (cost) *cost += log(w->outcomes[best]);

    // If a node was linked, recompute its embeddings as deprel has changed
    if (child >= 0)
      for (size_t i = 0; i < embeddings.size(); i++) {
        values[i].extract(t.nodes[child], w->word);
        w->embeddings[child][i] = embeddings[i].lookup_word(w->word, w->word_buffer);
      }
  }

  if (cost && transitions)
    *cost = *cost / transitions * (t.nodes.size() - 1);

  // Store workspace
  workspaces.push(w);
}

void parser_nn::parse_beam_search(tree& t, unsigned beam_size, double* cost) const {
  assert(system);

  // Retrieve or create workspace
  workspace* w = workspaces.pop();
  if (!w) w = new workspace(single_root);

  // Allocate and initialize configuration
  for (int i = 0; i < 2; i++) {
    while (w->bs_confs[i].size() < beam_size) w->bs_confs[i].emplace_back(single_root);
    while (w->bs_confs[i].size() > beam_size) w->bs_confs[i].pop_back();
    w->bs_confs_size[i] = 0;
  }
  w->bs_confs[0][0].cost = 0;
  w->bs_confs[0][0].conf.init(&t);
  w->bs_confs[0][0].save_tree();
  w->bs_confs_size[0] = 1;

  // Compute embeddings of all nodes
  if (w->embeddings.size() < t.nodes.size()) w->embeddings.resize(t.nodes.size());
  if (w->embeddings_values.size() < t.nodes.size()) w->embeddings_values.resize(t.nodes.size());
  for (size_t i = 0; i < t.nodes.size(); i++) {
    if (w->embeddings[i].size() < embeddings.size()) w->embeddings[i].resize(embeddings.size());
    if (w->embeddings_values[i].size() < embeddings.size()) w->embeddings_values[i].resize(embeddings.size());
    for (size_t j = 0; j < embeddings.size(); j++) {
      values[j].extract(t.nodes[i], w->embeddings_values[i][j]);
      w->embeddings[i][j] = embeddings[j].lookup_word(w->embeddings_values[i][j], w->word_buffer);
    }
  }

  // Compute which transitions to perform and perform them
  size_t iteration = 0;
  for (bool all_final = false; !all_final; iteration++) {
    all_final = true;
    w->bs_alternatives.clear();

    for (size_t c = 0; c < w->bs_confs_size[iteration & 1]; c++) {
      auto& bs_conf = w->bs_confs[iteration & 1][c];

      if (bs_conf.conf.final()) {
        if (w->bs_alternatives.size() == beam_size) {
          if (bs_conf.cost <= w->bs_alternatives[0].cost) continue;
          pop_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
          w->bs_alternatives.pop_back();
        }
        w->bs_alternatives.emplace_back(&bs_conf, -1, bs_conf.cost);
        push_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
        continue;
      }
      all_final = false;

      bs_conf.refresh_tree();
      // Update embeddings for all nodes
      for (size_t i = 0; i < t.nodes.size(); i++)
        for (size_t j = 0; j < embeddings.size(); j++) {
          values[j].extract(t.nodes[i], w->word);
          if (w->word != w->embeddings_values[i][j]) {
            w->embeddings[i][j] = embeddings[j].lookup_word(w->word, w->word_buffer);
            w->embeddings_values[i][j].assign(w->word);
          }
        }

      // Extract nodes from the configuration
      nodes.extract(bs_conf.conf, w->extracted_nodes);
      w->extracted_embeddings.resize(w->extracted_nodes.size());
      for (size_t i = 0; i < w->extracted_nodes.size(); i++)
        w->extracted_embeddings[i] = w->extracted_nodes[i] >= 0 ? &w->embeddings[w->extracted_nodes[i]] : nullptr;

      // Classify using neural network
      network.propagate(embeddings, w->extracted_embeddings, w->network_buffer, w->outcomes, &embeddings_cache);

      // Store all alternatives
      for (unsigned i = 0; i < w->outcomes.size(); i++)
        if (system->applicable(bs_conf.conf, i)) {
          double cost = (bs_conf.cost * iteration + log(w->outcomes[i])) / (iteration + 1);
          if (w->bs_alternatives.size() == beam_size) {
            if (cost <= w->bs_alternatives[0].cost) continue;
            pop_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
            w->bs_alternatives.pop_back();
          }
          w->bs_alternatives.emplace_back(&bs_conf, i, cost);
          push_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
        }
    }

    w->bs_confs_size[(iteration + 1) & 1] = 0;
    for (auto&& alternative : w->bs_alternatives) {
      auto& bs_conf_new = w->bs_confs[(iteration + 1) & 1][w->bs_confs_size[(iteration + 1) & 1]++];
      bs_conf_new = *alternative.bs_conf;
      bs_conf_new.cost = alternative.cost;
      if (alternative.transition >= 0) {
        bs_conf_new.refresh_tree();
        system->perform(bs_conf_new.conf, alternative.transition);
        bs_conf_new.save_tree();
      }
    }
  }

  // Return the best tree
  size_t best = 0;
  for (size_t i = 1; i < w->bs_confs_size[iteration & 1]; i++)
    if (w->bs_confs[iteration & 1][i].cost > w->bs_confs[iteration & 1][best].cost)
      best = i;
  w->bs_confs[iteration & 1][best].refresh_tree();

  if (cost) *cost = w->bs_confs[iteration & 1][best].cost * (t.nodes.size() - 1);

  // Store workspace
  workspaces.push(w);
}

void parser_nn::workspace::beam_size_configuration::refresh_tree() {
  for (auto&& node : conf.t->nodes) node.children.clear();
  for (size_t i = 0; i < conf.t->nodes.size(); i++) {
    conf.t->nodes[i].head = heads[i];
    conf.t->nodes[i].deprel = deprels[i];
    if (heads[i] >= 0) conf.t->nodes[heads[i]].children.push_back(i);
  }
}

void parser_nn::workspace::beam_size_configuration::save_tree() {
  if (conf.t->nodes.size() > heads.size()) heads.resize(conf.t->nodes.size());
  if (conf.t->nodes.size() > deprels.size()) deprels.resize(conf.t->nodes.size());
  for (size_t i = 0; i < conf.t->nodes.size(); i++) {
    heads[i] = conf.t->nodes[i].head;
    deprels[i] = conf.t->nodes[i].deprel;
  }
}

void parser_nn::load(binary_decoder& data, unsigned cache) {
  string description, error;

  version = versioned ? data.next_1B() : 1;
  if (!(version >= 1 && version <= VERSION_LATEST))
    throw binary_decoder_error("Unrecognized version of the parser_nn model");

  single_root = version >= 2 ? data.next_1B() : false;

  // Load labels
  labels.resize(data.next_2B());
  for (auto&& label : labels)
    data.next_str(label);

  // Load transition system
  string system_name;
  data.next_str(system_name);
  system.reset(transition_system::create(system_name, labels));
  if (!system) throw binary_decoder_error("Cannot load transition system");

  // Load node extractor
  data.next_str(description);
  if (!nodes.create(description, error))
    throw binary_decoder_error(error.c_str());

  // Load value extractors and embeddings
  values.resize(data.next_2B());
  for (auto&& value : values) {
    data.next_str(description);
    if (!value.create(description, error))
      throw binary_decoder_error(error.c_str());
  }

  embeddings.resize(values.size());
  for (auto&& embedding : embeddings)
    embedding.load(data);

  // Load the network
  network.load(data);
  network.generate_tanh_cache();
  network.generate_embeddings_cache(embeddings, embeddings_cache, cache);
}

} // namespace parsito

/////////
// File: parsito/parser/parser_nn_trainer.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class parser_nn_trainer {
 public:
  static void train(const string& transition_system_name, const string& transition_oracle_name, bool single_root,
                    const string& embeddings_description, const string& nodes_description, const network_parameters& parameters,
                    unsigned number_of_threads, const vector<tree>& train, const vector<tree>& heldout, binary_encoder& enc);
};

} // namespace parsito

/////////
// File: parsito/parser/parser_nn_trainer.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void parser_nn_trainer::train(const string& transition_system_name, const string& transition_oracle_name, bool single_root,
                              const string& embeddings_description, const string& nodes_description, const network_parameters& parameters,
                              unsigned /*number_of_threads*/, const vector<tree>& train, const vector<tree>& heldout, binary_encoder& enc) {
  if (train.empty()) training_failure("No training data was given!");

  // Random generator with fixed seed for reproducibility
  mt19937 generator(42);

  // Check that all non-root nodes have heads and nonempty deprel
  for (auto&& tree : train)
    for (auto&& node : tree.nodes)
      if (node.id) {
        if (node.head < 0) training_failure("The node '" << node.form << "' with id " << node.id << " has no head set!");
        if (node.deprel.empty()) training_failure("The node '" << node.form << "' with id " << node.id << " has no deprel set!");
      }

  // Create parser instance to be trained
  parser_nn parser(true); parser.version = parser_nn::VERSION_LATEST;

  // Generate labels for transition system
  unordered_set<string> labels_set;
  for (auto&& tree : train)
    for (auto&& node : tree.nodes)
      if (node.id && !labels_set.count(node.deprel)) {
        labels_set.insert(node.deprel);
        parser.labels.push_back(node.deprel);
      }

  // If single_root, check that exactly root nodes have "root" deprel
  if (single_root) {
    for (auto&& tree : train) {
      unsigned roots = 0;
      for (auto&& node : tree.nodes)
        if (node.id) {
          if (node.head == 0 && node.deprel != "root")
            training_failure("When single root is required, every root node must have 'root' deprel!");
          if (node.head != 0 && node.deprel == "root")
            training_failure("When single root is required, any non-root cannot have 'root' deprel!");
          roots += node.head == 0;
        }
      if (roots != 1)
        training_failure("When single root is required, every training tree must have single root!");
    }

    // Make sure (in case input is really small) there is "root" deprel plus another one
    if (!labels_set.count("root"))
      training_failure("When single root is required, the deprel 'root' must be present!");
    if (labels_set.size() <= 1)
      training_failure("When single root is required, deprel different from 'root' must exist!");
  }

  // Create transition system and transition oracle
  parser.system.reset(transition_system::create(transition_system_name, parser.labels));
  if (!parser.system) training_failure("Cannot create transition system '" << transition_system_name << "'!");

  unique_ptr<transition_oracle> oracle(parser.system->oracle(transition_oracle_name));
  if (!oracle) training_failure("Cannot create transition oracle '" << transition_oracle_name << "' for transition system '" << transition_system_name << "'!");

  // Create node_extractor
  string error;
  if (!parser.nodes.create(nodes_description, error)) training_failure(error);

  // Load value_extractors and embeddings
  vector<string> value_names;
  vector<string_piece> lines, tokens;
  split(embeddings_description, '\n', lines);
  for (auto&& line : lines) {
    // Ignore empty lines and comments
    if (!line.len || line.str[0] == '#') continue;

    split(line, ' ', tokens);
    if (!(tokens.size() >= 3 && tokens.size() <= 6))
      training_failure("Expected 3 to 6 columns on embedding description line '" << line << "'!");

    value_names.emplace_back(string(tokens[0].str, tokens[0].len));
    parser.values.emplace_back();
    if (!parser.values.back().create(tokens[0], error)) training_failure(error);

    int dimension = parse_int(tokens[1], "embedding dimension");
    int min_count = parse_int(tokens[2], "minimum frequency count");
    unsigned updatable_index = 0;
    unsigned embeddings_from_file = 0;
    string embeddings_from_file_comment;
    vector<pair<string, vector<float>>> weights;
    unordered_set<string> weights_set;

    // Compute words and counts present in the training data
    string word;
    unordered_map<string, int> word_counts;
    for (auto&& tree : train)
      for (auto&& node : tree.nodes)
        if (node.id) {
          parser.values.back().extract(node, word);
          word_counts[word]++;
        }

    // Load embedding if it was given
    if (tokens.size() >= 4) {
      int update_weights = tokens.size() >= 5 ? parse_int(tokens[4], "update weights") : 1;
      int max_embeddings = tokens.size() >= 6 ? parse_int(tokens[5], "maximum embeddings count") : numeric_limits<int>::max();
      ifstream in(string(tokens[3].str, tokens[3].len));
      if (!in.is_open()) training_failure("Cannot load '" << tokens[0] << "' embedding from file '" << tokens[3] << "'!");

      // Load first line containing dictionary size and dimensions
      string line;
      vector<string_piece> parts;
      if (!getline(in, line)) training_failure("Cannot read first line from embedding file '" << tokens[3] << "'!");
      split(line, ' ', parts);
      if (parts.size() != 2) training_failure("Expected two numbers on the first line of embedding file '" << tokens[3] << "'!");
      int file_dimension = parse_int(parts[1], "embedding file dimension");

      if (file_dimension < dimension) training_failure("The embedding file '" << tokens[3] << "' has lower dimension than required!");

      // Generate random projection when smaller dimension is required
      vector<vector<float>> projection;
      if (file_dimension > dimension) {
        embeddings_from_file_comment = "[dim" + to_string(file_dimension) + "->" + to_string(dimension) + "]";

        uniform_real_distribution<double> uniform(0, 1);
        projection.resize(dimension);
        for (auto&& row : projection) {
          row.resize(file_dimension);
          for (auto&& weight : row) weight = uniform(generator);

          double sum = 0;
          for (auto&& weight : row) sum += weight;
          for (auto&& weight : row) weight /= sum;
        }
      }

      // Load input embedding
      vector<double> input_weights(file_dimension);
      vector<float> projected_weights(dimension);
      while (getline(in, line) && int(weights.size()) < max_embeddings) {
        split(line, ' ', parts);
        if (!parts.empty() && !parts.back().len) parts.pop_back(); // Ignore space at the end of line
        if (int(parts.size()) != file_dimension + 1) training_failure("Wrong number of values on line '" << line << "' of embedding file '" << tokens[3]);
        for (int i = 0; i < file_dimension; i++)
          input_weights[i] = parse_double(parts[1 + i], "embedding weight");

        string word(parts[0].str, parts[0].len);

        // For update_weights == 2, ignore embeddings for unknown words
        if (update_weights == 2 && !word_counts.count(word))
          continue;

        for (int i = 0; i < dimension; i++)
          if (file_dimension == dimension) {
            projected_weights[i] = input_weights[i];
          } else {
            projected_weights[i] = 0;
            for (int j = 0; j < file_dimension; j++)
              projected_weights[i] += projection[i][j] * input_weights[j];
          }

        if (!weights_set.count(word)) {
          weights.emplace_back(word, projected_weights);
          weights_set.insert(word);
        }
      }
      embeddings_from_file = weights.size();
      updatable_index = update_weights ? 0 : embeddings_from_file;
    }

    // Add embedding for non-present word with min_count, sorted by count
    {
      vector<pair<int, string>> count_words;
      for (auto&& word_count : word_counts)
        if (word_count.second >= min_count && !weights_set.count(word_count.first))
          count_words.emplace_back(word_count.second, word_count.first);

      sort(count_words.rbegin(), count_words.rend());

      vector<float> word_weights(dimension);
      uniform_real_distribution<float> uniform(-1, 1);
      for (auto&& count_word : count_words) {
        for (auto&& word_weight : word_weights)
          word_weight = uniform(generator);

        weights.emplace_back(count_word.second, word_weights);
      }
    }

    // If there are unknown words in the training data, create initial embedding
    vector<float> unknown_weights(dimension);
    if (min_count > 1) {
      uniform_real_distribution<float> uniform(-1, 1);

      for (auto&& weight : unknown_weights)
        weight = uniform(generator);
    }

    // Add the embedding
    parser.embeddings.emplace_back();
    parser.embeddings.back().create(dimension, updatable_index, weights, unknown_weights);

    // Count the cover of this embedding
    string buffer;
    unsigned words_total = 0, words_covered = 0, words_covered_from_file = 0;
    for (auto&& tree : train)
      for (auto&& node : tree.nodes)
        if (node.id) {
          parser.values.back().extract(node, word);
          words_total++;
          int word_id = parser.embeddings.back().lookup_word(word, buffer);
          words_covered += word_id != parser.embeddings.back().unknown_word();
          words_covered_from_file += word_id != parser.embeddings.back().unknown_word() && unsigned(word_id) < embeddings_from_file;
        }

    cerr << "Initialized '" << tokens[0] << "' embedding with " << embeddings_from_file << embeddings_from_file_comment
         << "," << weights.size() << " words and " << fixed << setprecision(1) << 100. * words_covered_from_file / words_total
         << "%," << 100. * words_covered / words_total << "% coverage." << endl;
  }

  // Train the network
  unsigned total_dimension = 0, total_nodes = 0;
  for (auto&& embedding : parser.embeddings) total_dimension += embedding.dimension;
  for (auto&& tree : train) total_nodes += tree.nodes.size() - 1;
  auto scaled_parameters = parameters;
  scaled_parameters.l1_regularization /= train.size();
  scaled_parameters.l2_regularization /= total_nodes;
  neural_network_trainer network_trainer(parser.network, total_dimension * parser.nodes.node_count(), parser.system->transition_count(), scaled_parameters, generator);

  neural_network heldout_best_network;
  unsigned heldout_best_correct_labelled = 0, heldout_best_iteration = 0;

  vector<int> permutation;
  for (size_t i = 0; i < train.size(); i++)
    permutation.push_back(permutation.size());

  for (int iteration = 1; network_trainer.next_iteration(); iteration++) {
    // Train on training data
    shuffle(permutation.begin(), permutation.end(), generator);

    atomic<unsigned> atomic_index(0);
    atomic<double> atomic_logprob(0);
    auto training = [&]() {
      tree t;
      configuration conf(single_root);
      string word, word_buffer;
      vector<vector<int>> nodes_embeddings;
      vector<int> extracted_nodes;
      vector<const vector<int>*> extracted_embeddings;
      neural_network_trainer::workspace workspace;
      double logprob = 0;

      // Data for structured prediction
      tree t_eval;
      configuration conf_eval(single_root);
      vector<vector<int>> nodes_embeddings_eval;
      vector<int>  extracted_nodes_eval;
      vector<const vector<int>*>  extracted_embeddings_eval;
      vector<unsigned> transitions_eval;
      vector<float> hidden_layer_eval, outcomes_eval;

      for (unsigned current_index; (current_index = atomic_index++) < permutation.size();) {
        const tree& gold = train[permutation[current_index]];
        t = gold;
        t.unlink_all_nodes();
        conf.init(&t);

        // Compute embeddings
        if (t.nodes.size() > nodes_embeddings.size()) nodes_embeddings.resize(t.nodes.size());
        for (size_t i = 0; i < t.nodes.size(); i++) {
          nodes_embeddings[i].resize(parser.embeddings.size());
          for (size_t j = 0; j < parser.embeddings.size(); j++) {
            parser.values[j].extract(t.nodes[i], word);
            nodes_embeddings[i][j] = parser.embeddings[j].lookup_word(word, word_buffer);
          }
        }

        // Create tree oracle
        auto tree_oracle = oracle->create_tree_oracle(gold);

        // Train the network
        while (!conf.final()) {
          // Extract nodes
          parser.nodes.extract(conf, extracted_nodes);
          extracted_embeddings.resize(extracted_nodes.size());
          for (size_t i = 0; i < extracted_nodes.size(); i++)
            extracted_embeddings[i] = extracted_nodes[i] >= 0 ? &nodes_embeddings[extracted_nodes[i]] : nullptr;

          // Propagate
          network_trainer.propagate(parser.embeddings, extracted_embeddings, workspace);

          // Find most probable applicable transition
          int network_best = -1;
          for (unsigned i = 0; i < workspace.outcomes.size(); i++)
            if (parser.system->applicable(conf, i) && (network_best < 0 || workspace.outcomes[i] > workspace.outcomes[network_best]))
              network_best = i;

          // Apply the oracle
          auto prediction = tree_oracle->predict(conf, network_best, iteration);

          // If the best transition is applicable, train on it
          if (parser.system->applicable(conf, prediction.best)) {
            // Update logprob
            if (workspace.outcomes[prediction.best])
              logprob += log(workspace.outcomes[prediction.best]);

            // Backpropagate the chosen outcome
            network_trainer.backpropagate(parser.embeddings, extracted_embeddings, prediction.best, workspace);
          }

          // Emergency break if the to_follow transition is not applicable
          if (!parser.system->applicable(conf, prediction.to_follow))
            break;

          // Follow the chosen outcome
          int child = parser.system->perform(conf, prediction.to_follow);

          // If a node was linked, recompute its embeddings as deprel has changed
          if (child >= 0)
            for (size_t i = 0; i < parser.embeddings.size(); i++) {
              parser.values[i].extract(t.nodes[child], word);
              nodes_embeddings[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
            }
        }
        network_trainer.finalize_sentence();

        // Structured prediction
        if (parameters.structured_interval && (current_index % parameters.structured_interval) == 0) {
          uniform_int_distribution<size_t> train_distribution(0, train.size() - 1);
          const tree& gold = train[train_distribution(generator)];
          t = gold;
          t.unlink_all_nodes();
          conf.init(&t);

          // Compute embeddings
          if (t.nodes.size() > nodes_embeddings.size()) nodes_embeddings.resize(t.nodes.size());
          for (size_t i = 0; i < t.nodes.size(); i++) {
            nodes_embeddings[i].resize(parser.embeddings.size());
            for (size_t j = 0; j < parser.embeddings.size(); j++) {
              parser.values[j].extract(t.nodes[i], word);
              nodes_embeddings[i][j] = parser.embeddings[j].lookup_word(word, word_buffer);
            }
          }

          // Create tree oracle
          auto tree_oracle = oracle->create_tree_oracle(gold);

          // Train the network
          while (!conf.final()) {
            // Extract nodes
            parser.nodes.extract(conf, extracted_nodes);
            extracted_embeddings.resize(extracted_nodes.size());
            for (size_t i = 0; i < extracted_nodes.size(); i++)
              extracted_embeddings[i] = extracted_nodes[i] >= 0 ? &nodes_embeddings[extracted_nodes[i]] : nullptr;

            // Find the best transition
            int best = 0;
            int best_uas = -1;
            tree_oracle->interesting_transitions(conf, transitions_eval);
            for (auto&& transition : transitions_eval) {
              t_eval = t;
              conf_eval = conf;
              conf_eval.t = &t_eval;
              nodes_embeddings_eval = nodes_embeddings;

              // Perform probed transition
              int child = parser.system->perform(conf_eval, transition);
              if (child >= 0)
                for (size_t i = 0; i < parser.embeddings.size(); i++) {
                  parser.values[i].extract(t_eval.nodes[child], word);
                  nodes_embeddings_eval[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
                }

              // Train the network
              while (!conf_eval.final()) {
                // Extract nodes
                parser.nodes.extract(conf_eval, extracted_nodes_eval);
                extracted_embeddings_eval.resize(extracted_nodes_eval.size());
                for (size_t i = 0; i < extracted_nodes_eval.size(); i++)
                  extracted_embeddings_eval[i] = extracted_nodes_eval[i] >= 0 ? &nodes_embeddings_eval[extracted_nodes_eval[i]] : nullptr;

                // Classify using neural network
                parser.network.propagate(parser.embeddings, extracted_embeddings_eval, hidden_layer_eval, outcomes_eval, nullptr, false);

                // Find most probable applicable transition
                int network_best = -1;
                for (unsigned i = 0; i < outcomes_eval.size(); i++)
                  if (parser.system->applicable(conf_eval, i) && (network_best < 0 || outcomes_eval[i] > outcomes_eval[network_best]))
                    network_best = i;

                // Perform the best transition
                int child = parser.system->perform(conf_eval, network_best);

                // If a node was linked, recompute its embeddings as deprel has changed
                if (child >= 0)
                  for (size_t i = 0; i < parser.embeddings.size(); i++) {
                    parser.values[i].extract(t_eval.nodes[child], word);
                    nodes_embeddings_eval[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
                  }
              }

              int uas = 0;
              for (unsigned i = 1; i < gold.nodes.size(); i++)
                uas += gold.nodes[i].head == t_eval.nodes[i].head;

              if (uas > best_uas) best = transition, best_uas = uas;
            }

            // Propagate
            network_trainer.propagate(parser.embeddings, extracted_embeddings, workspace);

            // Backpropagate for the best transition
            if (workspace.outcomes[best])
              logprob += log(workspace.outcomes[best]);
            network_trainer.backpropagate(parser.embeddings, extracted_embeddings, best, workspace);

            //              // Find most probable applicable transition when following network outcome
            //              int network_best = -1;
            //              for (unsigned i = 0; i < workspace.outcomes.size(); i++)
            //                if (parser.system->applicable(conf, i) && (network_best < 0 || workspace.outcomes[i] > workspace.outcomes[network_best]))
            //                  network_best = i;

            // Follow the best outcome
            int child = parser.system->perform(conf, /*network_*/best);

            // If a node was linked, recompute its embeddings as deprel has changed
            if (child >= 0)
              for (size_t i = 0; i < parser.embeddings.size(); i++) {
                parser.values[i].extract(t.nodes[child], word);
                nodes_embeddings[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
              }
          }
          network_trainer.finalize_sentence();
        }
      }
      for (double old_atomic_logprob = atomic_logprob; atomic_logprob.compare_exchange_weak(old_atomic_logprob, old_atomic_logprob + logprob); ) {}
    };

    cerr << "Iteration " << iteration << ": ";
    training();
    cerr << "training logprob " << scientific << setprecision(4) << atomic_logprob;

    // Evaluate heldout data if present
    if (!heldout.empty()) {
      tree t;
      unsigned total = 0, correct_unlabelled = 0, correct_labelled = 0;
      for (auto&& gold : heldout) {
        t = gold;
        t.unlink_all_nodes();
        parser.parse(t);
        for (size_t i = 1; i < t.nodes.size(); i++) {
          total++;
          correct_unlabelled += t.nodes[i].head == gold.nodes[i].head;
          correct_labelled += t.nodes[i].head == gold.nodes[i].head && t.nodes[i].deprel == gold.nodes[i].deprel;
        }
      }

      cerr << ", heldout UAS " << fixed << setprecision(2) << (100. * correct_unlabelled / total) << "%, LAS " << (100. * correct_labelled / total) << "%";

      if (parameters.early_stopping && correct_labelled > heldout_best_correct_labelled) {
        heldout_best_network = parser.network;
        heldout_best_correct_labelled = correct_labelled;
        heldout_best_iteration = iteration;
      }
    }

    cerr << endl;
  }

  if (parameters.early_stopping && heldout_best_iteration > 0) {
    cerr << "Using early stopping -- choosing network from iteration " << heldout_best_iteration << endl;
    parser.network = heldout_best_network;
  }

  // Encode version
  enc.add_1B(parser.version);

  // Encode single_root
  enc.add_1B(single_root);

  // Encode transition system
  enc.add_2B(parser.labels.size());
  for (auto&& label : parser.labels)
    enc.add_str(label);
  enc.add_str(transition_system_name);

  // Encode nodes selector
  enc.add_str(nodes_description);

  // Encode value extractors and embeddings
  enc.add_2B(value_names.size());
  for (auto&& value_name : value_names)
    enc.add_str(value_name);
  for (auto&& embedding : parser.embeddings)
    embedding.save(enc);

  // Encode the network
  network_trainer.save_network(enc);
}

} // namespace parsito

/////////
// File: parsito/transition/transition.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Left arc
bool transition_left_arc::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else
    return conf.stack.size() >= 2 && conf.stack[conf.stack.size() - 2];
}

int transition_left_arc::perform(configuration& conf) const {
  assert(applicable(conf));

  int parent = conf.stack.back(); conf.stack.pop_back();
  int child = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(parent);
  conf.t->set_head(child, parent, label);
  return child;
}

// Right arc
bool transition_right_arc::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return conf.stack.size() == 2 && conf.buffer.empty();
  else if (conf.single_root) // && !label_is_root
    return conf.stack.size() > 2;
  else
    return conf.stack.size() >= 2;
}

int transition_right_arc::perform(configuration& conf) const {
  assert(applicable(conf));

  int child = conf.stack.back(); conf.stack.pop_back();
  int parent = conf.stack.back();
  conf.t->set_head(child, parent, label);
  return child;
}

// Shift
bool transition_shift::applicable(const configuration& conf) const {
  return !conf.buffer.empty();
}

int transition_shift::perform(configuration& conf) const {
  assert(applicable(conf));

  conf.stack.push_back(conf.buffer.back());
  conf.buffer.pop_back();
  return -1;
}

// Swap
bool transition_swap::applicable(const configuration& conf) const {
  return conf.stack.size() >= 2 && conf.stack[conf.stack.size() - 2] && conf.stack[conf.stack.size() - 2] < conf.stack[conf.stack.size() - 1];
}

int transition_swap::perform(configuration& conf) const {
  assert(applicable(conf));

  int top = conf.stack.back(); conf.stack.pop_back();
  int to_buffer = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(top);
  conf.buffer.push_back(to_buffer);
  return -1;
}

// Left arc 2
bool transition_left_arc_2::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else
    return conf.stack.size() >= 3 && conf.stack[conf.stack.size() - 3];
}

int transition_left_arc_2::perform(configuration& conf) const {
  assert(applicable(conf));

  int parent = conf.stack.back(); conf.stack.pop_back();
  int ignore = conf.stack.back(); conf.stack.pop_back();
  int child = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(ignore);
  conf.stack.push_back(parent);
  conf.t->set_head(child, parent, label);
  return child;
}

// Right arc 2
bool transition_right_arc_2::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else if (conf.single_root) // && !label_is_root
    return conf.stack.size() >= 4;
  else
    return conf.stack.size() >= 3;
}

int transition_right_arc_2::perform(configuration& conf) const {
  assert(applicable(conf));

  int child = conf.stack.back(); conf.stack.pop_back();
  int to_buffer = conf.stack.back(); conf.stack.pop_back();
  int parent = conf.stack.back();
  conf.buffer.push_back(to_buffer);
  conf.t->set_head(child, parent, label);
  return child;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_link2.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_link2 : public transition_system {
 public:
  transition_system_link2(const vector<string>& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system_projective.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_projective : public transition_system {
 public:
  transition_system_projective(const vector<string>& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system_swap.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_swap : public transition_system {
 public:
  transition_system_swap(const vector<string>& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

unsigned transition_system::transition_count() const {
  return transitions.size();
}

bool transition_system::applicable(const configuration& conf, unsigned transition) const {
  assert(transition < transitions.size());

  return transitions[transition]->applicable(conf);
}

int transition_system::perform(configuration& conf, unsigned transition) const {
  assert(transition < transitions.size());

  return transitions[transition]->perform(conf);
}

transition_system* transition_system::create(const string& name, const vector<string>& labels) {
  if (name == "projective") return new transition_system_projective(labels);
  if (name == "swap") return new transition_system_swap(labels);
  if (name == "link2") return new transition_system_link2(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_link2.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_link2::transition_system_link2(const vector<string>& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
    transitions.emplace_back(new transition_left_arc_2(label));
    transitions.emplace_back(new transition_right_arc_2(label));
  }
}

// Static oracle
class transition_system_link2_oracle_static : public transition_oracle {
 public:
  transition_system_link2_oracle_static(const vector<string>& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector<string>& labels, unsigned root_label, const tree& gold) : labels(labels), root_label(root_label), gold(gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const override;
   private:
    const vector<string>& labels;
    unsigned root_label;
    const tree& gold;
  };

  virtual unique_ptr<tree_oracle> create_tree_oracle(const tree& gold) const override;
 private:
  const vector<string>& labels;
  unsigned root_label;
};

unique_ptr<transition_oracle::tree_oracle> transition_system_link2_oracle_static::create_tree_oracle(const tree& gold) const {
  return unique_ptr<transition_oracle::tree_oracle>(new tree_oracle_static(labels, root_label, gold));
}

void transition_system_link2_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const {
  transitions.clear();

  // Shift
  if (!conf.buffer.empty()) transitions.push_back(0);

  // Arcs
  unsigned parents[4] = {1, 2, 1, 3};
  unsigned children[4] = {2, 1, 3, 1};
  for (int direction = 0; direction < 4; direction++)
    if (conf.stack.size() >= parents[direction] && conf.stack.size() >= children[direction]) {
      int parent = conf.stack[conf.stack.size() - parents[direction]];
      int child = conf.stack[conf.stack.size() - children[direction]];

      // Allow arc_2 only when seeing golden edge.
      if (direction >= 2 && gold.nodes[child].head != parent) continue;

      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2 && direction < 2) ||
              (i != root_label && conf.stack.size() > 3 && direction >= 2))
            transitions.push_back(1 + 4*i + direction);
    }
}

transition_oracle::predicted_transition transition_system_link2_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Arcs
  unsigned parents[4] = {1, 2, 1, 3};
  unsigned children[4] = {2, 1, 3, 1};
  for (int direction = 0; direction < 4; direction++)
    if (conf.stack.size() >= parents[direction] && conf.stack.size() >= children[direction]) {
      int parent = conf.stack[conf.stack.size() - parents[direction]];
      int child = conf.stack[conf.stack.size() - children[direction]];

      if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
        for (size_t i = 0; i < labels.size(); i++)
          if (gold.nodes[child].deprel == labels[i])
            return predicted_transition(1 + 4*i + direction, 1 + 4*i + direction);

        assert(!"label was not found");
      }
    }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Oracle factory method
transition_oracle* transition_system_link2::oracle(const string& name) const {
  if (name == "static") return new transition_system_link2_oracle_static(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_projective.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_projective::transition_system_projective(const vector<string>& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
  }
}

// Static oracle
class transition_system_projective_oracle_static : public transition_oracle {
 public:
  transition_system_projective_oracle_static(const vector<string>& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector<string>& labels, unsigned root_label, const tree& gold) : labels(labels), root_label(root_label), gold(gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const override;
   private:
    const vector<string>& labels;
    unsigned root_label;
    const tree& gold;
  };

  virtual unique_ptr<tree_oracle> create_tree_oracle(const tree& gold) const override;
 private:
  const vector<string>& labels;
  unsigned root_label;
};

unique_ptr<transition_oracle::tree_oracle> transition_system_projective_oracle_static::create_tree_oracle(const tree& gold) const {
  return unique_ptr<transition_oracle::tree_oracle>(new tree_oracle_static(labels, root_label, gold));
}

void transition_system_projective_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const {
  transitions.clear();
  if (!conf.buffer.empty()) transitions.push_back(0);
  if (conf.stack.size() >= 2)
    for (int direction = 0; direction < 2; direction++) {
      int child = conf.stack[conf.stack.size() - 2 + direction];
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2))
            transitions.push_back(1 + 2*i + direction);
    }
}

transition_oracle::predicted_transition transition_system_projective_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Use left if appropriate
  if (conf.stack.size() >= 2) {
    int parent = conf.stack[conf.stack.size() - 1];
    int child = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(1 + 2*i, 1 + 2*i);

      assert(!"label was not found");
    }
  }

  // Use right if appropriate
  if (conf.stack.size() >= 2) {
    int child = conf.stack[conf.stack.size() - 1];
    int parent = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent &&
        (conf.buffer.empty() || gold.nodes[child].children.empty() || gold.nodes[child].children.back() < conf.buffer.back())) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(1 + 2*i + 1, 1 + 2*i + 1);

      assert(!"label was not found");
    }
  }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Dynamic oracle
class transition_system_projective_oracle_dynamic : public transition_oracle {
 public:
  transition_system_projective_oracle_dynamic(const vector<string>& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_dynamic : public transition_oracle::tree_oracle {
   public:
    tree_oracle_dynamic(const vector<string>& labels, unsigned root_label, const tree& gold) : labels(labels), gold(gold), oracle_static(labels, root_label, gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const override;
   private:
    const vector<string>& labels;
    const tree& gold;
    transition_system_projective_oracle_static::tree_oracle_static oracle_static;
  };

  virtual unique_ptr<tree_oracle> create_tree_oracle(const tree& gold) const override;
 private:
  const vector<string>& labels;
  unsigned root_label;
};

unique_ptr<transition_oracle::tree_oracle> transition_system_projective_oracle_dynamic::create_tree_oracle(const tree& gold) const {
  return unique_ptr<transition_oracle::tree_oracle>(new tree_oracle_dynamic(labels, root_label, gold));
}

void transition_system_projective_oracle_dynamic::tree_oracle_dynamic::interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const {
  oracle_static.interesting_transitions(conf, transitions);
}

transition_oracle::predicted_transition transition_system_projective_oracle_dynamic::tree_oracle_dynamic::predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const {
  // Use static oracle in the first iteration
  if (iteration <= 1)
    return oracle_static.predict(conf, network_outcome, iteration);

  // Use dynamic programming to compute transition leading to best parse tree

  // Start by computing the right stack
  vector<int> right_stack;

  unordered_set<int> right_stack_inserted;
  if (!conf.buffer.empty()) {
    int buffer_start = conf.buffer.back();
    for (size_t i = conf.buffer.size(); i--; ) {
      const auto& node = conf.buffer[i];
      bool to_right_stack = gold.nodes[node].head < buffer_start;
      for (auto&& child : gold.nodes[node].children)
        to_right_stack |= child < buffer_start || right_stack_inserted.count(child);
      if (to_right_stack) {
        right_stack.push_back(node);
        right_stack_inserted.insert(node);
      }
    }
  }

  // Fill the array T from the 2014 Goldberg paper
  class t_representation {
   public:
    t_representation(const vector<int>& stack, const vector<int>& right_stack, const tree& gold, const vector<string>& labels)
        : stack(stack), right_stack(right_stack), gold(gold), labels(labels) {
      for (int i = 0; i < 2; i++) {
        costs[i].reserve((stack.size() + right_stack.size()) * (stack.size() + right_stack.size()));
        transitions[i].reserve((stack.size() + right_stack.size()) * (stack.size() + right_stack.size()));
      }
    }

    void prepare(unsigned diagonal) {
      costs[diagonal & 1].assign((diagonal + 1) * (diagonal + 1), gold.nodes.size() + 1);
      transitions[diagonal & 1].assign((diagonal + 1) * (diagonal + 1), -1);
    }

    int& cost(unsigned i, unsigned j, unsigned h) { return costs[(i+j) & 1][i * (i+j+1) + h]; }
    int& transition(unsigned i, unsigned j, unsigned h) { return transitions[(i+j) & 1][i * (i+j+1) + h]; }

    int node(unsigned i, unsigned /*j*/, unsigned h) const { return h <= i ? stack[stack.size() - 1 - i + h] : right_stack[h - i - 1]; }
    int edge_cost(int parent, int child) const { return gold.nodes[child].head != parent; }
    int which_arc_transition(int parent, int child) const {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return 1 + 2*i + (child > parent);
      assert(!"label was not found");
      return 0; // To keep VS 2015 happy and warning-free
    }

   private:
    const vector<int>& stack;
    const vector<int>& right_stack;
    const tree& gold;
    const vector<string>& labels;
    vector<int> costs[2], transitions[2];
  } t(conf.stack, right_stack, gold, labels);

  t.prepare(0);
  t.cost(0, 0, 0) = 0;
  for (unsigned diagonal = 0; diagonal < conf.stack.size() + right_stack.size(); diagonal++) {
    t.prepare(diagonal + 1);
    for (unsigned i = diagonal > right_stack.size() ? diagonal - right_stack.size() : 0; i <= diagonal && i < conf.stack.size(); i++) {
      unsigned j = diagonal - i;

      // Try extending stack
      if (i+1 < conf.stack.size())
        for (unsigned h = 0; h <= diagonal; h++) {
          int h_node = t.node(i, j, h), new_node = t.node(i+1, j, 0);
          if (new_node && t.cost(i, j, h) + t.edge_cost(h_node, new_node) < t.cost(i+1, j, h+1) + (t.transition(i, j, h) != 0)) {
            t.cost(i+1, j, h+1) = t.cost(i, j, h) + t.edge_cost(h_node, new_node);
            t.transition(i+1, j, h+1) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : t.which_arc_transition(h_node, new_node);
          }
          if (t.cost(i, j, h) + t.edge_cost(new_node, h_node) < t.cost(i+1, j, 0) + (t.transition(i, j, h) != 0)) {
            t.cost(i+1, j, 0) = t.cost(i, j, h) + t.edge_cost(new_node, h_node);
            t.transition(i+1, j, 0) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : t.which_arc_transition(new_node, h_node);
          }
        }

      // Try extending right_stack
      if (j+1 < right_stack.size() + 1)
        for (unsigned h = 0; h <= diagonal; h++) {
          int h_node = t.node(i, j, h), new_node = t.node(i, j+1, diagonal+1);
          if (t.cost(i, j, h) + t.edge_cost(h_node, new_node) < t.cost(i, j+1, h) + (t.transition(i, j, h) > 0)) {
            t.cost(i, j+1, h) = t.cost(i, j, h) + t.edge_cost(h_node, new_node);
            t.transition(i, j+1, h) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : 0;
          }
          if (h_node && t.cost(i, j, h) + t.edge_cost(new_node, h_node) < t.cost(i, j+1, diagonal+1) + (t.transition(i, j, h) > 0)) {
            t.cost(i, j+1, diagonal+1) = t.cost(i, j, h) + t.edge_cost(new_node, h_node);
            t.transition(i, j+1, diagonal+1) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : 0;
          }
        }
    }
  }

  return predicted_transition(t.transition(conf.stack.size() - 1, right_stack.size(), 0), network_outcome);
}

// Oracle factory method
transition_oracle* transition_system_projective::oracle(const string& name) const {
  if (name == "static") return new transition_system_projective_oracle_static(labels);
  if (name == "dynamic") return new transition_system_projective_oracle_dynamic(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_swap.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_swap::transition_system_swap(const vector<string>& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  transitions.emplace_back(new transition_swap());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
  }
}

// Static oracle
class transition_system_swap_oracle_static : public transition_oracle {
 public:
  transition_system_swap_oracle_static(const vector<string>& labels, bool lazy) : labels(labels), lazy(lazy) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector<string>& labels, unsigned root_label, const tree& gold, vector<int>&& projective_order, vector<int>&& projective_components)
        : labels(labels), root_label(root_label), gold(gold), projective_order(projective_order), projective_components(projective_components) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const override;
   private:
    const vector<string>& labels;
    unsigned root_label;
    const tree& gold;
    const vector<int> projective_order;
    const vector<int> projective_components;
  };

  virtual unique_ptr<tree_oracle> create_tree_oracle(const tree& gold) const override;
 private:
  void create_projective_order(const tree& gold, int node, vector<int>& projective_order, int& projective_index) const;
  void create_projective_component(const tree& gold, int node, vector<int>& projective_components, int component_index) const;

  const vector<string>& labels;
  bool lazy;
  unsigned root_label;
};

unique_ptr<transition_oracle::tree_oracle> transition_system_swap_oracle_static::create_tree_oracle(const tree& gold) const {
  vector<int> projective_order(gold.nodes.size());
  int projective_index;
  create_projective_order(gold, 0, projective_order, projective_index);

  vector<int> projective_components;
  if (lazy) {
    tree_oracle_static projective_oracle(labels, root_label, gold, vector<int>(), vector<int>());
    configuration conf(false);
    tree t = gold;
    transition_system_swap system(labels);

    conf.init(&t);
    while (!conf.final()) {
      auto prediction = projective_oracle.predict(conf, 0, 0);
      if (!system.applicable(conf, prediction.to_follow)) break;
      system.perform(conf, prediction.to_follow);
    }

    projective_components.assign(gold.nodes.size(), 0);
    for (auto&& node : conf.stack)
      if (node)
        create_projective_component(t, node, projective_components, node);
  }

  return unique_ptr<transition_oracle::tree_oracle>(new tree_oracle_static(labels, root_label, gold, move(projective_order), move(projective_components)));
}

void transition_system_swap_oracle_static::create_projective_order(const tree& gold, int node, vector<int>& projective_order, int& projective_index) const {
  unsigned child_index = 0;
  while (child_index < gold.nodes[node].children.size() && gold.nodes[node].children[child_index] < node)
    create_projective_order(gold, gold.nodes[node].children[child_index++], projective_order, projective_index);
  projective_order[node] = projective_index++;
  while (child_index < gold.nodes[node].children.size())
    create_projective_order(gold, gold.nodes[node].children[child_index++], projective_order, projective_index);
}

void transition_system_swap_oracle_static::create_projective_component(const tree& gold, int node, vector<int>& projective_components, int component_index) const {
  projective_components[node] = component_index;
  for (auto&& child : gold.nodes[node].children)
    create_projective_component(gold, child, projective_components, component_index);
}

void transition_system_swap_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector<unsigned>& transitions) const {
  transitions.clear();
  if (!conf.buffer.empty()) transitions.push_back(0);
  if (conf.stack.size() >= 2) {
    // Swap
    if (!projective_order.empty()) {
      int last = conf.stack[conf.stack.size() - 1];
      int prev = conf.stack[conf.stack.size() - 2];
      if (projective_order[last] < projective_order[prev] &&
          (projective_components.empty() ||
           (conf.buffer.empty() || projective_components[last] != projective_components[conf.buffer.back()])))
        transitions.push_back(1);
    }

    // Arcs
    for (int direction = 0; direction < 2; direction++) {
      int child = conf.stack[conf.stack.size() - 2 + direction];
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2))
            transitions.push_back(2 + 2*i + direction);
    }
  }
}

transition_oracle::predicted_transition transition_system_swap_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Use left if appropriate
  if (conf.stack.size() >= 2) {
    int parent = conf.stack[conf.stack.size() - 1];
    int child = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(2 + 2*i, 2 + 2*i);

      assert(!"label was not found");
    }
  }

  // Use right if appropriate
  if (conf.stack.size() >= 2) {
    int child = conf.stack[conf.stack.size() - 1];
    int parent = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(2 + 2*i + 1, 2 + 2*i + 1);

      assert(!"label was not found");
    }
  }

  // Use swap if appropriate
  if (conf.stack.size() >= 2 && !projective_order.empty()) {
    int last = conf.stack[conf.stack.size() - 1];
    int prev = conf.stack[conf.stack.size() - 2];
    if (projective_order[last] < projective_order[prev] &&
        (projective_components.empty() ||
         (conf.buffer.empty() || projective_components[last] != projective_components[conf.buffer.back()])))
      return predicted_transition(1, 1);
  }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Oracle factory method
transition_oracle* transition_system_swap::oracle(const string& name) const {
  if (name == "static_eager") return new transition_system_swap_oracle_static(labels, false);
  if (name == "static_lazy") return new transition_system_swap_oracle_static(labels, true);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/tree/tree.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

const string tree::root_form = "<root>";

tree::tree() {
  clear();
}

bool tree::empty() {
  return nodes.size() == 1;
}

void tree::clear() {
  nodes.clear();
  node& root = add_node(root_form);
  root.lemma = root.upostag = root.xpostag = root.feats = root_form;
}

node& tree::add_node(const string& form) {
  nodes.emplace_back(nodes.size(), form);
  return nodes.back();
}

void tree::set_head(int id, int head, const string& deprel) {
  assert(id >= 0 && id < int(nodes.size()));
  assert(head < int(nodes.size()));

  // Remove existing head
  if (nodes[id].head >= 0) {
    auto& children = nodes[nodes[id].head].children;
    for (size_t i = children.size(); i && children[i-1] >= id; i--)
      if (children[i-1] == id) {
        children.erase(children.begin() + i - 1);
        break;
      }
  }

  // Set new head
  nodes[id].head = head;
  nodes[id].deprel = deprel;
  if (head >= 0) {
    auto& children = nodes[head].children;
    size_t i = children.size();
    while (i && children[i-1] > id) i--;
    if (!i || children[i-1] < id) children.insert(children.begin() + i, id);
  }
}

void tree::unlink_all_nodes() {
  for (auto&& node : nodes) {
    node.head = -1;
    node.deprel.clear();
    node.children.clear();
  }
}

} // namespace parsito

/////////
// File: parsito/tree/tree_format.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input format
class tree_input_format {
 public:
  virtual ~tree_input_format() {}

  virtual bool read_block(istream& in, string& block) const = 0;
  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_tree(tree& t) = 0;
  const string& last_error() const;

  // Static factory methods
  static tree_input_format* new_input_format(const string& name);
  static tree_input_format* new_conllu_input_format();

 protected:
  string error;
};

// Output format
class tree_output_format {
 public:
  virtual ~tree_output_format() {}

  virtual void write_tree(const tree& t, string& output, const tree_input_format* additional_info = nullptr) const = 0;

  // Static factory methods
  static tree_output_format* new_output_format(const string& name);
  static tree_output_format* new_conllu_output_format();
};

} // namespace parsito

/////////
// File: parsito/tree/tree_format_conllu.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input CoNLL-U format
class tree_input_format_conllu : public tree_input_format {
 public:
  virtual bool read_block(istream& in, string& block) const override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_tree(tree& t) override;

 private:
  friend class tree_output_format_conllu;
  vector<string_piece> comments;
  vector<pair<int, string_piece>> multiword_tokens;

  string_piece text;
  string text_copy;
};

// Output CoNLL-U format
class tree_output_format_conllu : public tree_output_format {
 public:
  virtual void write_tree(const tree& t, string& output, const tree_input_format* additional_info = nullptr) const override;

 private:
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
};

} // namespace parsito

/////////
// File: parsito/tree/tree_format.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

const string& tree_input_format::last_error() const {
  return error;
}

// Input Static factory methods
tree_input_format* tree_input_format::new_conllu_input_format() {
  return new tree_input_format_conllu();
}

tree_input_format* tree_input_format::new_input_format(const string& name) {
  if (name == "conllu") return new_conllu_input_format();
  return nullptr;
}

// Output static factory methods
tree_output_format* tree_output_format::new_conllu_output_format() {
  return new tree_output_format_conllu();
}

tree_output_format* tree_output_format::new_output_format(const string& name) {
  if (name == "conllu") return new_conllu_output_format();
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/tree/tree_format_conllu.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input CoNLL-U format

bool tree_input_format_conllu::read_block(istream& in, string& block) const {
  return bool(getpara(in, block));
}

void tree_input_format_conllu::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool tree_input_format_conllu::next_tree(tree& t) {
  error.clear();
  t.clear();
  comments.clear();
  multiword_tokens.clear();
  int last_multiword_token = 0;

  vector<string_piece> tokens, parts;
  while (text.len) {
    // Read line
    string_piece line(text.str, 0);
    while (line.len < text.len && line.str[line.len] != '\n') line.len++;
    text.str += line.len + (line.len < text.len);
    text.len -= line.len + (line.len < text.len);

    // Empty lines denote end of tree, unless at the beginning
    if (!line.len) {
      if (t.empty()) continue;
      break;
    }

    if (*line.str == '#') {
      // Store comments at the beginning and ignore the rest
      if (t.empty()) comments.push_back(line);
      continue;
    }

    // Parse another tree node
    split(line, '\t', tokens);
    if (tokens.size() != 10)
      return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' does not contain 10 columns!") , false;

    // Store and skip multiword tokens
    if (memchr(tokens[0].str, '-', tokens[0].len)) {
      split(tokens[0], '-', parts);
      if (parts.size() != 2)
        return error.assign("Cannot parse ID of multiword token '").append(line.str, line.len).append("'!") , false;
      int from, to;
      if (!parse_int(parts[0], "CoNLL-U id", from, error) || !parse_int(parts[1], "CoNLL-U id", to, error))
        return false;
      if (from != int(t.nodes.size()))
        return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (to < from)
        return error.assign("Incorrect range '").append(tokens[0].str, tokens[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (from <= last_multiword_token)
        return error.assign("Multiword token '").append(line.str, line.len).append("' overlaps with the previous one!"), false;
      last_multiword_token = to;
      multiword_tokens.emplace_back(from, line);
      continue;
    }

    // Parse node ID and head
    int id;
    if (!parse_int(tokens[0], "CoNLL-U id", id, error))
      return false;
    if (id != int(t.nodes.size()))
      return error.assign("Incorrect ID '").append(tokens[0].str, tokens[0].len).append("' of CoNLL-U line '").append(line.str, line.len).append("'!"), false;

    int head;
    if (tokens[6].len == 1 && tokens[6].str[0] == '_') {
      head = -1;
    } else {
      if (!parse_int(tokens[6], "CoNLL-U head", head, error))
        return false;
      if (head < 0)
        return error.assign("Numeric head value '").append(tokens[0].str, tokens[0].len).append("' cannot be negative!"), false;
    }

    // Add new node
    auto& node = t.add_node(string(tokens[1].str, tokens[1].len));
    if (!(tokens[2].len == 1 && tokens[2].str[0] == '_')) node.lemma.assign(tokens[2].str, tokens[2].len);
    if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) node.upostag.assign(tokens[3].str, tokens[3].len);
    if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) node.xpostag.assign(tokens[4].str, tokens[4].len);
    if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) node.feats.assign(tokens[5].str, tokens[5].len);
    node.head = head;
    if (!(tokens[7].len == 1 && tokens[7].str[0] == '_')) node.deprel.assign(tokens[7].str, tokens[7].len);
    if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) node.deps.assign(tokens[8].str, tokens[8].len);
    if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) node.misc.assign(tokens[9].str, tokens[9].len);
  }

  // Check that we got word for the last multiword token
  if (last_multiword_token >= int(t.nodes.size()))
    return error.assign("There are words missing for multiword token '").append(multiword_tokens.back().second.str, multiword_tokens.back().second.len).append("'!"), false;

  // Set heads correctly
  for (auto&& node : t.nodes)
    if (node.id && node.head >= 0) {
      if (node.head >= int(t.nodes.size()))
        return error.assign("Node ID '").append(to_string(node.id)).append("' form '").append(node.form).append("' has too large head: '").append(to_string(node.head)).append("'!"), false;
      t.set_head(node.id, node.head, node.deprel);
    }

  return !t.empty();
}

// Output CoNLL-U format

const string tree_output_format_conllu::underscore = "_";

void tree_output_format_conllu::write_tree(const tree& t, string& output, const tree_input_format* additional_info) const {
  output.clear();

  // Try casting input format to CoNLL-U
  auto input_conllu = dynamic_cast<const tree_input_format_conllu*>(additional_info);
  size_t input_conllu_multiword_tokens = 0;

  // Comments if present
  if (input_conllu)
    for (auto&& comment : input_conllu->comments)
      output.append(comment.str, comment.len).push_back('\n');

  // Print out the tokens
  for (int i = 1 /*skip the root node*/; i < int(t.nodes.size()); i++) {
    // Write multiword token if present
    if (input_conllu && input_conllu_multiword_tokens < input_conllu->multiword_tokens.size() &&
        i == input_conllu->multiword_tokens[input_conllu_multiword_tokens].first) {
      output.append(input_conllu->multiword_tokens[input_conllu_multiword_tokens].second.str,
                    input_conllu->multiword_tokens[input_conllu_multiword_tokens].second.len).push_back('\n');
      input_conllu_multiword_tokens++;
    }

    // Write the token
    output.append(to_string(i)).push_back('\t');
    output.append(t.nodes[i].form).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].lemma)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].upostag)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].xpostag)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].feats)).push_back('\t');
    output.append(t.nodes[i].head < 0 ? "_" : to_string(t.nodes[i].head)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].deprel)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].deps)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].misc)).push_back('\n');
  }
  output.push_back('\n');
}

} // namespace parsito

/////////
// File: parsito/version/version.h
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct version {
  unsigned major;
  unsigned minor;
  unsigned patch;
  std::string prerelease;

  // Returns current version.
  static version current();

  // Returns multi-line formated version and copyright string.
  static string version_and_copyright(const string& other_libraries = string());
};

} // namespace parsito

/////////
// File: parsito/version/version.cpp
/////////

// This file is part of Parsito <http://github.com/ufal/parsito/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Returns current version.
version version::current() {
  return {1, 1, 1, "devel"};
}

// Returns multi-line formated version and copyright string.
string version::version_and_copyright(const string& other_libraries) {
  ostringstream info;

  auto parsito = version::current();
  auto unilib = unilib::version::current();

  info << "Parsito version " << parsito.major << '.' << parsito.minor << '.' << parsito.patch
       << (parsito.prerelease.empty() ? "" : "-") << parsito.prerelease
       << " (using UniLib " << unilib.major << '.' << unilib.minor << '.' << unilib.patch
       << (other_libraries.empty() ? "" : " and ") << other_libraries << ")\n"
          "Copyright 2015 by Institute of Formal and Applied Linguistics, Faculty of\n"
          "Mathematics and Physics, Charles University in Prague, Czech Republic.";

  return info.str();
}

} // namespace parsito

/////////
// File: sentence/input_format.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string input_format::CONLLU_V1 = "v1";
const string input_format::CONLLU_V2 = "v2";
const string input_format::GENERIC_TOKENIZER_NORMALIZED_SPACES = "normalized_spaces";
const string input_format::GENERIC_TOKENIZER_PRESEGMENTED = "presegmented";
const string input_format::GENERIC_TOKENIZER_RANGES = "ranges";

// CoNLL-U input format
class input_format_conllu : public input_format {
 public:
  input_format_conllu(unsigned version) : version(version) {}

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unsigned version;
  string_piece text;
  string text_copy;

  static const string columns[10];
};

const string input_format_conllu::columns[10] = {"ID", "FORM", "LEMMA",
  "UPOS", "XPOS", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC"};

bool input_format_conllu::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void input_format_conllu::reset_document(string_piece /*id*/) {
  set_text("");
}

void input_format_conllu::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_conllu::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();
  int last_multiword_token = 0;

  vector<string_piece> tokens, parts;
  while (text.len) {
    // Read line
    string_piece line(text.str, 0);
    while (line.len < text.len && (line.str[line.len] != '\r' && line.str[line.len] != '\n')) line.len++;

    text.str += line.len, text.len -= line.len;
    if (text.len >= 2 && text.str[0] == '\r' && text.str[1] == '\n')
      text.str += 2, text.len -= 2;
    else if (text.len && *text.str == '\n')
      text.str++, text.len--;

    // Empty lines denote end of tree, unless at the beginning
    if (!line.len) {
      if (s.empty()) continue;
      break;
    }

    if (*line.str == '#') {
      // Store comments at the beginning and ignore the rest
      if (s.empty()) s.comments.emplace_back(line.str, line.len);
      continue;
    }

    // Parse the line
    split(line, '\t', tokens);
    if (tokens.size() != 10)
      return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' does not contain 10 columns!") , false;

    // Check that no column is empty and contains no spaces (except FORM and LEMMA in version >= 2)
    for (int i = 0; i < 10; i++) {
      if (!tokens[i].len)
        return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' contains empty column ").append(columns[i]).append("!"), false;
      if ((version < 2 || (i != 1 && i != 2)) && memchr(tokens[i].str, ' ', tokens[i].len) != NULL)
        return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' contains spaces in column ").append(columns[i]).append("!"), false;
    }

    // Handle multiword tokens
    if (memchr(tokens[0].str, '-', tokens[0].len)) {
      split(tokens[0], '-', parts);
      if (parts.size() != 2)
        return error.assign("Cannot parse ID of multiword token '").append(line.str, line.len).append("'!") , false;
      int from, to;
      if (!parse_int(parts[0], "CoNLL-U id", from, error) || !parse_int(parts[1], "CoNLL-U id", to, error))
        return false;
      if (from != int(s.words.size()))
        return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (to < from)
        return error.assign("Incorrect range '").append(tokens[0].str, tokens[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (from <= last_multiword_token)
        return error.assign("Multiword token '").append(line.str, line.len).append("' overlaps with the previous one!"), false;
      last_multiword_token = to;
      for (int i = 2; i < 9; i++)
        if (tokens[i].len != 1 || tokens[i].str[0] != '_')
          return error.assign("Column ").append(columns[i]).append(" of an multi-word token '").append(line.str, line.len).append("' is not an empty!"), false;
      s.multiword_tokens.emplace_back(from, to, tokens[1], tokens[9].len == 1 && tokens[9].str[0] == '_' ? string_piece() : tokens[9]);
      continue;
    }

    // Handle empty nodes
    if (version >= 2)
      if (memchr(tokens[0].str, '.', tokens[0].len)) {
        split(tokens[0], '.', parts);
        if (parts.size() != 2)
          return error.assign("Cannot parse ID of empty node '").append(line.str, line.len).append("'!") , false;
        int id, index;
        if (!parse_int(parts[0], "CoNLL-U empty node id", id, error) || !parse_int(parts[1], "CoNLL-U empty node index", index, error))
          return false;
        if (id != int(s.words.size()) - 1)
          return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of empty node token '").append(line.str, line.len).append("'!"), false;
        if (!((s.empty_nodes.empty() && index == 1) || (!s.empty_nodes.empty() && s.empty_nodes.back().id < id && index == 1) ||
             (!s.empty_nodes.empty() && s.empty_nodes.back().id == id && index == s.empty_nodes.back().index + 1)))
          return error.assign("Incorrect ID index '").append(parts[1].str, parts[1].len).append("' of empty node token '").append(line.str, line.len).append("'!"), false;
        for (int i = 6; i < 8; i++)
          if (tokens[i].len != 1 || tokens[i].str[0] != '_')
            return error.assign("Column ").append(columns[i]).append(" of an empty node token '").append(line.str, line.len).append("' is not an empty!"), false;

        s.empty_nodes.emplace_back(id, index);
        s.empty_nodes.back().form.assign(tokens[1].str, tokens[1].len);
        s.empty_nodes.back().lemma.assign(tokens[2].str, tokens[2].len);
        if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) s.empty_nodes.back().upostag.assign(tokens[3].str, tokens[3].len);
        if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) s.empty_nodes.back().xpostag.assign(tokens[4].str, tokens[4].len);
        if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) s.empty_nodes.back().feats.assign(tokens[5].str, tokens[5].len);
        if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) s.empty_nodes.back().deps.assign(tokens[8].str, tokens[8].len);
        if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) s.empty_nodes.back().misc.assign(tokens[9].str, tokens[9].len);
        continue;
      }

    // Parse word ID and head
    int id;
    if (!parse_int(tokens[0], "CoNLL-U id", id, error))
      return false;
    if (id != int(s.words.size()))
      return error.assign("Incorrect ID '").append(tokens[0].str, tokens[0].len).append("' of CoNLL-U line '").append(line.str, line.len).append("'!"), false;

    int head;
    if (tokens[6].len == 1 && tokens[6].str[0] == '_') {
      head = -1;
    } else {
      if (!parse_int(tokens[6], "CoNLL-U head", head, error))
        return false;
      if (head < 0)
        return error.assign("Numeric head value '").append(tokens[0].str, tokens[0].len).append("' cannot be negative!"), false;
    }

    // Add new word
    auto& word = s.add_word(tokens[1]);
    word.lemma.assign(tokens[2].str, tokens[2].len);
    if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) word.upostag.assign(tokens[3].str, tokens[3].len);
    if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) word.xpostag.assign(tokens[4].str, tokens[4].len);
    if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) word.feats.assign(tokens[5].str, tokens[5].len);
    word.head = head;
    if (!(tokens[7].len == 1 && tokens[7].str[0] == '_')) word.deprel.assign(tokens[7].str, tokens[7].len);
    if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) word.deps.assign(tokens[8].str, tokens[8].len);
    if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) word.misc.assign(tokens[9].str, tokens[9].len);
  }

  // Check that we got word for the last multiword token
  if (last_multiword_token >= int(s.words.size()))
    return error.assign("There are words missing for multiword token '").append(s.multiword_tokens.back().form).append("'!"), false;

  // Set heads correctly
  for (auto&& word : s.words)
    if (word.id && word.head >= 0) {
      if (word.head >= int(s.words.size()))
        return error.assign("Node ID '").append(to_string(word.id)).append("' form '").append(word.form).append("' has too large head: '").append(to_string(word.head)).append("'!"), false;
      s.set_head(word.id, word.head, word.deprel);
    }

  return !s.empty();
}

// Horizontal input format
class input_format_horizontal : public input_format {
 public:
  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_horizontal::read_block(istream& is, string& block) const {
  if (getline(is, block))
    return block.push_back('\n'), true;
  return false;
}

void input_format_horizontal::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
}

void input_format_horizontal::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_horizontal::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  // Skip spaces and newlines
  while (text.len && (*text.str == ' ' || *text.str == '\t' || *text.str == '\r' || *text.str == '\n')) {
    preceeding_newlines += *text.str == '\n';
    text.str++, text.len--;
  }

  // Read space (and tab) separated words
  while (text.len && *text.str != '\r' && *text.str != '\n') {
    string_piece word = text;

    // Slurp the word
    while (text.len && *text.str != ' ' && *text.str != '\t' && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;
    word.len = text.str - word.str;
    s.add_word(word);

    // Replace &nbsp;s by regular spaces
    if (s.words.back().form.find("\302\240") != string::npos) {
      string& form = s.words.back().form;
      size_t form_len = 0;
      for (size_t i = 0; i < form.size(); i++) {
        if (form_len && form[form_len-1] == '\302' && form[i] == '\240')
          form[form_len - 1] = ' ';
        else
          form[form_len++] = form[i];
      }
      form.resize(form_len);
    }

    // Skip spaces
    while (text.len && (*text.str == ' ' || *text.str == '\t'))
      text.str++, text.len--;
  }

  if (!s.empty()) {
    // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = 0;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));
  }

  return !s.empty();
}

// Vertical input format
class input_format_vertical : public input_format {
 public:
  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_vertical::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void input_format_vertical::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
}

void input_format_vertical::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_vertical::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  // Skip tabs and newlines
  while (text.len && (*text.str == '\t' || *text.str == '\r' || *text.str == '\n')) {
    preceeding_newlines += *text.str == '\n';
    text.str++, text.len--;
  }

  // Read first word without tabs on every line
  while (text.len && *text.str != '\r' && *text.str != '\n') {
    string_piece word = text;

    // Slurp the word
    while (text.len && *text.str != '\t' && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;
    word.len = text.str - word.str;
    s.add_word(word);

    // Skip spaces till end of line
    while (text.len && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;

    // Skip one new line
    if (text.len >= 2 && text.str[0] == '\r' && text.str[1] == '\n')
      text.str += 2, text.len -= 2;
    else if (text.len && *text.str == '\n')
      text.str++, text.len--;

    // Skip tabs on the beginning of the line
    while (text.len && *text.str == '\t')
      text.str++, text.len--;
  }

  if (!s.empty()) {
    // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = 0;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));
  }

  return !s.empty();
}

// Presegmented tokenizer
class input_format_presegmented_tokenizer : public input_format {
 public:
  input_format_presegmented_tokenizer(input_format* tokenizer) : tokenizer(tokenizer) {}

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unique_ptr<input_format> tokenizer;
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_presegmented_tokenizer::read_block(istream& is, string& block) const {
  if (getline(is, block))
    return block.push_back('\n'), true;
  return false;
}

void input_format_presegmented_tokenizer::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  tokenizer->reset_document();
  set_text("");
}

void input_format_presegmented_tokenizer::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_presegmented_tokenizer::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  sentence partial;
  unsigned following_newlines = 0;
  while (text.len && s.empty()) {
    // Move next line from `text' to `line', including leading and following newlines
    string_piece line(text.str, 0);
    while (line.len < text.len && (line.str[line.len] == '\n' || line.str[line.len] == '\r')) {
      preceeding_newlines += line.str[line.len] == '\n';
      line.len++;
    }
    while (line.len < text.len && (line.str[line.len] != '\n' && line.str[line.len] != '\r'))
      line.len++;
    while (line.len < text.len && (line.str[line.len] == '\n' || line.str[line.len] == '\r')) {
      following_newlines += line.str[line.len] == '\n';
      line.len++;
    }
    text.str += line.len, text.len -= line.len;

    // Add all tokens from the line to `s'
    tokenizer->set_text(line, false);
    while (tokenizer->next_sentence(partial, error)) {
      // Append words
      size_t words = s.words.size() - 1;
      for (size_t i = 1; i < partial.words.size(); i++) {
        s.words.push_back(move(partial.words[i]));
        s.words.back().id += words;
        if (s.words.back().head > 0) s.words.back().head += words;
      }

      // Append multiword_tokens
      for (auto&& multiword_token : partial.multiword_tokens) {
        s.multiword_tokens.push_back(move(multiword_token));
        s.multiword_tokens.back().id_first += words;
        s.multiword_tokens.back().id_last += words;
      }

      // Append empty nodes
      for (auto&& empty_node : partial.empty_nodes) {
        s.empty_nodes.push_back(move(empty_node));
        s.empty_nodes.back().id += words;
      }
    }
    if (!error.empty()) return false;

    if (s.empty()) {
      preceeding_newlines += following_newlines;
      following_newlines = 0;
    }
  }

  if (!s.empty()) {
  // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = following_newlines;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));

    // Fill "# text" comment
    s.comments.emplace_back("# text = ");
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j].form : (const token&)s.words[i].form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;

      s.comments.back().append(tok.form);
      if (i+1 < s.words.size() && tok.get_space_after()) s.comments.back().push_back(' ');
    }
  }

  return !s.empty();
}

// Static factory methods
input_format* input_format::new_conllu_input_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  unsigned version = 2;
  if (parsed_options.count(CONLLU_V1))
    version = 1;
  if (parsed_options.count(CONLLU_V2))
    version = 2;

  return new input_format_conllu(version);
}

input_format* input_format::new_generic_tokenizer_input_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  bool normalized_spaces = parsed_options.count(GENERIC_TOKENIZER_NORMALIZED_SPACES);
  bool token_ranges = parsed_options.count(GENERIC_TOKENIZER_RANGES);

  input_format* result = new morphodita_tokenizer_wrapper(morphodita::tokenizer::new_generic_tokenizer(), nullptr, normalized_spaces, token_ranges);
  return (parsed_options.count(GENERIC_TOKENIZER_PRESEGMENTED) && result) ? input_format::new_presegmented_tokenizer(result) : result;
}

input_format* input_format::new_horizontal_input_format(const string& /*options*/) {
  return new input_format_horizontal();
}

input_format* input_format::new_vertical_input_format(const string& /*options*/) {
  return new input_format_vertical();
}

input_format* input_format::new_input_format(const string& name) {
  size_t equal = name.find('=');
  size_t name_len = equal != string::npos ? equal : name.size();
  size_t option_offset = equal != string::npos ? equal + 1 : name.size();

  if (name.compare(0, name_len, "conllu") == 0) return new_conllu_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "generic_tokenizer") == 0) return new_generic_tokenizer_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "horizontal") == 0) return new_horizontal_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "vertical") == 0) return new_vertical_input_format(name.substr(option_offset));
  return nullptr;
}

input_format* input_format::new_presegmented_tokenizer(input_format* tokenizer) {
  return new input_format_presegmented_tokenizer(tokenizer);
}

/////////
// File: utils/xml_encoded.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Print xml content while encoding <>& and optionally " using XML entities.
class xml_encoded {
 public:
  xml_encoded(string_piece str, bool encode_quot = false) : str(str), encode_quot(encode_quot) {}

  friend ostream& operator<<(ostream& os, xml_encoded data);
 private:
  string_piece str;
  bool encode_quot;
};

inline ostream& operator<<(ostream& os, xml_encoded data);

//
// Definitions
//

ostream& operator<<(ostream& os, xml_encoded data) {
  string_piece& str = data.str;
  const char* to_print = str.str;

  while (str.len) {
    while (str.len && *str.str != '<' && *str.str != '>' && *str.str != '&' && (!data.encode_quot || *str.str != '"'))
      str.str++, str.len--;

    if (str.len) {
      if (to_print < str.str) os.write(to_print, str.str - to_print);
      os << (*str.str == '<' ? "&lt;" : *str.str == '>' ? "&gt;" : *str.str == '&' ? "&amp;" : "&quot;");
      str.str++, str.len--;
      to_print = str.str;
    }
  }

  if (to_print < str.str) os.write(to_print, str.str - to_print);

  return os;
}

} // namespace utils

/////////
// File: sentence/output_format.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string output_format::CONLLU_V1 = "v1";
const string output_format::CONLLU_V2 = "v2";
const string output_format::HORIZONTAL_PARAGRAPHS = "paragraphs";
const string output_format::PLAINTEXT_NORMALIZED_SPACES = "normalized_spaces";
const string output_format::VERTICAL_PARAGRAPHS = "paragraphs";

// CoNLL-U output format
class output_format_conllu : public output_format {
 public:
  output_format_conllu(unsigned version) : version(version) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;

 private:
  unsigned version;
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
  ostream& write_with_spaces(ostream& os, const string& str);
};

const string output_format_conllu::underscore = "_";

void output_format_conllu::write_sentence(const sentence& s, ostream& os) {
  // Comments
  for (auto&& comment : s.comments)
    os << comment << '\n';

  // Words and multiword tokens
  size_t multiword_token = 0, empty_node = 0;
  for (int i = 0; i < int(s.words.size()); i++) {
    // Write non-root nodes
    if (i > 0) {
      // Multiword token if present
      if (multiword_token < s.multiword_tokens.size() &&
          i == s.multiword_tokens[multiword_token].id_first) {
        os << s.multiword_tokens[multiword_token].id_first << '-'
           << s.multiword_tokens[multiword_token].id_last << '\t';
        write_with_spaces(os, s.multiword_tokens[multiword_token].form) << "\t_\t_\t_\t_\t_\t_\t_\t"
           << underscore_on_empty(s.multiword_tokens[multiword_token].misc) << '\n';
        multiword_token++;
      }

      // Write the word
      os << i << '\t';
      write_with_spaces(os, s.words[i].form) << '\t';
      write_with_spaces(os, underscore_on_empty(s.words[i].lemma)) << '\t'
         << underscore_on_empty(s.words[i].upostag) << '\t'
         << underscore_on_empty(s.words[i].xpostag) << '\t'
         << underscore_on_empty(s.words[i].feats) << '\t';
      if (s.words[i].head < 0) os << '_'; else os << s.words[i].head; os << '\t'
         << underscore_on_empty(s.words[i].deprel) << '\t'
         << underscore_on_empty(s.words[i].deps) << '\t'
         << underscore_on_empty(s.words[i].misc) << '\n';
    }

    // Empty nodes
    if (version >= 2)
      for (; empty_node < s.empty_nodes.size() && i == s.empty_nodes[empty_node].id; empty_node++) {
        os << i << '.' << s.empty_nodes[empty_node].index << '\t'
           << s.empty_nodes[empty_node].form << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].lemma) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].upostag) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].xpostag) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].feats) << '\t'
           << "_\t"
           << "_\t"
           << underscore_on_empty(s.empty_nodes[empty_node].deps) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].misc) << '\n';
      }
  }
  os << endl;
}

ostream& output_format_conllu::write_with_spaces(ostream& os, const string& str) {
  if (version >= 2 || str.find(' ') == string::npos)
    os << str;
  else
    for (auto&& chr : str)
      os << (chr == ' ' ? '_' : chr);

  return os;
}

// EPE output format
class output_format_epe : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  class json_builder {
   public:
    json_builder& object() { comma(); json.push_back('{'); stack.push_back('}'); return *this; }
    json_builder& array() { comma(); json.push_back('['); stack.push_back(']'); return *this; }
    json_builder& close() { if (!stack.empty()) { json.push_back(stack.back()); stack.pop_back(); } comma_needed = true; return *this; }
    json_builder& key(string_piece name) { comma(); string(name); json.push_back(':'); return *this; }
    json_builder& value(string_piece value) { comma(); string(value); comma_needed=true; return *this; }
    json_builder& value(size_t value) { comma(); number(value); comma_needed=true; return *this; }
    json_builder& value_true() { comma(); json.push_back('t'); json.push_back('r'); json.push_back('u'); json.push_back('e'); comma_needed=true; return *this; }

    string_piece current() const { return string_piece(json.data(), json.size()); }
    void clear() { json.clear(); stack.clear(); comma_needed=false; }

   private:
    void comma() {
      if (comma_needed) {
        json.push_back(',');
        json.push_back(' ');
      }
      comma_needed = false;
    }
    void string(string_piece str) {
      json.push_back('"');
      for (; str.len; str.str++, str.len--)
        switch (*str.str) {
          case '"': json.push_back('\\'); json.push_back('\"'); break;
          case '\\': json.push_back('\\'); json.push_back('\\'); break;
          case '\b': json.push_back('\\'); json.push_back('b'); break;
          case '\f': json.push_back('\\'); json.push_back('f'); break;
          case '\n': json.push_back('\\'); json.push_back('n'); break;
          case '\r': json.push_back('\\'); json.push_back('r'); break;
          case '\t': json.push_back('\\'); json.push_back('t'); break;
          default:
            if (((unsigned char)*str.str) < 32) {
              json.push_back('u'); json.push_back('0'); json.push_back('0'); json.push_back('0' + (*str.str >> 4)); json.push_back("0123456789ABCDEF"[*str.str & 0xF]);
            } else {
              json.push_back(*str.str);
            }
        }
      json.push_back('"');
    }
    void number(size_t value) {
      size_t start_size = json.size();
      for (; value || start_size == json.size(); value /= 10)
        json.push_back('0' + (value % 10));
      reverse(json.begin() + start_size, json.end());
    }

    std::vector<char> json;
    std::vector<char> stack;
    bool comma_needed = false;
  } json;

  vector<string_piece> feats;
  size_t sentences = 0;
};

void output_format_epe::write_sentence(const sentence& s, ostream& os) {
  json.object().key("id").value(++sentences).key("nodes").array();

  for (size_t i = 1; i < s.words.size(); i++) {
    json.object().key("id").value(i).key("form").value(s.words[i].form);

    size_t start, end;
    if (s.words[i].get_token_range(start, end))
      json.key("start").value(start).key("end").value(end);
    if (s.words[i].head == 0)
      json.key("top").value_true();

    json.key("properties").object()
        .key("lemma").value(s.words[i].lemma)
        .key("upos").value(s.words[i].upostag)
        .key("xpos").value(s.words[i].xpostag);
    split(s.words[i].feats, '|', feats);
    for (auto&& feat : feats) {
      string_piece key(feat.str, 0);
      while (key.len < feat.len && key.str[key.len] != '=')
        key.len++;
      if (key.len + 1 < feat.len)
        json.key(key).value(string_piece(key.str + key.len + 1, feat.len - key.len - 1));
    }
    json.close();

    if (!s.words[i].children.empty()) {
      json.key("edges").array();
      for (auto&& child : s.words[i].children)
        json.object().key("label").value(s.words[child].deprel).key("target").value(child).close();
      json.close();
    }

    json.close();
  }
  json.close().close();

  string_piece current = json.current();
  os.write(current.str, current.len).put('\n');
  json.clear();
}

void output_format_epe::finish_document(ostream& /*os*/) {
  sentences = 0;
}

// Matxin output format
class output_format_matxin : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  void write_node(const sentence& s, int node, string& pad, ostream& os);

  int sentences = 0;
};

void output_format_matxin::write_sentence(const sentence& s, ostream& os) {
  if (!sentences) {
    os << "<corpus>";
  }
  os << "\n<SENTENCE ord=\"" << ++sentences << "\" alloc=\"" << 0 << "\">\n";

  string pad;
  for (auto&& node : s.words[0].children)
    write_node(s, node, pad, os);

  os << "</SENTENCE>" << endl;
}

void output_format_matxin::finish_document(ostream& os) {
  os << "</corpus>\n";

  sentences = 0;
}

void output_format_matxin::write_node(const sentence& s, int node, string& pad, ostream& os) {
  // <NODE ord="%d" alloc="%d" form="%s" lem="%s" mi="%s" si="%s">
  pad.push_back(' ');

  os << pad << "<NODE ord=\"" << node << "\" alloc=\"" << 0
     << "\" form=\"" << xml_encoded(s.words[node].form, true)
     << "\" lem=\"" << xml_encoded(s.words[node].lemma, true)
     << "\" mi=\"" << xml_encoded(s.words[node].feats, true)
     << "\" si=\"" << xml_encoded(s.words[node].deprel, true) << '"';

  if (s.words[node].children.empty()) {
    os << "/>\n";
  } else {
    os << ">\n";
    for (auto&& child : s.words[node].children)
      write_node(s, child, pad, os);
    os << pad << "</NODE>\n";
  }

  pad.pop_back();
}

// Horizontal output format
class output_format_horizontal : public output_format {
 public:
  output_format_horizontal(bool paragraphs) : paragraphs(paragraphs), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }

 private:
  bool paragraphs;
  bool empty;
};

void output_format_horizontal::write_sentence(const sentence& s, ostream& os) {
  if (paragraphs && !empty && (s.get_new_doc() || s.get_new_par()))
    os << '\n';
  empty = false;

  string line;
  for (size_t i = 1; i < s.words.size(); i++) {
    // Append word, but replace spaces by &nbsp;s
    for (auto&& chr : s.words[i].form)
      if (chr == ' ')
        line.append("\302\240");
      else
        line.push_back(chr);

    if (i+1 < s.words.size())
      line.push_back(' ');
  }
  os << line << endl;
}

// Plaintext output format
class output_format_plaintext : public output_format {
 public:
  output_format_plaintext(bool normalized): normalized(normalized), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }
 private:
  bool normalized;
  bool empty;
};

void output_format_plaintext::write_sentence(const sentence& s, ostream& os) {
  if (normalized) {
    if (!empty && (s.get_new_doc() || s.get_new_par()))
      os << '\n';
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      os << tok.form;
      if (i+1 < s.words.size() && tok.get_space_after())
        os << ' ';
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << endl;
  } else {
    string spaces;
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      tok.get_spaces_before(spaces); os << spaces;
      tok.get_spaces_in_token(spaces); os << (!spaces.empty() ? spaces : tok.form);
      tok.get_spaces_after(spaces); os << spaces;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << flush;
  }
  empty = false;
}

// Vertical output format
class output_format_vertical : public output_format {
 public:
  output_format_vertical(bool paragraphs) : paragraphs(paragraphs), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }

 private:
  bool paragraphs;
  bool empty;
};

void output_format_vertical::write_sentence(const sentence& s, ostream& os) {
  if (paragraphs && !empty && (s.get_new_doc() || s.get_new_par()))
    os << '\n';
  empty = false;

  for (size_t i = 1; i < s.words.size(); i++)
    os << s.words[i].form << '\n';
  os << endl;
}

// Static factory methods
output_format* output_format::new_conllu_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  unsigned version = 2;
  if (parsed_options.count(CONLLU_V1))
    version = 1;
  if (parsed_options.count(CONLLU_V2))
    version = 2;

  return new output_format_conllu(version);
}

output_format* output_format::new_epe_output_format(const string& /*options*/) {
  return new output_format_epe();
}

output_format* output_format::new_matxin_output_format(const string& /*options*/) {
  return new output_format_matxin();
}

output_format* output_format::new_horizontal_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_horizontal(parsed_options.count(HORIZONTAL_PARAGRAPHS));
}

output_format* output_format::new_plaintext_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_plaintext(parsed_options.count(PLAINTEXT_NORMALIZED_SPACES));
}

output_format* output_format::new_vertical_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_vertical(parsed_options.count(VERTICAL_PARAGRAPHS));
}

output_format* output_format::new_output_format(const string& name) {
  size_t equal = name.find('=');
  size_t name_len = equal != string::npos ? equal : name.size();
  size_t option_offset = equal != string::npos ? equal + 1 : name.size();

  if (name.compare(0, name_len, "conllu") == 0) return new_conllu_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "epe") == 0) return new_epe_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "matxin") == 0) return new_matxin_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "horizontal") == 0) return new_horizontal_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "plaintext") == 0) return new_plaintext_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "vertical") == 0) return new_vertical_output_format(name.substr(option_offset));
  return nullptr;
}

/////////
// File: sentence/sentence.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string sentence::root_form = "<root>";

sentence::sentence() {
  clear();
}

bool sentence::empty() {
  return words.size() == 1;
}

void sentence::clear() {
  words.clear();
  multiword_tokens.clear();
  empty_nodes.clear();
  comments.clear();

  word& root = add_word(root_form);
  root.lemma = root.upostag = root.xpostag = root.feats = root_form;
}

word& sentence::add_word(string_piece form) {
  words.emplace_back(words.size(), form);
  return words.back();
}

void sentence::set_head(int id, int head, const string& deprel) {
  assert(id >= 0 && id < int(words.size()));
  assert(head < int(words.size()));

  // Remove existing head
  if (words[id].head >= 0) {
    auto& children = words[words[id].head].children;
    for (size_t i = children.size(); i && children[i-1] >= id; i--)
      if (children[i-1] == id) {
        children.erase(children.begin() + i - 1);
        break;
      }
  }

  // Set new head
  words[id].head = head;
  words[id].deprel = deprel;
  if (head >= 0) {
    auto& children = words[head].children;
    size_t i = children.size();
    while (i && children[i-1] > id) i--;
    if (!i || children[i-1] < id) children.insert(children.begin() + i, id);
  }
}

void sentence::unlink_all_words() {
  for (auto&& word : words) {
    word.head = -1;
    word.deprel.clear();
    word.children.clear();
  }
}

bool sentence::get_new_doc(string* id) const {
  if (get_comment("newdoc id", id))
    return true;
  return get_comment("newdoc", id);
}

void sentence::set_new_doc(bool new_doc, string_piece id) {
  remove_comment("newdoc");
  remove_comment("newdoc id");

  if (new_doc && id.len)
    set_comment("newdoc id", id);
  else if (new_doc)
    set_comment("newdoc");
}

bool sentence::get_new_par(string* id) const {
  if (get_comment("newpar id", id))
    return true;
  return get_comment("newpar", id);
}

void sentence::set_new_par(bool new_par, string_piece id) {
  remove_comment("newpar");
  remove_comment("newpar id");

  if (new_par && id.len)
    set_comment("newpar id", id);
  else if (new_par)
    set_comment("newpar");
}

bool sentence::get_sent_id(string& id) const {
  id.clear();

  return get_comment("sent_id", &id);
}

void sentence::set_sent_id(string_piece id) {
  remove_comment("sent_id");

  if (id.len)
    set_comment("sent_id", id);
}

bool sentence::get_text(string& text) const {
  text.clear();

  return get_comment("text", &text);
}

void sentence::set_text(string_piece text) {
  remove_comment("text");

  if (text.len)
    set_comment("text", text);
}

bool sentence::get_comment(string_piece name, string* value) const {
  for (auto&& comment : comments)
    if (comment[0] == '#') {
      // Skip spaces
      unsigned j = 1;
      while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;

      // Try matching the name
      if (j + name.len <= comment.size() && comment.compare(j, name.len, name.str, name.len) == 0) {
        j += name.len;
        while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;
        if (j < comment.size() && comment[j] == '=') {
          //We have a value
          j++;
          while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;
          if (value) value->assign(comment, j, comment.size() - j);
        } else {
          // No value
          if (value) value->clear();
        }

        return true;
      }
    }

  return false;
}

void sentence::remove_comment(string_piece name) {
  for (unsigned i = comments.size(); i--; )
    if (comments[i][0] == '#') {
      // Skip spaces
      unsigned j = 1;
      while (j < comments[i].size() && (comments[i][j] == ' ' || comments[i][j] == '\t')) j++;

      // Remove matching comments
      if (j + name.len <= comments[i].size() && comments[i].compare(j, name.len, name.str, name.len) == 0)
        comments.erase(comments.begin() + i);
    }
}

void sentence::set_comment(string_piece name, string_piece value) {
  remove_comment(name);

  string comment;
  comment.append("# ").append(name.str, name.len);
  if (value.len) {
    comment.append(" = ");
    for (size_t i = 0; i < value.len; i++)
      comment.push_back(value.str[i] == '\r' || value.str[i] == '\n' ? ' ' : value.str[i]);
  }
  comments.push_back(move(comment));
}

/////////
// File: sentence/token.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2017 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

token::token(string_piece form, string_piece misc) {
  if (form.len) this->form.assign(form.str, form.len);
  if (misc.len) this->misc.assign(misc.str, misc.len);
}

// CoNLL-U defined SpaceAfter=No feature
bool token::get_space_after() const {
  string_piece value;

  return !(get_misc_field("SpaceAfter", value) && value.len == 2 && memcmp(value.str, "No", 2) == 0);
}

void token::set_space_after(bool space_after) {
  if (space_after)
    remove_misc_field("SpaceAfter");
  else
    start_misc_field("SpaceAfter").append("No");
}

// UDPipe-specific all-spaces-preserving SpacesBefore and SpacesAfter features
void token::get_spaces_before(string& spaces_before) const {
  string_piece value;

  if (get_misc_field("SpacesBefore", value))
    unescape_spaces(value, spaces_before);
  else
    spaces_before.clear();
}

void token::set_spaces_before(string_piece spaces_before) {
  if (spaces_before.len == 0)
    remove_misc_field("SpacesBefore");
  else
    append_escaped_spaces(spaces_before, start_misc_field("SpacesBefore"));
}

void token::get_spaces_after(string& spaces_after) const {
  string_piece value;

  if (get_misc_field("SpacesAfter", value))
    unescape_spaces(value, spaces_after);
  else
    spaces_after.assign(get_space_after() ? " " : "");
}

void token::set_spaces_after(string_piece spaces_after) {
  if (spaces_after.len == 0) {
    set_space_after(false);
    remove_misc_field("SpacesAfter");
  } else if (spaces_after.len == 1 && spaces_after.str[0] == ' ') {
    set_space_after(true);
    remove_misc_field("SpacesAfter");
  } else {
    set_space_after(true);
    append_escaped_spaces(spaces_after, start_misc_field("SpacesAfter"));
  }
}

void token::get_spaces_in_token(string& spaces_in_token) const {
  string_piece value;

  if (get_misc_field("SpacesInToken", value))
    unescape_spaces(value, spaces_in_token);
  else
    spaces_in_token.clear();
}

void token::set_spaces_in_token(string_piece spaces_in_token) {
  if (spaces_in_token.len == 0)
    remove_misc_field("SpacesInToken");
  else
    append_escaped_spaces(spaces_in_token, start_misc_field("SpacesInToken"));
}

// UDPipe-specific TokenRange feature
bool token::get_token_range(size_t& start, size_t& end) const {
  string_piece value;

  if (!get_misc_field("TokenRange", value)) return false;

  start = 0;
  while (value.len && value.str[0] >= '0' && value.str[0] <= '9') {
    if (start > (numeric_limits<size_t>::max() - (value.str[0] - '0')) / 10)
      return false;
    start = 10 * start + (value.str[0] - '0');
    value.str++, value.len--;
  }

  if (value.len == 0 || value.str[0] != ':') return false;
  value.str++, value.len--;

  end = 0;
  while (value.len && value.str[0] >= '0' && value.str[0] <= '9') {
    if (end > (numeric_limits<size_t>::max() - (value.str[0] - '0')) / 10)
      return false;
    end = 10 * end + (value.str[0] - '0');
    value.str++, value.len--;
  }

  return true;
}

void token::set_token_range(size_t start, size_t end) {
  if (start == size_t(string::npos))
    remove_misc_field("TokenRange");
  else
    start_misc_field("TokenRange").append(to_string(start)).append(1, ':').append(to_string(end));
}

// Private MISC field helpers
bool token::get_misc_field(string_piece name, string_piece& value) const {
  for (size_t index = 0; index < misc.size(); ) {
    if (misc.compare(index, name.len, name.str, name.len) == 0 && misc[index + name.len] == '=') {
      index += name.len + 1;
      value.str = misc.c_str() + index;
      value.len = misc.find('|', index);
      value.len = (value.len == size_t(string::npos) ? misc.size() : value.len) - index;
      return true;
    }
    index = misc.find('|', index);
    if (index != size_t(string::npos)) index++;
  }
  return false;
}

void token::remove_misc_field(string_piece name) {
  for (size_t index = 0; index < misc.size(); )
    if (misc.compare(index, name.len, name.str, name.len) == 0 && misc[index + name.len] == '=') {
      size_t end_index = misc.find('|', index + name.len + 1);
      if (end_index == size_t(string::npos)) end_index = misc.size();

      // Be careful to delete at most one neighboring '|'
      if (index)
        misc.erase(index - 1, end_index - (index - 1));
      else
        misc.erase(index, end_index + (end_index < misc.size() ? 1 : 0) - index);
    } else {
      index = misc.find('|', index);
      if (index != size_t(string::npos)) index++;
    }
}

string& token::start_misc_field(string_piece name) {
  remove_misc_field(name);
  if (!misc.empty()) misc.push_back('|');
  misc.append(name.str, name.len).push_back('=');
  return misc;
}

void token::append_escaped_spaces(string_piece spaces, string& escaped_spaces) const {
  for (unsigned i = 0; i < spaces.len; i++)
    switch (spaces.str[i]) {
      case ' ':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('s'); break;
      case '|':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('p'); break;
      case '\t':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('t'); break;
      case '\r':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('r'); break;
      case '\n':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('n'); break;
      case '\\':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('\\'); break;
      default:
        escaped_spaces.push_back(spaces.str[i]);
    }
}

void token::unescape_spaces(string_piece escaped_spaces, string& spaces) const {
  spaces.clear();

  for (unsigned i = 0; i < escaped_spaces.len; i++)
    if (escaped_spaces.str[i] != '\\' || i+1 >= escaped_spaces.len)
      spaces.push_back(escaped_spaces.str[i]);
    else switch (escaped_spaces.str[++i]) {
      case 's':
        spaces.push_back(' '); break;
      case 'p':
        spaces.push_back('|'); break;
      case 't':
        spaces.push_back('\t'); break;
      case 'r':
        spaces.push_back('\r'); break;
      case 'n':
        spaces.push_back('\n'); break;
      case '\\':
        spaces.push_back('\\'); break;
      default:
        spaces.push_back(escaped_spaces.str[i - 1]);
        spaces.push_back(escaped_spaces.str[i]);
    }
}

/////////
// File: tokenizer/detokenizer.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class detokenizer {
 public:
  detokenizer(const string& plain_text);

  void detokenize(sentence& s) const;
 private:
  enum { LOWERCASE, CATEGORIZE, TOTAL };

  int difference(const string& left, const string& right, bool separate, int mode) const;

  static string perform_lowercase(const string& input);
  static string perform_categorize(const string& input);
  bool has_letters(const string& word) const;
  bool only_digits(const string& word) const;

  class suffix_array {
   public:
    suffix_array(const string& str);
    suffix_array(suffix_array&& other) = default;

    unsigned count(const string& data) const;

   private:
    vector<unsigned> sa;

    struct suffix_compare {
      suffix_compare(const string& str) : str(str) {}
      bool operator()(unsigned a, unsigned b) const { return str.compare(a, string::npos, str, b, string::npos) < 0; }
     private:
      const string& str;
    } suffix_comparator;

    struct suffix_lower_find {
      suffix_lower_find(const string& str) : str(str) {}
      bool operator()(unsigned a, const string& data) const { return str.compare(a, data.size(), data) < 0; }

     private:
      const string& str;
    } suffix_lower_finder;

    struct suffix_upper_find {
      suffix_upper_find(const string& str) : str(str) {}
      bool operator()(const string& data, unsigned a) const { return str.compare(a, data.size(), data) > 0; }

     private:
      const string& str;
    } suffix_upper_finder;
  };

  string data_lowercased, data_categorized;
  suffix_array sa_lowercased, sa_categorized;
};

/////////
// File: tokenizer/detokenizer.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

detokenizer::detokenizer(const string& plain_text)
    : data_lowercased(perform_lowercase(plain_text)), data_categorized(perform_categorize(plain_text)),
    sa_lowercased(data_lowercased), sa_categorized(data_categorized) {}

void detokenizer::detokenize(sentence& s) const {
  token* previous_tok = nullptr;
  for (size_t i = 1, j = 0; i < s.words.size(); i++) {
    token* tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (token*)&s.multiword_tokens[j] : (token*)&s.words[i];

    if (previous_tok) {
      // Should we add SpaceAfter=No to the previous form?
      int score = difference(previous_tok->form, tok->form, true, LOWERCASE);
      if (!score) score = has_letters(previous_tok->form) && has_letters(tok->form) ? -1 : 0;
      if (!score) score = only_digits(previous_tok->form) && only_digits(tok->form) ? -1 : 0;
      if (!score) score = difference(previous_tok->form, tok->form, false, LOWERCASE);
      if (!score) score = difference(previous_tok->form, tok->form, false, CATEGORIZE);
      if (!score) score = difference(previous_tok->form, tok->form, true, CATEGORIZE);

      if (score > 0)
        previous_tok->set_space_after(false);
    }

    // Remove the SpaceAfter attribute on current token
    tok->set_space_after(true);
    previous_tok = tok;

    if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
      i = s.multiword_tokens[j++].id_last;
  }
}

int detokenizer::difference(const string& left, const string& right, bool separate, int mode) const {
  auto& func = mode == LOWERCASE ? perform_lowercase : perform_categorize;
  auto& sa = mode == LOWERCASE ? sa_lowercased : sa_categorized;

  string left_mapped = func(left);
  string right_mapped = func(right);
  string pattern;

  pattern.assign(separate?" ":"").append(left_mapped).append(right_mapped).append(separate?" ":"");
  int together = sa.count(pattern);

  pattern.assign(separate?" ":"").append(left_mapped).append(" ").append(right_mapped).append(separate?" ":"");
  int apart = sa.count(pattern);

  return together - apart;
}

string detokenizer::perform_lowercase(const string& input) {
  using namespace unilib;

  string output;
  for (auto&& chr : utf8::decoder(input))
    utf8::append(output, unicode::lowercase(chr));
  return output;
}

string detokenizer::perform_categorize(const string& input) {
  using namespace unilib;

  string output;
  for (auto&& chr : utf8::decoder(input)) {
    auto category = unicode::category(chr);
    if (category & unicode::C) output.push_back('C');
    if (category & unicode::L) output.push_back('L');
    if (category & unicode::M) output.push_back('M');
    if (category & unicode::N) output.push_back('N');
    if (category & unicode::Pc) output.push_back('c');
    if (category & unicode::Pd) output.push_back('d');
    if (category & unicode::Pe) output.push_back('e');
    if (category & unicode::Pf) output.push_back('f');
    if (category & unicode::Pi) output.push_back('i');
    if (category & unicode::Po) output.push_back('o');
    if (category & unicode::Ps) output.push_back('s');
    if (category & unicode::S) output.push_back('S');
    if (category & unicode::Zl) output.push_back('Z');
    if (category & unicode::Zp) output.push_back('z');
    if (category & unicode::Zs) output.push_back(' ');
  }
  return output;
}

bool detokenizer::has_letters(const string& word) const {
  using namespace unilib;

  for (auto&& chr : utf8::decoder(word))
    if (unicode::category(chr) & unicode::L)
      return true;
  return false;
}

bool detokenizer::only_digits(const string& word) const {
  using namespace unilib;

  for (auto&& chr : utf8::decoder(word))
    if (unicode::category(chr) & ~unicode::N)
      return false;
  return true;
}

detokenizer::suffix_array::suffix_array(const string& str) : suffix_comparator(str), suffix_lower_finder(str), suffix_upper_finder(str) {
  sa.reserve(str.size());
  for (unsigned i = 0; i < str.size(); i++)
    sa.push_back(i);

  sort(sa.begin(), sa.end(), suffix_comparator);
}

unsigned detokenizer::suffix_array::count(const string& data) const {
  auto lower_it = lower_bound(sa.begin(), sa.end(), data, suffix_lower_finder);
  auto upper_it = upper_bound(sa.begin(), sa.end(), data, suffix_upper_finder);
  return upper_it - lower_it;
}

/////////
// File: tokenizer/morphodita_tokenizer_wrapper.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

morphodita_tokenizer_wrapper::morphodita_tokenizer_wrapper(morphodita::tokenizer* tokenizer, const multiword_splitter* splitter,
                                                           bool normalized_spaces, bool token_ranges)
  : tokenizer(tokenizer), splitter(splitter), normalized_spaces(normalized_spaces), token_ranges(token_ranges) {}

bool morphodita_tokenizer_wrapper::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void morphodita_tokenizer_wrapper::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
  unicode_offset = 0;
  text_unicode_length = 0;
  saved_spaces.clear();
}

void morphodita_tokenizer_wrapper::set_text(string_piece text, bool make_copy) {
  // Start by skipping spaces and copying them to saved_spaces
  string_piece following;
  for (char32_t chr;
       text.len && (following = text, chr = unilib::utf8::decode(following.str, following.len),
                    (unilib::unicode::category(chr) & unilib::unicode::Zs) || chr == '\r' || chr == '\n' || chr == '\t');
       text = following, unicode_offset++)
    saved_spaces.append(text.str, following.str - text.str);

  // Offset unicode_offset by length of previous text, update text_unicode_length for the new text
  unicode_offset += text_unicode_length;
  text_unicode_length = 0;
  for (following = text; following.len; unilib::utf8::decode(following.str, following.len))
    text_unicode_length++;

  // Copy the text to local storage if needed
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }

  // Store the text locally and in the morphodita::tokenizer
  this->text = text;
  tokenizer->set_text(this->text, false);

}

bool morphodita_tokenizer_wrapper::next_sentence(sentence& s, string& error) {
  unsigned following_newlines = 0;

  s.clear();
  error.clear();

  if (tokenizer->next_sentence(&forms, token_ranges ? &tokens : nullptr)) {
    // The forms returned by GRU tokenizer *should not* start/end with spaces,
    // but we trim them anyway (including all "remove empty forms/sentences" machinery).
    for (size_t i = 0; i < forms.size(); i++) {
      while (forms[i].len && (forms[i].str[0] == '\r' || forms[i].str[0] == '\n' ||
                              forms[i].str[0] == '\t' || forms[i].str[0] == ' '))
        forms[i].str++, forms[i].len--;
      while (forms[i].len && (forms[i].str[forms[i].len-1] == '\r' || forms[i].str[forms[i].len-1] == '\n' ||
                              forms[i].str[forms[i].len-1] == '\t' || forms[i].str[forms[i].len-1] == ' '))
        forms[i].len--;
      if (!forms[i].len)
        forms.erase(forms.begin() + i--);
    }
    if (!forms.size()) return next_sentence(s, error);

    for (size_t i = 0; i < forms.size(); i++) {
      // The form might contain spaces, even '\r', '\n' or '\t',
      // which we change to space. We also normalize multiple spaces to one.
      tok.form.clear();
      for (size_t j = 0; j < forms[i].len; j++) {
        char chr = forms[i].str[j];
        if (chr == '\r' || chr == '\n' || chr == '\t') chr = ' ';
        if (chr != ' ' || tok.form.empty() || tok.form.back() != ' ')
          tok.form.push_back(chr);
      }

      // Track pre-sentence spaces and store SpacesBefore
      if (i == 0) {
        if (forms[0].str > text.str)
          saved_spaces.append(text.str, forms[0].str - text.str);
        preceeding_newlines += count(saved_spaces.begin(), saved_spaces.end(), '\n');
      }
      if (!normalized_spaces) {
        tok.set_spaces_before(i == 0 ? saved_spaces : "");
      }
      saved_spaces.clear();

      // Track post-sentence spaces and store SpaceAfter, SpacesInToken and SpacesAfter
      if (i+1 == forms.size()) {
        text.len -= forms[i].str + forms[i].len - text.str;
        text.str = forms[i].str + forms[i].len;

        string_piece following;
        for (char32_t chr; text.len && (following = text, chr = unilib::utf8::decode(following.str, following.len),
                                        (unilib::unicode::category(chr) & unilib::unicode::Zs) || chr == '\r' || chr == '\n' || chr == '\t'); text = following)
          saved_spaces.append(text.str, following.str - text.str);

        following_newlines += count(saved_spaces.begin(), saved_spaces.end(), '\n');
      }
      if (normalized_spaces) {
        tok.set_space_after(!(i+1 < forms.size() && forms[i+1].str == forms[i].str + forms[i].len));
      } else {
        tok.set_spaces_in_token(tok.form.size() != forms[i].len ? forms[i] : "");
        tok.set_spaces_after(i+1 == forms.size() ? saved_spaces : string_piece(forms[i].str + forms[i].len, forms[i+1].str - forms[i].str - forms[i].len));
      }
      saved_spaces.clear();

      // Store TokenRange if requested
      if (token_ranges)
        tok.set_token_range(unicode_offset + tokens[i].start, unicode_offset + tokens[i].start + tokens[i].length);

      if (splitter)
        splitter->append_token(tok.form, tok.misc, s);
      else
        s.add_word(tok.form).misc.assign(tok.misc);
    }

    // Mark new document if needed
    if (new_document) {
      s.set_new_doc(true, document_id);
      new_document = false;
    }

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = following_newlines;

    s.set_sent_id(to_string(sentence_id++));

    // Fill "# text" comment
    s.comments.emplace_back("# text = ");
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j].form : (const token&)s.words[i].form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;

      s.comments.back().append(tok.form);
      if (i+1 < s.words.size() && tok.get_space_after()) s.comments.back().push_back(' ');
    }

    return true;
  }

  // Save unused text parts.
  if (text.len) {
    saved_spaces.append(text.str, text.len);
    text.str += text.len;
    text.len = 0;
  }

  return false;
}

/////////
// File: tokenizer/multiword_splitter.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

void multiword_splitter::append_token(string_piece token, string_piece misc, sentence& s) const {
  using namespace unilib;

  // Buffer
  s.add_word();
  string& buffer = s.words.back().form;

  // Lowercase the token
  utf8::map(unicode::lowercase, token.str, token.len, buffer);
  reverse(buffer.begin(), buffer.end());

  // Try finding lowercased version in the full_rules
  size_t prefix_len = 0;
  auto it = full_rules.find(buffer);

  if (it == full_rules.end()) {
    if (version >= 2) {
      string& suffix = s.words.back().misc;
      // Try searching suffix_rules if needed
      while (suffix.size() + 1 < buffer.size()) {
        suffix.push_back(buffer[suffix.size()]);

        auto suffix_it = suffix_rules.find(suffix);
        if (suffix_it == suffix_rules.end())
          break;

        if (!suffix_it->second.words.empty()) {
          it = suffix_it;
          prefix_len = buffer.size() - suffix.size();
        }
      }
      suffix.clear();
    }

    if (!prefix_len) {
      // No match
      s.words.back().form.assign(token.str, token.len);
      if (misc.len) s.words.back().misc.assign(misc.str, misc.len);
      return;
    }
  }

  // Determine casing
  enum { UC_FIRST, UC_ALL, OTHER } casing = OTHER;

  if (unicode::category(utf8::first(token.str, token.len)) & unicode::Lut) {
    casing = UC_ALL;
    for (auto&& chr : utf8::decoder(token.str, token.len))
      if (unicode::category(chr) & (unicode::L & ~unicode::Lut)) { casing = UC_FIRST; break; }
  }

  // Fill the multiword token
  s.multiword_tokens.emplace_back(s.words.back().id, s.words.back().id + it->second.words.size() - 1, token, misc);

  s.words.back().form.clear();
  if (prefix_len) {
    // Note that prefix_len is measured in byte length of lowercased characters
    string_piece suffix(token);
    while (s.words.back().form.size() < prefix_len && suffix.len)
      utf8::append(s.words.back().form, unicode::lowercase(utf8::decode(suffix.str, suffix.len)));
    s.words.back().form.assign(token.str, token.len - suffix.len);
  }
  for (auto&& chr : utf8::decoder(it->second.words[0]))
    utf8::append(s.words.back().form, casing == UC_ALL || (casing == UC_FIRST && s.words.back().form.empty()) ? unicode::uppercase(chr) : chr);

  for (size_t i = 1; i < it->second.words.size(); i++)
    if (casing != UC_ALL) {
      s.add_word(it->second.words[i]);
    } else {
      s.add_word();
      utf8::map(unicode::uppercase, it->second.words[i], s.words.back().form);
    }
}

multiword_splitter* multiword_splitter::load(istream& is) {
  char version;
  if (!is.get(version)) return nullptr;
  if (!(version >= 1 && version <= VERSION_LATEST)) return nullptr;

  binary_decoder data;
  if (!compressor::load(is, data)) return nullptr;

  unique_ptr<multiword_splitter> splitter(new multiword_splitter(version));
  try {
    for (unsigned full_rules = data.next_4B(); full_rules; full_rules--) {
      string full_rule;
      data.next_str(full_rule);
      reverse(full_rule.begin(), full_rule.end());

      // Add the full_rule and its words
      auto& info = splitter->full_rules[full_rule];
      for (unsigned words = data.next_1B(); words; words--) {
        info.words.emplace_back();
        data.next_str(info.words.back());
      }
      if (info.words.empty()) return nullptr;
    }

    if (version >= 2)
      for (unsigned suffix_rules = data.next_4B(); suffix_rules; suffix_rules--) {
        string suffix_rule;
        data.next_str(suffix_rule);
        reverse(suffix_rule.begin(), suffix_rule.end());

        // Add the suffix_rule and its words
        auto& info = splitter->suffix_rules[suffix_rule];
        for (unsigned words = data.next_1B(); words; words--) {
          info.words.emplace_back();
          data.next_str(info.words.back());
        }
        if (info.words.empty()) return nullptr;

        // Add prefixes of the suffix with empty data
        if (!suffix_rule.empty())
          for (suffix_rule.pop_back(); !suffix_rule.empty(); suffix_rule.pop_back())
            splitter->suffix_rules[suffix_rule];
      }
  } catch (binary_decoder_error&) {
    return nullptr;
  }

  return data.is_end() ? splitter.release() : nullptr;
}

/////////
// File: tokenizer/multiword_splitter_trainer.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_splitter_trainer {
 public:
  static bool train(const vector<sentence>& data, ostream& os, string& error);
};

/////////
// File: tokenizer/multiword_splitter_trainer.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

bool multiword_splitter_trainer::train(const vector<sentence>& data, ostream& os, string& error) {
  using namespace unilib;
  error.clear();

  // Train
  struct rule_info {
    vector<string> words;
    unsigned count = 0;
  };
  map<string, rule_info> full_rules, suffix_rules;

  // Full rules
  string lc_form;
  vector<string> lc_words;
  for (auto&& sentence : data)
    for (auto&& multiword : sentence.multiword_tokens) {
      utf8::map(unicode::lowercase, multiword.form, lc_form);
      lc_words.clear();
      for (int i = multiword.id_first; i <= multiword.id_last; i++)
        utf8::map(unicode::lowercase, sentence.words[i].form, (lc_words.emplace_back(), lc_words.back()));

      auto& info = full_rules[lc_form];
      if (info.words.empty())
        info.words.assign(lc_words.begin(), lc_words.end());
      info.count += lc_words == info.words;
      if (!info.count) full_rules.erase(lc_form);
    }

  // Remove the full rules which trigger too negatively
  for (auto&& sentence : data)
    for (size_t i = 1, j = 0; i < sentence.words.size(); i++) {
      if (j < sentence.multiword_tokens.size() && sentence.multiword_tokens[j].id_first == int(i)) {
        i = sentence.multiword_tokens[j++].id_last;
        continue;
      }

      utf8::map(unicode::lowercase, sentence.words[i].form, lc_form);
      auto it = full_rules.find(lc_form);
      if (it != full_rules.end())
        if (!--it->second.count)
          full_rules.erase(it);
    }

  // Suffix rules
  for (auto&& full_rule : full_rules) {
    size_t prefix_match = 0;
    while (prefix_match < full_rule.first.size() && prefix_match < full_rule.second.words[0].size()) prefix_match++;
    for (; prefix_match; prefix_match--)
      if (((unsigned char)full_rule.first[prefix_match]) < 0x80 || ((unsigned char)full_rule.first[prefix_match]) >= 0xC0) {
        lc_form.assign(full_rule.first, prefix_match, string::npos);
        lc_words.assign(full_rule.second.words.begin(), full_rule.second.words.end());
        lc_words[0].erase(0, prefix_match);

        auto& info = suffix_rules[lc_form];
        if (info.words.empty())
          info.words.assign(lc_words.begin(), lc_words.end());
        info.count += lc_words == info.words;
        if (!info.count) suffix_rules.erase(lc_form);
      }
  }

  // Remove the suffix rules which trigger too negatively
  for (auto&& sentence : data)
    for (size_t i = 1, j = 0; i < sentence.words.size(); i++) {
      if (j < sentence.multiword_tokens.size() && sentence.multiword_tokens[j].id_first == int(i)) {
        i = sentence.multiword_tokens[j++].id_last;
        continue;
      }

      utf8::map(unicode::lowercase, sentence.words[i].form, lc_form);
      while (lc_form.size() > 1) {
        lc_form.erase(0, 1);
        auto it = suffix_rules.find(lc_form);
        if (it != suffix_rules.end()) {
          if (it->second.count <= 10)
            suffix_rules.erase(it);
          else
            it->second.count -= 10;
        }
      }
    }

  // Encode
  binary_encoder enc;
  enc.add_4B(full_rules.size());
  for (auto&& full_rule : full_rules) {
    enc.add_str(full_rule.first);
    enc.add_1B(full_rule.second.words.size());
    for (auto& word : full_rule.second.words)
      enc.add_str(word);
  }
  enc.add_4B(suffix_rules.size());
  for (auto&& suffix_rule : suffix_rules) {
    enc.add_str(suffix_rule.first);
    enc.add_1B(suffix_rule.second.words.size());
    for (auto& word : suffix_rule.second.words)
      enc.add_str(word);
  }

  // Save
  os.put(multiword_splitter::VERSION_LATEST);
  if (!compressor::save(os, enc)) return error.assign("Cannot encode multiword_splitter!"), false;

  return true;
}

/////////
// File: trainer/trainer.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class trainer {
 public:
  static bool train(const string& method, const vector<sentence>& train, const vector<sentence>& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error);

  static const string DEFAULT;
  static const string NONE;

 protected:
  static unsigned hyperparameter_integer(unsigned run, unsigned index, unsigned minimum, unsigned maximum);
  static double hyperparameter_uniform(unsigned run, unsigned index, double minimum, double maximum);
  static double hyperparameter_logarithmic(unsigned run, unsigned index, double minimum, double maximum);

 private:
  static double rnd(unsigned run, unsigned index);
};

/////////
// File: trainer/trainer_morphodita_parsito.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class trainer_morphodita_parsito : public trainer {
 public:
  static bool train(const vector<sentence>& training, const vector<sentence>& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error);

 private:
  static bool train_tokenizer(const vector<sentence>& training, const vector<sentence>& heldout,
                              const string& options, ostream& os, string& error);
  static bool train_tagger(const vector<sentence>& training, const vector<sentence>& heldout,
                           const string& options, ostream& os, string& error);
  static bool train_parser(const vector<sentence>& training, const vector<sentence>& heldout,
                           const string& options, const string& tagger_model, ostream& os, string& error);

  // Generic model methods
  enum model_type { TOKENIZER_MODEL, TAGGER_MODEL, PARSER_MODEL };
  static bool load_model(const string& data, model_type model, string_piece& range);
  static const string& model_normalize_form(string_piece form, string& output);
  static const string& model_normalize_lemma(string_piece lemma, string& output);
  static void model_fill_word_analysis(const morphodita::tagged_lemma& analysis, bool upostag, int lemma, bool xpostag, bool feats, word& word);

  // Tagger-specific model methods
  static bool train_tagger_model(const vector<sentence>& training, const vector<sentence>& heldout,
                                 unsigned model, unsigned models, const named_values::map& tagger, ostream& os, string& error);
  static bool can_combine_tag(const word& w, string& error);
  static const string& combine_tag(const word& w, bool xpostag, bool feats, string& combined_tag);
  static const string& most_frequent_tag(const vector<sentence>& data, const string& upostag, bool xpostag, bool feats, string& combined_tag);
  static const string& combine_lemma(const word& w, int use_lemma, string& combined_lemma, const unordered_set<string>& flat_lemmas = unordered_set<string>());

  // Generic options handling
  static const string& option_str(const named_values::map& options, const string& name, int model = -1);
  static bool option_int(const named_values::map& options, const string& name, int& value, string& error, int model = -1);
  static bool option_bool(const named_values::map& options, const string& name, bool& value, string& error, int model = -1);
  static bool option_double(const named_values::map& options, const string& name, double& value, string& error, int model = -1);

  // Various string data
  static const string empty_string;
  static const string tag_separators;
  static const string tagger_features_tagger;
  static const string tagger_features_lemmatizer;
  static const string parser_nodes;
};

/////////
// File: trainer/trainer.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string trainer::DEFAULT;
const string trainer::NONE = "none";

bool trainer::train(const string& method, const vector<sentence>& training, const vector<sentence>& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error) {
  error.clear();

  stringstream os_buffer;
  os_buffer.put(method.size());
  os_buffer.write(method.c_str(), method.size());

  try {
    if (method == "morphodita_parsito") {
      if (!trainer_morphodita_parsito::train(training, heldout, tokenizer, tagger, parser, os_buffer, error))
        return false;
    } else {
      error.assign("Unknown UDPipe method '").append(method).append("'!");
      return false;
    }
  } catch (training_error& e) {
    error.assign(e.what());
    return false;
  }

  os << os_buffer.rdbuf();
  return true;
}

unsigned trainer::hyperparameter_integer(unsigned run, unsigned index, unsigned minimum, unsigned maximum) {
  return minimum + int((maximum - minimum + 1) * rnd(run, index));
}

double trainer::hyperparameter_uniform(unsigned run, unsigned index, double minimum, double maximum) {
  return minimum + (maximum - minimum) * rnd(run, index);
}

double trainer::hyperparameter_logarithmic(unsigned run, unsigned index, double minimum, double maximum) {
  return exp(log(minimum) + (log(maximum) - log(minimum)) * rnd(run, index));
}

double trainer::rnd(unsigned run, unsigned index) {
  uint32_t state = 12345U;
  for (unsigned i = 0; i < 10; i++)
    state = state * 1103515245U + run * 19999999U + index * 1000000007U + 12345U;
  return (state >> 16) / double(1<<16);
}

/////////
// File: morphodita/tagger/elementary_features_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template <class Map>
inline bool elementary_features<Map>::save(ostream& os) {
  binary_encoder enc;

  enc.add_1B(maps.size());
  for (auto&& map : maps)
    map.save(enc);

  return compressor::save(os, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/feature_sequences_encoder.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template <class ElementaryFeatures, class Map>
void feature_sequences<ElementaryFeatures, Map>::parse(int window_size, istream& is) {
  unordered_map<string, elementary_feature_description> elementary_map;
  for (auto&& description : ElementaryFeatures::descriptions)
    if (!elementary_map.emplace(description.name, description).second)
      training_failure("Repeated elementary feature with name " << description.name << '!');

  string line;
  vector<string> tokens;
  while (getline(is, line)) {
    split(line, ',', tokens);
    if (tokens.empty()) training_failure("Feature sequence cannot be empty!");

    bool contains_only_current = false;
    sequences.emplace_back();
    for (auto&& token : tokens) {
      vector<string> parts;
      split(token, ' ', parts);
      if (parts.size() != 2) training_failure("Cannot parse feature sequence element '" << token << "'!");
      auto it = elementary_map.find(parts[0]);
      if (it == elementary_map.end()) training_failure("Unknown elementary feature '" << parts[0] << "' used in feature sequence '" << token << "'!");

      auto& desc = it->second;
      int sequence_index = parse_int(parts[1].c_str(), "sequence_index");
      if (desc.type == DYNAMIC && sequence_index != 0) training_failure("Nonzero sequence index " << sequence_index << " of dynamic elementary feature '" << desc.name << "'!");
      if (desc.type == PER_TAG && (sequence_index > 0 || sequence_index <= -window_size)) training_failure("Wrong sequence index " << sequence_index << " of per-tag elementary feature '" << desc.name << "'!");
      if (desc.range == ONLY_CURRENT && sequence_index != 0) training_failure("Nonzero sequence index " << sequence_index << " of elementary feature '" << desc.name << "' requiring zero offset!");

      sequences.back().elements.emplace_back(it->second.type, it->second.index, sequence_index);
      if (desc.type == DYNAMIC) sequences.back().dependant_range = max(sequences.back().dependant_range, window_size + 1);
      if (desc.type == PER_TAG) sequences.back().dependant_range = max(sequences.back().dependant_range, 1 - sequence_index);
      contains_only_current |= desc.range == ONLY_CURRENT;
    }
    if (contains_only_current && sequences.back().dependant_range > 1) training_failure("Feature sequence '" << line << "' contains both a non-local elementary feature and exclusively-local elementary feature!");
  }

  stable_sort(sequences.begin(), sequences.end(), [](const feature_sequence& a, const feature_sequence& b) { return a.dependant_range > b.dependant_range; });
  scores.resize(sequences.size());
}

template <class ElementaryFeatures, class Map>
inline bool feature_sequences<ElementaryFeatures, Map>::save(ostream& os) {
  if (!elementary.save(os)) return false;

  binary_encoder enc;
  enc.add_1B(sequences.size());
  for (auto&& sequence : sequences) {
    enc.add_4B(sequence.dependant_range);
    enc.add_1B(sequence.elements.size());
    for (auto&& element : sequence.elements) {
      enc.add_4B(element.type);
      enc.add_4B(element.elementary_index);
      enc.add_4B(element.sequence_index);
    }
  }

  enc.add_1B(scores.size());
  for (auto&& score : scores)
    score.save(enc);

  return compressor::save(os, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/training_maps.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class training_elementary_feature_map {
 public:
  inline elementary_feature_value value(const char* feature, int len) const;
  mutable unordered_map<string, elementary_feature_value> map = {{"", elementary_feature_empty}};
 private:
  mutable string key;
};

class training_feature_sequence_map {
 public:
  struct info {
    // We deliberately use feature_sequence*s*_score to check for overflow
    feature_sequences_score alpha = 0;
    feature_sequences_score gamma = 0;
    int last_gamma_update = 0;
  };

  inline feature_sequence_score score(const char* feature, int len) const;
  mutable unordered_map<string, info> map;
 private:
  mutable string key;
};

template <template <class> class ElementaryFeatures> using train_feature_sequences = feature_sequences<ElementaryFeatures<training_elementary_feature_map>, training_feature_sequence_map>;

// Definitions
elementary_feature_value training_elementary_feature_map::value(const char* feature, int len) const {
  key.assign(feature, len);
  return map.emplace(key, elementary_feature_empty + map.size()).first->second;
}

feature_sequence_score training_feature_sequence_map::score(const char* feature, int len) const {
  key.assign(feature, len);
  auto it = map.find(key);
  return it != map.end() ? it->second.alpha : 0;
}

} // namespace morphodita

/////////
// File: morphodita/tagger/feature_sequences_optimizer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class T>
class feature_sequences_optimizer;

template <template <class, class> class FeatureSequences, template <class> class ElementaryFeatures>
class feature_sequences_optimizer<FeatureSequences<ElementaryFeatures<training_elementary_feature_map>, training_feature_sequence_map>> {
 public:
  typedef FeatureSequences<ElementaryFeatures<training_elementary_feature_map>, training_feature_sequence_map> original_feature_sequences;
  typedef FeatureSequences<ElementaryFeatures<persistent_elementary_feature_map>, persistent_feature_sequence_map> optimized_feature_sequences;

  static void optimize(const original_feature_sequences& features, optimized_feature_sequences& optimized_features);
};

// Definitions
template <template <class, class> class FeatureSequences, template <class> class ElementaryFeatures>
void feature_sequences_optimizer<FeatureSequences<ElementaryFeatures<training_elementary_feature_map>, training_feature_sequence_map>>::optimize(const original_feature_sequences& features, optimized_feature_sequences& optimized_features) {
  const ElementaryFeatures<training_elementary_feature_map>& elementary = features.elementary;
  ElementaryFeatures<persistent_elementary_feature_map>& optimized_elementary = optimized_features.elementary;

  // Iterate over feature sequences of non-zero weight and count number of
  // occurences in corresponding elementary feature maps.
  // In order to be able to do so, precompute map_index for elements of features.sequences.
  vector<vector<int>> map_indices(features.sequences.size());
  for (unsigned i = 0; i < map_indices.size(); i++) {
    for (auto&& element : features.sequences[i].elements)
      for (auto&& description : decltype(features.elementary)::descriptions)
        if (element.type == description.type && element.elementary_index == description.index)
          map_indices[i].emplace_back(description.map_index);

    assert(map_indices[i].size() == features.sequences[i].elements.size());
  }

  struct count_info { elementary_feature_value ori = 0; int count = 0; };
  vector<vector<count_info>> counts(elementary.maps.size());
  vector<elementary_feature_value> elementary_ids;
  for (unsigned i = 0; i < features.sequences.size(); i++)
    for (auto&& element : features.scores[i].map)
      if (element.second.gamma) {
        elementary_ids.clear();
        for (const char* key = element.first.c_str(); key != element.first.c_str() + element.first.size(); assert(key <= element.first.c_str() + element.first.size()))
          elementary_ids.emplace_back(vli<elementary_feature_value>::decode(key));

        assert(elementary_ids.size() == features.sequences[i].elements.size());
        for (unsigned j = 0; j < elementary_ids.size(); j++) {
          if (map_indices[i][j] < 0) continue;
          if (elementary_ids[j] >= counts[map_indices[i][j]].size()) counts[map_indices[i][j]].resize(elementary_ids[j] + 1);
          counts[map_indices[i][j]][elementary_ids[j]].count++;
        }
      }

  // Sort counts by sizes decreasing
  for (auto&& count : counts) {
    if (elementary_feature_empty >= count.size()) count.resize(elementary_feature_empty + 1);
    count[elementary_feature_unknown].count = 0;
    count[elementary_feature_empty].count = 1;
    for (elementary_feature_value i = 0; i < count.size(); i++) count[i].ori = i;
    sort(count.begin() + elementary_feature_empty + 1, count.end(), [](const count_info& a, const count_info& b){ return a.count > b.count; });
  }

  // Create an elementary ids map
  vector<vector<elementary_feature_value>> elementary_ids_map(counts.size());
  for (unsigned i = 0; i < counts.size(); i++) {
    elementary_ids_map[i].resize(counts[i].size());
    for (elementary_feature_value j = 0; j < counts[i].size(); j++)
      elementary_ids_map[i][counts[i][j].ori] = counts[i][j].count ? j : elementary_feature_unknown;
  }

  // Make optimized elementary maps by applying elementary ids map
  optimized_elementary.maps.clear();
  for (unsigned i = 0; i < elementary.maps.size(); i++) {
    unordered_map<string, elementary_feature_value> mapped_ids;
    for (auto&& element : elementary.maps[i].map)
      if (element.second < elementary_ids_map[i].size() && elementary_ids_map[i][element.second] != elementary_feature_unknown)
        mapped_ids.emplace(element.first, elementary_ids_map[i][element.second]);

    optimized_elementary.maps.emplace_back(persistent_unordered_map(mapped_ids, 1, [](binary_encoder& enc, int id) {
      enc.add_4B(id);
    }));
  }

  // Remap keys in feature sequences by applying elementary_ids_map to appropriate subkeys
  optimized_features.sequences = features.sequences;
  optimized_features.scores.clear();
  vector<char> key_buffer;
  for (unsigned i = 0; i < features.sequences.size(); i++) {
    decltype(features.scores[i].map) updated_map;
    for (auto&& element : features.scores[i].map)
      if (element.second.gamma) {
        elementary_ids.clear();
        for (const char* key = element.first.c_str(); key < element.first.c_str() + element.first.size(); )
          elementary_ids.emplace_back(vli<elementary_feature_value>::decode(key));

        assert(elementary_ids.size() == features.sequences[i].elements.size());
        for (unsigned j = 0; j < elementary_ids.size(); j++) {
          if (map_indices[i][j] < 0) continue;
          assert(elementary_ids[j] < elementary_ids_map[map_indices[i][j]].size() && elementary_ids_map[map_indices[i][j]][elementary_ids[j]] != elementary_feature_unknown);
          elementary_ids[j] = elementary_ids_map[map_indices[i][j]][elementary_ids[j]];
        }

        key_buffer.resize(elementary_ids.size() * vli<elementary_feature_value>::max_length());
        char* key = key_buffer.data();
        for (unsigned j = 0; j < elementary_ids.size(); j++)
          vli<elementary_feature_value>::encode(elementary_ids[j], key);

        updated_map.emplace(string(key_buffer.data(), key - key_buffer.data()), element.second);
      }

    optimized_features.scores.emplace_back(persistent_unordered_map(updated_map, 1, [](binary_encoder& enc, const training_feature_sequence_map::info& info) {
      assert(feature_sequence_score(info.gamma) == info.gamma);
      enc.add_4B(info.gamma);
    }));
  }

  // Original code which only dropped feature sequences with gamma == 0
  // optimized_elementary.maps.clear();
  // for (auto&& map : elementary.maps)
  //   optimized_elementary.maps.emplace_back(persistent_unordered_map(map.map, 1, [](binary_encoder& enc, elementary_feature_value value) {
  //     enc.add_4B(value);
  //   }));
  //
  // optimized_features.sequences = features.sequences;
  // optimized_features.scores.clear();
  // for (auto&& score : features.scores) {
  //   decltype(score.map) pruned_map;
  //   for (auto&& element : score.map)
  //     if (element.second.gamma)
  //       pruned_map.insert(element);
  //
  //   optimized_features.scores.emplace_back(persistent_unordered_map(pruned_map, 1, [](binary_encoder& enc, const training_feature_sequence_map::info& info) {
  //     enc.add_4B(info.gamma);
  //   }));
  // }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger_trainer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class TaggerTrainer>
class tagger_trainer {
 public:
  struct sentence {
    vector<string> words;
    vector<string_piece> forms;
    vector<vector<tagged_lemma>> analyses;
    vector<tagged_lemma> gold;
    vector<int> gold_index;
  };

  static void train(int decoding_order, int window_size, int iterations, istream& in_morpho_dict, bool use_guesser, istream& in_feature_templates, bool prune_features, istream& in_train, istream& in_heldout, bool early_stopping, ostream& out_tagger);

 private:
  static double load_data(istream& is, const morpho& d, bool use_guesser, vector<sentence>& sentences, bool add_gold);
};

// Definitions
template <class TaggerTrainer>
void tagger_trainer<TaggerTrainer>::train(int decoding_order, int window_size, int iterations, istream& in_morpho_dict, bool use_guesser, istream& in_feature_templates, bool prune_features, istream& in_train, istream& in_heldout, bool early_stopping, ostream& out_tagger) {
//  cerr << "Loading dictionary: ";
  unique_ptr<morpho> d(morpho::load(in_morpho_dict));
  if (!d) training_failure("Cannot load dictionary!");
//  cerr << "done" << endl;
  if (!in_morpho_dict.seekg(0, istream::beg)) training_failure("Cannot seek in dictionary file to the beginning!");

  vector<sentence> train_data;
//  cerr << "Loading train data: ";
//  cerr << "done, matched " << fixed << setprecision(2) << 100 * load_data(in_train, *d, use_guesser, train_data, true) << '%' << endl;
  load_data(in_train, *d, use_guesser, train_data, true);

  vector<sentence> heldout_data;
  if (in_heldout) {
//    cerr << "Loading heldout data: ";
//    cerr << "done, matched " << fixed << setprecision(2) << 100 * load_data(in_heldout, *d, use_guesser, heldout_data, false) << '%' << endl;
    load_data(in_heldout, *d, use_guesser, heldout_data, false);
  }

  // Encode morphological dictionary
//  cerr << "Encoding morphological dictionary." << endl;
  out_tagger << in_morpho_dict.rdbuf();
  out_tagger.put(use_guesser);

  // Train and encode the tagger
  TaggerTrainer::train(decoding_order, window_size, iterations, train_data, heldout_data, early_stopping, prune_features, in_feature_templates, out_tagger);
}

template <class TaggerTrainer>
double tagger_trainer<TaggerTrainer>::load_data(istream& is, const morpho& d, bool use_guesser, vector<sentence>& sentences, bool add_gold) {
  sentences.clear();

  int forms = 0, forms_matched = 0;

  string line;
  vector<string> tokens;
  sentences.emplace_back();
  while (getline(is, line)) {
    if (line.empty()) {
      if (!sentences.back().words.empty())
        sentences.emplace_back();
      continue;
    }

    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("The tagger data line '" << line << "' does not contain three columns!");

    // Add form to sentence
    forms++;
    sentence& s = sentences.back();
    s.words.emplace_back(tokens[0]);
    s.forms.emplace_back(string_piece(s.words.back().c_str(), d.raw_form_len(s.words.back())));
    s.gold.emplace_back(tokens[1], tokens[2]);
    s.gold_index.emplace_back(-1);

    // Analyse
    s.analyses.emplace_back();
    d.analyze(tokens[0], use_guesser ? morpho::GUESSER : morpho::NO_GUESSER, s.analyses.back());

    // Locate gold analysis
    for (size_t i = 0; i < s.analyses.back().size(); i++)
      if (s.analyses.back()[i].lemma == s.gold.back().lemma && s.analyses.back()[i].tag == s.gold.back().tag) {
        s.gold_index.back() = i;
        forms_matched++;
        break;
      }
    if (s.gold_index.back() == -1 && add_gold) {
      s.gold_index.back() = s.analyses.back().size();
      s.analyses.back().emplace_back(tokens[1], tokens[2]);
    }
  }
  if (!sentences.empty() && sentences.back().words.empty()) sentences.pop_back();

  return forms_matched / double(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/perceptron_tagger_trainer.h
/////////

// This file is part of MorphoDiTa <http://github.com/ufal/morphodita/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template <class FeatureSequences>
class perceptron_tagger_trainer {
 public:
  typedef typename tagger_trainer<perceptron_tagger_trainer<FeatureSequences>>::sentence sentence;

  static void train(int decoding_order, int window_size, int iterations, const vector<sentence>& train, const vector<sentence>& heldout, bool early_stopping, bool prune_features, istream& in_feature_templates, ostream& out_tagger);

 private:
  static void train_viterbi(int decoding_order, int window_size, int iterations, const vector<sentence>& train, const vector<sentence>& heldout, bool early_stopping, bool prune_features, FeatureSequences& features);
};

// Definitions
template <class FeatureSequences>
void perceptron_tagger_trainer<FeatureSequences>::train(int decoding_order, int window_size, int iterations, const vector<sentence>& train, const vector<sentence>& heldout, bool early_stopping, bool prune_features, istream& in_feature_templates, ostream& out_tagger) {
  FeatureSequences features;

//  cerr << "Parsing feature templates..." << endl;
  features.parse(window_size, in_feature_templates);

//  cerr << "Training tagger..." << endl;
  train_viterbi(decoding_order, window_size, iterations, train, heldout, early_stopping, prune_features, features);

//  cerr << "Encoding tagger..." << endl;
  typedef feature_sequences_optimizer<FeatureSequences> optimizer;
  typename optimizer::optimized_feature_sequences optimized_features;
  optimizer::optimize(features, optimized_features);
  if (!optimized_features.save(out_tagger)) training_failure("Cannot save feature sequences!");
}

template <class FeatureSequences>
void perceptron_tagger_trainer<FeatureSequences>::train_viterbi(int decoding_order, int window_size, int iterations, const vector<sentence>& train, const vector<sentence>& heldout, bool early_stopping, bool prune_features, FeatureSequences& features) {
  int best_correct = 0, best_iteration = -1;
  FeatureSequences best_features;

  viterbi<FeatureSequences> decoder(features, decoding_order, window_size);
  typename decltype(decoder)::cache decoder_cache(decoder);

  typename FeatureSequences::cache feature_sequences_cache(features);
  typename FeatureSequences::dynamic_features decoded_dynamic_features, gold_dynamic_features;
  vector<string> decoded_feature_sequences_keys, gold_feature_sequences_keys;

  vector<int> window(window_size);

  // Initialize feature sequences for the gold decoding only if requested
  if (prune_features)
    for (unsigned s = 0; s < train.size(); s++) {
      auto& sentence = train[s];
      features.initialize_sentence(sentence.forms, sentence.analyses, feature_sequences_cache);
      for (int i = 0; i < int(sentence.forms.size()); i++) {
        window.assign(window_size, -1);
        for (int j = 0; j < window_size && i - j >= 0; j++) window[j] = sentence.gold_index[i - j];

        features.compute_dynamic_features(i, window[0], &gold_dynamic_features, gold_dynamic_features, feature_sequences_cache);
        features.feature_keys(i, window.data(), 0, gold_dynamic_features, gold_feature_sequences_keys, feature_sequences_cache);

        for (unsigned f = 0; f < features.scores.size(); f++)
          if (!gold_feature_sequences_keys[f].empty())
            features.scores[f].map[gold_feature_sequences_keys[f]];
      }
    }

  // Train for given number of iterations
  for (int i = 0; i < iterations; i++) {
    // Train
    int train_correct = 0, train_total = 0;
    cerr << "Iteration " << i + 1 << ": ";

    vector<int> tags;
    for (unsigned s = 0; s < train.size(); s++) {
      auto& sentence = train[s];

      // Run Viterbi
      if (tags.size() < sentence.forms.size()) tags.resize(2 * sentence.forms.size());
      decoder.tag(sentence.forms, sentence.analyses, decoder_cache, tags);

      // Compute feature sequence keys or decoded result and gold result and update alpha & gamma
      features.initialize_sentence(sentence.forms, sentence.analyses, feature_sequences_cache);
      for (int i = 0; i < int(sentence.forms.size()); i++) {
        train_correct += tags[i] == sentence.gold_index[i];
        train_total++;

        window.assign(window_size, -1);
        for (int j = 0; j < window_size && i - j >= 0; j++) window[j] = tags[i - j];
        features.compute_dynamic_features(i, window[0], &decoded_dynamic_features, decoded_dynamic_features, feature_sequences_cache);
        features.feature_keys(i, window.data(), 0, decoded_dynamic_features, decoded_feature_sequences_keys, feature_sequences_cache);

        for (int j = 0; j < window_size && i - j >= 0; j++) window[j] = sentence.gold_index[i - j];
        features.compute_dynamic_features(i, window[0], &gold_dynamic_features, gold_dynamic_features, feature_sequences_cache);
        features.feature_keys(i, window.data(), 0, gold_dynamic_features, gold_feature_sequences_keys, feature_sequences_cache);

        for (unsigned f = 0; f < features.scores.size(); f++) {
          if (decoded_feature_sequences_keys[f] != gold_feature_sequences_keys[f]) {
            if (!decoded_feature_sequences_keys[f].empty()) {
              auto it = features.scores[f].map.find(decoded_feature_sequences_keys[f]);
              if (it == features.scores[f].map.end() && !prune_features) it = features.scores[f].map.emplace(decoded_feature_sequences_keys[f], typename decltype(features.scores[f].map)::mapped_type()).first;
              if (it != features.scores[f].map.end()) {
                auto& decoded_info = it->second;
                decoded_info.gamma += decoded_info.alpha * (s - decoded_info.last_gamma_update);
                decoded_info.last_gamma_update = s;
                decoded_info.alpha--;
              }
            }

            if (!gold_feature_sequences_keys[f].empty()) {
              auto it = features.scores[f].map.find(gold_feature_sequences_keys[f]);
              if (it == features.scores[f].map.end() && !prune_features) it = features.scores[f].map.emplace(gold_feature_sequences_keys[f], typename decltype(features.scores[f].map)::mapped_type()).first;
              if (it != features.scores[f].map.end()) {
                auto& gold_info = it->second;
                gold_info.gamma += gold_info.alpha * (s - gold_info.last_gamma_update);
                gold_info.last_gamma_update = s;
                gold_info.alpha++;
              }
            }
          }
        }
      }
    }

    // Finalize incremental gamma updates
    for (auto&& score : features.scores)
      for (auto&& element : score.map) {
        element.second.gamma += element.second.alpha * (train.size() - element.second.last_gamma_update);
        element.second.last_gamma_update = 0;
      }
    cerr << "done, accuracy " << fixed << setprecision(2) << train_correct * 100 / double(train_total) << '%';

    // If we have any heldout data, compute accuracy and if requested store best tagger configuration
    if (!heldout.empty()) {
      enum { TAGS, LEMMAS, BOTH, TOTAL };
      int heldout_correct[TOTAL] = {}, heldout_total = 0;

      typedef feature_sequences_optimizer<FeatureSequences> optimizer;
      typename optimizer::optimized_feature_sequences frozen_features;
      optimizer::optimize(features, frozen_features);
      viterbi<decltype(frozen_features)> frozen_decoder(frozen_features, decoding_order, window_size);
      typename decltype(frozen_decoder)::cache frozen_decoder_cache(frozen_decoder);

      for (auto&& sentence : heldout) {
        if (tags.size() < sentence.forms.size()) tags.resize(sentence.forms.size() * 2);
        frozen_decoder.tag(sentence.forms, sentence.analyses, frozen_decoder_cache, tags);

        for (unsigned i = 0; i < sentence.forms.size(); i++) {
          heldout_correct[TAGS] += sentence.gold[i].tag == sentence.analyses[i][tags[i]].tag;
          heldout_correct[LEMMAS] += sentence.gold[i].lemma == sentence.analyses[i][tags[i]].lemma;
          heldout_correct[BOTH] += sentence.gold[i].tag == sentence.analyses[i][tags[i]].tag && sentence.gold[i].lemma == sentence.analyses[i][tags[i]].lemma;
          heldout_total++;
        }
      }

      if (early_stopping && heldout_correct[BOTH] > best_correct) {
        best_correct = heldout_correct[BOTH];
        best_iteration = i;
        best_features = features;
      }

      cerr << ", heldout accuracy " << fixed << setprecision(2)
          << 100 * heldout_correct[TAGS] / double(heldout_total) << "%t/"
          << 100 * heldout_correct[LEMMAS] / double(heldout_total) << "%l/"
          << 100 * heldout_correct[BOTH] / double(heldout_total) << "%b";
    }
    cerr << endl;
  }

  if (early_stopping && best_iteration >= 0) {
    cerr << "Chosen tagger model from iteration " << best_iteration + 1 << endl;
    features = best_features;
  }
}

} // namespace morphodita

/////////
// File: utils/options.h
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

class options {
 public:
  typedef unordered_map<string, string> map;

  struct value {
    enum allowed_t { NONE, ANY, SET };
    allowed_t allowed;
    unordered_set<string> set;

    value(initializer_list<string> set) : allowed(SET), set(set) {}
    static const value none;
    static const value any;

   private:
    value(allowed_t allowed) : allowed(allowed) {}
  };

  // Parse options according to allowed map. If successful, argv is reordered so
  // that non-option arguments are placed in argv[1] to argv[argc-1]. The '--'
  // indicates end of option arguments (as usual).  The allowed map contains
  // values allowed for every option. If empty, no value is allowed, if it
  // contains just an empty string, any value is allowed.
  static bool parse(const unordered_map<string, value>& allowed, int& argc, char**& argv, map& options);
};

} // namespace utils

/////////
// File: version/version.h
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class version {
 public:
  unsigned major;
  unsigned minor;
  unsigned patch;
  std::string prerelease;

  // Returns current version.
  static version current();

  // Returns multi-line formated version and copyright string.
  static string version_and_copyright(const string& other_libraries = string());
};

/////////
// File: trainer/trainer_morphodita_parsito.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

bool trainer_morphodita_parsito::train(const vector<sentence>& training, const vector<sentence>& heldout,
                                       const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error) {
  error.clear();

  // Save model version info
  os.put(model_morphodita_parsito::VERSION_LATEST);
  // Add sentinel required since version 2
  os.put(0x7F).put(0x7F);

  // Check input data
  for (auto&& sentence : training)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;
  for (auto&& sentence : heldout)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;

  if (!train_tokenizer(training, heldout, tokenizer, os, error)) return false;
  string tagger_model;
  {
    ostringstream os_tagger;
    if (!train_tagger(training, heldout, tagger, os_tagger, error)) return false;
    tagger_model.assign(os_tagger.str());
    os.write(tagger_model.data(), tagger_model.size());
  }
  if (!train_parser(training, heldout, parser, tagger_model, os, error)) return false;

  return true;
}

bool trainer_morphodita_parsito::train_tokenizer(const vector<sentence>& training, const vector<sentence>& heldout,
                                                 const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Tokenizer options
    named_values::map tokenizer;
    if (!named_values::parse(options, tokenizer, error)) return false;
    int run = 0; if (!option_int(tokenizer, "run", run, error)) return false;

    if (tokenizer.count("from_model")) {
      // Use specified tokenizer model
      string_piece tokenizer_data;
      if (!load_model(tokenizer["from_model"], TOKENIZER_MODEL, tokenizer_data))
        return error.assign("Cannot load model from which the tokenizer should be used!"), false;

      cerr << "Using tokenizer from given model." << endl;
      os.write(tokenizer_data.str, tokenizer_data.len);
    } else {
      os.put(1);
      const string& model = option_str(tokenizer, "model");

      // Tokenizer itself
      if (model == "generic") {
        os.put(morphodita::tokenizer_id::GENERIC);
        morphodita::generic_tokenizer_factory_encoder::encode(morphodita::generic_tokenizer::LATEST, os);
      } else if (model.empty() || model == "gru") {
        // Create a detokenizator if required
        unique_ptr<detokenizer> detokenizer;
        if (tokenizer.count("detokenize")) {
          detokenizer.reset(new udpipe::detokenizer(tokenizer["detokenize"]));
          if (!detokenizer) return error.assign("Cannot create detokenizer!"), false;
        }

        // Prepare training data for the gru_tokenizer
        vector<morphodita::tokenized_sentence> sentences;
        bool spaces_in_training = false;
        for (size_t training_sentence = 0; training_sentence < training.size(); training_sentence++) {
          sentence s = training[training_sentence];
          if (detokenizer) detokenizer->detokenize(s);

          auto& sentence = (sentences.emplace_back(), sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ?
                (const token&)s.multiword_tokens[j] : (const token&)s.words[i];

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(tok.form)) {
              sentence.sentence.push_back(chr);
              if (unilib::unicode::category(chr) & unilib::unicode::Zs) spaces_in_training = true;
            }
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            if (tok.get_space_after()) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          if (training_sentence + 1 < training.size() && (training[training_sentence + 1].get_new_doc() || training[training_sentence + 1].get_new_par()))
            sentence.sentence.append(2, '\n');
        }

        // Heldout data
        vector<morphodita::tokenized_sentence> heldout_sentences;

        bool detokenize_handout = true; if (!option_bool(tokenizer, "detokenize_handout", detokenize_handout, error)) return false;
        for (size_t heldout_sentence = 0; heldout_sentence < heldout.size(); heldout_sentence++) {
          sentence s = heldout[heldout_sentence];
          if (detokenizer && detokenize_handout) detokenizer->detokenize(s);

          auto& sentence = (heldout_sentences.emplace_back(), heldout_sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ?
                (const token&)s.multiword_tokens[j] : (const token&)s.words[i];

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(tok.form))
              sentence.sentence.push_back(chr);
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            if (tok.get_space_after()) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          if (heldout_sentence + 1 < heldout.size() && (heldout[heldout_sentence + 1].get_new_doc() || heldout[heldout_sentence + 1].get_new_par()))
            sentence.sentence.append(2, '\n');
        }

        // Options
        bool tokenize_url = true; if (!option_bool(tokenizer, "tokenize_url", tokenize_url, error)) return false;
        int segment_size = 50; // if (!option_int(tokenizer, "segment_size", segment_size, error)) return false;
        bool allow_spaces = spaces_in_training; if (!option_bool(tokenizer, "allow_spaces", allow_spaces, error)) return false;
        int dimension = 24; if (!option_int(tokenizer, "dimension", dimension, error)) return false;
        int epochs = 100; if (!option_int(tokenizer, "epochs", epochs, error)) return false;
        int batch_size = run <= 1 ? 50 : 50 + 50 * hyperparameter_integer(run, 1, 0, 1);
        if (!option_int(tokenizer, "batch_size", batch_size, error)) return false;
        double learning_rate = run <= 1 ? 0.005 : hyperparameter_logarithmic(run, 2, 0.0005, 0.01);
        if (!option_double(tokenizer, "learning_rate", learning_rate, error)) return false;
        double learning_rate_final = 0.0; // if (!option_double(tokenizer, "learning_rate_final", learning_rate_final, error)) return false;
        double dropout = 0.1; if (!option_double(tokenizer, "dropout", dropout, error)) return false;
        double initialization_range = 0.5; if (!option_double(tokenizer, "initialization_range", initialization_range, error)) return false;
        bool early_stopping = !heldout_sentences.empty(); if (!option_bool(tokenizer, "early_stopping", early_stopping, error)) return false;

        if (run >= 1) cerr << "Random search run " << run << ", batch_size=" << batch_size
                           << ", learning_rate=" << fixed << setprecision(8) << learning_rate << endl;

        cerr << "Training tokenizer with the following options: " << "tokenize_url=" << (tokenize_url ? 1 : 0)
             << ", allow_spaces=" << (allow_spaces ? 1 : 0) << ", dimension=" << dimension << endl
             << "  epochs=" << epochs << ", batch_size=" << batch_size << ", learning_rate=" << fixed << setprecision(4) << learning_rate
             << ", dropout=" << dropout << ", early_stopping=" << (early_stopping ? 1 : 0) << endl;

        // Train and encode gru_tokenizer
        os.put(morphodita::tokenizer_ids::GRU);
        if (!morphodita::gru_tokenizer_trainer::train(tokenize_url ? morphodita::gru_tokenizer_trainer::URL_EMAIL_LATEST : 0,
                                                      segment_size, allow_spaces, dimension, epochs, batch_size, learning_rate,
                                                      learning_rate_final, dropout, initialization_range, early_stopping,
                                                      sentences, heldout_sentences, os, error))
          return false;
      } else {
        return error.assign("Unknown tokenizer model '").append(model).append("'!"), false;
      }

      // Multiword splitter
      if (!multiword_splitter_trainer::train(training, os, error)) return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_tagger(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Parse options
    named_values::map tagger;
    if (!named_values::parse(options, tagger, error)) return false;

    if (tagger.count("from_model")) {
      // Use specified tokenizer model(s)
      int model_index = 1, taggers_total = 0;
      string model_name = "from_model";
      vector<string_piece> taggers_data;
      do {
        taggers_data.emplace_back();
        if (!load_model(tagger[model_name], TAGGER_MODEL, taggers_data.back()))
          return error.assign("Cannot load model from which the tagger should be used!"), false;
        if (taggers_data.back().str[0])
          taggers_total += taggers_data.back().str[0];
        else
          taggers_data.pop_back();
        model_name = "from_model_" + to_string(++model_index);
      } while (tagger.count(model_name));
      if (taggers_total < 0 || taggers_total > 4) return error.assign("Cannot create more than four tagger models!"), false;

      cerr << "Using tagger from given model(s)." << endl;
      os.put(taggers_total);
      for (auto&& tagger_data : taggers_data)
        os.write(tagger_data.str + 1, tagger_data.len - 1);
    } else {
      // Create MorphoDiTa model(s)
      int models = 1; if (!option_int(tagger, "models", models, error)) return false;
      if (models <= 0) return error.assign("Number of tagger models cannot be negative or zero!"), false;
      if (models > 4) return error.assign("Cannot create more than four tagger models!"), false;

      os.put(models);
      for (int model = 0; model < models; model++)
        if (!train_tagger_model(training, heldout, model, models, tagger, os, error))
          return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_parser(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, const string& tagger_model, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Create Parsito model
    named_values::map parser;
    if (!named_values::parse(options, parser, error)) return false;
    int run = 0; if (!option_int(parser, "run", run, error)) return false;

    if (parser.count("from_model")) {
      // Use specified parser model
      string_piece parser_data;
      if (!load_model(parser["from_model"], PARSER_MODEL, parser_data))
        return error.assign("Cannot load model from which the parser should be used!"), false;

      cerr << "Using parser from given model." << endl;
      os.write(parser_data.str, parser_data.len);
    } else {
      os.put(1);

      // Parsito options
      string transition_system = parser.count("transition_system") ? parser["transition_system"] : "projective";
      string transition_oracle = parser.count("transition_oracle") ? parser["transition_oracle"] :
          transition_system == "projective" ? "dynamic" :
          transition_system == "swap" ? "static_lazy" :
          "static";

      int embedding_upostag = 20; if (!option_int(parser, "embedding_upostag", embedding_upostag, error)) return false;
      int embedding_feats = 20; if (!option_int(parser, "embedding_feats", embedding_feats, error)) return false;
      int embedding_xpostag = 0; if (!option_int(parser, "embedding_xpostag", embedding_xpostag, error)) return false;
      int embedding_form = 50; if (!option_int(parser, "embedding_form", embedding_form, error)) return false;
      int embedding_form_mincount = 2; if (!option_int(parser, "embedding_form_mincount", embedding_form_mincount, error)) return false;
      int embedding_lemma = 0; if (!option_int(parser, "embedding_lemma", embedding_lemma, error)) return false;
      int embedding_lemma_mincount = 2; if (!option_int(parser, "embedding_lemma_mincount", embedding_lemma_mincount, error)) return false;
      int embedding_deprel = 20; if (!option_int(parser, "embedding_deprel", embedding_deprel, error)) return false;
      string embeddings;
      if (embedding_upostag) embeddings.append("universal_tag ").append(to_string(embedding_upostag)).append(" 1\n");
      if (embedding_feats) embeddings.append("feats ").append(to_string(embedding_feats)).append(" 1\n");
      if (embedding_xpostag) embeddings.append("tag ").append(to_string(embedding_xpostag)).append(" 1\n");
      if (embedding_form) {
        embeddings.append("form ").append(to_string(embedding_form)).append(" ").append(to_string(embedding_form_mincount));
        if (!option_str(parser, "embedding_form_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_form_file"));
        embeddings.push_back('\n');
      }
      if (embedding_lemma) {
        embeddings.append("lemma ").append(to_string(embedding_lemma)).append(" ").append(to_string(embedding_lemma_mincount));
        if (!option_str(parser, "embedding_lemma_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_lemma_file"));
        embeddings.push_back('\n');
      }
      if (embedding_deprel) embeddings.append("deprel ").append(to_string(embedding_deprel)).append(" 1\n");

      bool single_root = true; if (!option_bool(parser, "single_root", single_root, error)) return false;
      int iterations = 10; if (!option_int(parser, "iterations", iterations, error)) return false;
      int hidden_layer = 200; if (!option_int(parser, "hidden_layer", hidden_layer, error)) return false;
      int batch_size = 10; if (!option_int(parser, "batch_size", batch_size, error)) return false;
      int structured_interval = run <= 1 ? 8 : hyperparameter_integer(run,1,0,2) == 2 ? 0 : 8 + 2*hyperparameter_integer(run,1,0,2);
      if (!option_int(parser, "structured_interval", structured_interval, error)) return false;
      double learning_rate = run <= 1 ? 0.02 : hyperparameter_logarithmic(run, 2, 0.005, 0.04);
      if (!option_double(parser, "learning_rate", learning_rate, error)) return false;
      double learning_rate_final = 0.001; if (!option_double(parser, "learning_rate_final", learning_rate_final, error)) return false;
      double l2 = run <= 1 ? 0.5 : hyperparameter_uniform(run, 3, 0.2, 0.6);
      if (!option_double(parser, "l2", l2, error)) return false;
      bool early_stopping = !heldout.empty(); if (!option_bool(parser, "early_stopping", early_stopping, error)) return false;

      if (run >= 1) cerr << "Random search run " << run << ", structured_interval=" << structured_interval
                         << ", learning_rate=" << fixed << setprecision(8) << learning_rate
                         << ", l2=" << l2 << endl;

      // Prepare data in the correct format
      parsito::network_parameters parameters;
      parameters.iterations = iterations;
      parameters.structured_interval = structured_interval;
      parameters.hidden_layer = hidden_layer;
      parameters.hidden_layer_type = parsito::activation_function::TANH;
      parameters.trainer.algorithm = parsito::network_trainer::SGD;
      parameters.trainer.learning_rate = learning_rate;
      parameters.trainer.learning_rate_final = learning_rate_final;
      parameters.trainer.momentum = 0;
      parameters.trainer.epsilon = 0;
      parameters.batch_size = batch_size;
      parameters.initialization_range = 0.1f;
      parameters.l1_regularization = 0;
      parameters.l2_regularization = l2;
      parameters.maxnorm_regularization = 0;
      parameters.dropout_hidden = 0;
      parameters.dropout_input = 0;
      parameters.early_stopping = early_stopping;

      // Tag the input if required
      unique_ptr<model> tagger;
      bool use_gold_tags = false; if (!option_bool(parser, "use_gold_tags", use_gold_tags, error)) return false;
      if (!use_gold_tags && !tagger_model.empty() && tagger_model[0]) {
        stringstream tagger_description;
        tagger_description.put(model_morphodita_parsito::VERSION_LATEST).put(0x7F).put(0x7F).put(0).write(tagger_model.data(), tagger_model.size()).put(0);
        tagger.reset(model_morphodita_parsito::load(tagger_description));
        if (!tagger) return error.assign("Cannot load the tagger model for parser training data generation!"), false;
      }

      // Training data
      sentence tagged;
      vector<parsito::tree> train_trees;
      for (auto&& sentence : training) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        train_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          train_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, train_trees.back().nodes.back().form);
          train_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          train_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          train_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          train_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          train_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      // Heldout data
      vector<parsito::tree> heldout_trees;
      for (auto&& sentence : heldout) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        heldout_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          heldout_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, heldout_trees.back().nodes.back().form);
          heldout_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          heldout_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          heldout_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          heldout_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          heldout_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      cerr << "Parser transition options: system=" << transition_system << ", oracle=" << transition_oracle
           << ", structured_interval=" << structured_interval << ", single_root=" << (single_root ? 1 : 0) << endl
           << "Parser uses lemmas/upos/xpos/feats: " << (tagger ? "automatically generated by tagger" : "from gold data") << endl
           << "Parser embeddings options: upostag=" << embedding_upostag << ", feats=" << embedding_feats << ", xpostag=" << embedding_xpostag
           << ", form=" << embedding_form << ", lemma=" << embedding_lemma << ", deprel=" << embedding_deprel << endl
           << "  form mincount=" << embedding_form_mincount << ", precomputed form embeddings=" << (parser["embedding_form_file"].empty() ? "none" : parser["embedding_form_file"]) << endl
           << "  lemma mincount=" << embedding_lemma_mincount << ", precomputed lemma embeddings=" << (parser["embedding_lemma_file"].empty() ? "none" : parser["embedding_lemma_file"]) << endl
           << "Parser network options: iterations=" << iterations << ", hidden_layer=" << hidden_layer << ", batch_size=" << batch_size << "," << endl
           << "  learning_rate=" << fixed << setprecision(4) << learning_rate << ", learning_rate_final=" << learning_rate_final
           << ", l2=" << l2 << ", early_stopping=" << (early_stopping ? 1 : 0) << endl;

      // Train the parser
      binary_encoder enc;
      enc.add_str("nn_versioned");
      parsito::parser_nn_trainer::train(transition_system, transition_oracle, single_root, embeddings, parser_nodes,
                                        parameters, 1, train_trees, heldout_trees, enc);
      compressor::save(os, enc);
    }
  }

  return true;
}

bool trainer_morphodita_parsito::load_model(const string& data, model_type model, string_piece& range) {
  istringstream is(data);

  // Check that it is morphodita_parsito model.
  char len;
  if (!is.get(len)) return false;
  string name(len, ' ');
  if (!is.read(&name[0], len)) return false;
  if (name != "morphodita_parsito") return false;

  char version;
  if (!is.get(version)) return false;
  if (!(version >= 1 && version <= model_morphodita_parsito::VERSION_LATEST)) return false;

  // Because UDPipe 1.0 does not check the model version,
  // a specific sentinel was added since version 2 so that
  // loading of such model fail on UDPipe 1.0
  if (version >= 2) {
    char sentinel;
    if (!is.get(sentinel) || sentinel != 0x7F) return false;
    if (!is.get(sentinel) || sentinel != 0x7F) return false;
  }

  // Tokenizer
  {
    if (model == TOKENIZER_MODEL) range.str = data.data() + is.tellg();
    char tokenizer; if (!is.get(tokenizer)) return false;
    unique_ptr<morphodita::tokenizer_factory> tokenizer_factory(tokenizer ? morphodita::tokenizer_factory::load(is) : nullptr);
    if (tokenizer && !tokenizer_factory) return false;
    unique_ptr<multiword_splitter> splitter(tokenizer ? multiword_splitter::load(is) : nullptr);
    if (model == TOKENIZER_MODEL) return range.len = is.tellg() - streampos(range.str - data.data()), true;
  }

  // Tagger
  {
    if (model == TAGGER_MODEL) range.str = data.data() + is.tellg();
    char taggers; if (!is.get(taggers)) return false;
    for (char i = 0; i < taggers; i++) {
      char lemma; if (!is.get(lemma)) return false;
      char xpostag; if (!is.get(xpostag)) return false;
      char feats; if (!is.get(feats)) return false;
      unique_ptr<morphodita::tagger> tagger(morphodita::tagger::load(is));
      if (!tagger) return false;
    }
    if (model == TAGGER_MODEL) return range.len = is.tellg() - streampos(range.str - data.data()), true;
  }

  // Parser
  {
    if (model == PARSER_MODEL) range.str = data.data() + is.tellg();
    char parser;
    if (!is.get(parser)) return false;
    unique_ptr<parsito::parser> parser_model(parser ? parsito::parser::load(is) : nullptr);
    if (parser && !parser_model) return false;
    if (model == PARSER_MODEL) return range.len = is.tellg() - streampos(range.str - data.data()), true;
  }

  return false;
}

const string& trainer_morphodita_parsito::model_normalize_form(string_piece form, string& output) {
  return model_morphodita_parsito(model_morphodita_parsito::VERSION_LATEST).normalize_form(form, output);
}

const string& trainer_morphodita_parsito::model_normalize_lemma(string_piece lemma, string& output) {
  return model_morphodita_parsito(model_morphodita_parsito::VERSION_LATEST).normalize_lemma(lemma, output);
}

void trainer_morphodita_parsito::model_fill_word_analysis(const morphodita::tagged_lemma& analysis, bool upostag, int lemma, bool xpostag, bool feats, word& word) {
  model_morphodita_parsito(model_morphodita_parsito::VERSION_LATEST).fill_word_analysis(analysis, upostag, lemma, xpostag, feats, word);
}

// Tagger model helper functions

bool trainer_morphodita_parsito::train_tagger_model(const vector<sentence>& training, const vector<sentence>& heldout,
                                                    unsigned model, unsigned models, const named_values::map& tagger,
                                                    ostream& os, string& error) {
  unique_ptr<input_format> conllu_input_format(input_format::new_conllu_input_format());

  int run = 0; if (!option_int(tagger, "run", run, error, model)) return false;

  bool have_lemma = false;
  for (auto&& sentence : training)
    for (size_t i = 1; !have_lemma && i < sentence.words.size(); i++)
      if (!sentence.words[i].lemma.empty() && sentence.words[i].lemma != "_")
        have_lemma = true;
  bool use_lemma_flag = model == 1 || models == 1; if (!option_bool(tagger, "use_lemma", use_lemma_flag, error, model)) return false;
  int lemma_encoding = 2; if (!option_int(tagger, "dictionary_lemma_encoding", lemma_encoding, error, model)) return false;
  int use_lemma = have_lemma && use_lemma_flag ? lemma_encoding : 0;
  bool use_xpostag = model == 0; if (!option_bool(tagger, "use_xpostag", use_xpostag, error, model)) return false;
  bool use_feats = model == 0; if (!option_bool(tagger, "use_feats", use_feats, error, model)) return false;

  bool provide_lemma = model == 1 || models == 1; if (!option_bool(tagger, "provide_lemma", provide_lemma, error, model)) return false;
  bool provide_xpostag = model == 0; if (!option_bool(tagger, "provide_xpostag", provide_xpostag, error, model)) return false;
  bool provide_feats = model == 0; if (!option_bool(tagger, "provide_feats", provide_feats, error, model)) return false;
  os.put(char(provide_lemma ? use_lemma : 0));
  os.put(char(provide_xpostag && use_xpostag));
  os.put(char(provide_feats && use_feats));

  cerr << "Tagger model " << model+1 << " columns: " << "lemma use=" << (use_lemma ? 1 : 0) << "/provide=" << (provide_lemma ? 1 : 0)
       << ", xpostag use=" << (use_xpostag ? 1 : 0) << "/provide=" << (provide_xpostag ? 1 : 0)
       << ", feats use=" << (use_feats ? 1 : 0) << "/provide=" << (provide_feats ? 1 : 0) << endl;

  // Start by creating the morphological dictionary
  stringstream morpho_description;
  string normalized_form, combined_tag, combined_lemma;

  // Generic options
  const string& dictionary_model = option_str(tagger, "dictionary_model", model);
  if (!dictionary_model.empty()) {
    // Use specified morphological dictionary
    cerr << "Using given morphological dictionary for tagger model " << model+1 << "." << endl;
    morpho_description << dictionary_model;
  } else {
    // Create the morphological dictionary and guesser from data
    cerr << "Creating morphological dictionary for tagger model " << model+1 << "." << endl;

    // Dictionary options
    int dictionary_suffix_len = 8; if (!option_int(tagger, "dictionary_suffix_len", dictionary_suffix_len, error, model)) return false;
    unordered_set<string> flat_lemmas;
    if (!option_str(tagger, "dictionary_flat_lemmas", model).empty()) {
      vector<string> lemmas;
      split(option_str(tagger, "dictionary_flat_lemmas", model), ',', lemmas);
      for (auto&& lemma : lemmas) {
        if (lemma.find('~') != string::npos)
          return error.assign("Dictionary_flat_lemmas cannot contain '~' character!"), false;
        flat_lemmas.insert(lemma);
      }
    } else {
      flat_lemmas.insert("greek.expression");
    }

    if (!option_str(tagger, "dictionary", model).empty())
      return error.assign("The tagger 'dictionary' option is no longer supported, use 'dictionary_file' instead!"), false;
    const string& dictionary_file = option_str(tagger, "dictionary_file", model);
    int max_form_analyses = 0; if (!option_int(tagger, "dictionary_max_form_analyses", max_form_analyses, error, model)) return false;

    cerr << "Tagger model " << model+1 << " dictionary options: " << "max_form_analyses=" << max_form_analyses
         << ", custom dictionary_file=" << (dictionary_file.empty() ? "none" : dictionary_file) << endl;

    // Guesser options
    int guesser_suffix_len = 4; if (!option_int(tagger, "guesser_suffix_len", guesser_suffix_len, error, model)) return false;
    int guesser_suffix_rules = run <= 1 ? 8 : 5 + hyperparameter_integer(run, 1, 0, 7);
    if (!option_int(tagger, "guesser_suffix_rules", guesser_suffix_rules, error, model)) return false;
    int guesser_prefixes_max = provide_lemma ? 4 : 0; if (!option_int(tagger, "guesser_prefixes_max", guesser_prefixes_max, error, model)) return false;
    int guesser_prefix_min_count = 10; if (!option_int(tagger, "guesser_prefix_min_count", guesser_prefix_min_count, error, model)) return false;
    int guesser_enrich_dictionary = run <= 1 ? 6 : 3 + hyperparameter_integer(run, 2, 0, 7);
    if (!dictionary_file.empty()) guesser_enrich_dictionary = 0;
    if (!option_int(tagger, "guesser_enrich_dictionary", guesser_enrich_dictionary, error, model)) return false;

    if (run >= 1) cerr << "Random search run " << run << ", guesser_suffix_rules=" << guesser_suffix_rules
                       << ", guesser_enrich_dictionary=" << guesser_enrich_dictionary << endl;

    cerr << "Tagger model " << model+1 << " guesser options: " << "suffix_rules=" << guesser_suffix_rules
         << ", prefixes_max=" << guesser_prefixes_max << ", prefix_min_count=" << guesser_prefix_min_count
         << ", enrich_dictionary=" << guesser_enrich_dictionary << endl;

    // Start by generating statistical guesser
    stringstream guesser_description;
    {
      stringstream guesser_input;
      for (auto&& sentence : training) {
        for (size_t i = 1; i < sentence.words.size(); i++)
          guesser_input << model_normalize_form(sentence.words[i].form, normalized_form) << '\t'
              << combine_lemma(sentence.words[i], use_lemma, combined_lemma, flat_lemmas) << '\t'
              << combine_tag(sentence.words[i], use_xpostag, use_feats, combined_tag) << '\n';
        guesser_input << '\n';
      }
      morphodita::morpho_statistical_guesser_trainer::train(guesser_input, guesser_suffix_len, guesser_suffix_rules, guesser_prefixes_max, guesser_prefix_min_count, guesser_description);
    }

    // Generate morphological dictionary data from the input
    unordered_set<string> dictionary_entries;
    {
      unordered_map<string, unordered_map<string, int>> entries;
      string entry;
      for (auto&& sentence : training)
        for (size_t i = 1; i < sentence.words.size(); i++) {
          model_normalize_form(sentence.words[i].form, normalized_form);
          entry.assign(combine_lemma(sentence.words[i], use_lemma, combined_lemma, flat_lemmas))
              .append("\t").append(combine_tag(sentence.words[i], use_xpostag, use_feats, combined_tag))
              .append("\t").append(normalized_form);
          entries[normalized_form][entry]++;
        }

      vector<pair<int, string>> analyses;
      for (auto&& form_analyses : entries) {
        analyses.clear();
        for (auto&& analysis : form_analyses.second)
          analyses.emplace_back(analysis.second, analysis.first);
        if (max_form_analyses && int(analyses.size()) > max_form_analyses) {
          sort(analyses.begin(), analyses.end(), greater<pair<int, string>>());
          analyses.resize(max_form_analyses);
        }
        for (auto&& analysis : analyses)
          dictionary_entries.insert(analysis.second);
      }
    }
    morphodita::generic_morpho_encoder::tags dictionary_special_tags;
    dictionary_special_tags.unknown_tag = "~X";
    dictionary_special_tags.number_tag = most_frequent_tag(training, "NUM", use_xpostag, use_feats, combined_tag);
    dictionary_special_tags.punctuation_tag = most_frequent_tag(training, "PUNCT", use_xpostag, use_feats, combined_tag);
    dictionary_special_tags.symbol_tag = most_frequent_tag(training, "SYM", use_xpostag, use_feats, combined_tag);

    // Append given dictionary_file if given
    if (!dictionary_file.empty()) {
      ifstream is(dictionary_file);
      if (!is.is_open()) return error.assign("Cannot open dictionary_file '").append(dictionary_file).append("'!"), false;

      vector<string_piece> dictionary_parts;
      word entry;
      string entry_encoded, line;
      while (getline(is, line)) {
        // Skip empty lines
        if (line.empty()) continue;

        split(line, '\t', dictionary_parts);

        if (dictionary_parts.size() != 5)
          return error.assign("Dictionary line '").append(line).append("' does not contain 5 tab-separated columns!"), false;

        model_normalize_form(dictionary_parts[0], entry.form);
        entry.lemma.assign(dictionary_parts[1].str, dictionary_parts[1].len == 1 && dictionary_parts[1].str[0] == '_' ? 0 : dictionary_parts[1].len);
        entry.upostag.assign(dictionary_parts[2].str, dictionary_parts[2].len == 1 && dictionary_parts[2].str[0] == '_' ? 0 : dictionary_parts[2].len);
        entry.xpostag.assign(dictionary_parts[3].str, dictionary_parts[3].len == 1 && dictionary_parts[3].str[0] == '_' ? 0 : dictionary_parts[3].len);
        entry.feats.assign(dictionary_parts[4].str, dictionary_parts[4].len == 1 && dictionary_parts[4].str[0] == '_' ? 0 : dictionary_parts[4].len);

        entry_encoded.assign(combine_lemma(entry, use_lemma, combined_lemma, flat_lemmas))
            .append("\t").append(combine_tag(entry, use_xpostag, use_feats, combined_tag))
            .append("\t").append(entry.form);
        dictionary_entries.insert(entry_encoded);
      }
    }

    // Enrich the dictionary if required
    if (guesser_enrich_dictionary) {
      // Create temporary morphology using only the guesser
      stringstream empty_data, guesser_description_copy(guesser_description.str()), guesser_only_morphology;
      guesser_only_morphology.put(morphodita::morpho_ids::GENERIC);
      morphodita::generic_morpho_encoder::encode(empty_data, dictionary_suffix_len, dictionary_special_tags, guesser_description_copy, guesser_only_morphology);

      unique_ptr<morphodita::morpho> guesser_only_morpho(morphodita::morpho::load(guesser_only_morphology));
      if (!guesser_only_morpho) return error.assign("Cannot create temporary guesser-only morphology!"), false;

      string entry;
      unordered_set<string> analyzed_forms;
      vector<morphodita::tagged_lemma> analyses;
      for (auto&& sentence : training)
        for (size_t i = 1; i < sentence.words.size(); i++) {
          const auto& form = model_normalize_form(sentence.words[i].form, normalized_form);
          if (!analyzed_forms.count(form)) {
            guesser_only_morpho->analyze(form, morphodita::morpho::GUESSER, analyses);

            int to_add = guesser_enrich_dictionary;
            for (auto&& analyse : analyses) {
              entry.assign(analyse.lemma).push_back('\t');
              entry.append(analyse.tag).push_back('\t');
              entry.append(form);
              if (dictionary_entries.insert(entry).second)
                if (!--to_add)
                  break;
            }
            analyzed_forms.insert(form);
          }
        }
    }

    // Create the dictionary
    vector<string> sorted_dictionary(dictionary_entries.begin(), dictionary_entries.end());
    sort(sorted_dictionary.begin(), sorted_dictionary.end());

    stringstream morpho_input;
    for (auto&& entry : sorted_dictionary)
      morpho_input << entry << '\n';

    morpho_description.put(morphodita::morpho_ids::GENERIC);
    morphodita::generic_morpho_encoder::encode(morpho_input, dictionary_suffix_len, dictionary_special_tags, guesser_description, morpho_description);
  }

  // Measure dictionary accuracy if required
  const string& dictionary_accuracy = option_str(tagger, "dictionary_accuracy", model);
  if (!dictionary_accuracy.empty()) {
    unique_ptr<morphodita::morpho> morpho(morphodita::morpho::load(morpho_description));
    if (!morpho) return error.assign("Cannot create temporary morphology for evaluating accuracy!"), false;
    morpho_description.seekg(0, ios::beg);

    // Measure dictionary accuracy on given data
    unsigned words = 0, total_analyses = 0, upostag = 0, xpostag = 0, feats = 0, all_tags = 0, lemma = 0;

    word w;
    vector<morphodita::tagged_lemma> analyses;
    conllu_input_format->set_text(dictionary_accuracy.c_str());
    for (sentence sentence; conllu_input_format->next_sentence(sentence, error); )
      for (size_t i = 1; i < sentence.words.size(); i++) {
        morpho->analyze(model_normalize_form(sentence.words[i].form, normalized_form), morphodita::morpho::GUESSER, analyses);
        unsigned upostag_ok = 0, xpostag_ok = 0, feats_ok = 0, all_tags_ok = 0, lemma_ok = 0;
        for (auto&& analysis : analyses) {
          w.lemma.assign("_");
          model_fill_word_analysis(analysis, true, use_lemma, true, true, w);
          upostag_ok |= int(sentence.words[i].upostag == w.upostag);
          xpostag_ok |= int(sentence.words[i].xpostag == w.xpostag);
          feats_ok |= int(sentence.words[i].feats == w.feats);
          all_tags_ok |= int(sentence.words[i].upostag == w.upostag && sentence.words[i].xpostag == w.xpostag && sentence.words[i].feats == w.feats);
          lemma_ok |= int(sentence.words[i].lemma == w.lemma);
        }
        words++;
        total_analyses += analyses.size();
        upostag += upostag_ok;
        xpostag += xpostag_ok;
        feats += feats_ok;
        all_tags += all_tags_ok;
        lemma += lemma_ok;
      }
    if (!error.empty()) return false;

    cerr << "Dictionary accuracy for tagging model " << model+1 << " - forms: " << words
         << ", analyses per form: " << fixed << setprecision(2) << total_analyses / double(words)
         << ", upostag: " << setprecision(1) << 100. * upostag / words << "%, xpostag: " << 100. * xpostag / words
         << "%, feats: " << 100. * feats / words << "%, all tags: " << 100. * all_tags / words << "%, lemma: " << 100. * lemma / words << '%' << endl;
  }

  // Tagger options
  double tagger_order = 3; if (!option_double(tagger, "order", tagger_order, error, model)) return false;
  morphodita::tagger_id tagger_id;
  if (tagger_order == 2) tagger_id = morphodita::tagger_ids::CONLLU2;
  else if (tagger_order == 2.5) tagger_id = morphodita::tagger_ids::CONLLU2_3;
  else if (tagger_order == 3) tagger_id = morphodita::tagger_ids::CONLLU3;
  else return error.assign("The tagger_order can be only 2, 2.5 or 3!"), false;

  int tagger_iterations = 20; if (!option_int(tagger, "iterations", tagger_iterations, error, model)) return false;
  bool tagger_prune_features = false; if (!option_bool(tagger, "prune_features", tagger_prune_features, error, model)) return false;
  bool tagger_early_stopping = true; if (!option_bool(tagger, "early_stopping", tagger_early_stopping, error, model)) return false;
  const string& tagger_feature_templates =
      option_str(tagger, "templates", model) == "tagger" ? tagger_features_tagger :
      option_str(tagger, "templates", model) == "lemmatizer" ? tagger_features_lemmatizer :
      !option_str(tagger, "templates", model).empty() ? option_str(tagger, "templates", model) :
      model == 1 ? tagger_features_lemmatizer : tagger_features_tagger;
  if (heldout.empty()) tagger_early_stopping = false;

  cerr << "Tagger model " << model+1 << " options: iterations=" << tagger_iterations
       << ", early_stopping=" << (tagger_early_stopping ? 1 : 0) << ", templates="
       << (tagger_feature_templates == tagger_features_tagger ? "tagger" :
           tagger_feature_templates == tagger_features_lemmatizer ? "lemmatizer" : "custom") << endl;

  // Train the tagger
  cerr << "Training tagger model " << model+1 << "." << endl;
  stringstream input, heldout_input, feature_templates_input(tagger_feature_templates);
  for (auto&& sentence : training) {
    for (size_t i = 1; i < sentence.words.size(); i++)
      input << model_normalize_form(sentence.words[i].form, normalized_form) << '\t'
          << combine_lemma(sentence.words[i], use_lemma, combined_lemma) << '\t'
          << combine_tag(sentence.words[i], use_xpostag, use_feats, combined_tag) << '\n';
    input << '\n';
  }

  for (auto&& sentence : heldout) {
    for (size_t i = 1; i < sentence.words.size(); i++)
      heldout_input << model_normalize_form(sentence.words[i].form, normalized_form) << '\t'
          << combine_lemma(sentence.words[i], use_lemma, combined_lemma) << '\t'
          << combine_tag(sentence.words[i], use_xpostag, use_feats, combined_tag) << '\n';
    heldout_input << '\n';
  }

  os.put(tagger_id);
  morphodita::tagger_trainer<morphodita::perceptron_tagger_trainer<morphodita::train_feature_sequences<morphodita::conllu_elementary_features>>>::train(morphodita::tagger_ids::decoding_order(tagger_id), morphodita::tagger_ids::window_size(tagger_id), tagger_iterations, morpho_description, true, feature_templates_input, tagger_prune_features, input, heldout_input, tagger_early_stopping, os);

  return true;
}

bool trainer_morphodita_parsito::can_combine_tag(const word& w, string& error) {
  error.clear();

  unsigned separator = 0;
  while (separator < tag_separators.size() &&
         (w.upostag.find(tag_separators[separator]) != string::npos || w.xpostag.find(tag_separators[separator]) != string::npos))
    separator++;

  if (separator >= tag_separators.size()) {
    error.assign("Cannot find tag separating character, UPOSTAG and XPOSTAG contain all of '").append(tag_separators).append("'!");
    return false;
  }
  return true;
}

const string& trainer_morphodita_parsito::combine_tag(const word& w, bool xpostag, bool feats, string& combined_tag) {
  unsigned separator = 0;
  while (separator < tag_separators.size() &&
         (w.upostag.find(tag_separators[separator]) != string::npos || w.xpostag.find(tag_separators[separator]) != string::npos))
    separator++;
  if (separator >= tag_separators.size())
    // Should not happen, as can_combine_tag was called before
    separator = 0;

  combined_tag.assign(1, tag_separators[separator]);
  combined_tag.append(w.upostag);
  if (xpostag || feats) {
    combined_tag.push_back(tag_separators[separator]);
    if (xpostag) combined_tag.append(w.xpostag);
    if (feats) combined_tag.push_back(tag_separators[separator]);
    if (feats) combined_tag.append(w.feats);
  }

  return combined_tag;
}

const string& trainer_morphodita_parsito::most_frequent_tag(const vector<sentence>& data, const string& upostag, bool xpostag, bool feats, string& combined_tag) {
  unordered_map<string, unsigned> counts;

  for (auto&& sentence : data)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (sentence.words[i].upostag == upostag)
        counts[combine_tag(sentence.words[i], xpostag, feats, combined_tag)]++;

  combined_tag.assign("~").append(upostag);
  unsigned best = 0;
  for (auto&& tags : counts)
    if (tags.second > best) {
      best = tags.second;
      combined_tag.assign(tags.first);
    }
  return combined_tag;
}

const string& trainer_morphodita_parsito::combine_lemma(const word& w, int use_lemma, string& combined_lemma, const unordered_set<string>& flat_lemmas) {
  switch (use_lemma) {
    case 0:
      return model_normalize_form(w.form, combined_lemma);
    case 1:
      model_normalize_lemma(w.lemma, combined_lemma);
      if (flat_lemmas.count(w.lemma) || flat_lemmas.count(combined_lemma))
        return model_normalize_form(w.form, combined_lemma);
      return combined_lemma;
    default: /*2*/
      if (w.lemma == "")
        return model_normalize_form(w.form, combined_lemma), combined_lemma.insert(0, "~~");
      else if (w.lemma == "_")
        return model_normalize_form(w.form, combined_lemma), combined_lemma.insert(0, "~_~");

      model_normalize_lemma(w.lemma, combined_lemma);
      if (flat_lemmas.count(w.lemma) || flat_lemmas.count(combined_lemma)) {
        string normalized_form;
        model_normalize_form(w.form, normalized_form);
        return combined_lemma.insert(0, "~").append("~").append(normalized_form);
      }
      return combined_lemma;
  }
}

// Generic options handling

const string& trainer_morphodita_parsito::option_str(const named_values::map& options, const string& name, int model) {
  string indexed_name(name);
  if (model >= 0 && model < 9) indexed_name.append("_").push_back('1' + model);

  return options.count(indexed_name) ? options.at(indexed_name) : options.count(name) ? options.at(name) : empty_string;
}

bool trainer_morphodita_parsito::option_int(const named_values::map& options, const string& name, int& value, string& error, int model) {
  string indexed_name(name);
  if (model >= 0 && model < 9) indexed_name.append("_").push_back('1' + model);

  if (options.count(indexed_name))
    return parse_int(options.at(indexed_name), name.c_str(), value, error);
  if (options.count(name))
    return parse_int(options.at(name), name.c_str(), value, error);
  return true;
}

bool trainer_morphodita_parsito::option_bool(const named_values::map& options, const string& name, bool& value, string& error, int model) {
  string indexed_name(name);
  if (model >= 0 && model < 9) indexed_name.append("_").push_back('1' + model);

  if (options.count(indexed_name) || options.count(name)) {
    int int_value;
    if (!parse_int(options.count(indexed_name) ? options.at(indexed_name) : options.at(name), name.c_str(), int_value, error))
      return false;
    value = int_value != 0;
  }
  return true;
}

bool trainer_morphodita_parsito::option_double(const named_values::map& options, const string& name, double& value, string& error, int model) {
  string indexed_name(name);
  if (model >= 0 && model < 9) indexed_name.append("_").push_back('1' + model);

  if (options.count(indexed_name))
    return parse_double(options.at(indexed_name), name.c_str(), value, error);
  if (options.count(name))
    return parse_double(options.at(name), name.c_str(), value, error);
  return true;
}

// Various string data

const string trainer_morphodita_parsito::empty_string;

const string trainer_morphodita_parsito::tag_separators = "~!@#$%^&*()/";

const string trainer_morphodita_parsito::tagger_features_tagger =
  "Tag 0\n"
  "Tag 0,Tag -1\n"
  "Tag 0,TagUPos -1\n"
  "Tag 0,Tag -1,Tag -2\n"
  "Tag 0,TagUPos -1,TagUPos -2\n"
  "Tag 0,Tag -2\n"
  "Tag 0,Form 0\n"
  "Tag 0,Form 0,Form -1\n"
  "Tag 0,Form -1\n"
  "Tag 0,Form -2\n"
  "Tag 0,Form -1,Form -2\n"
  "Tag 0,Form 1\n"
  "Tag 0,Form 1,Form 2\n"
  "Tag 0,PreviousVerbTag 0\n"
  "Tag 0,PreviousVerbForm 0\n"
  "Tag 0,FollowingVerbTag 0\n"
  "Tag 0,FollowingVerbForm 0\n"
  "Tag 0,Lemma -1\n"
  "Tag 0,Form 1\n"
  "Lemma 0,Tag -1\n"
  "Tag 0,Prefix1 0\n"
  "Tag 0,Prefix2 0\n"
  "Tag 0,Prefix3 0\n"
  "Tag 0,Prefix4 0\n"
  "Tag 0,Prefix5 0\n"
  "Tag 0,Prefix6 0\n"
  "Tag 0,Prefix7 0\n"
  "Tag 0,Prefix8 0\n"
  "Tag 0,Prefix9 0\n"
  "Tag 0,Suffix1 0\n"
  "Tag 0,Suffix2 0\n"
  "Tag 0,Suffix3 0\n"
  "Tag 0,Suffix4 0\n"
  "Tag 0,Suffix5 0\n"
  "Tag 0,Suffix6 0\n"
  "Tag 0,Suffix7 0\n"
  "Tag 0,Suffix8 0\n"
  "Tag 0,Suffix9 0\n"
  "TagUPos 0\n"
  "TagUPos 0,TagUPos -1\n"
  "TagUPos 0,TagUPos -1,TagUPos -2\n"
  "TagCase 0,TagCase -1\n"
  "TagCase 0,TagCase -1,TagCase -2\n"
  "TagGender 0,TagGender -1\n"
  "TagGender 0,TagGender -1,TagGender -2\n"
  "TagUPos 0,Prefix1 0\n"
  "TagUPos 0,Prefix2 0\n"
  "TagUPos 0,Prefix3 0\n"
  "TagUPos 0,Prefix4 0\n"
  "TagUPos 0,Prefix5 0\n"
  "TagUPos 0,Prefix6 0\n"
  "TagUPos 0,Prefix7 0\n"
  "TagUPos 0,Prefix8 0\n"
  "TagUPos 0,Prefix9 0\n"
  "TagUPos 0,Suffix1 0\n"
  "TagUPos 0,Suffix2 0\n"
  "TagUPos 0,Suffix3 0\n"
  "TagUPos 0,Suffix4 0\n"
  "TagUPos 0,Suffix5 0\n"
  "TagUPos 0,Suffix6 0\n"
  "TagUPos 0,Suffix7 0\n"
  "TagUPos 0,Suffix8 0\n"
  "TagUPos 0,Suffix9 0\n"
  "Tag 0,Num 0\n"
  "Tag 0,Cap 0\n"
  "Tag 0,Dash 0\n"
  "TagNegative 0,Prefix1 0\n"
  "TagNegative 0,Prefix2 0\n"
  "TagNegative 0,Prefix3 0\n"
  "TagCase 0,Suffix1 0\n"
  "TagCase 0,Suffix2 0\n"
  "TagCase 0,Suffix3 0\n"
  "TagCase 0,Suffix4 0\n"
  "TagCase 0,Suffix5 0\n";

const string trainer_morphodita_parsito::tagger_features_lemmatizer =
  "Tag 0\n"
  "Tag 0,Tag -1\n"
  "Tag 0,Tag -1,Tag -2\n"
  "Tag 0,Tag -2\n"
  "Tag 0,Form 0\n"
  "Tag 0,Form 0,Form -1\n"
  "Tag 0,Form -1\n"
  "Tag 0,Form -2\n"
  "Tag 0,PreviousVerbTag 0\n"
  "Tag 0,PreviousVerbForm 0\n"
  "Tag 0,FollowingVerbTag 0\n"
  "Tag 0,FollowingVerbForm 0\n"
  "Tag 0,Lemma -1\n"
  "Tag 0,Form 1\n"
  "Lemma 0\n"
  "Lemma 0,Tag -1\n"
  "Lemma 0,Tag -1,Tag -2\n"
  "Lemma 0,Tag -2\n"
  "Lemma 0,Form -1\n"
  "Lemma 0,Form -1,Form -2\n"
  "Lemma 0,Form -2\n"
  "Lemma 0,PreviousVerbTag 0\n"
  "Lemma 0,PreviousVerbForm 0\n"
  "Lemma 0,FollowingVerbTag 0\n"
  "Lemma 0,FollowingVerbForm 0\n"
  "Lemma 0,Form 1\n"
  "Tag 0,Prefix1 0\n"
  "Tag 0,Prefix2 0\n"
  "Tag 0,Prefix3 0\n"
  "Tag 0,Prefix4 0\n"
  "Tag 0,Prefix5 0\n"
  "Tag 0,Suffix1 0\n"
  "Tag 0,Suffix2 0\n"
  "Tag 0,Suffix3 0\n"
  "Tag 0,Suffix4 0\n"
  "Tag 0,Suffix5 0\n"
  "Tag 0,Num 0\n"
  "Tag 0,Cap 0\n"
  "Tag 0,Dash 0\n";

const string trainer_morphodita_parsito::parser_nodes =
  "stack 0\n"
  "stack 1\n"
  "stack 2\n"
  "buffer 0\n"
  "buffer 1\n"
  "buffer 2\n"
  "stack 0,child 0\n"
  "stack 0,child 1\n"
  "stack 0,child -2\n"
  "stack 0,child -1\n"
  "stack 1,child 0\n"
  "stack 1,child 1\n"
  "stack 1,child -2\n"
  "stack 1,child -1\n"
  "stack 0,child 0,child 0\n"
  "stack 0,child -1,child -1\n"
  "stack 1,child 0,child 0\n"
  "stack 1,child -1,child -1\n";

/////////
// File: trainer/training_failure.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

training_error::training_error() : runtime_error(message_collector.str()) {
  message_collector.str(string());
}

ostringstream training_error::message_collector;

/////////
// File: unilib/unicode.cpp
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

const char32_t unicode::CHARS;

const int32_t unicode::DEFAULT_CAT;

const uint8_t unicode::category_index[unicode::CHARS >> 8] = {
  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,17,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,33,41,42,43,44,45,46,47,48,39,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,49,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,50,51,17,17,17,52,17,53,54,55,56,57,58,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,59,60,60,60,60,60,60,60,60,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,17,62,63,17,64,65,66,67,68,69,70,71,72,17,73,74,75,76,77,78,79,80,79,81,82,83,84,85,86,87,88,89,79,90,79,79,79,79,79,17,17,17,91,92,93,79,79,79,79,79,79,79,79,79,79,17,17,17,17,94,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,17,17,95,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,17,17,96,97,79,79,79,98,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,99,79,79,79,79,79,79,79,79,79,79,79,100,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,101,102,103,104,105,106,107,108,39,39,109,79,79,79,79,79,79,79,79,79,79,79,79,79,110,79,79,79,79,79,111,79,112,113,114,115,39,116,117,118,119,120,79,79,79,79,79,79,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
    17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,121,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,122,123,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,124,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,17,17,125,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,
    79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,
    79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,
    79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,
    79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,126,127,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,79,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,
    61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,128,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,128
};

const uint8_t unicode::category_block[][256] = {
  {_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Zs,_Po,_Po,_Po,_Sc,_Po,_Po,_Po,_Ps,_Pe,_Po,_Sm,_Po,_Pd,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Sm,_Sm,_Sm,_Po,_Po,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ps,_Po,_Pe,_Sk,_Pc,_Sk,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ps,_Sm,_Pe,_Sm,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Cc,_Zs,_Po,_Sc,_Sc,_Sc,_Sc,_So,_Po,_Sk,_So,_Lo,_Pi,_Sm,_Cf,_So,_Sk,_So,_Sm,_No,_No,_Sk,_Ll,_Po,_Po,_Sk,_No,_Lo,_Pf,_No,_No,_No,_Po,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll},
  {_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Lu,_Lu,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Lu,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Ll,_Lo,_Lu,_Ll,_Ll,_Ll,_Lo,_Lo,_Lo,_Lo,_Lu,_Lt,_Ll,_Lu,_Lt,_Ll,_Lu,_Lt,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Lt,_Ll,_Lu,_Ll,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll},
  {_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Ll,_Lu,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lo,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Sk,_Sk,_Sk,_Sk,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Lm,_Lm,_Lm,_Lm,_Lm,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Lm,_Sk,_Lm,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk},
  {_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lu,_Ll,_Lu,_Ll,_Lm,_Sk,_Lu,_Ll,_Cn,_Cn,_Lm,_Ll,_Ll,_Ll,_Po,_Lu,_Cn,_Cn,_Cn,_Cn,_Sk,_Sk,_Lu,_Po,_Lu,_Lu,_Lu,_Cn,_Lu,_Cn,_Lu,_Lu,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Ll,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Sm,_Lu,_Ll,_Lu,_Lu,_Ll,_Ll,_Lu,_Lu,_Lu},
  {_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Me,_Me,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll},
  {_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Lm,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Po,_Pd,_Cn,_Cn,_So,_So,_Sc,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Pd,_Mn,_Po,_Mn,_Mn,_Po,_Mn,_Mn,_Po,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Sm,_Sm,_Sm,_Po,_Po,_Sc,_Po,_Po,_So,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Cf,_Cn,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Po,_Po,_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cf,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lm,_Lm,_Mn,_Mn,_So,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_So,_So,_Lo},
  {_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cf,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lm,_Lm,_So,_Po,_Po,_Po,_Lm,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Lm,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lm,_Mn,_Mn,_Mn,_Lm,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Cn,_Cn,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn},
  {_Mn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mn,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Mn,_Mc,_Mc,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Mc,_Mc,_Cn,_Cn,_Mc,_Mc,_Mn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Sc,_Sc,_No,_No,_No,_No,_No,_No,_So,_Sc,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Mn,_Mn,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Cn,_Mn,_Cn,_Mc,_Mc,_Mc,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Cn,_Cn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Mn,_Mn,_Lo,_Lo,_Lo,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Mc,_Cn,_Mc,_Mc,_Mn,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Sc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Mn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Lo,_Mc,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Mc,_Mc,_Cn,_Cn,_Mc,_Mc,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mc,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_So,_Lo,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Mc,_Mc,_Mn,_Mc,_Mc,_Cn,_Cn,_Cn,_Mc,_Mc,_Mc,_Cn,_Mc,_Mc,_Mc,_Mn,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_No,_No,_So,_So,_So,_So,_So,_So,_Sc,_So,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mn,_Mc,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Mn,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Cn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Cn,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_So,_Cn,_Mn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Lo,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Cn,_Mn,_Mc,_Mc,_Cn,_Mc,_Mc,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Cn,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Mn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Cn,_Mc,_Mc,_Mc,_Cn,_Mc,_Mc,_Mc,_Mn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_So,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Mn,_Cn,_Cn,_Cn,_Cn,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Cn,_Mn,_Cn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Mc,_Mc,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Sc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lm,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_So,_So,_So,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_So,_Po,_So,_So,_So,_Mn,_Mn,_So,_So,_So,_So,_So,_So,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_Mn,_So,_Mn,_So,_Mn,_Ps,_Pe,_Ps,_Pe,_Mc,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_Po,_Po,_Po,_Po,_Po,_So,_So,_So,_So,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Po,_Po,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Lo,_Mc,_Mc,_Mc,_Lo,_Lo,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mc,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Lo,_Mc,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Mc,_Mc,_Mc,_Mn,_So,_So,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Cn,_Cn,_Cn,_Cn,_Cn,_Lu,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Lm,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn},
  {_Pd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Zs,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Ps,_Pe,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Po,_Po,_Nl,_Nl,_Nl,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Po,_Po,_Lm,_Po,_Po,_Po,_Sc,_Lo,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Po,_Po,_Po,_Po,_Po,_Po,_Pd,_Po,_Po,_Po,_Po,_Mn,_Mn,_Mn,_Cf,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mc,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Mc,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_So,_Cn,_Cn,_Cn,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mc,_Mc,_Mn,_Cn,_Cn,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mc,_Mn,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Mn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Lm,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Me,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mn,_Mn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Mc,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Lo,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mn,_Mn,_Mc,_Mc,_Mc,_Mn,_Mc,_Mn,_Mn,_Mn,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Po,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mn,_Lo,_Lo,_Cn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn},
  {_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Lu,_Cn,_Lu,_Cn,_Lu,_Cn,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Lt,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lt,_Sk,_Ll,_Sk,_Sk,_Sk,_Ll,_Ll,_Ll,_Cn,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lt,_Sk,_Sk,_Sk,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Cn,_Sk,_Sk,_Sk,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Sk,_Sk,_Sk,_Cn,_Cn,_Ll,_Ll,_Ll,_Cn,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lt,_Sk,_Sk,_Cn},
  {_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Zs,_Cf,_Cf,_Cf,_Cf,_Cf,_Pd,_Pd,_Pd,_Pd,_Pd,_Pd,_Po,_Po,_Pi,_Pf,_Ps,_Pi,_Pi,_Pf,_Ps,_Pi,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Zl,_Zp,_Cf,_Cf,_Cf,_Cf,_Cf,_Zs,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Pi,_Pf,_Po,_Po,_Po,_Po,_Pc,_Pc,_Po,_Po,_Po,_Sm,_Ps,_Pe,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Sm,_Po,_Pc,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Zs,_Cf,_Cf,_Cf,_Cf,_Cf,_Cn,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_No,_Lm,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_Sm,_Sm,_Sm,_Ps,_Pe,_Lm,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Sm,_Sm,_Sm,_Ps,_Pe,_Cn,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Cn,_Cn,_Cn,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Sc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Me,_Me,_Me,_Me,_Mn,_Me,_Me,_Me,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_Lu,_So,_So,_So,_So,_Lu,_So,_So,_Ll,_Lu,_Lu,_Lu,_Ll,_Ll,_Lu,_Lu,_Lu,_Ll,_So,_Lu,_So,_So,_Sm,_Lu,_Lu,_Lu,_Lu,_Lu,_So,_So,_So,_So,_So,_So,_Lu,_So,_Lu,_So,_Lu,_So,_Lu,_Lu,_Lu,_Lu,_So,_Ll,_Lu,_Lu,_Lu,_Lu,_Ll,_Lo,_Lo,_Lo,_Lo,_Ll,_So,_So,_Ll,_Ll,_Lu,_Lu,_Sm,_Sm,_Sm,_Sm,_Sm,_Lu,_Ll,_Ll,_Ll,_Ll,_So,_Sm,_So,_So,_Ll,_So,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Lu,_Ll,_Nl,_Nl,_Nl,_Nl,_No,_So,_So,_Cn,_Cn,_Cn,_Cn,_Sm,_Sm,_Sm,_Sm,_Sm,_So,_So,_So,_So,_So,_Sm,_Sm,_So,_So,_So,_So,_Sm,_So,_So,_Sm,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_So,_So,_Sm,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm},
  {_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm},
  {_So,_So,_So,_So,_So,_So,_So,_So,_Ps,_Pe,_Ps,_Pe,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_So,_So,_So,_So,_So,_So,_So,_Ps,_Pe,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Ps,_Pe,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Ps,_Pe,_Ps,_Pe,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Ps,_Pe,_Sm,_Sm},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_So,_So,_Sm,_Sm,_Sm,_Sm,_Sm,_Sm,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Lu,_Ll,_Lu,_Lu,_Lu,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Ll,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lm,_Lm,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_So,_So,_So,_So,_So,_So,_Lu,_Ll,_Lu,_Ll,_Mn,_Mn,_Mn,_Lu,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_No,_Po,_Po},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Ll,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lm,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn},
  {_Po,_Po,_Pi,_Pf,_Pi,_Pf,_Po,_Po,_Po,_Pi,_Pf,_Po,_Pi,_Pf,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Pd,_Po,_Po,_Pd,_Po,_Pi,_Pf,_Po,_Po,_Pi,_Pf,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Po,_Po,_Po,_Po,_Po,_Lm,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Pd,_Pd,_Po,_Po,_Po,_Po,_Pd,_Po,_Ps,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn},
  {_Zs,_Po,_Po,_Po,_So,_Lm,_Lo,_Nl,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_So,_So,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Pd,_Ps,_Pe,_Pe,_So,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Pd,_Lm,_Lm,_Lm,_Lm,_Lm,_So,_So,_Nl,_Nl,_Nl,_Lm,_Lo,_Po,_So,_So,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Mn,_Sk,_Sk,_Lm,_Lm,_Lo,_Pd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Lm,_Lm,_Lm,_Lo},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_So,_So,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_No,_No,_No,_No,_No,_No,_No,_No,_So,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Po,_Po},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Po,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lo,_Mn,_Me,_Me,_Me,_Po,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Lm,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lm,_Lm,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Sk,_Sk,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lm,_Sk,_Sk,_Lu,_Ll,_Lu,_Ll,_Lo,_Lu,_Ll,_Lu,_Ll,_Ll,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Ll,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Lu,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lm,_Lm,_Ll,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mn,_Mn,_Mc,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_So,_So,_Sc,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Po,_Po,_Lo,_Po,_Lo,_Cn,_Cn},
  {_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Lm,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Po,_Po,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_So,_So,_So,_Lo,_Mc,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Lo,_Mn,_Mn,_Mn,_Lo,_Lo,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Lo,_Mn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lm,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mn,_Mn,_Mc,_Mc,_Po,_Po,_Lo,_Lm,_Lm,_Mc,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sk,_Lm,_Lm,_Lm,_Lm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mn,_Mc,_Mc,_Mn,_Mc,_Mc,_Po,_Mc,_Mn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn},
  {_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs,_Cs},
  {_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Sm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Sk,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Pe,_Ps,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Sc,_So,_Cn,_Cn},
  {_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Ps,_Pe,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Pd,_Pd,_Pc,_Pc,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Po,_Po,_Ps,_Pe,_Po,_Po,_Po,_Po,_Pc,_Pc,_Pc,_Po,_Po,_Po,_Cn,_Po,_Po,_Po,_Po,_Pd,_Ps,_Pe,_Ps,_Pe,_Ps,_Pe,_Po,_Po,_Po,_Sm,_Pd,_Sm,_Sm,_Sm,_Cn,_Po,_Sc,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cf},
  {_Cn,_Po,_Po,_Po,_Sc,_Po,_Po,_Po,_Ps,_Pe,_Po,_Sm,_Po,_Pd,_Po,_Po,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Sm,_Sm,_Sm,_Po,_Po,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ps,_Po,_Pe,_Sk,_Pc,_Sk,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ps,_Sm,_Pe,_Sm,_Ps,_Pe,_Po,_Ps,_Pe,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lm,_Lm,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Sc,_Sc,_Sm,_Sk,_So,_Sc,_Sc,_Cn,_So,_Sm,_Sm,_Sm,_Sm,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cf,_Cf,_Cf,_So,_So,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_No,_No,_No,_No,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_No,_No,_So,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Nl,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Nl,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Nl,_Nl,_Nl,_Nl,_Nl,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Po,_No,_No,_No,_No,_No,_No,_No,_No,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_So,_So,_No,_No,_No,_No,_No,_No,_No,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_No,_No,_Lo,_Lo,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No},
  {_Lo,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Mn,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_No,_No,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_So,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mc,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Po,_Po,_Cf,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mn,_Mn,_Mn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Po,_Po,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mc,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Lo,_Lo,_Lo,_Lo,_Po,_Po,_Po,_Po,_Po,_Mn,_Mn,_Mn,_Po,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Lo,_Po,_Lo,_Po,_Po,_Po,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mc,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mn,_Mn,_Mc,_Mc,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Lo,_Mc,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Cn,_Cn,_Mc,_Mc,_Cn,_Cn,_Mc,_Mc,_Mc,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mc,_Mn,_Mn,_Lo,_Lo,_Po,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Mc,_Mc,_Mc,_Mc,_Mn,_Mn,_Mc,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Po,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mc,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mc,_Mn,_Mn,_Po,_Po,_Po,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mc,_Mn,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mc,_Mc,_Mn,_Mn,_Mn,_Mn,_Mc,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_No,_Po,_Po,_Po,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Nl,_Cn,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_Cn,_Cn,_Cn,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Po,_Po,_Po,_Po,_Po,_So,_So,_So,_So,_Lm,_Lm,_Lm,_Lm,_Po,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Cn,_No,_No,_No,_No,_No,_No,_No,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Lm,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_So,_Mn,_Mn,_Po,_Cf,_Cf,_Cf,_Cf,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mc,_Mc,_Mn,_Mn,_Mn,_So,_So,_So,_Mc,_Mc,_Mc,_Mc,_Mc,_Mc,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_So,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_Mn,_Mn,_Mn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_Mn,_Mn,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Cn,_Lu,_Lu,_Cn,_Cn,_Lu,_Cn,_Cn,_Lu,_Lu,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Cn,_Ll,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll},
  {_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Lu,_Cn,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Cn,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Cn,_Cn,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Ll,_Ll,_Ll,_Ll},
  {_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Lu,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Sm,_Ll,_Ll,_Ll,_Ll,_Ll,_Ll,_Lu,_Ll,_Cn,_Cn,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd,_Nd},
  {_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_So,_So,_So,_So,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Mn,_So,_So,_Po,_Po,_Po,_Po,_Po,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Cn,_Cn,_Cn,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Cn,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Sm,_Sm,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_No,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Sk,_Sk,_Sk,_Sk,_Sk},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_So,_So,_So,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_So,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Lo,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Cn,_Cf,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cf,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Mn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn,_Cn},
  {_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Co,_Cn,_Cn}
};

const uint8_t unicode::othercase_index[unicode::CHARS >> 8] = {
  0,1,2,3,4,5,6,6,6,6,6,6,6,6,6,6,7,6,6,8,6,6,6,6,6,6,6,6,6,9,10,11,6,12,6,6,13,6,6,6,6,6,6,6,14,15,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,16,17,6,6,6,18,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,19,6,6,6,6,20,6,6,6,6,6,6,6,21,6,6,6,6,6,6,6,6,6,6,6,22,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
    6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
    6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
    6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
    6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6
};

const char32_t unicode::othercase_block[][256] = {
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24833,25089,25345,25601,25857,26113,26369,26625,26881,27137,27393,27649,27905,28161,28417,28673,28929,29185,29441,29697,29953,30209,30465,30721,30977,31233,0,0,0,0,0,0,16642,16898,17154,17410,17666,17922,18178,18434,18690,18946,19202,19458,19714,19970,20226,20482,20738,20994,21250,21506,21762,22018,22274,22530,22786,23042,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,236546,0,0,0,0,0,0,0,0,0,0,57345,57601,57857,58113,58369,58625,58881,59137,59393,59649,59905,60161,60417,60673,60929,61185,61441,61697,61953,62209,62465,62721,62977,0,63489,63745,64001,64257,64513,64769,65025,0,49154,49410,49666,49922,50178,50434,50690,50946,51202,51458,51714,51970,52226,52482,52738,52994,53250,53506,53762,54018,54274,54530,54786,0,55298,55554,55810,56066,56322,56578,56834,96258},
  {65793,65538,66305,66050,66817,66562,67329,67074,67841,67586,68353,68098,68865,68610,69377,69122,69889,69634,70401,70146,70913,70658,71425,71170,71937,71682,72449,72194,72961,72706,73473,73218,73985,73730,74497,74242,75009,74754,75521,75266,76033,75778,76545,76290,77057,76802,77569,77314,26881,18690,78593,78338,79105,78850,79617,79362,0,80385,80130,80897,80642,81409,81154,81921,81666,82433,82178,82945,82690,83457,83202,83969,83714,0,84737,84482,85249,84994,85761,85506,86273,86018,86785,86530,87297,87042,87809,87554,88321,88066,88833,88578,89345,89090,89857,89602,90369,90114,90881,90626,91393,91138,91905,91650,92417,92162,92929,92674,93441,93186,93953,93698,94465,94210,94977,94722,95489,95234,96001,95746,65281,96769,96514,97281,97026,97793,97538,21250,148226,152321,99073,98818,99585,99330,152577,100353,100098,153089,153345,101377,101122,0,122113,153857,154369,102913,102658,155649,156417,128514,157953,157697,104705,104450,146690,0,159489,160257,139266,161025,106753,106498,107265,107010,107777,107522,163841,108545,108290,164609,0,0,109825,109570,165889,110593,110338,166401,166657,111617,111362,112129,111874,168449,112897,112642,0,0,113921,113666,0,128770,0,0,0,0,115973,116227,115716,116741,116995,116484,117509,117763,117252,118273,118018,118785,118530,119297,119042,119809,119554,120321,120066,120833,120578,121345,121090,121857,121602,101890,122625,122370,123137,122882,123649,123394,124161,123906,124673,124418,125185,124930,125697,125442,126209,125954,126721,126466,0,127493,127747,127236,128257,128002,103681,114433,129281,129026,129793,129538,130305,130050,130817,130562},
  {131329,131074,131841,131586,132353,132098,132865,132610,133377,133122,133889,133634,134401,134146,134913,134658,135425,135170,135937,135682,136449,136194,136961,136706,137473,137218,137985,137730,138497,138242,139009,138754,105985,0,140033,139778,140545,140290,141057,140802,141569,141314,142081,141826,142593,142338,143105,142850,143617,143362,144129,143874,0,0,0,0,0,0,2909441,146433,146178,104961,2909697,2915842,2916098,147969,147714,98305,166145,166913,149249,148994,149761,149506,150273,150018,150785,150530,151297,151042,2912002,2911490,2912258,98562,99842,0,100610,100866,0,102146,0,102402,10988290,0,0,0,103170,10988546,0,103426,0,10980610,10988034,0,104194,103938,0,2908674,10988802,0,0,105474,0,2911746,105730,0,0,106242,0,0,0,0,0,0,0,2909186,0,0,108034,0,0,108802,0,0,0,10989826,110082,148482,110850,111106,148738,0,0,0,0,0,112386,0,0,0,0,0,0,0,0,0,0,10990082,10989570,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,235778,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,225537,225282,226049,225794,0,0,227073,226818,0,0,0,261378,261634,261890,0,258817,0,0,0,0,0,0,240641,0,240897,241153,241409,0,248833,0,249089,249345,0,241921,242177,242433,242689,242945,243201,243457,243713,243969,244225,244481,244737,244993,245249,245505,245761,246017,0,246529,246785,247041,247297,247553,247809,248065,248321,248577,230914,231426,231682,231938,0,233730,233986,234242,234498,234754,235010,235266,235522,235778,236034,236290,236546,236802,237058,237314,237570,237826,238338,238338,238594,238850,239106,239362,239618,239874,240130,240386,232450,232962,233218,251649,233986,235522,0,0,0,239106,237570,249602,252161,251906,252673,252418,253185,252930,253697,253442,254209,253954,254721,254466,255233,254978,255745,255490,256257,256002,256769,256514,257281,257026,257793,257538,236034,237826,260354,229122,243713,234754,0,260097,259842,258561,260865,260610,0,228097,228353,228609},
  {282625,282881,283137,283393,283649,283905,284161,284417,284673,284929,285185,285441,285697,285953,286209,286465,274433,274689,274945,275201,275457,275713,275969,276225,276481,276737,276993,277249,277505,277761,278017,278273,278529,278785,279041,279297,279553,279809,280065,280321,280577,280833,281089,281345,281601,281857,282113,282369,266242,266498,266754,267010,267266,267522,267778,268034,268290,268546,268802,269058,269314,269570,269826,270082,270338,270594,270850,271106,271362,271618,271874,272130,272386,272642,272898,273154,273410,273666,273922,274178,262146,262402,262658,262914,263170,263426,263682,263938,264194,264450,264706,264962,265218,265474,265730,265986,286977,286722,287489,287234,288001,287746,288513,288258,289025,288770,289537,289282,290049,289794,290561,290306,291073,290818,291585,291330,292097,291842,292609,292354,293121,292866,293633,293378,294145,293890,294657,294402,295169,294914,0,0,0,0,0,0,0,0,297729,297474,298241,297986,298753,298498,299265,299010,299777,299522,300289,300034,300801,300546,301313,301058,301825,301570,302337,302082,302849,302594,303361,303106,303873,303618,304385,304130,304897,304642,305409,305154,305921,305666,306433,306178,306945,306690,307457,307202,307969,307714,308481,308226,308993,308738,309505,309250,310017,309762,310529,310274,311041,310786,315137,311809,311554,312321,312066,312833,312578,313345,313090,313857,313602,314369,314114,314881,314626,311298,315649,315394,316161,315906,316673,316418,317185,316930,317697,317442,318209,317954,318721,318466,319233,318978,319745,319490,320257,320002,320769,320514,321281,321026,321793,321538,322305,322050,322817,322562,323329,323074,323841,323586,324353,324098,324865,324610,325377,325122,325889,325634,326401,326146,326913,326658,327425,327170},
  {327937,327682,328449,328194,328961,328706,329473,329218,329985,329730,330497,330242,331009,330754,331521,331266,332033,331778,332545,332290,333057,332802,333569,333314,334081,333826,334593,334338,335105,334850,335617,335362,336129,335874,336641,336386,337153,336898,337665,337410,338177,337922,338689,338434,339201,338946,339713,339458,0,352513,352769,353025,353281,353537,353793,354049,354305,354561,354817,355073,355329,355585,355841,356097,356353,356609,356865,357121,357377,357633,357889,358145,358401,358657,358913,359169,359425,359681,359937,360193,360449,360705,360961,361217,361473,361729,361985,0,0,0,0,0,0,0,0,0,0,340226,340482,340738,340994,341250,341506,341762,342018,342274,342530,342786,343042,343298,343554,343810,344066,344322,344578,344834,345090,345346,345602,345858,346114,346370,346626,346882,347138,347394,347650,347906,348162,348418,348674,348930,349186,349442,349698,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2949121,2949377,2949633,2949889,2950145,2950401,2950657,2950913,2951169,2951425,2951681,2951937,2952193,2952449,2952705,2952961,2953217,2953473,2953729,2953985,2954241,2954497,2954753,2955009,2955265,2955521,2955777,2956033,2956289,2956545,2956801,2957057,2957313,2957569,2957825,2958081,2958337,2958593,0,2959105,0,0,0,0,0,2960641,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11235329,11235585,11235841,11236097,11236353,11236609,11236865,11237121,11237377,11237633,11237889,11238145,11238401,11238657,11238913,11239169,11239425,11239681,11239937,11240193,11240449,11240705,11240961,11241217,11241473,11241729,11241985,11242241,11242497,11242753,11243009,11243265,11243521,11243777,11244033,11244289,11244545,11244801,11245057,11245313,11245569,11245825,11246081,11246337,11246593,11246849,11247105,11247361,11247617,11247873,11248129,11248385,11248641,11248897,11249153,11249409,11249665,11249921,11250177,11250433,11250689,11250945,11251201,11251457,11251713,11251969,11252225,11252481,11252737,11252993,11253249,11253505,11253761,11254017,11254273,11254529,11254785,11255041,11255297,11255553,1308673,1308929,1309185,1309441,1309697,1309953,0,0,1306626,1306882,1307138,1307394,1307650,1307906,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10976514,0,0,0,2908930,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {1966337,1966082,1966849,1966594,1967361,1967106,1967873,1967618,1968385,1968130,1968897,1968642,1969409,1969154,1969921,1969666,1970433,1970178,1970945,1970690,1971457,1971202,1971969,1971714,1972481,1972226,1972993,1972738,1973505,1973250,1974017,1973762,1974529,1974274,1975041,1974786,1975553,1975298,1976065,1975810,1976577,1976322,1977089,1976834,1977601,1977346,1978113,1977858,1978625,1978370,1979137,1978882,1979649,1979394,1980161,1979906,1980673,1980418,1981185,1980930,1981697,1981442,1982209,1981954,1982721,1982466,1983233,1982978,1983745,1983490,1984257,1984002,1984769,1984514,1985281,1985026,1985793,1985538,1986305,1986050,1986817,1986562,1987329,1987074,1987841,1987586,1988353,1988098,1988865,1988610,1989377,1989122,1989889,1989634,1990401,1990146,1990913,1990658,1991425,1991170,1991937,1991682,1992449,1992194,1992961,1992706,1993473,1993218,1993985,1993730,1994497,1994242,1995009,1994754,1995521,1995266,1996033,1995778,1996545,1996290,1997057,1996802,1997569,1997314,1998081,1997826,1998593,1998338,1999105,1998850,1999617,1999362,2000129,1999874,2000641,2000386,2001153,2000898,2001665,2001410,2002177,2001922,2002689,2002434,2003201,2002946,2003713,2003458,2004225,2003970,0,0,0,0,0,1990658,0,0,57089,0,2007297,2007042,2007809,2007554,2008321,2008066,2008833,2008578,2009345,2009090,2009857,2009602,2010369,2010114,2010881,2010626,2011393,2011138,2011905,2011650,2012417,2012162,2012929,2012674,2013441,2013186,2013953,2013698,2014465,2014210,2014977,2014722,2015489,2015234,2016001,2015746,2016513,2016258,2017025,2016770,2017537,2017282,2018049,2017794,2018561,2018306,2019073,2018818,2019585,2019330,2020097,2019842,2020609,2020354,2021121,2020866,2021633,2021378,2022145,2021890,2022657,2022402,2023169,2022914,2023681,2023426,2024193,2023938,2024705,2024450,2025217,2024962,2025729,2025474,2026241,2025986,2026753,2026498,2027265,2027010,2027777,2027522,2028289,2028034,2028801,2028546,2029313,2029058,2029825,2029570,2030337,2030082,2030849,2030594,2031361,
    2031106},
  {2033666,2033922,2034178,2034434,2034690,2034946,2035202,2035458,2031617,2031873,2032129,2032385,2032641,2032897,2033153,2033409,2037762,2038018,2038274,2038530,2038786,2039042,0,0,2035713,2035969,2036225,2036481,2036737,2036993,0,0,2041858,2042114,2042370,2042626,2042882,2043138,2043394,2043650,2039809,2040065,2040321,2040577,2040833,2041089,2041345,2041601,2045954,2046210,2046466,2046722,2046978,2047234,2047490,2047746,2043905,2044161,2044417,2044673,2044929,2045185,2045441,2045697,2050050,2050306,2050562,2050818,2051074,2051330,0,0,2048001,2048257,2048513,2048769,2049025,2049281,0,0,0,2054402,0,2054914,0,2055426,0,2055938,0,2052353,0,2052865,0,2053377,0,2053889,2058242,2058498,2058754,2059010,2059266,2059522,2059778,2060034,2056193,2056449,2056705,2056961,2057217,2057473,2057729,2057985,2079234,2079490,2082818,2083074,2083330,2083586,2087426,2087682,2095106,2095362,2091522,2091778,2095618,2095874,0,0,2066434,2066690,2066946,2067202,2067458,2067714,2067970,2068226,2064385,2064641,2064897,2065153,2065409,2065665,2065921,2066177,2070530,2070786,2071042,2071298,2071554,2071810,2072066,2072322,2068481,2068737,2068993,2069249,2069505,2069761,2070017,2070273,2074626,2074882,2075138,2075394,2075650,2075906,2076162,2076418,2072577,2072833,2073089,2073345,2073601,2073857,2074113,2074369,2078722,2078978,0,2079746,0,0,0,0,2076673,2076929,2060289,2060545,2077441,0,235778,0,0,0,0,2083842,0,0,0,0,2060801,2061057,2061313,2061569,2081537,0,0,0,2086914,2087170,0,0,0,0,0,0,2084865,2085121,2061825,2062081,0,0,0,0,2091010,2091266,0,0,0,2092034,0,0,2088961,2089217,2062849,2063105,2090241,0,0,0,0,0,0,2096130,0,0,0,0,2062337,2062593,2063361,2063617,2093825,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,248065,0,0,0,27393,58625,0,0,0,0,0,0,2182657,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2175490,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2191361,2191617,2191873,2192129,2192385,2192641,2192897,2193153,2193409,2193665,2193921,2194177,2194433,2194689,2194945,2195201,2187266,2187522,2187778,2188034,2188290,2188546,2188802,2189058,2189314,2189570,2189826,2190082,2190338,2190594,2190850,2191106,0,0,0,2196481,2196226,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2412545,2412801,2413057,2413313,2413569,2413825,2414081,2414337,2414593,2414849,2415105,2415361,2415617,2415873,2416129,2416385,2416641,2416897,2417153,2417409,2417665,2417921,2418177,2418433,2418689,2418945,2405890,2406146,2406402,2406658,2406914,2407170,2407426,2407682,2407938,2408194,2408450,2408706,2408962,2409218,2409474,2409730,2409986,2410242,2410498,2410754,2411010,2411266,2411522,2411778,2412034,2412290,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {2895873,2896129,2896385,2896641,2896897,2897153,2897409,2897665,2897921,2898177,2898433,2898689,2898945,2899201,2899457,2899713,2899969,2900225,2900481,2900737,2900993,2901249,2901505,2901761,2902017,2902273,2902529,2902785,2903041,2903297,2903553,2903809,2904065,2904321,2904577,2904833,2905089,2905345,2905601,2905857,2906113,2906369,2906625,2906881,2907137,2907393,2907649,0,2883586,2883842,2884098,2884354,2884610,2884866,2885122,2885378,2885634,2885890,2886146,2886402,2886658,2886914,2887170,2887426,2887682,2887938,2888194,2888450,2888706,2888962,2889218,2889474,2889730,2889986,2890242,2890498,2890754,2891010,2891266,2891522,2891778,2892034,2892290,2892546,2892802,2893058,2893314,2893570,2893826,2894082,2894338,2894594,2894850,2895106,2895362,0,2908417,2908162,158465,1932545,163073,145922,146946,2910209,2909954,2910721,2910466,2911233,2910978,151809,160001,151553,152065,0,2913025,2912770,0,2913793,2913538,0,0,0,0,0,0,0,147201,147457,2916609,2916354,2917121,2916866,2917633,2917378,2918145,2917890,2918657,2918402,2919169,2918914,2919681,2919426,2920193,2919938,2920705,2920450,2921217,2920962,2921729,2921474,2922241,2921986,2922753,2922498,2923265,2923010,2923777,2923522,2924289,2924034,2924801,2924546,2925313,2925058,2925825,2925570,2926337,2926082,2926849,2926594,2927361,2927106,2927873,2927618,2928385,2928130,2928897,2928642,2929409,2929154,2929921,2929666,2930433,2930178,2930945,2930690,2931457,2931202,2931969,2931714,2932481,2932226,2932993,2932738,2933505,2933250,2934017,2933762,2934529,2934274,2935041,2934786,2935553,2935298,2936065,2935810,2936577,2936322,2937089,2936834,2937601,2937346,2938113,2937858,2938625,2938370,2939137,2938882,2939649,2939394,2940161,2939906,2940673,2940418,2941185,2940930,2941697,2941442,0,0,0,0,0,0,0,2944001,2943746,2944513,2944258,0,0,0,2945793,2945538,0,0,0,0,0,0,0,0,0,0,0,0},
  {1089538,1089794,1090050,1090306,1090562,1090818,1091074,1091330,1091586,1091842,1092098,1092354,1092610,1092866,1093122,1093378,1093634,1093890,1094146,1094402,1094658,1094914,1095170,1095426,1095682,1095938,1096194,1096450,1096706,1096962,1097218,1097474,1097730,1097986,1098242,1098498,1098754,1099010,0,1099522,0,0,0,0,0,1101058,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10895617,10895362,10896129,10895874,10896641,10896386,10897153,10896898,10897665,10897410,10898177,10897922,10898689,10898434,10899201,10898946,10899713,10899458,10900225,10899970,10900737,10900482,10901249,10900994,10901761,10901506,10902273,10902018,10902785,10902530,10903297,10903042,10903809,10903554,10904321,10904066,10904833,10904578,10905345,10905090,10905857,10905602,10906369,10906114,10906881,10906626,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10912001,10911746,10912513,10912258,10913025,10912770,10913537,10913282,10914049,10913794,10914561,10914306,10915073,10914818,10915585,10915330,10916097,10915842,10916609,10916354,10917121,10916866,10917633,10917378,10918145,10917890,10918657,10918402,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10953473,10953218,10953985,10953730,10954497,10954242,10955009,10954754,10955521,10955266,10956033,10955778,10956545,10956290,0,0,10957569,10957314,10958081,10957826,10958593,10958338,10959105,10958850,10959617,10959362,10960129,10959874,10960641,10960386,10961153,10960898,10961665,10961410,10962177,10961922,10962689,10962434,10963201,10962946,10963713,10963458,10964225,10963970,10964737,10964482,10965249,10964994,10965761,10965506,10966273,10966018,10966785,10966530,10967297,10967042,10967809,10967554,10968321,10968066,10968833,10968578,10969345,10969090,10969857,10969602,10970369,10970114,10970881,10970626,10971393,10971138,10971905,10971650,10972417,10972162,10972929,10972674,0,0,0,0,0,0,0,0,0,10975745,10975490,10976257,10976002,1931521,10977025,10976770,10977537,10977282,10978049,10977794,10978561,10978306,10979073,10978818,0,0,0,10980353,10980098,156929,0,0,10981633,10981378,10982145,10981890,0,0,10983169,10982914,10983681,10983426,10984193,10983938,10984705,10984450,10985217,10984962,10985729,10985474,10986241,10985986,10986753,10986498,10987265,10987010,10987777,10987522,157185,154625,155905,158721,0,0,171521,165633,171265,11227905,10990849,10990594,10991361,10991106,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10990338,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1286146,1286402,1286658,1286914,1287170,1287426,1287682,1287938,1288194,1288450,1288706,1288962,1289218,1289474,1289730,1289986,1290242,1290498,1290754,1291010,1291266,1291522,1291778,1292034,1292290,1292546,1292802,1293058,1293314,1293570,1293826,1294082,1294338,1294594,1294850,1295106,1295362,1295618,1295874,1296130,1296386,1296642,1296898,1297154,1297410,1297666,1297922,1298178,1298434,1298690,1298946,1299202,1299458,1299714,1299970,1300226,1300482,1300738,1300994,1301250,1301506,1301762,1302018,1302274,1302530,1302786,1303042,1303298,1303554,1303810,1304066,1304322,1304578,1304834,1305090,1305346,1305602,1305858,1306114,1306370,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16728321,16728577,16728833,16729089,16729345,16729601,16729857,16730113,16730369,16730625,16730881,16731137,16731393,16731649,16731905,16732161,16732417,16732673,16732929,16733185,16733441,16733697,16733953,16734209,16734465,16734721,0,0,0,0,0,0,16720130,16720386,16720642,16720898,16721154,16721410,16721666,16721922,16722178,16722434,16722690,16722946,16723202,16723458,16723714,16723970,16724226,16724482,16724738,16724994,16725250,16725506,16725762,16726018,16726274,16726530,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {17049601,17049857,17050113,17050369,17050625,17050881,17051137,17051393,17051649,17051905,17052161,17052417,17052673,17052929,17053185,17053441,17053697,17053953,17054209,17054465,17054721,17054977,17055233,17055489,17055745,17056001,17056257,17056513,17056769,17057025,17057281,17057537,17057793,17058049,17058305,17058561,17058817,17059073,17059329,17059585,17039362,17039618,17039874,17040130,17040386,17040642,17040898,17041154,17041410,17041666,17041922,17042178,17042434,17042690,17042946,17043202,17043458,17043714,17043970,17044226,17044482,17044738,17044994,17045250,17045506,17045762,17046018,17046274,17046530,17046786,17047042,17047298,17047554,17047810,17048066,17048322,17048578,17048834,17049090,17049346,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17612801,17613057,17613313,17613569,17613825,17614081,17614337,17614593,17614849,17615105,17615361,17615617,17615873,17616129,17616385,17616641,17616897,17617153,17617409,17617665,17617921,17618177,17618433,17618689,17618945,17619201,17619457,17619713,17619969,17620225,17620481,17620737,17620993,17621249,17621505,17621761,17622017,17622273,17622529,17622785,17623041,17623297,17623553,17623809,17624065,17624321,17624577,17624833,17625089,17625345,17625601,0,0,0,0,0,0,0,0,0,0,0,0,0,17596418,17596674,17596930,17597186,17597442,17597698,17597954,17598210,17598466,17598722,17598978,17599234,17599490,17599746,17600002,17600258,17600514,17600770,17601026,17601282,17601538,17601794,17602050,17602306,17602562,17602818,17603074,17603330,17603586,17603842,17604098,17604354,17604610,17604866,17605122,17605378,17605634,17605890,17606146,17606402,17606658,17606914,17607170,17607426,17607682,17607938,17608194,17608450,17608706,17608962,17609218,0,0,0,0,0,0,0,0,0,0,0,0,0},
  {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18399233,18399489,18399745,18400001,18400257,18400513,18400769,18401025,18401281,18401537,18401793,18402049,18402305,18402561,18402817,18403073,18403329,18403585,18403841,18404097,18404353,18404609,18404865,18405121,18405377,18405633,18405889,18406145,18406401,18406657,18406913,18407169,18391042,18391298,18391554,18391810,18392066,18392322,18392578,18392834,18393090,18393346,18393602,18393858,18394114,18394370,18394626,18394882,18395138,18395394,18395650,18395906,18396162,18396418,18396674,18396930,18397186,18397442,18397698,18397954,18398210,18398466,18398722,18398978,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
};

} // namespace unilib

/////////
// File: unilib/utf8.cpp
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

bool utf8::valid(const char* str) {
  for (; *str; str++)
    if (((unsigned char)*str) >= 0x80) {
      if (((unsigned char)*str) < 0xC0) return false;
      else if (((unsigned char)*str) < 0xE0) {
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else if (((unsigned char)*str) < 0xF0) {
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else if (((unsigned char)*str) < 0xF8) {
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else return false;
    }
  return true;
}

bool utf8::valid(const char* str, size_t len) {
  for (; len > 0; str++, len--)
    if (((unsigned char)*str) >= 0x80) {
      if (((unsigned char)*str) < 0xC0) return false;
      else if (((unsigned char)*str) < 0xE0) {
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else if (((unsigned char)*str) < 0xF0) {
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else if (((unsigned char)*str) < 0xF8) {
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
        str++; if (!--len || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return false;
      } else return false;
    }
  return true;
}

void utf8::decode(const char* str, std::u32string& decoded) {
  decoded.clear();

  for (char32_t chr; (chr = decode(str)); )
    decoded.push_back(chr);
}

void utf8::decode(const char* str, size_t len, std::u32string& decoded) {
  decoded.clear();

  while (len)
    decoded.push_back(decode(str, len));
}

void utf8::encode(const std::u32string& str, std::string& encoded) {
  encoded.clear();

  for (auto&& chr : str)
    append(encoded, chr);
}

const char utf8::REPLACEMENT_CHAR;

} // namespace unilib

/////////
// File: unilib/version.cpp
/////////

// This file is part of UniLib <http://github.com/ufal/unilib/>.
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.1.1
// Unicode version: 8.0.0

namespace unilib {

// Returns current version.
version version::current() {
  return {3, 1, 1, ""};
}

} // namespace unilib

/////////
// File: utils/compressor_load.cpp
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

// Start of LZMA compression library by Igor Pavlov
namespace lzma {

// Types.h -- Basic types
// 2010-10-09 : Igor Pavlov : Public domain
#ifndef UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H
#define UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H

#define SZ_OK 0

#define SZ_ERROR_DATA 1
#define SZ_ERROR_MEM 2
#define SZ_ERROR_CRC 3
#define SZ_ERROR_UNSUPPORTED 4
#define SZ_ERROR_PARAM 5
#define SZ_ERROR_INPUT_EOF 6
#define SZ_ERROR_OUTPUT_EOF 7
#define SZ_ERROR_READ 8
#define SZ_ERROR_WRITE 9
#define SZ_ERROR_PROGRESS 10
#define SZ_ERROR_FAIL 11
#define SZ_ERROR_THREAD 12

#define SZ_ERROR_ARCHIVE 16
#define SZ_ERROR_NO_ARCHIVE 17

typedef int SRes;

#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif

/* The following interfaces use first parameter as pointer to structure */

struct IByteIn
{
  uint8_t (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */
};

struct IByteOut
{
  void (*Write)(void *p, uint8_t b);
};

struct ISeqInStream
{
  SRes (*Read)(void *p, void *buf, size_t *size);
    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
       (output(*size) < input(*size)) is allowed */
};

/* it can return SZ_ERROR_INPUT_EOF */
SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size);
SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType);
SRes SeqInStream_ReadByte(ISeqInStream *stream, uint8_t *buf);

struct ISeqOutStream
{
  size_t (*Write)(void *p, const void *buf, size_t size);
    /* Returns: result - the number of actually written bytes.
       (result < size) means error */
};

enum ESzSeek
{
  SZ_SEEK_SET = 0,
  SZ_SEEK_CUR = 1,
  SZ_SEEK_END = 2
};

struct ISeekInStream
{
  SRes (*Read)(void *p, void *buf, size_t *size);  /* same as ISeqInStream::Read */
  SRes (*Seek)(void *p, int64_t *pos, ESzSeek origin);
};

struct ILookInStream
{
  SRes (*Look)(void *p, const void **buf, size_t *size);
    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
       (output(*size) > input(*size)) is not allowed
       (output(*size) < input(*size)) is allowed */
  SRes (*Skip)(void *p, size_t offset);
    /* offset must be <= output(*size) of Look */

  SRes (*Read)(void *p, void *buf, size_t *size);
    /* reads directly (without buffer). It's same as ISeqInStream::Read */
  SRes (*Seek)(void *p, int64_t *pos, ESzSeek origin);
};

SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size);
SRes LookInStream_SeekTo(ILookInStream *stream, uint64_t offset);

/* reads via ILookInStream::Read */
SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType);
SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size);

#define LookToRead_BUF_SIZE (1 << 14)

struct CLookToRead
{
  ILookInStream s;
  ISeekInStream *realStream;
  size_t pos;
  size_t size;
  uint8_t buf[LookToRead_BUF_SIZE];
};

void LookToRead_CreateVTable(CLookToRead *p, int lookahead);
void LookToRead_Init(CLookToRead *p);

struct CSecToLook
{
  ISeqInStream s;
  ILookInStream *realStream;
};

void SecToLook_CreateVTable(CSecToLook *p);

struct CSecToRead
{
  ISeqInStream s;
  ILookInStream *realStream;
};

void SecToRead_CreateVTable(CSecToRead *p);

struct ICompressProgress
{
  SRes (*Progress)(void *p, uint64_t inSize, uint64_t outSize);
    /* Returns: result. (result != SZ_OK) means break.
       Value (uint64_t)(int64_t)-1 for size means unknown value. */
};

struct ISzAlloc
{
  void *(*Alloc)(void *p, size_t size);
  void (*Free)(void *p, void *address); /* address can be 0 */
};

#define IAlloc_Alloc(p, size) (p)->Alloc((p), size)
#define IAlloc_Free(p, a) (p)->Free((p), a)

#endif // UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H

// LzmaDec.h -- LZMA Decoder
// 2009-02-07 : Igor Pavlov : Public domain

/* #define _LZMA_PROB32 */
/* _LZMA_PROB32 can increase the speed on some CPUs,
   but memory usage for CLzmaDec::probs will be doubled in that case */

#ifdef _LZMA_PROB32
#define CLzmaProb uint32_t
#else
#define CLzmaProb uint16_t
#endif

/* ---------- LZMA Properties ---------- */

#define LZMA_PROPS_SIZE 5

struct CLzmaProps
{
  unsigned lc, lp, pb;
  uint32_t dicSize;
};

/* LzmaProps_Decode - decodes properties
Returns:
  SZ_OK
  SZ_ERROR_UNSUPPORTED - Unsupported properties
*/

SRes LzmaProps_Decode(CLzmaProps *p, const uint8_t *data, unsigned size);

/* ---------- LZMA Decoder state ---------- */

/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case.
   Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */

#define LZMA_REQUIRED_INPUT_MAX 20

struct CLzmaDec
{
  CLzmaProps prop;
  CLzmaProb *probs;
  uint8_t *dic;
  const uint8_t *buf;
  uint32_t range, code;
  size_t dicPos;
  size_t dicBufSize;
  uint32_t processedPos;
  uint32_t checkDicSize;
  unsigned state;
  uint32_t reps[4];
  unsigned remainLen;
  int needFlush;
  int needInitState;
  uint32_t numProbs;
  unsigned tempBufSize;
  uint8_t tempBuf[LZMA_REQUIRED_INPUT_MAX];
};

#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; }

void LzmaDec_Init(CLzmaDec *p);

/* There are two types of LZMA streams:
     0) Stream with end mark. That end mark adds about 6 bytes to compressed size.
     1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */

enum ELzmaFinishMode
{
  LZMA_FINISH_ANY,   /* finish at any point */
  LZMA_FINISH_END    /* block must be finished at the end */
};

/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!!

   You must use LZMA_FINISH_END, when you know that current output buffer
   covers last bytes of block. In other cases you must use LZMA_FINISH_ANY.

   If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK,
   and output value of destLen will be less than output buffer size limit.
   You can check status result also.

   You can use multiple checks to test data integrity after full decompression:
     1) Check Result and "status" variable.
     2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize.
     3) Check that output(srcLen) = compressedSize, if you know real compressedSize.
        You must use correct finish mode in that case. */

enum ELzmaStatus
{
  LZMA_STATUS_NOT_SPECIFIED,               /* use main error code instead */
  LZMA_STATUS_FINISHED_WITH_MARK,          /* stream was finished with end mark. */
  LZMA_STATUS_NOT_FINISHED,                /* stream was not finished */
  LZMA_STATUS_NEEDS_MORE_INPUT,            /* you must provide more input bytes */
  LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK  /* there is probability that stream was finished without end mark */
};

/* ELzmaStatus is used only as output value for function call */

/* ---------- Interfaces ---------- */

/* There are 3 levels of interfaces:
     1) Dictionary Interface
     2) Buffer Interface
     3) One Call Interface
   You can select any of these interfaces, but don't mix functions from different
   groups for same object. */

/* There are two variants to allocate state for Dictionary Interface:
     1) LzmaDec_Allocate / LzmaDec_Free
     2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs
   You can use variant 2, if you set dictionary buffer manually.
   For Buffer Interface you must always use variant 1.

LzmaDec_Allocate* can return:
  SZ_OK
  SZ_ERROR_MEM         - Memory allocation error
  SZ_ERROR_UNSUPPORTED - Unsupported properties
*/

SRes LzmaDec_AllocateProbs(CLzmaDec *p, const uint8_t *props, unsigned propsSize, ISzAlloc *alloc);
void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc);

SRes LzmaDec_Allocate(CLzmaDec *state, const uint8_t *prop, unsigned propsSize, ISzAlloc *alloc);
void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc);

/* ---------- Dictionary Interface ---------- */

/* You can use it, if you want to eliminate the overhead for data copying from
   dictionary to some other external buffer.
   You must work with CLzmaDec variables directly in this interface.

   STEPS:
     LzmaDec_Constr()
     LzmaDec_Allocate()
     for (each new stream)
     {
       LzmaDec_Init()
       while (it needs more decompression)
       {
         LzmaDec_DecodeToDic()
         use data from CLzmaDec::dic and update CLzmaDec::dicPos
       }
     }
     LzmaDec_Free()
*/

/* LzmaDec_DecodeToDic

   The decoding to internal dictionary buffer (CLzmaDec::dic).
   You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!!

finishMode:
  It has meaning only if the decoding reaches output limit (dicLimit).
  LZMA_FINISH_ANY - Decode just dicLimit bytes.
  LZMA_FINISH_END - Stream must be finished after dicLimit.

Returns:
  SZ_OK
    status:
      LZMA_STATUS_FINISHED_WITH_MARK
      LZMA_STATUS_NOT_FINISHED
      LZMA_STATUS_NEEDS_MORE_INPUT
      LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
  SZ_ERROR_DATA - Data error
*/

SRes LzmaDec_DecodeToDic(CLzmaDec *p, size_t dicLimit,
    const uint8_t *src, size_t *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);

/* ---------- Buffer Interface ---------- */

/* It's zlib-like interface.
   See LzmaDec_DecodeToDic description for information about STEPS and return results,
   but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need
   to work with CLzmaDec variables manually.

finishMode:
  It has meaning only if the decoding reaches output limit (*destLen).
  LZMA_FINISH_ANY - Decode just destLen bytes.
  LZMA_FINISH_END - Stream must be finished after (*destLen).
*/

SRes LzmaDec_DecodeToBuf(CLzmaDec *p, uint8_t *dest, size_t *destLen,
    const uint8_t *src, size_t *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);

/* ---------- One Call Interface ---------- */

/* LzmaDecode

finishMode:
  It has meaning only if the decoding reaches output limit (*destLen).
  LZMA_FINISH_ANY - Decode just destLen bytes.
  LZMA_FINISH_END - Stream must be finished after (*destLen).

Returns:
  SZ_OK
    status:
      LZMA_STATUS_FINISHED_WITH_MARK
      LZMA_STATUS_NOT_FINISHED
      LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
  SZ_ERROR_DATA - Data error
  SZ_ERROR_MEM  - Memory allocation error
  SZ_ERROR_UNSUPPORTED - Unsupported properties
  SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
*/

SRes LzmaDecode(uint8_t *dest, size_t *destLen, const uint8_t *src, size_t *srcLen,
    const uint8_t *propData, unsigned propSize, ELzmaFinishMode finishMode,
    ELzmaStatus *status, ISzAlloc *alloc);

// LzmaDec.c -- LZMA Decoder
// 2009-09-20 : Igor Pavlov : Public domain

#define kNumTopBits 24
#define kTopValue ((uint32_t)1 << kNumTopBits)

#define kNumBitModelTotalBits 11
#define kBitModelTotal (1 << kNumBitModelTotalBits)
#define kNumMoveBits 5

#define RC_INIT_SIZE 5

#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); }

#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound)
#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits));
#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits));
#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \
  { UPDATE_0(p); i = (i + i); A0; } else \
  { UPDATE_1(p); i = (i + i) + 1; A1; }
#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;)

#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); }
#define TREE_DECODE(probs, limit, i) \
  { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; }

/* #define _LZMA_SIZE_OPT */

#ifdef _LZMA_SIZE_OPT
#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i)
#else
#define TREE_6_DECODE(probs, i) \
  { i = 1; \
  TREE_GET_BIT(probs, i); \
  TREE_GET_BIT(probs, i); \
  TREE_GET_BIT(probs, i); \
  TREE_GET_BIT(probs, i); \
  TREE_GET_BIT(probs, i); \
  TREE_GET_BIT(probs, i); \
  i -= 0x40; }
#endif

#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); }

#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound)
#define UPDATE_0_CHECK range = bound;
#define UPDATE_1_CHECK range -= bound; code -= bound;
#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \
  { UPDATE_0_CHECK; i = (i + i); A0; } else \
  { UPDATE_1_CHECK; i = (i + i) + 1; A1; }
#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;)
#define TREE_DECODE_CHECK(probs, limit, i) \
  { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; }

#define kNumPosBitsMax 4
#define kNumPosStatesMax (1 << kNumPosBitsMax)

#define kLenNumLowBits 3
#define kLenNumLowSymbols (1 << kLenNumLowBits)
#define kLenNumMidBits 3
#define kLenNumMidSymbols (1 << kLenNumMidBits)
#define kLenNumHighBits 8
#define kLenNumHighSymbols (1 << kLenNumHighBits)

#define LenChoice 0
#define LenChoice2 (LenChoice + 1)
#define LenLow (LenChoice2 + 1)
#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits))
#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits))
#define kNumLenProbs (LenHigh + kLenNumHighSymbols)

#define kNumStates 12
#define kNumLitStates 7

#define kStartPosModelIndex 4
#define kEndPosModelIndex 14
#define kNumFullDistances (1 << (kEndPosModelIndex >> 1))

#define kNumPosSlotBits 6
#define kNumLenToPosStates 4

#define kNumAlignBits 4
#define kAlignTableSize (1 << kNumAlignBits)

#define kMatchMinLen 2
#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols)

#define IsMatch 0
#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax))
#define IsRepG0 (IsRep + kNumStates)
#define IsRepG1 (IsRepG0 + kNumStates)
#define IsRepG2 (IsRepG1 + kNumStates)
#define IsRep0Long (IsRepG2 + kNumStates)
#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax))
#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits))
#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex)
#define LenCoder (Align + kAlignTableSize)
#define RepLenCoder (LenCoder + kNumLenProbs)
#define Literal (RepLenCoder + kNumLenProbs)

#define LZMA_BASE_SIZE 1846
#define LZMA_LIT_SIZE 768

#define LzmaProps_GetNumProbs(p) ((uint32_t)LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((p)->lc + (p)->lp)))

#if Literal != LZMA_BASE_SIZE
StopCompilingDueBUG
#endif

#define LZMA_DIC_MIN (1 << 12)

/* First LZMA-symbol is always decoded.
And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization
Out:
  Result:
    SZ_OK - OK
    SZ_ERROR_DATA - Error
  p->remainLen:
    < kMatchSpecLenStart : normal remain
    = kMatchSpecLenStart : finished
    = kMatchSpecLenStart + 1 : Flush marker
    = kMatchSpecLenStart + 2 : State Init Marker
*/

static int LzmaDec_DecodeReal(CLzmaDec *p, size_t limit, const uint8_t *bufLimit)
{
  CLzmaProb *probs = p->probs;

  unsigned state = p->state;
  uint32_t rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3];
  unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1;
  unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1;
  unsigned lc = p->prop.lc;

  uint8_t *dic = p->dic;
  size_t dicBufSize = p->dicBufSize;
  size_t dicPos = p->dicPos;

  uint32_t processedPos = p->processedPos;
  uint32_t checkDicSize = p->checkDicSize;
  unsigned len = 0;

  const uint8_t *buf = p->buf;
  uint32_t range = p->range;
  uint32_t code = p->code;

  do
  {
    CLzmaProb *prob;
    uint32_t bound;
    unsigned ttt;
    unsigned posState = processedPos & pbMask;

    prob = probs + IsMatch + (state << kNumPosBitsMax) + posState;
    IF_BIT_0(prob)
    {
      unsigned symbol;
      UPDATE_0(prob);
      prob = probs + Literal;
      if (checkDicSize != 0 || processedPos != 0)
        prob += (LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) +
        (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc))));

      if (state < kNumLitStates)
      {
        state -= (state < 4) ? state : 3;
        symbol = 1;
        do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100);
      }
      else
      {
        unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)];
        unsigned offs = 0x100;
        state -= (state < 10) ? 3 : 6;
        symbol = 1;
        do
        {
          unsigned bit;
          CLzmaProb *probLit;
          matchByte <<= 1;
          bit = (matchByte & offs);
          probLit = prob + offs + bit + symbol;
          GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit)
        }
        while (symbol < 0x100);
      }
      dic[dicPos++] = (uint8_t)symbol;
      processedPos++;
      continue;
    }
    else
    {
      UPDATE_1(prob);
      prob = probs + IsRep + state;
      IF_BIT_0(prob)
      {
        UPDATE_0(prob);
        state += kNumStates;
        prob = probs + LenCoder;
      }
      else
      {
        UPDATE_1(prob);
        if (checkDicSize == 0 && processedPos == 0)
          return SZ_ERROR_DATA;
        prob = probs + IsRepG0 + state;
        IF_BIT_0(prob)
        {
          UPDATE_0(prob);
          prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState;
          IF_BIT_0(prob)
          {
            UPDATE_0(prob);
            dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)];
            dicPos++;
            processedPos++;
            state = state < kNumLitStates ? 9 : 11;
            continue;
          }
          UPDATE_1(prob);
        }
        else
        {
          uint32_t distance;
          UPDATE_1(prob);
          prob = probs + IsRepG1 + state;
          IF_BIT_0(prob)
          {
            UPDATE_0(prob);
            distance = rep1;
          }
          else
          {
            UPDATE_1(prob);
            prob = probs + IsRepG2 + state;
            IF_BIT_0(prob)
            {
              UPDATE_0(prob);
              distance = rep2;
            }
            else
            {
              UPDATE_1(prob);
              distance = rep3;
              rep3 = rep2;
            }
            rep2 = rep1;
          }
          rep1 = rep0;
          rep0 = distance;
        }
        state = state < kNumLitStates ? 8 : 11;
        prob = probs + RepLenCoder;
      }
      {
        unsigned limit, offset;
        CLzmaProb *probLen = prob + LenChoice;
        IF_BIT_0(probLen)
        {
          UPDATE_0(probLen);
          probLen = prob + LenLow + (posState << kLenNumLowBits);
          offset = 0;
          limit = (1 << kLenNumLowBits);
        }
        else
        {
          UPDATE_1(probLen);
          probLen = prob + LenChoice2;
          IF_BIT_0(probLen)
          {
            UPDATE_0(probLen);
            probLen = prob + LenMid + (posState << kLenNumMidBits);
            offset = kLenNumLowSymbols;
            limit = (1 << kLenNumMidBits);
          }
          else
          {
            UPDATE_1(probLen);
            probLen = prob + LenHigh;
            offset = kLenNumLowSymbols + kLenNumMidSymbols;
            limit = (1 << kLenNumHighBits);
          }
        }
        TREE_DECODE(probLen, limit, len);
        len += offset;
      }

      if (state >= kNumStates)
      {
        uint32_t distance;
        prob = probs + PosSlot +
            ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits);
        TREE_6_DECODE(prob, distance);
        if (distance >= kStartPosModelIndex)
        {
          unsigned posSlot = (unsigned)distance;
          int numDirectBits = (int)(((distance >> 1) - 1));
          distance = (2 | (distance & 1));
          if (posSlot < kEndPosModelIndex)
          {
            distance <<= numDirectBits;
            prob = probs + SpecPos + distance - posSlot - 1;
            {
              uint32_t mask = 1;
              unsigned i = 1;
              do
              {
                GET_BIT2(prob + i, i, ; , distance |= mask);
                mask <<= 1;
              }
              while (--numDirectBits != 0);
            }
          }
          else
          {
            numDirectBits -= kNumAlignBits;
            do
            {
              NORMALIZE
              range >>= 1;

              {
                uint32_t t;
                code -= range;
                t = (0 - ((uint32_t)code >> 31)); /* (uint32_t)((int32_t)code >> 31) */
                distance = (distance << 1) + (t + 1);
                code += range & t;
              }
              /*
              distance <<= 1;
              if (code >= range)
              {
                code -= range;
                distance |= 1;
              }
              */
            }
            while (--numDirectBits != 0);
            prob = probs + Align;
            distance <<= kNumAlignBits;
            {
              unsigned i = 1;
              GET_BIT2(prob + i, i, ; , distance |= 1);
              GET_BIT2(prob + i, i, ; , distance |= 2);
              GET_BIT2(prob + i, i, ; , distance |= 4);
              GET_BIT2(prob + i, i, ; , distance |= 8);
            }
            if (distance == (uint32_t)0xFFFFFFFF)
            {
              len += kMatchSpecLenStart;
              state -= kNumStates;
              break;
            }
          }
        }
        rep3 = rep2;
        rep2 = rep1;
        rep1 = rep0;
        rep0 = distance + 1;
        if (checkDicSize == 0)
        {
          if (distance >= processedPos)
            return SZ_ERROR_DATA;
        }
        else if (distance >= checkDicSize)
          return SZ_ERROR_DATA;
        state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3;
      }

      len += kMatchMinLen;

      if (limit == dicPos)
        return SZ_ERROR_DATA;
      {
        size_t rem = limit - dicPos;
        unsigned curLen = ((rem < len) ? (unsigned)rem : len);
        size_t pos = (dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0);

        processedPos += curLen;

        len -= curLen;
        if (pos + curLen <= dicBufSize)
        {
          uint8_t *dest = dic + dicPos;
          ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos;
          const uint8_t *lim = dest + curLen;
          dicPos += curLen;
          do
            *(dest) = (uint8_t)*(dest + src);
          while (++dest != lim);
        }
        else
        {
          do
          {
            dic[dicPos++] = dic[pos];
            if (++pos == dicBufSize)
              pos = 0;
          }
          while (--curLen != 0);
        }
      }
    }
  }
  while (dicPos < limit && buf < bufLimit);
  NORMALIZE;
  p->buf = buf;
  p->range = range;
  p->code = code;
  p->remainLen = len;
  p->dicPos = dicPos;
  p->processedPos = processedPos;
  p->reps[0] = rep0;
  p->reps[1] = rep1;
  p->reps[2] = rep2;
  p->reps[3] = rep3;
  p->state = state;

  return SZ_OK;
}

static void LzmaDec_WriteRem(CLzmaDec *p, size_t limit)
{
  if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart)
  {
    uint8_t *dic = p->dic;
    size_t dicPos = p->dicPos;
    size_t dicBufSize = p->dicBufSize;
    unsigned len = p->remainLen;
    uint32_t rep0 = p->reps[0];
    if (limit - dicPos < len)
      len = (unsigned)(limit - dicPos);

    if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len)
      p->checkDicSize = p->prop.dicSize;

    p->processedPos += len;
    p->remainLen -= len;
    while (len-- != 0)
    {
      dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)];
      dicPos++;
    }
    p->dicPos = dicPos;
  }
}

static int LzmaDec_DecodeReal2(CLzmaDec *p, size_t limit, const uint8_t *bufLimit)
{
  do
  {
    size_t limit2 = limit;
    if (p->checkDicSize == 0)
    {
      uint32_t rem = p->prop.dicSize - p->processedPos;
      if (limit - p->dicPos > rem)
        limit2 = p->dicPos + rem;
    }
    RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit));
    if (p->processedPos >= p->prop.dicSize)
      p->checkDicSize = p->prop.dicSize;
    LzmaDec_WriteRem(p, limit);
  }
  while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart);

  if (p->remainLen > kMatchSpecLenStart)
  {
    p->remainLen = kMatchSpecLenStart;
  }
  return 0;
}

enum ELzmaDummy
{
  DUMMY_ERROR, /* unexpected end of input stream */
  DUMMY_LIT,
  DUMMY_MATCH,
  DUMMY_REP
};

static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const uint8_t *buf, size_t inSize)
{
  uint32_t range = p->range;
  uint32_t code = p->code;
  const uint8_t *bufLimit = buf + inSize;
  CLzmaProb *probs = p->probs;
  unsigned state = p->state;
  ELzmaDummy res;

  {
    CLzmaProb *prob;
    uint32_t bound;
    unsigned ttt;
    unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1);

    prob = probs + IsMatch + (state << kNumPosBitsMax) + posState;
    IF_BIT_0_CHECK(prob)
    {
      UPDATE_0_CHECK

      /* if (bufLimit - buf >= 7) return DUMMY_LIT; */

      prob = probs + Literal;
      if (p->checkDicSize != 0 || p->processedPos != 0)
        prob += (LZMA_LIT_SIZE *
          ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) +
          (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc))));

      if (state < kNumLitStates)
      {
        unsigned symbol = 1;
        do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100);
      }
      else
      {
        unsigned matchByte = p->dic[p->dicPos - p->reps[0] +
            ((p->dicPos < p->reps[0]) ? p->dicBufSize : 0)];
        unsigned offs = 0x100;
        unsigned symbol = 1;
        do
        {
          unsigned bit;
          CLzmaProb *probLit;
          matchByte <<= 1;
          bit = (matchByte & offs);
          probLit = prob + offs + bit + symbol;
          GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit)
        }
        while (symbol < 0x100);
      }
      res = DUMMY_LIT;
    }
    else
    {
      unsigned len;
      UPDATE_1_CHECK;

      prob = probs + IsRep + state;
      IF_BIT_0_CHECK(prob)
      {
        UPDATE_0_CHECK;
        state = 0;
        prob = probs + LenCoder;
        res = DUMMY_MATCH;
      }
      else
      {
        UPDATE_1_CHECK;
        res = DUMMY_REP;
        prob = probs + IsRepG0 + state;
        IF_BIT_0_CHECK(prob)
        {
          UPDATE_0_CHECK;
          prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState;
          IF_BIT_0_CHECK(prob)
          {
            UPDATE_0_CHECK;
            NORMALIZE_CHECK;
            return DUMMY_REP;
          }
          else
          {
            UPDATE_1_CHECK;
          }
        }
        else
        {
          UPDATE_1_CHECK;
          prob = probs + IsRepG1 + state;
          IF_BIT_0_CHECK(prob)
          {
            UPDATE_0_CHECK;
          }
          else
          {
            UPDATE_1_CHECK;
            prob = probs + IsRepG2 + state;
            IF_BIT_0_CHECK(prob)
            {
              UPDATE_0_CHECK;
            }
            else
            {
              UPDATE_1_CHECK;
            }
          }
        }
        state = kNumStates;
        prob = probs + RepLenCoder;
      }
      {
        unsigned limit, offset;
        CLzmaProb *probLen = prob + LenChoice;
        IF_BIT_0_CHECK(probLen)
        {
          UPDATE_0_CHECK;
          probLen = prob + LenLow + (posState << kLenNumLowBits);
          offset = 0;
          limit = 1 << kLenNumLowBits;
        }
        else
        {
          UPDATE_1_CHECK;
          probLen = prob + LenChoice2;
          IF_BIT_0_CHECK(probLen)
          {
            UPDATE_0_CHECK;
            probLen = prob + LenMid + (posState << kLenNumMidBits);
            offset = kLenNumLowSymbols;
            limit = 1 << kLenNumMidBits;
          }
          else
          {
            UPDATE_1_CHECK;
            probLen = prob + LenHigh;
            offset = kLenNumLowSymbols + kLenNumMidSymbols;
            limit = 1 << kLenNumHighBits;
          }
        }
        TREE_DECODE_CHECK(probLen, limit, len);
        len += offset;
      }

      if (state < 4)
      {
        unsigned posSlot;
        prob = probs + PosSlot +
            ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) <<
            kNumPosSlotBits);
        TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot);
        if (posSlot >= kStartPosModelIndex)
        {
          int numDirectBits = ((posSlot >> 1) - 1);

          /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */

          if (posSlot < kEndPosModelIndex)
          {
            prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1;
          }
          else
          {
            numDirectBits -= kNumAlignBits;
            do
            {
              NORMALIZE_CHECK
              range >>= 1;
              code -= range & (((code - range) >> 31) - 1);
              /* if (code >= range) code -= range; */
            }
            while (--numDirectBits != 0);
            prob = probs + Align;
            numDirectBits = kNumAlignBits;
          }
          {
            unsigned i = 1;
            do
            {
              GET_BIT_CHECK(prob + i, i);
            }
            while (--numDirectBits != 0);
          }
        }
      }
    }
  }
  NORMALIZE_CHECK;
  return res;
}

static void LzmaDec_InitRc(CLzmaDec *p, const uint8_t *data)
{
  p->code = ((uint32_t)data[1] << 24) | ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 8) | ((uint32_t)data[4]);
  p->range = 0xFFFFFFFF;
  p->needFlush = 0;
}

void LzmaDec_InitDicAndState(CLzmaDec *p, bool initDic, bool initState)
{
  p->needFlush = 1;
  p->remainLen = 0;
  p->tempBufSize = 0;

  if (initDic)
  {
    p->processedPos = 0;
    p->checkDicSize = 0;
    p->needInitState = 1;
  }
  if (initState)
    p->needInitState = 1;
}

void LzmaDec_Init(CLzmaDec *p)
{
  p->dicPos = 0;
  LzmaDec_InitDicAndState(p, true, true);
}

static void LzmaDec_InitStateReal(CLzmaDec *p)
{
  uint32_t numProbs = Literal + ((uint32_t)LZMA_LIT_SIZE << (p->prop.lc + p->prop.lp));
  uint32_t i;
  CLzmaProb *probs = p->probs;
  for (i = 0; i < numProbs; i++)
    probs[i] = kBitModelTotal >> 1;
  p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1;
  p->state = 0;
  p->needInitState = 0;
}

SRes LzmaDec_DecodeToDic(CLzmaDec *p, size_t dicLimit, const uint8_t *src, size_t *srcLen,
    ELzmaFinishMode finishMode, ELzmaStatus *status)
{
  size_t inSize = *srcLen;
  (*srcLen) = 0;
  LzmaDec_WriteRem(p, dicLimit);

  *status = LZMA_STATUS_NOT_SPECIFIED;

  while (p->remainLen != kMatchSpecLenStart)
  {
      int checkEndMarkNow;

      if (p->needFlush != 0)
      {
        for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--)
          p->tempBuf[p->tempBufSize++] = *src++;
        if (p->tempBufSize < RC_INIT_SIZE)
        {
          *status = LZMA_STATUS_NEEDS_MORE_INPUT;
          return SZ_OK;
        }
        if (p->tempBuf[0] != 0)
          return SZ_ERROR_DATA;

        LzmaDec_InitRc(p, p->tempBuf);
        p->tempBufSize = 0;
      }

      checkEndMarkNow = 0;
      if (p->dicPos >= dicLimit)
      {
        if (p->remainLen == 0 && p->code == 0)
        {
          *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK;
          return SZ_OK;
        }
        if (finishMode == LZMA_FINISH_ANY)
        {
          *status = LZMA_STATUS_NOT_FINISHED;
          return SZ_OK;
        }
        if (p->remainLen != 0)
        {
          *status = LZMA_STATUS_NOT_FINISHED;
          return SZ_ERROR_DATA;
        }
        checkEndMarkNow = 1;
      }

      if (p->needInitState)
        LzmaDec_InitStateReal(p);

      if (p->tempBufSize == 0)
      {
        size_t processed;
        const uint8_t *bufLimit;
        if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow)
        {
          int dummyRes = LzmaDec_TryDummy(p, src, inSize);
          if (dummyRes == DUMMY_ERROR)
          {
            memcpy(p->tempBuf, src, inSize);
            p->tempBufSize = (unsigned)inSize;
            (*srcLen) += inSize;
            *status = LZMA_STATUS_NEEDS_MORE_INPUT;
            return SZ_OK;
          }
          if (checkEndMarkNow && dummyRes != DUMMY_MATCH)
          {
            *status = LZMA_STATUS_NOT_FINISHED;
            return SZ_ERROR_DATA;
          }
          bufLimit = src;
        }
        else
          bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX;
        p->buf = src;
        if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0)
          return SZ_ERROR_DATA;
        processed = (size_t)(p->buf - src);
        (*srcLen) += processed;
        src += processed;
        inSize -= processed;
      }
      else
      {
        unsigned rem = p->tempBufSize, lookAhead = 0;
        while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize)
          p->tempBuf[rem++] = src[lookAhead++];
        p->tempBufSize = rem;
        if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow)
        {
          int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem);
          if (dummyRes == DUMMY_ERROR)
          {
            (*srcLen) += lookAhead;
            *status = LZMA_STATUS_NEEDS_MORE_INPUT;
            return SZ_OK;
          }
          if (checkEndMarkNow && dummyRes != DUMMY_MATCH)
          {
            *status = LZMA_STATUS_NOT_FINISHED;
            return SZ_ERROR_DATA;
          }
        }
        p->buf = p->tempBuf;
        if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0)
          return SZ_ERROR_DATA;
        lookAhead -= (rem - (unsigned)(p->buf - p->tempBuf));
        (*srcLen) += lookAhead;
        src += lookAhead;
        inSize -= lookAhead;
        p->tempBufSize = 0;
      }
  }
  if (p->code == 0)
    *status = LZMA_STATUS_FINISHED_WITH_MARK;
  return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA;
}

SRes LzmaDec_DecodeToBuf(CLzmaDec *p, uint8_t *dest, size_t *destLen, const uint8_t *src, size_t *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)
{
  size_t outSize = *destLen;
  size_t inSize = *srcLen;
  *srcLen = *destLen = 0;
  for (;;)
  {
    size_t inSizeCur = inSize, outSizeCur, dicPos;
    ELzmaFinishMode curFinishMode;
    SRes res;
    if (p->dicPos == p->dicBufSize)
      p->dicPos = 0;
    dicPos = p->dicPos;
    if (outSize > p->dicBufSize - dicPos)
    {
      outSizeCur = p->dicBufSize;
      curFinishMode = LZMA_FINISH_ANY;
    }
    else
    {
      outSizeCur = dicPos + outSize;
      curFinishMode = finishMode;
    }

    res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status);
    src += inSizeCur;
    inSize -= inSizeCur;
    *srcLen += inSizeCur;
    outSizeCur = p->dicPos - dicPos;
    memcpy(dest, p->dic + dicPos, outSizeCur);
    dest += outSizeCur;
    outSize -= outSizeCur;
    *destLen += outSizeCur;
    if (res != 0)
      return res;
    if (outSizeCur == 0 || outSize == 0)
      return SZ_OK;
  }
}

void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc)
{
  alloc->Free(alloc, p->probs);
  p->probs = 0;
}

static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc)
{
  alloc->Free(alloc, p->dic);
  p->dic = 0;
}

void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc)
{
  LzmaDec_FreeProbs(p, alloc);
  LzmaDec_FreeDict(p, alloc);
}

SRes LzmaProps_Decode(CLzmaProps *p, const uint8_t *data, unsigned size)
{
  uint32_t dicSize;
  uint8_t d;

  if (size < LZMA_PROPS_SIZE)
    return SZ_ERROR_UNSUPPORTED;
  else
    dicSize = data[1] | ((uint32_t)data[2] << 8) | ((uint32_t)data[3] << 16) | ((uint32_t)data[4] << 24);

  if (dicSize < LZMA_DIC_MIN)
    dicSize = LZMA_DIC_MIN;
  p->dicSize = dicSize;

  d = data[0];
  if (d >= (9 * 5 * 5))
    return SZ_ERROR_UNSUPPORTED;

  p->lc = d % 9;
  d /= 9;
  p->pb = d / 5;
  p->lp = d % 5;

  return SZ_OK;
}

static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc)
{
  uint32_t numProbs = LzmaProps_GetNumProbs(propNew);
  if (p->probs == 0 || numProbs != p->numProbs)
  {
    LzmaDec_FreeProbs(p, alloc);
    p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb));
    p->numProbs = numProbs;
    if (p->probs == 0)
      return SZ_ERROR_MEM;
  }
  return SZ_OK;
}

SRes LzmaDec_AllocateProbs(CLzmaDec *p, const uint8_t *props, unsigned propsSize, ISzAlloc *alloc)
{
  CLzmaProps propNew;
  RINOK(LzmaProps_Decode(&propNew, props, propsSize));
  RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc));
  p->prop = propNew;
  return SZ_OK;
}

SRes LzmaDec_Allocate(CLzmaDec *p, const uint8_t *props, unsigned propsSize, ISzAlloc *alloc)
{
  CLzmaProps propNew;
  size_t dicBufSize;
  RINOK(LzmaProps_Decode(&propNew, props, propsSize));
  RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc));
  dicBufSize = propNew.dicSize;
  if (p->dic == 0 || dicBufSize != p->dicBufSize)
  {
    LzmaDec_FreeDict(p, alloc);
    p->dic = (uint8_t *)alloc->Alloc(alloc, dicBufSize);
    if (p->dic == 0)
    {
      LzmaDec_FreeProbs(p, alloc);
      return SZ_ERROR_MEM;
    }
  }
  p->dicBufSize = dicBufSize;
  p->prop = propNew;
  return SZ_OK;
}

SRes LzmaDecode(uint8_t *dest, size_t *destLen, const uint8_t *src, size_t *srcLen,
    const uint8_t *propData, unsigned propSize, ELzmaFinishMode finishMode,
    ELzmaStatus *status, ISzAlloc *alloc)
{
  CLzmaDec p;
  SRes res;
  size_t inSize = *srcLen;
  size_t outSize = *destLen;
  *srcLen = *destLen = 0;
  if (inSize < RC_INIT_SIZE)
    return SZ_ERROR_INPUT_EOF;

  LzmaDec_Construct(&p);
  res = LzmaDec_AllocateProbs(&p, propData, propSize, alloc);
  if (res != 0)
    return res;
  p.dic = dest;
  p.dicBufSize = outSize;

  LzmaDec_Init(&p);

  *srcLen = inSize;
  res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status);

  if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT)
    res = SZ_ERROR_INPUT_EOF;

  (*destLen) = p.dicPos;
  LzmaDec_FreeProbs(&p, alloc);
  return res;
}

} // namespace lzma
// End of LZMA compression library by Igor Pavlov

#ifndef UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H
#define UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H
static void *LzmaAlloc(void* /*p*/, size_t size) { return new char[size]; }
static void LzmaFree(void* /*p*/, void *address) { delete[] (char*) address; }
static lzma::ISzAlloc lzmaAllocator = { LzmaAlloc, LzmaFree };
#endif // UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H

bool compressor::load(istream& is, binary_decoder& data) {
  uint32_t uncompressed_len, compressed_len, poor_crc;
  unsigned char props_encoded[LZMA_PROPS_SIZE];

  if (!is.read((char *) &uncompressed_len, sizeof(uncompressed_len))) return false;
  if (!is.read((char *) &compressed_len, sizeof(compressed_len))) return false;
  if (!is.read((char *) &poor_crc, sizeof(poor_crc))) return false;
  if (poor_crc != uncompressed_len * 19991 + compressed_len * 199999991 + 1234567890) return false;
  if (!is.read((char *) props_encoded, sizeof(props_encoded))) return false;

  vector<unsigned char> compressed(compressed_len);
  if (!is.read((char *) compressed.data(), compressed_len)) return false;

  lzma::ELzmaStatus status;
  size_t uncompressed_size = uncompressed_len, compressed_size = compressed_len;
  auto res = lzma::LzmaDecode(data.fill(uncompressed_len), &uncompressed_size, compressed.data(), &compressed_size, props_encoded, LZMA_PROPS_SIZE, lzma::LZMA_FINISH_ANY, &status, &lzmaAllocator);
  if (res != SZ_OK || uncompressed_size != uncompressed_len || compressed_size != compressed_len) return false;

  return true;
}

} // namespace utils

/////////
// File: utils/compressor_save.cpp
/////////

// This file is part of UFAL C++ Utils <http://github.com/ufal/cpp_utils/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

// Start of LZMA compression library by Igor Pavlov
namespace lzma {

// Types.h -- Basic types
// 2010-10-09 : Igor Pavlov : Public domain
#ifndef UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H
#define UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H

#define SZ_OK 0

#define SZ_ERROR_DATA 1
#define SZ_ERROR_MEM 2
#define SZ_ERROR_CRC 3
#define SZ_ERROR_UNSUPPORTED 4
#define SZ_ERROR_PARAM 5
#define SZ_ERROR_INPUT_EOF 6
#define SZ_ERROR_OUTPUT_EOF 7
#define SZ_ERROR_READ 8
#define SZ_ERROR_WRITE 9
#define SZ_ERROR_PROGRESS 10
#define SZ_ERROR_FAIL 11
#define SZ_ERROR_THREAD 12

#define SZ_ERROR_ARCHIVE 16
#define SZ_ERROR_NO_ARCHIVE 17

typedef int SRes;

#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif

/* The following interfaces use first parameter as pointer to structure */

struct IByteIn
{
  uint8_t (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */
};

struct IByteOut
{
  void (*Write)(void *p, uint8_t b);
};

struct ISeqInStream
{
  SRes (*Read)(void *p, void *buf, size_t *size);
    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
       (output(*size) < input(*size)) is allowed */
};

/* it can return SZ_ERROR_INPUT_EOF */
SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size);
SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType);
SRes SeqInStream_ReadByte(ISeqInStream *stream, uint8_t *buf);

struct ISeqOutStream
{
  size_t (*Write)(void *p, const void *buf, size_t size);
    /* Returns: result - the number of actually written bytes.
       (result < size) means error */
};

enum ESzSeek
{
  SZ_SEEK_SET = 0,
  SZ_SEEK_CUR = 1,
  SZ_SEEK_END = 2
};

struct ISeekInStream
{
  SRes (*Read)(void *p, void *buf, size_t *size);  /* same as ISeqInStream::Read */
  SRes (*Seek)(void *p, int64_t *pos, ESzSeek origin);
};

struct ILookInStream
{
  SRes (*Look)(void *p, const void **buf, size_t *size);
    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
       (output(*size) > input(*size)) is not allowed
       (output(*size) < input(*size)) is allowed */
  SRes (*Skip)(void *p, size_t offset);
    /* offset must be <= output(*size) of Look */

  SRes (*Read)(void *p, void *buf, size_t *size);
    /* reads directly (without buffer). It's same as ISeqInStream::Read */
  SRes (*Seek)(void *p, int64_t *pos, ESzSeek origin);
};

SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size);
SRes LookInStream_SeekTo(ILookInStream *stream, uint64_t offset);

/* reads via ILookInStream::Read */
SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType);
SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size);

#define LookToRead_BUF_SIZE (1 << 14)

struct CLookToRead
{
  ILookInStream s;
  ISeekInStream *realStream;
  size_t pos;
  size_t size;
  uint8_t buf[LookToRead_BUF_SIZE];
};

void LookToRead_CreateVTable(CLookToRead *p, int lookahead);
void LookToRead_Init(CLookToRead *p);

struct CSecToLook
{
  ISeqInStream s;
  ILookInStream *realStream;
};

void SecToLook_CreateVTable(CSecToLook *p);

struct CSecToRead
{
  ISeqInStream s;
  ILookInStream *realStream;
};

void SecToRead_CreateVTable(CSecToRead *p);

struct ICompressProgress
{
  SRes (*Progress)(void *p, uint64_t inSize, uint64_t outSize);
    /* Returns: result. (result != SZ_OK) means break.
       Value (uint64_t)(int64_t)-1 for size means unknown value. */
};

struct ISzAlloc
{
  void *(*Alloc)(void *p, size_t size);
  void (*Free)(void *p, void *address); /* address can be 0 */
};

#define IAlloc_Alloc(p, size) (p)->Alloc((p), size)
#define IAlloc_Free(p, a) (p)->Free((p), a)

#endif // UFAL_CPPUTILS_COMPRESSOR_LZMA_TYPES_H

// LzHash.h -- HASH functions for LZ algorithms
// 2009-02-07 : Igor Pavlov : Public domain

#define kHash2Size (1 << 10)
#define kHash3Size (1 << 16)
#define kHash4Size (1 << 20)

#define kFix3HashSize (kHash2Size)
#define kFix4HashSize (kHash2Size + kHash3Size)
#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size)

#define HASH2_CALC hashValue = cur[0] | ((uint32_t)cur[1] << 8);

#define HASH3_CALC { \
  uint32_t temp = p->crc[cur[0]] ^ cur[1]; \
  hash2Value = temp & (kHash2Size - 1); \
  hashValue = (temp ^ ((uint32_t)cur[2] << 8)) & p->hashMask; }

#define HASH4_CALC { \
  uint32_t temp = p->crc[cur[0]] ^ cur[1]; \
  hash2Value = temp & (kHash2Size - 1); \
  hash3Value = (temp ^ ((uint32_t)cur[2] << 8)) & (kHash3Size - 1); \
  hashValue = (temp ^ ((uint32_t)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & p->hashMask; }

#define HASH5_CALC { \
  uint32_t temp = p->crc[cur[0]] ^ cur[1]; \
  hash2Value = temp & (kHash2Size - 1); \
  hash3Value = (temp ^ ((uint32_t)cur[2] << 8)) & (kHash3Size - 1); \
  hash4Value = (temp ^ ((uint32_t)cur[2] << 8) ^ (p->crc[cur[3]] << 5)); \
  hashValue = (hash4Value ^ (p->crc[cur[4]] << 3)) & p->hashMask; \
  hash4Value &= (kHash4Size - 1); }

/* #define HASH_ZIP_CALC hashValue = ((cur[0] | ((uint32_t)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */
#define HASH_ZIP_CALC hashValue = ((cur[2] | ((uint32_t)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF;

#define MT_HASH2_CALC \
  hash2Value = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1);

#define MT_HASH3_CALC { \
  uint32_t temp = p->crc[cur[0]] ^ cur[1]; \
  hash2Value = temp & (kHash2Size - 1); \
  hash3Value = (temp ^ ((uint32_t)cur[2] << 8)) & (kHash3Size - 1); }

#define MT_HASH4_CALC { \
  uint32_t temp = p->crc[cur[0]] ^ cur[1]; \
  hash2Value = temp & (kHash2Size - 1); \
  hash3Value = (temp ^ ((uint32_t)cur[2] << 8)) & (kHash3Size - 1); \
  hash4Value = (temp ^ ((uint32_t)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); }

// LzFind.h -- Match finder for LZ algorithms
// 2009-04-22 : Igor Pavlov : Public domain

typedef uint32_t CLzRef;

struct CMatchFinder
{
  uint8_t *buffer;
  uint32_t pos;
  uint32_t posLimit;
  uint32_t streamPos;
  uint32_t lenLimit;

  uint32_t cyclicBufferPos;
  uint32_t cyclicBufferSize; /* it must be = (historySize + 1) */

  uint32_t matchMaxLen;
  CLzRef *hash;
  CLzRef *son;
  uint32_t hashMask;
  uint32_t cutValue;

  uint8_t *bufferBase;
  ISeqInStream *stream;
  int streamEndWasReached;

  uint32_t blockSize;
  uint32_t keepSizeBefore;
  uint32_t keepSizeAfter;

  uint32_t numHashBytes;
  int directInput;
  size_t directInputRem;
  int btMode;
  int bigHash;
  uint32_t historySize;
  uint32_t fixedHashSize;
  uint32_t hashSizeSum;
  uint32_t numSons;
  SRes result;
  uint32_t crc[256];
};

#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)
#define Inline_MatchFinder_GetIndexByte(p, index) ((p)->buffer[(int32_t)(index)])

#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos)

int MatchFinder_NeedMove(CMatchFinder *p);
uint8_t *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p);
void MatchFinder_MoveBlock(CMatchFinder *p);
void MatchFinder_ReadIfRequired(CMatchFinder *p);

void MatchFinder_Construct(CMatchFinder *p);

/* Conditions:
     historySize <= 3 GB
     keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB
*/
int MatchFinder_Create(CMatchFinder *p, uint32_t historySize,
    uint32_t keepAddBufferBefore, uint32_t matchMaxLen, uint32_t keepAddBufferAfter,
    ISzAlloc *alloc);
void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc);
void MatchFinder_Normalize3(uint32_t subValue, CLzRef *items, uint32_t numItems);
void MatchFinder_ReduceOffsets(CMatchFinder *p, uint32_t subValue);

uint32_t * GetMatchesSpec1(uint32_t lenLimit, uint32_t curMatch, uint32_t pos, const uint8_t *buffer, CLzRef *son,
    uint32_t _cyclicBufferPos, uint32_t _cyclicBufferSize, uint32_t _cutValue,
    uint32_t *distances, uint32_t maxLen);

/*
Conditions:
  Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func.
  Mf_GetPointerToCurrentPos_Func's result must be used only before any other function
*/

typedef void (*Mf_Init_Func)(void *object);
typedef uint8_t (*Mf_GetIndexByte_Func)(void *object, int32_t index);
typedef uint32_t (*Mf_GetNumAvailableBytes_Func)(void *object);
typedef const uint8_t * (*Mf_GetPointerToCurrentPos_Func)(void *object);
typedef uint32_t (*Mf_GetMatches_Func)(void *object, uint32_t *distances);
typedef void (*Mf_Skip_Func)(void *object, uint32_t);

struct IMatchFinder
{
  Mf_Init_Func Init;
  Mf_GetIndexByte_Func GetIndexByte;
  Mf_GetNumAvailableBytes_Func GetNumAvailableBytes;
  Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos;
  Mf_GetMatches_Func GetMatches;
  Mf_Skip_Func Skip;
};

void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);

void MatchFinder_Init(CMatchFinder *p);
uint32_t Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances);
uint32_t Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances);
void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, uint32_t num);
void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, uint32_t num);

// LzFind.c -- Match finder for LZ algorithms
// 2009-04-22 : Igor Pavlov : Public domain

#define kEmptyHashValue 0
#define kMaxValForNormalize ((uint32_t)0xFFFFFFFF)
#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */
#define kNormalizeMask (~(kNormalizeStepMin - 1))
#define kMaxHistorySize ((uint32_t)3 << 30)

#define kStartMaxLen 3

static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc)
{
  if (!p->directInput)
  {
    alloc->Free(alloc, p->bufferBase);
    p->bufferBase = 0;
  }
}

/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */

static int LzInWindow_Create(CMatchFinder *p, uint32_t keepSizeReserv, ISzAlloc *alloc)
{
  uint32_t blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv;
  if (p->directInput)
  {
    p->blockSize = blockSize;
    return 1;
  }
  if (p->bufferBase == 0 || p->blockSize != blockSize)
  {
    LzInWindow_Free(p, alloc);
    p->blockSize = blockSize;
    p->bufferBase = (uint8_t *)alloc->Alloc(alloc, (size_t)blockSize);
  }
  return (p->bufferBase != 0);
}

uint8_t *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; }
uint8_t MatchFinder_GetIndexByte(CMatchFinder *p, int32_t index) { return p->buffer[index]; }

uint32_t MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; }

void MatchFinder_ReduceOffsets(CMatchFinder *p, uint32_t subValue)
{
  p->posLimit -= subValue;
  p->pos -= subValue;
  p->streamPos -= subValue;
}

static void MatchFinder_ReadBlock(CMatchFinder *p)
{
  if (p->streamEndWasReached || p->result != SZ_OK)
    return;
  if (p->directInput)
  {
    uint32_t curSize = 0xFFFFFFFF - p->streamPos;
    if (curSize > p->directInputRem)
      curSize = (uint32_t)p->directInputRem;
    p->directInputRem -= curSize;
    p->streamPos += curSize;
    if (p->directInputRem == 0)
      p->streamEndWasReached = 1;
    return;
  }
  for (;;)
  {
    uint8_t *dest = p->buffer + (p->streamPos - p->pos);
    size_t size = (p->bufferBase + p->blockSize - dest);
    if (size == 0)
      return;
    p->result = p->stream->Read(p->stream, dest, &size);
    if (p->result != SZ_OK)
      return;
    if (size == 0)
    {
      p->streamEndWasReached = 1;
      return;
    }
    p->streamPos += (uint32_t)size;
    if (p->streamPos - p->pos > p->keepSizeAfter)
      return;
  }
}

void MatchFinder_MoveBlock(CMatchFinder *p)
{
  memmove(p->bufferBase,
    p->buffer - p->keepSizeBefore,
    (size_t)(p->streamPos - p->pos + p->keepSizeBefore));
  p->buffer = p->bufferBase + p->keepSizeBefore;
}

int MatchFinder_NeedMove(CMatchFinder *p)
{
  if (p->directInput)
    return 0;
  /* if (p->streamEndWasReached) return 0; */
  return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter);
}

void MatchFinder_ReadIfRequired(CMatchFinder *p)
{
  if (p->streamEndWasReached)
    return;
  if (p->keepSizeAfter >= p->streamPos - p->pos)
    MatchFinder_ReadBlock(p);
}

static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p)
{
  if (MatchFinder_NeedMove(p))
    MatchFinder_MoveBlock(p);
  MatchFinder_ReadBlock(p);
}

static void MatchFinder_SetDefaultSettings(CMatchFinder *p)
{
  p->cutValue = 32;
  p->btMode = 1;
  p->numHashBytes = 4;
  p->bigHash = 0;
}

#define kCrcPoly 0xEDB88320

void MatchFinder_Construct(CMatchFinder *p)
{
  uint32_t i;
  p->bufferBase = 0;
  p->directInput = 0;
  p->hash = 0;
  MatchFinder_SetDefaultSettings(p);

  for (i = 0; i < 256; i++)
  {
    uint32_t r = i;
    int j;
    for (j = 0; j < 8; j++)
      r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1));
    p->crc[i] = r;
  }
}

static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc)
{
  alloc->Free(alloc, p->hash);
  p->hash = 0;
}

void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc)
{
  MatchFinder_FreeThisClassMemory(p, alloc);
  LzInWindow_Free(p, alloc);
}

static CLzRef* AllocRefs(uint32_t num, ISzAlloc *alloc)
{
  size_t sizeInBytes = (size_t)num * sizeof(CLzRef);
  if (sizeInBytes / sizeof(CLzRef) != num)
    return 0;
  return (CLzRef *)alloc->Alloc(alloc, sizeInBytes);
}

int MatchFinder_Create(CMatchFinder *p, uint32_t historySize,
    uint32_t keepAddBufferBefore, uint32_t matchMaxLen, uint32_t keepAddBufferAfter,
    ISzAlloc *alloc)
{
  uint32_t sizeReserv;
  if (historySize > kMaxHistorySize)
  {
    MatchFinder_Free(p, alloc);
    return 0;
  }
  sizeReserv = historySize >> 1;
  if (historySize > ((uint32_t)2 << 30))
    sizeReserv = historySize >> 2;
  sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19);

  p->keepSizeBefore = historySize + keepAddBufferBefore + 1;
  p->keepSizeAfter = matchMaxLen + keepAddBufferAfter;
  /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */
  if (LzInWindow_Create(p, sizeReserv, alloc))
  {
    uint32_t newCyclicBufferSize = historySize + 1;
    uint32_t hs;
    p->matchMaxLen = matchMaxLen;
    {
      p->fixedHashSize = 0;
      if (p->numHashBytes == 2)
        hs = (1 << 16) - 1;
      else
      {
        hs = historySize - 1;
        hs |= (hs >> 1);
        hs |= (hs >> 2);
        hs |= (hs >> 4);
        hs |= (hs >> 8);
        hs >>= 1;
        hs |= 0xFFFF; /* don't change it! It's required for Deflate */
        if (hs > (1 << 24))
        {
          if (p->numHashBytes == 3)
            hs = (1 << 24) - 1;
          else
            hs >>= 1;
        }
      }
      p->hashMask = hs;
      hs++;
      if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size;
      if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size;
      if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size;
      hs += p->fixedHashSize;
    }

    {
      uint32_t prevSize = p->hashSizeSum + p->numSons;
      uint32_t newSize;
      p->historySize = historySize;
      p->hashSizeSum = hs;
      p->cyclicBufferSize = newCyclicBufferSize;
      p->numSons = (p->btMode ? newCyclicBufferSize * 2 : newCyclicBufferSize);
      newSize = p->hashSizeSum + p->numSons;
      if (p->hash != 0 && prevSize == newSize)
        return 1;
      MatchFinder_FreeThisClassMemory(p, alloc);
      p->hash = AllocRefs(newSize, alloc);
      if (p->hash != 0)
      {
        p->son = p->hash + p->hashSizeSum;
        return 1;
      }
    }
  }
  MatchFinder_Free(p, alloc);
  return 0;
}

static void MatchFinder_SetLimits(CMatchFinder *p)
{
  uint32_t limit = kMaxValForNormalize - p->pos;
  uint32_t limit2 = p->cyclicBufferSize - p->cyclicBufferPos;
  if (limit2 < limit)
    limit = limit2;
  limit2 = p->streamPos - p->pos;
  if (limit2 <= p->keepSizeAfter)
  {
    if (limit2 > 0)
      limit2 = 1;
  }
  else
    limit2 -= p->keepSizeAfter;
  if (limit2 < limit)
    limit = limit2;
  {
    uint32_t lenLimit = p->streamPos - p->pos;
    if (lenLimit > p->matchMaxLen)
      lenLimit = p->matchMaxLen;
    p->lenLimit = lenLimit;
  }
  p->posLimit = p->pos + limit;
}

void MatchFinder_Init(CMatchFinder *p)
{
  uint32_t i;
  for (i = 0; i < p->hashSizeSum; i++)
    p->hash[i] = kEmptyHashValue;
  p->cyclicBufferPos = 0;
  p->buffer = p->bufferBase;
  p->pos = p->streamPos = p->cyclicBufferSize;
  p->result = SZ_OK;
  p->streamEndWasReached = 0;
  MatchFinder_ReadBlock(p);
  MatchFinder_SetLimits(p);
}

static uint32_t MatchFinder_GetSubValue(CMatchFinder *p)
{
  return (p->pos - p->historySize - 1) & kNormalizeMask;
}

void MatchFinder_Normalize3(uint32_t subValue, CLzRef *items, uint32_t numItems)
{
  uint32_t i;
  for (i = 0; i < numItems; i++)
  {
    uint32_t value = items[i];
    if (value <= subValue)
      value = kEmptyHashValue;
    else
      value -= subValue;
    items[i] = value;
  }
}

static void MatchFinder_Normalize(CMatchFinder *p)
{
  uint32_t subValue = MatchFinder_GetSubValue(p);
  MatchFinder_Normalize3(subValue, p->hash, p->hashSizeSum + p->numSons);
  MatchFinder_ReduceOffsets(p, subValue);
}

static void MatchFinder_CheckLimits(CMatchFinder *p)
{
  if (p->pos == kMaxValForNormalize)
    MatchFinder_Normalize(p);
  if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos)
    MatchFinder_CheckAndMoveAndRead(p);
  if (p->cyclicBufferPos == p->cyclicBufferSize)
    p->cyclicBufferPos = 0;
  MatchFinder_SetLimits(p);
}

static uint32_t * Hc_GetMatchesSpec(uint32_t lenLimit, uint32_t curMatch, uint32_t pos, const uint8_t *cur, CLzRef *son,
    uint32_t _cyclicBufferPos, uint32_t _cyclicBufferSize, uint32_t cutValue,
    uint32_t *distances, uint32_t maxLen)
{
  son[_cyclicBufferPos] = curMatch;
  for (;;)
  {
    uint32_t delta = pos - curMatch;
    if (cutValue-- == 0 || delta >= _cyclicBufferSize)
      return distances;
    {
      const uint8_t *pb = cur - delta;
      curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)];
      if (pb[maxLen] == cur[maxLen] && *pb == *cur)
      {
        uint32_t len = 0;
        while (++len != lenLimit)
          if (pb[len] != cur[len])
            break;
        if (maxLen < len)
        {
          *distances++ = maxLen = len;
          *distances++ = delta - 1;
          if (len == lenLimit)
            return distances;
        }
      }
    }
  }
}

uint32_t * GetMatchesSpec1(uint32_t lenLimit, uint32_t curMatch, uint32_t pos, const uint8_t *cur, CLzRef *son,
    uint32_t _cyclicBufferPos, uint32_t _cyclicBufferSize, uint32_t cutValue,
    uint32_t *distances, uint32_t maxLen)
{
  CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
  CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
  uint32_t len0 = 0, len1 = 0;
  for (;;)
  {
    uint32_t delta = pos - curMatch;
    if (cutValue-- == 0 || delta >= _cyclicBufferSize)
    {
      *ptr0 = *ptr1 = kEmptyHashValue;
      return distances;
    }
    {
      CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
      const uint8_t *pb = cur - delta;
      uint32_t len = (len0 < len1 ? len0 : len1);
      if (pb[len] == cur[len])
      {
        if (++len != lenLimit && pb[len] == cur[len])
          while (++len != lenLimit)
            if (pb[len] != cur[len])
              break;
        if (maxLen < len)
        {
          *distances++ = maxLen = len;
          *distances++ = delta - 1;
          if (len == lenLimit)
          {
            *ptr1 = pair[0];
            *ptr0 = pair[1];
            return distances;
          }
        }
      }
      if (pb[len] < cur[len])
      {
        *ptr1 = curMatch;
        ptr1 = pair + 1;
        curMatch = *ptr1;
        len1 = len;
      }
      else
      {
        *ptr0 = curMatch;
        ptr0 = pair;
        curMatch = *ptr0;
        len0 = len;
      }
    }
  }
}

static void SkipMatchesSpec(uint32_t lenLimit, uint32_t curMatch, uint32_t pos, const uint8_t *cur, CLzRef *son,
    uint32_t _cyclicBufferPos, uint32_t _cyclicBufferSize, uint32_t cutValue)
{
  CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
  CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
  uint32_t len0 = 0, len1 = 0;
  for (;;)
  {
    uint32_t delta = pos - curMatch;
    if (cutValue-- == 0 || delta >= _cyclicBufferSize)
    {
      *ptr0 = *ptr1 = kEmptyHashValue;
      return;
    }
    {
      CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
      const uint8_t *pb = cur - delta;
      uint32_t len = (len0 < len1 ? len0 : len1);
      if (pb[len] == cur[len])
      {
        while (++len != lenLimit)
          if (pb[len] != cur[len])
            break;
        {
          if (len == lenLimit)
          {
            *ptr1 = pair[0];
            *ptr0 = pair[1];
            return;
          }
        }
      }
      if (pb[len] < cur[len])
      {
        *ptr1 = curMatch;
        ptr1 = pair + 1;
        curMatch = *ptr1;
        len1 = len;
      }
      else
      {
        *ptr0 = curMatch;
        ptr0 = pair;
        curMatch = *ptr0;
        len0 = len;
      }
    }
  }
}

#define MOVE_POS \
  ++p->cyclicBufferPos; \
  p->buffer++; \
  if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p);

#define MOVE_POS_RET MOVE_POS return offset;

static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; }

#define GET_MATCHES_HEADER2(minLen, ret_op) \
  uint32_t lenLimit; uint32_t hashValue; const uint8_t *cur; uint32_t curMatch; \
  lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \
  cur = p->buffer;

#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0)
#define SKIP_HEADER(minLen)        GET_MATCHES_HEADER2(minLen, continue)

#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue

#define GET_MATCHES_FOOTER(offset, maxLen) \
  offset = (uint32_t)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \
  distances + offset, maxLen) - distances); MOVE_POS_RET;

#define SKIP_FOOTER \
  SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS;

static uint32_t Bt2_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t offset;
  GET_MATCHES_HEADER(2)
  HASH2_CALC;
  curMatch = p->hash[hashValue];
  p->hash[hashValue] = p->pos;
  offset = 0;
  GET_MATCHES_FOOTER(offset, 1)
}

uint32_t Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t offset;
  GET_MATCHES_HEADER(3)
  HASH_ZIP_CALC;
  curMatch = p->hash[hashValue];
  p->hash[hashValue] = p->pos;
  offset = 0;
  GET_MATCHES_FOOTER(offset, 2)
}

static uint32_t Bt3_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t hash2Value, delta2, maxLen, offset;
  GET_MATCHES_HEADER(3)

  HASH3_CALC;

  delta2 = p->pos - p->hash[hash2Value];
  curMatch = p->hash[kFix3HashSize + hashValue];

  p->hash[hash2Value] =
  p->hash[kFix3HashSize + hashValue] = p->pos;

  maxLen = 2;
  offset = 0;
  if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur)
  {
    for (; maxLen != lenLimit; maxLen++)
      if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen])
        break;
    distances[0] = maxLen;
    distances[1] = delta2 - 1;
    offset = 2;
    if (maxLen == lenLimit)
    {
      SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p));
      MOVE_POS_RET;
    }
  }
  GET_MATCHES_FOOTER(offset, maxLen)
}

static uint32_t Bt4_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t hash2Value, hash3Value, delta2, delta3, maxLen, offset;
  GET_MATCHES_HEADER(4)

  HASH4_CALC;

  delta2 = p->pos - p->hash[                hash2Value];
  delta3 = p->pos - p->hash[kFix3HashSize + hash3Value];
  curMatch = p->hash[kFix4HashSize + hashValue];

  p->hash[                hash2Value] =
  p->hash[kFix3HashSize + hash3Value] =
  p->hash[kFix4HashSize + hashValue] = p->pos;

  maxLen = 1;
  offset = 0;
  if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur)
  {
    distances[0] = maxLen = 2;
    distances[1] = delta2 - 1;
    offset = 2;
  }
  if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur)
  {
    maxLen = 3;
    distances[offset + 1] = delta3 - 1;
    offset += 2;
    delta2 = delta3;
  }
  if (offset != 0)
  {
    for (; maxLen != lenLimit; maxLen++)
      if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen])
        break;
    distances[offset - 2] = maxLen;
    if (maxLen == lenLimit)
    {
      SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p));
      MOVE_POS_RET;
    }
  }
  if (maxLen < 3)
    maxLen = 3;
  GET_MATCHES_FOOTER(offset, maxLen)
}

static uint32_t Hc4_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t hash2Value, hash3Value, delta2, delta3, maxLen, offset;
  GET_MATCHES_HEADER(4)

  HASH4_CALC;

  delta2 = p->pos - p->hash[                hash2Value];
  delta3 = p->pos - p->hash[kFix3HashSize + hash3Value];
  curMatch = p->hash[kFix4HashSize + hashValue];

  p->hash[                hash2Value] =
  p->hash[kFix3HashSize + hash3Value] =
  p->hash[kFix4HashSize + hashValue] = p->pos;

  maxLen = 1;
  offset = 0;
  if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur)
  {
    distances[0] = maxLen = 2;
    distances[1] = delta2 - 1;
    offset = 2;
  }
  if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur)
  {
    maxLen = 3;
    distances[offset + 1] = delta3 - 1;
    offset += 2;
    delta2 = delta3;
  }
  if (offset != 0)
  {
    for (; maxLen != lenLimit; maxLen++)
      if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen])
        break;
    distances[offset - 2] = maxLen;
    if (maxLen == lenLimit)
    {
      p->son[p->cyclicBufferPos] = curMatch;
      MOVE_POS_RET;
    }
  }
  if (maxLen < 3)
    maxLen = 3;
  offset = (uint32_t)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
    distances + offset, maxLen) - (distances));
  MOVE_POS_RET
}

uint32_t Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, uint32_t *distances)
{
  uint32_t offset;
  GET_MATCHES_HEADER(3)
  HASH_ZIP_CALC;
  curMatch = p->hash[hashValue];
  p->hash[hashValue] = p->pos;
  offset = (uint32_t)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
    distances, 2) - (distances));
  MOVE_POS_RET
}

static void Bt2_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    SKIP_HEADER(2)
    HASH2_CALC;
    curMatch = p->hash[hashValue];
    p->hash[hashValue] = p->pos;
    SKIP_FOOTER
  }
  while (--num != 0);
}

void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    SKIP_HEADER(3)
    HASH_ZIP_CALC;
    curMatch = p->hash[hashValue];
    p->hash[hashValue] = p->pos;
    SKIP_FOOTER
  }
  while (--num != 0);
}

static void Bt3_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    uint32_t hash2Value;
    SKIP_HEADER(3)
    HASH3_CALC;
    curMatch = p->hash[kFix3HashSize + hashValue];
    p->hash[hash2Value] =
    p->hash[kFix3HashSize + hashValue] = p->pos;
    SKIP_FOOTER
  }
  while (--num != 0);
}

static void Bt4_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    uint32_t hash2Value, hash3Value;
    SKIP_HEADER(4)
    HASH4_CALC;
    curMatch = p->hash[kFix4HashSize + hashValue];
    p->hash[                hash2Value] =
    p->hash[kFix3HashSize + hash3Value] = p->pos;
    p->hash[kFix4HashSize + hashValue] = p->pos;
    SKIP_FOOTER
  }
  while (--num != 0);
}

static void Hc4_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    uint32_t hash2Value, hash3Value;
    SKIP_HEADER(4)
    HASH4_CALC;
    curMatch = p->hash[kFix4HashSize + hashValue];
    p->hash[                hash2Value] =
    p->hash[kFix3HashSize + hash3Value] =
    p->hash[kFix4HashSize + hashValue] = p->pos;
    p->son[p->cyclicBufferPos] = curMatch;
    MOVE_POS
  }
  while (--num != 0);
}

void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, uint32_t num)
{
  do
  {
    SKIP_HEADER(3)
    HASH_ZIP_CALC;
    curMatch = p->hash[hashValue];
    p->hash[hashValue] = p->pos;
    p->son[p->cyclicBufferPos] = curMatch;
    MOVE_POS
  }
  while (--num != 0);
}

void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable)
{
  vTable->Init = (Mf_Init_Func)MatchFinder_Init;
  vTable->GetIndexByte = (Mf_GetIndexByte_Func)MatchFinder_GetIndexByte;
  vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes;
  vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos;
  if (!p->btMode)
  {
    vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches;
    vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip;
  }
  else if (p->numHashBytes == 2)
  {
    vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches;
    vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip;
  }
  else if (p->numHashBytes == 3)
  {
    vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches;
    vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip;
  }
  else
  {
    vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches;
    vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip;
  }
}

//  LzmaEnc.h -- LZMA Encoder
// 2009-02-07 : Igor Pavlov : Public domain

#define LZMA_PROPS_SIZE 5

struct CLzmaEncProps
{
  int level;       /*  0 <= level <= 9 */
  uint32_t dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version
                      (1 << 12) <= dictSize <= (1 << 30) for 64-bit version
                       default = (1 << 24) */
  int lc;          /* 0 <= lc <= 8, default = 3 */
  int lp;          /* 0 <= lp <= 4, default = 0 */
  int pb;          /* 0 <= pb <= 4, default = 2 */
  int algo;        /* 0 - fast, 1 - normal, default = 1 */
  int fb;          /* 5 <= fb <= 273, default = 32 */
  int btMode;      /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */
  int numHashBytes; /* 2, 3 or 4, default = 4 */
  uint32_t mc;        /* 1 <= mc <= (1 << 30), default = 32 */
  unsigned writeEndMark;  /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */
  int numThreads;  /* 1 or 2, default = 2 */
};

void LzmaEncProps_Init(CLzmaEncProps *p);
void LzmaEncProps_Normalize(CLzmaEncProps *p);
uint32_t LzmaEncProps_GetDictSize(const CLzmaEncProps *props2);

/* ---------- CLzmaEncHandle Interface ---------- */

/* LzmaEnc_* functions can return the following exit codes:
Returns:
  SZ_OK           - OK
  SZ_ERROR_MEM    - Memory allocation error
  SZ_ERROR_PARAM  - Incorrect paramater in props
  SZ_ERROR_WRITE  - Write callback error.
  SZ_ERROR_PROGRESS - some break from progress callback
  SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/

typedef void * CLzmaEncHandle;

CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc);
void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props);
SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, uint8_t *properties, size_t *size);
SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream,
    ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_MemEncode(CLzmaEncHandle p, uint8_t *dest, size_t *destLen, const uint8_t *src, size_t srcLen,
    int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);

/* ---------- One Call Interface ---------- */

/* LzmaEncode
Return code:
  SZ_OK               - OK
  SZ_ERROR_MEM        - Memory allocation error
  SZ_ERROR_PARAM      - Incorrect paramater
  SZ_ERROR_OUTPUT_EOF - output buffer overflow
  SZ_ERROR_THREAD     - errors in multithreading functions (only for Mt version)
*/

SRes LzmaEncode(uint8_t *dest, size_t *destLen, const uint8_t *src, size_t srcLen,
    const CLzmaEncProps *props, uint8_t *propsEncoded, size_t *propsSize, int writeEndMark,
    ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig);

// LzmaEnc.c -- LZMA Encoder
// 2010-04-16 : Igor Pavlov : Public domain

#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1)

#define kBlockSize (9 << 10)
#define kUnpackBlockSize (1 << 18)
#define kMatchArraySize (1 << 21)
#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX)

#define kNumMaxDirectBits (31)

#define kNumTopBits 24
#define kTopValue ((uint32_t)1 << kNumTopBits)

#define kNumBitModelTotalBits 11
#define kBitModelTotal (1 << kNumBitModelTotalBits)
#define kNumMoveBits 5
#define kProbInitValue (kBitModelTotal >> 1)

#define kNumMoveReducingBits 4
#define kNumBitPriceShiftBits 4
#define kBitPrice (1 << kNumBitPriceShiftBits)

void LzmaEncProps_Init(CLzmaEncProps *p)
{
  p->level = 5;
  p->dictSize = p->mc = 0;
  p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1;
  p->writeEndMark = 0;
}

void LzmaEncProps_Normalize(CLzmaEncProps *p)
{
  int level = p->level;
  if (level < 0) level = 5;
  p->level = level;
  if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)));
  if (p->lc < 0) p->lc = 3;
  if (p->lp < 0) p->lp = 0;
  if (p->pb < 0) p->pb = 2;
  if (p->algo < 0) p->algo = (level < 5 ? 0 : 1);
  if (p->fb < 0) p->fb = (level < 7 ? 32 : 64);
  if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1);
  if (p->numHashBytes < 0) p->numHashBytes = 4;
  if (p->mc == 0)  p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1);
  if (p->numThreads < 0)
    p->numThreads = 1;
}

uint32_t LzmaEncProps_GetDictSize(const CLzmaEncProps *props2)
{
  CLzmaEncProps props = *props2;
  LzmaEncProps_Normalize(&props);
  return props.dictSize;
}

/* #define LZMA_LOG_BSR */
/* Define it for Intel's CPU */

#ifdef LZMA_LOG_BSR

#define kDicLogSizeMaxCompress 30

#define BSR2_RET(pos, res) { unsigned long i; _BitScanReverse(&i, (pos)); res = (i + i) + ((pos >> (i - 1)) & 1); }

uint32_t GetPosSlot1(uint32_t pos)
{
  uint32_t res;
  BSR2_RET(pos, res);
  return res;
}
#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); }
#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); }

#else

//#define kNumLogBits (9 + (int)sizeof(size_t) / 2)
#define kNumLogBits (9 + (int)sizeof(uint32_t) / 2)
#define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7)

void LzmaEnc_FastPosInit(uint8_t *g_FastPos)
{
  int c = 2, slotFast;
  g_FastPos[0] = 0;
  g_FastPos[1] = 1;

  for (slotFast = 2; slotFast < kNumLogBits * 2; slotFast++)
  {
    uint32_t k = (1 << ((slotFast >> 1) - 1));
    uint32_t j;
    for (j = 0; j < k; j++, c++)
      g_FastPos[c] = (uint8_t)slotFast;
  }
}

#define BSR2_RET(pos, res) { uint32_t i = 6 + ((kNumLogBits - 1) & \
  (0 - (((((uint32_t)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \
  res = p->g_FastPos[pos >> i] + (i * 2); }
/*
#define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \
  p->g_FastPos[pos >> 6] + 12 : \
  p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; }
*/

#define GetPosSlot1(pos) p->g_FastPos[pos]
#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); }
#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); }

#endif

#define LZMA_NUM_REPS 4

typedef unsigned CState;

struct COptimal
{
  uint32_t price;

  CState state;
  int prev1IsChar;
  int prev2;

  uint32_t posPrev2;
  uint32_t backPrev2;

  uint32_t posPrev;
  uint32_t backPrev;
  uint32_t backs[LZMA_NUM_REPS];
};

#define kNumOpts (1 << 12)

#define kNumLenToPosStates 4
#define kNumPosSlotBits 6
#define kDicLogSizeMin 0
#define kDicLogSizeMax 32
#define kDistTableSizeMax (kDicLogSizeMax * 2)

#define kNumAlignBits 4
#define kAlignTableSize (1 << kNumAlignBits)
#define kAlignMask (kAlignTableSize - 1)

#define kStartPosModelIndex 4
#define kEndPosModelIndex 14
#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex)

#define kNumFullDistances (1 << (kEndPosModelIndex >> 1))

#ifdef _LZMA_PROB32
#define CLzmaProb uint32_t
#else
#define CLzmaProb uint16_t
#endif

#define LZMA_PB_MAX 4
#define LZMA_LC_MAX 8
#define LZMA_LP_MAX 4

#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX)

#define kLenNumLowBits 3
#define kLenNumLowSymbols (1 << kLenNumLowBits)
#define kLenNumMidBits 3
#define kLenNumMidSymbols (1 << kLenNumMidBits)
#define kLenNumHighBits 8
#define kLenNumHighSymbols (1 << kLenNumHighBits)

#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols)

#define LZMA_MATCH_LEN_MIN 2
#define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1)

#define kNumStates 12

struct CLenEnc
{
  CLzmaProb choice;
  CLzmaProb choice2;
  CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits];
  CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits];
  CLzmaProb high[kLenNumHighSymbols];
};

struct CLenPriceEnc
{
  CLenEnc p;
  uint32_t prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal];
  uint32_t tableSize;
  uint32_t counters[LZMA_NUM_PB_STATES_MAX];
};

struct CRangeEnc
{
  uint32_t range;
  uint8_t cache;
  uint64_t low;
  uint64_t cacheSize;
  uint8_t *buf;
  uint8_t *bufLim;
  uint8_t *bufBase;
  ISeqOutStream *outStream;
  uint64_t processed;
  SRes res;
};

struct CSaveState
{
  CLzmaProb *litProbs;

  CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX];
  CLzmaProb isRep[kNumStates];
  CLzmaProb isRepG0[kNumStates];
  CLzmaProb isRepG1[kNumStates];
  CLzmaProb isRepG2[kNumStates];
  CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX];

  CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits];
  CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex];
  CLzmaProb posAlignEncoder[1 << kNumAlignBits];

  CLenPriceEnc lenEnc;
  CLenPriceEnc repLenEnc;

  uint32_t reps[LZMA_NUM_REPS];
  uint32_t state;
};

struct CLzmaEnc
{
  IMatchFinder matchFinder;
  void *matchFinderObj;

  CMatchFinder matchFinderBase;

  uint32_t optimumEndIndex;
  uint32_t optimumCurrentIndex;

  uint32_t longestMatchLength;
  uint32_t numPairs;
  uint32_t numAvail;
  COptimal opt[kNumOpts];

  #ifndef LZMA_LOG_BSR
  uint8_t g_FastPos[1 << kNumLogBits];
  #endif

  uint32_t ProbPrices[kBitModelTotal >> kNumMoveReducingBits];
  uint32_t matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1];
  uint32_t numFastBytes;
  uint32_t additionalOffset;
  uint32_t reps[LZMA_NUM_REPS];
  uint32_t state;

  uint32_t posSlotPrices[kNumLenToPosStates][kDistTableSizeMax];
  uint32_t distancesPrices[kNumLenToPosStates][kNumFullDistances];
  uint32_t alignPrices[kAlignTableSize];
  uint32_t alignPriceCount;

  uint32_t distTableSize;

  unsigned lc, lp, pb;
  unsigned lpMask, pbMask;

  CLzmaProb *litProbs;

  CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX];
  CLzmaProb isRep[kNumStates];
  CLzmaProb isRepG0[kNumStates];
  CLzmaProb isRepG1[kNumStates];
  CLzmaProb isRepG2[kNumStates];
  CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX];

  CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits];
  CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex];
  CLzmaProb posAlignEncoder[1 << kNumAlignBits];

  CLenPriceEnc lenEnc;
  CLenPriceEnc repLenEnc;

  unsigned lclp;

  bool fastMode;

  CRangeEnc rc;

  bool writeEndMark;
  uint64_t nowPos64;
  uint32_t matchPriceCount;
  bool finished;
  bool multiThread;

  SRes result;
  uint32_t dictSize;
  uint32_t matchFinderCycles;

  int needInit;

  CSaveState saveState;
};

void LzmaEnc_SaveState(CLzmaEncHandle pp)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  CSaveState *dest = &p->saveState;
  int i;
  dest->lenEnc = p->lenEnc;
  dest->repLenEnc = p->repLenEnc;
  dest->state = p->state;

  for (i = 0; i < kNumStates; i++)
  {
    memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i]));
    memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i]));
  }
  for (i = 0; i < kNumLenToPosStates; i++)
    memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i]));
  memcpy(dest->isRep, p->isRep, sizeof(p->isRep));
  memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0));
  memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1));
  memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2));
  memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders));
  memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder));
  memcpy(dest->reps, p->reps, sizeof(p->reps));
  memcpy(dest->litProbs, p->litProbs, (0x300 << p->lclp) * sizeof(CLzmaProb));
}

void LzmaEnc_RestoreState(CLzmaEncHandle pp)
{
  CLzmaEnc *dest = (CLzmaEnc *)pp;
  const CSaveState *p = &dest->saveState;
  int i;
  dest->lenEnc = p->lenEnc;
  dest->repLenEnc = p->repLenEnc;
  dest->state = p->state;

  for (i = 0; i < kNumStates; i++)
  {
    memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i]));
    memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i]));
  }
  for (i = 0; i < kNumLenToPosStates; i++)
    memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i]));
  memcpy(dest->isRep, p->isRep, sizeof(p->isRep));
  memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0));
  memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1));
  memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2));
  memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders));
  memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder));
  memcpy(dest->reps, p->reps, sizeof(p->reps));
  memcpy(dest->litProbs, p->litProbs, (0x300 << dest->lclp) * sizeof(CLzmaProb));
}

SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  CLzmaEncProps props = *props2;
  LzmaEncProps_Normalize(&props);

  if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX ||
      props.dictSize > ((uint32_t)1 << kDicLogSizeMaxCompress) || props.dictSize > ((uint32_t)1 << 30))
    return SZ_ERROR_PARAM;
  p->dictSize = props.dictSize;
  p->matchFinderCycles = props.mc;
  {
    unsigned fb = props.fb;
    if (fb < 5)
      fb = 5;
    if (fb > LZMA_MATCH_LEN_MAX)
      fb = LZMA_MATCH_LEN_MAX;
    p->numFastBytes = fb;
  }
  p->lc = props.lc;
  p->lp = props.lp;
  p->pb = props.pb;
  p->fastMode = (props.algo == 0);
  p->matchFinderBase.btMode = props.btMode;
  {
    uint32_t numHashBytes = 4;
    if (props.btMode)
    {
      if (props.numHashBytes < 2)
        numHashBytes = 2;
      else if (props.numHashBytes < 4)
        numHashBytes = props.numHashBytes;
    }
    p->matchFinderBase.numHashBytes = numHashBytes;
  }

  p->matchFinderBase.cutValue = props.mc;

  p->writeEndMark = props.writeEndMark;

  return SZ_OK;
}

static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4,  5,  6,   4, 5};
static const int kMatchNextStates[kNumStates]   = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10};
static const int kRepNextStates[kNumStates]     = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11};
static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11};

#define IsCharState(s) ((s) < 7)

#define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1)

#define kInfinityPrice (1 << 30)

static void RangeEnc_Construct(CRangeEnc *p)
{
  p->outStream = 0;
  p->bufBase = 0;
}

#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize)

#define RC_BUF_SIZE (1 << 16)
static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc)
{
  if (p->bufBase == 0)
  {
    p->bufBase = (uint8_t *)alloc->Alloc(alloc, RC_BUF_SIZE);
    if (p->bufBase == 0)
      return 0;
    p->bufLim = p->bufBase + RC_BUF_SIZE;
  }
  return 1;
}

static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc)
{
  alloc->Free(alloc, p->bufBase);
  p->bufBase = 0;
}

static void RangeEnc_Init(CRangeEnc *p)
{
  /* Stream.Init(); */
  p->low = 0;
  p->range = 0xFFFFFFFF;
  p->cacheSize = 1;
  p->cache = 0;

  p->buf = p->bufBase;

  p->processed = 0;
  p->res = SZ_OK;
}

static void RangeEnc_FlushStream(CRangeEnc *p)
{
  size_t num;
  if (p->res != SZ_OK)
    return;
  num = p->buf - p->bufBase;
  if (num != p->outStream->Write(p->outStream, p->bufBase, num))
    p->res = SZ_ERROR_WRITE;
  p->processed += num;
  p->buf = p->bufBase;
}

static void RangeEnc_ShiftLow(CRangeEnc *p)
{
  if ((uint32_t)p->low < (uint32_t)0xFF000000 || (int)(p->low >> 32) != 0)
  {
    uint8_t temp = p->cache;
    do
    {
      uint8_t *buf = p->buf;
      *buf++ = (uint8_t)(temp + (uint8_t)(p->low >> 32));
      p->buf = buf;
      if (buf == p->bufLim)
        RangeEnc_FlushStream(p);
      temp = 0xFF;
    }
    while (--p->cacheSize != 0);
    p->cache = (uint8_t)((uint32_t)p->low >> 24);
  }
  p->cacheSize++;
  p->low = (uint32_t)p->low << 8;
}

static void RangeEnc_FlushData(CRangeEnc *p)
{
  int i;
  for (i = 0; i < 5; i++)
    RangeEnc_ShiftLow(p);
}

static void RangeEnc_EncodeDirectBits(CRangeEnc *p, uint32_t value, int numBits)
{
  do
  {
    p->range >>= 1;
    p->low += p->range & (0 - ((value >> --numBits) & 1));
    if (p->range < kTopValue)
    {
      p->range <<= 8;
      RangeEnc_ShiftLow(p);
    }
  }
  while (numBits != 0);
}

static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, uint32_t symbol)
{
  uint32_t ttt = *prob;
  uint32_t newBound = (p->range >> kNumBitModelTotalBits) * ttt;
  if (symbol == 0)
  {
    p->range = newBound;
    ttt += (kBitModelTotal - ttt) >> kNumMoveBits;
  }
  else
  {
    p->low += newBound;
    p->range -= newBound;
    ttt -= ttt >> kNumMoveBits;
  }
  *prob = (CLzmaProb)ttt;
  if (p->range < kTopValue)
  {
    p->range <<= 8;
    RangeEnc_ShiftLow(p);
  }
}

static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, uint32_t symbol)
{
  symbol |= 0x100;
  do
  {
    RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1);
    symbol <<= 1;
  }
  while (symbol < 0x10000);
}

static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, uint32_t symbol, uint32_t matchByte)
{
  uint32_t offs = 0x100;
  symbol |= 0x100;
  do
  {
    matchByte <<= 1;
    RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1);
    symbol <<= 1;
    offs &= ~(matchByte ^ symbol);
  }
  while (symbol < 0x10000);
}

void LzmaEnc_InitPriceTables(uint32_t *ProbPrices)
{
  uint32_t i;
  for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits))
  {
    const int kCyclesBits = kNumBitPriceShiftBits;
    uint32_t w = i;
    uint32_t bitCount = 0;
    int j;
    for (j = 0; j < kCyclesBits; j++)
    {
      w = w * w;
      bitCount <<= 1;
      while (w >= ((uint32_t)1 << 16))
      {
        w >>= 1;
        bitCount++;
      }
    }
    ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount);
  }
}

#define GET_PRICE(prob, symbol) \
  p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits];

#define GET_PRICEa(prob, symbol) \
  ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits];

#define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits]
#define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits]

#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits]
#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits]

static uint32_t LitEnc_GetPrice(const CLzmaProb *probs, uint32_t symbol, uint32_t *ProbPrices)
{
  uint32_t price = 0;
  symbol |= 0x100;
  do
  {
    price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1);
    symbol <<= 1;
  }
  while (symbol < 0x10000);
  return price;
}

static uint32_t LitEnc_GetPriceMatched(const CLzmaProb *probs, uint32_t symbol, uint32_t matchByte, uint32_t *ProbPrices)
{
  uint32_t price = 0;
  uint32_t offs = 0x100;
  symbol |= 0x100;
  do
  {
    matchByte <<= 1;
    price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1);
    symbol <<= 1;
    offs &= ~(matchByte ^ symbol);
  }
  while (symbol < 0x10000);
  return price;
}

static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, uint32_t symbol)
{
  uint32_t m = 1;
  int i;
  for (i = numBitLevels; i != 0;)
  {
    uint32_t bit;
    i--;
    bit = (symbol >> i) & 1;
    RangeEnc_EncodeBit(rc, probs + m, bit);
    m = (m << 1) | bit;
  }
}

static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, uint32_t symbol)
{
  uint32_t m = 1;
  int i;
  for (i = 0; i < numBitLevels; i++)
  {
    uint32_t bit = symbol & 1;
    RangeEnc_EncodeBit(rc, probs + m, bit);
    m = (m << 1) | bit;
    symbol >>= 1;
  }
}

static uint32_t RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, uint32_t symbol, uint32_t *ProbPrices)
{
  uint32_t price = 0;
  symbol |= (1 << numBitLevels);
  while (symbol != 1)
  {
    price += GET_PRICEa(probs[symbol >> 1], symbol & 1);
    symbol >>= 1;
  }
  return price;
}

static uint32_t RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, uint32_t symbol, uint32_t *ProbPrices)
{
  uint32_t price = 0;
  uint32_t m = 1;
  int i;
  for (i = numBitLevels; i != 0; i--)
  {
    uint32_t bit = symbol & 1;
    symbol >>= 1;
    price += GET_PRICEa(probs[m], bit);
    m = (m << 1) | bit;
  }
  return price;
}

static void LenEnc_Init(CLenEnc *p)
{
  unsigned i;
  p->choice = p->choice2 = kProbInitValue;
  for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++)
    p->low[i] = kProbInitValue;
  for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++)
    p->mid[i] = kProbInitValue;
  for (i = 0; i < kLenNumHighSymbols; i++)
    p->high[i] = kProbInitValue;
}

static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, uint32_t symbol, uint32_t posState)
{
  if (symbol < kLenNumLowSymbols)
  {
    RangeEnc_EncodeBit(rc, &p->choice, 0);
    RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol);
  }
  else
  {
    RangeEnc_EncodeBit(rc, &p->choice, 1);
    if (symbol < kLenNumLowSymbols + kLenNumMidSymbols)
    {
      RangeEnc_EncodeBit(rc, &p->choice2, 0);
      RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols);
    }
    else
    {
      RangeEnc_EncodeBit(rc, &p->choice2, 1);
      RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols);
    }
  }
}

static void LenEnc_SetPrices(CLenEnc *p, uint32_t posState, uint32_t numSymbols, uint32_t *prices, uint32_t *ProbPrices)
{
  uint32_t a0 = GET_PRICE_0a(p->choice);
  uint32_t a1 = GET_PRICE_1a(p->choice);
  uint32_t b0 = a1 + GET_PRICE_0a(p->choice2);
  uint32_t b1 = a1 + GET_PRICE_1a(p->choice2);
  uint32_t i = 0;
  for (i = 0; i < kLenNumLowSymbols; i++)
  {
    if (i >= numSymbols)
      return;
    prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices);
  }
  for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++)
  {
    if (i >= numSymbols)
      return;
    prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices);
  }
  for (; i < numSymbols; i++)
    prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices);
}

static void LenPriceEnc_UpdateTable(CLenPriceEnc *p, uint32_t posState, uint32_t *ProbPrices)
{
  LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices);
  p->counters[posState] = p->tableSize;
}

static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, uint32_t numPosStates, uint32_t *ProbPrices)
{
  uint32_t posState;
  for (posState = 0; posState < numPosStates; posState++)
    LenPriceEnc_UpdateTable(p, posState, ProbPrices);
}

static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, uint32_t symbol, uint32_t posState, bool updatePrice, uint32_t *ProbPrices)
{
  LenEnc_Encode(&p->p, rc, symbol, posState);
  if (updatePrice)
    if (--p->counters[posState] == 0)
      LenPriceEnc_UpdateTable(p, posState, ProbPrices);
}

static void MovePos(CLzmaEnc *p, uint32_t num)
{
  if (num != 0)
  {
    p->additionalOffset += num;
    p->matchFinder.Skip(p->matchFinderObj, num);
  }
}

static uint32_t ReadMatchDistances(CLzmaEnc *p, uint32_t *numDistancePairsRes)
{
  uint32_t lenRes = 0, numPairs;
  p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj);
  numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches);
  if (numPairs > 0)
  {
    lenRes = p->matches[numPairs - 2];
    if (lenRes == p->numFastBytes)
    {
      const uint8_t *pby = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1;
      uint32_t distance = p->matches[numPairs - 1] + 1;
      uint32_t numAvail = p->numAvail;
      if (numAvail > LZMA_MATCH_LEN_MAX)
        numAvail = LZMA_MATCH_LEN_MAX;
      {
        const uint8_t *pby2 = pby - distance;
        for (; lenRes < numAvail && pby[lenRes] == pby2[lenRes]; lenRes++);
      }
    }
  }
  p->additionalOffset++;
  *numDistancePairsRes = numPairs;
  return lenRes;
}

#define MakeAsChar(p) (p)->backPrev = (uint32_t)(-1); (p)->prev1IsChar = false;
#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = false;
#define IsShortRep(p) ((p)->backPrev == 0)

static uint32_t GetRepLen1Price(CLzmaEnc *p, uint32_t state, uint32_t posState)
{
  return
    GET_PRICE_0(p->isRepG0[state]) +
    GET_PRICE_0(p->isRep0Long[state][posState]);
}

static uint32_t GetPureRepPrice(CLzmaEnc *p, uint32_t repIndex, uint32_t state, uint32_t posState)
{
  uint32_t price;
  if (repIndex == 0)
  {
    price = GET_PRICE_0(p->isRepG0[state]);
    price += GET_PRICE_1(p->isRep0Long[state][posState]);
  }
  else
  {
    price = GET_PRICE_1(p->isRepG0[state]);
    if (repIndex == 1)
      price += GET_PRICE_0(p->isRepG1[state]);
    else
    {
      price += GET_PRICE_1(p->isRepG1[state]);
      price += GET_PRICE(p->isRepG2[state], repIndex - 2);
    }
  }
  return price;
}

static uint32_t GetRepPrice(CLzmaEnc *p, uint32_t repIndex, uint32_t len, uint32_t state, uint32_t posState)
{
  return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] +
    GetPureRepPrice(p, repIndex, state, posState);
}

static uint32_t Backward(CLzmaEnc *p, uint32_t *backRes, uint32_t cur)
{
  uint32_t posMem = p->opt[cur].posPrev;
  uint32_t backMem = p->opt[cur].backPrev;
  p->optimumEndIndex = cur;
  do
  {
    if (p->opt[cur].prev1IsChar)
    {
      MakeAsChar(&p->opt[posMem])
      p->opt[posMem].posPrev = posMem - 1;
      if (p->opt[cur].prev2)
      {
        p->opt[posMem - 1].prev1IsChar = false;
        p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2;
        p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2;
      }
    }
    {
      uint32_t posPrev = posMem;
      uint32_t backCur = backMem;

      backMem = p->opt[posPrev].backPrev;
      posMem = p->opt[posPrev].posPrev;

      p->opt[posPrev].backPrev = backCur;
      p->opt[posPrev].posPrev = cur;
      cur = posPrev;
    }
  }
  while (cur != 0);
  *backRes = p->opt[0].backPrev;
  p->optimumCurrentIndex  = p->opt[0].posPrev;
  return p->optimumCurrentIndex;
}

#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * 0x300)

static uint32_t GetOptimum(CLzmaEnc *p, uint32_t position, uint32_t *backRes)
{
  uint32_t numAvail, mainLen, numPairs, repMaxIndex, i, posState, lenEnd, len, cur;
  uint32_t matchPrice, repMatchPrice, normalMatchPrice;
  uint32_t reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS];
  uint32_t *matches;
  const uint8_t *data;
  uint8_t curByte, matchByte;
  if (p->optimumEndIndex != p->optimumCurrentIndex)
  {
    const COptimal *opt = &p->opt[p->optimumCurrentIndex];
    uint32_t lenRes = opt->posPrev - p->optimumCurrentIndex;
    *backRes = opt->backPrev;
    p->optimumCurrentIndex = opt->posPrev;
    return lenRes;
  }
  p->optimumCurrentIndex = p->optimumEndIndex = 0;

  if (p->additionalOffset == 0)
    mainLen = ReadMatchDistances(p, &numPairs);
  else
  {
    mainLen = p->longestMatchLength;
    numPairs = p->numPairs;
  }

  numAvail = p->numAvail;
  if (numAvail < 2)
  {
    *backRes = (uint32_t)(-1);
    return 1;
  }
  if (numAvail > LZMA_MATCH_LEN_MAX)
    numAvail = LZMA_MATCH_LEN_MAX;

  data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1;
  repMaxIndex = 0;
  for (i = 0; i < LZMA_NUM_REPS; i++)
  {
    uint32_t lenTest;
    const uint8_t *data2;
    reps[i] = p->reps[i];
    data2 = data - (reps[i] + 1);
    if (data[0] != data2[0] || data[1] != data2[1])
    {
      repLens[i] = 0;
      continue;
    }
    for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++);
    repLens[i] = lenTest;
    if (lenTest > repLens[repMaxIndex])
      repMaxIndex = i;
  }
  if (repLens[repMaxIndex] >= p->numFastBytes)
  {
    uint32_t lenRes;
    *backRes = repMaxIndex;
    lenRes = repLens[repMaxIndex];
    MovePos(p, lenRes - 1);
    return lenRes;
  }

  matches = p->matches;
  if (mainLen >= p->numFastBytes)
  {
    *backRes = matches[numPairs - 1] + LZMA_NUM_REPS;
    MovePos(p, mainLen - 1);
    return mainLen;
  }
  curByte = *data;
  matchByte = *(data - (reps[0] + 1));

  if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2)
  {
    *backRes = (uint32_t)-1;
    return 1;
  }

  p->opt[0].state = (CState)p->state;

  posState = (position & p->pbMask);

  {
    const CLzmaProb *probs = LIT_PROBS(position, *(data - 1));
    p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) +
        (!IsCharState(p->state) ?
          LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) :
          LitEnc_GetPrice(probs, curByte, p->ProbPrices));
  }

  MakeAsChar(&p->opt[1]);

  matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]);
  repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]);

  if (matchByte == curByte)
  {
    uint32_t shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState);
    if (shortRepPrice < p->opt[1].price)
    {
      p->opt[1].price = shortRepPrice;
      MakeAsShortRep(&p->opt[1]);
    }
  }
  lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]);

  if (lenEnd < 2)
  {
    *backRes = p->opt[1].backPrev;
    return 1;
  }

  p->opt[1].posPrev = 0;
  for (i = 0; i < LZMA_NUM_REPS; i++)
    p->opt[0].backs[i] = reps[i];

  len = lenEnd;
  do
    p->opt[len--].price = kInfinityPrice;
  while (len >= 2);

  for (i = 0; i < LZMA_NUM_REPS; i++)
  {
    uint32_t repLen = repLens[i];
    uint32_t price;
    if (repLen < 2)
      continue;
    price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState);
    do
    {
      uint32_t curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2];
      COptimal *opt = &p->opt[repLen];
      if (curAndLenPrice < opt->price)
      {
        opt->price = curAndLenPrice;
        opt->posPrev = 0;
        opt->backPrev = i;
        opt->prev1IsChar = false;
      }
    }
    while (--repLen >= 2);
  }

  normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]);

  len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2);
  if (len <= mainLen)
  {
    uint32_t offs = 0;
    while (len > matches[offs])
      offs += 2;
    for (; ; len++)
    {
      COptimal *opt;
      uint32_t distance = matches[offs + 1];

      uint32_t curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN];
      uint32_t lenToPosState = GetLenToPosState(len);
      if (distance < kNumFullDistances)
        curAndLenPrice += p->distancesPrices[lenToPosState][distance];
      else
      {
        uint32_t slot;
        GetPosSlot2(distance, slot);
        curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot];
      }
      opt = &p->opt[len];
      if (curAndLenPrice < opt->price)
      {
        opt->price = curAndLenPrice;
        opt->posPrev = 0;
        opt->backPrev = distance + LZMA_NUM_REPS;
        opt->prev1IsChar = false;
      }
      if (len == matches[offs])
      {
        offs += 2;
        if (offs == numPairs)
          break;
      }
    }
  }

  cur = 0;

  for (;;)
  {
    uint32_t numAvailFull, newLen, numPairs, posPrev, state, posState, startLen;
    uint32_t curPrice, curAnd1Price, matchPrice, repMatchPrice;
    bool nextIsChar;
    uint8_t curByte, matchByte;
    const uint8_t *data;
    COptimal *curOpt;
    COptimal *nextOpt;

    cur++;
    if (cur == lenEnd)
      return Backward(p, backRes, cur);

    newLen = ReadMatchDistances(p, &numPairs);
    if (newLen >= p->numFastBytes)
    {
      p->numPairs = numPairs;
      p->longestMatchLength = newLen;
      return Backward(p, backRes, cur);
    }
    position++;
    curOpt = &p->opt[cur];
    posPrev = curOpt->posPrev;
    if (curOpt->prev1IsChar)
    {
      posPrev--;
      if (curOpt->prev2)
      {
        state = p->opt[curOpt->posPrev2].state;
        if (curOpt->backPrev2 < LZMA_NUM_REPS)
          state = kRepNextStates[state];
        else
          state = kMatchNextStates[state];
      }
      else
        state = p->opt[posPrev].state;
      state = kLiteralNextStates[state];
    }
    else
      state = p->opt[posPrev].state;
    if (posPrev == cur - 1)
    {
      if (IsShortRep(curOpt))
        state = kShortRepNextStates[state];
      else
        state = kLiteralNextStates[state];
    }
    else
    {
      uint32_t pos;
      const COptimal *prevOpt;
      if (curOpt->prev1IsChar && curOpt->prev2)
      {
        posPrev = curOpt->posPrev2;
        pos = curOpt->backPrev2;
        state = kRepNextStates[state];
      }
      else
      {
        pos = curOpt->backPrev;
        if (pos < LZMA_NUM_REPS)
          state = kRepNextStates[state];
        else
          state = kMatchNextStates[state];
      }
      prevOpt = &p->opt[posPrev];
      if (pos < LZMA_NUM_REPS)
      {
        uint32_t i;
        reps[0] = prevOpt->backs[pos];
        for (i = 1; i <= pos; i++)
          reps[i] = prevOpt->backs[i - 1];
        for (; i < LZMA_NUM_REPS; i++)
          reps[i] = prevOpt->backs[i];
      }
      else
      {
        uint32_t i;
        reps[0] = (pos - LZMA_NUM_REPS);
        for (i = 1; i < LZMA_NUM_REPS; i++)
          reps[i] = prevOpt->backs[i - 1];
      }
    }
    curOpt->state = (CState)state;

    curOpt->backs[0] = reps[0];
    curOpt->backs[1] = reps[1];
    curOpt->backs[2] = reps[2];
    curOpt->backs[3] = reps[3];

    curPrice = curOpt->price;
    nextIsChar = false;
    data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1;
    curByte = *data;
    matchByte = *(data - (reps[0] + 1));

    posState = (position & p->pbMask);

    curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]);
    {
      const CLzmaProb *probs = LIT_PROBS(position, *(data - 1));
      curAnd1Price +=
        (!IsCharState(state) ?
          LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) :
          LitEnc_GetPrice(probs, curByte, p->ProbPrices));
    }

    nextOpt = &p->opt[cur + 1];

    if (curAnd1Price < nextOpt->price)
    {
      nextOpt->price = curAnd1Price;
      nextOpt->posPrev = cur;
      MakeAsChar(nextOpt);
      nextIsChar = true;
    }

    matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]);
    repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]);

    if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0))
    {
      uint32_t shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState);
      if (shortRepPrice <= nextOpt->price)
      {
        nextOpt->price = shortRepPrice;
        nextOpt->posPrev = cur;
        MakeAsShortRep(nextOpt);
        nextIsChar = true;
      }
    }
    numAvailFull = p->numAvail;
    {
      uint32_t temp = kNumOpts - 1 - cur;
      if (temp < numAvailFull)
        numAvailFull = temp;
    }

    if (numAvailFull < 2)
      continue;
    numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes);

    if (!nextIsChar && matchByte != curByte) /* speed optimization */
    {
      /* try Literal + rep0 */
      uint32_t temp;
      uint32_t lenTest2;
      const uint8_t *data2 = data - (reps[0] + 1);
      uint32_t limit = p->numFastBytes + 1;
      if (limit > numAvailFull)
        limit = numAvailFull;

      for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++);
      lenTest2 = temp - 1;
      if (lenTest2 >= 2)
      {
        uint32_t state2 = kLiteralNextStates[state];
        uint32_t posStateNext = (position + 1) & p->pbMask;
        uint32_t nextRepMatchPrice = curAnd1Price +
            GET_PRICE_1(p->isMatch[state2][posStateNext]) +
            GET_PRICE_1(p->isRep[state2]);
        /* for (; lenTest2 >= 2; lenTest2--) */
        {
          uint32_t curAndLenPrice;
          COptimal *opt;
          uint32_t offset = cur + 1 + lenTest2;
          while (lenEnd < offset)
            p->opt[++lenEnd].price = kInfinityPrice;
          curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext);
          opt = &p->opt[offset];
          if (curAndLenPrice < opt->price)
          {
            opt->price = curAndLenPrice;
            opt->posPrev = cur + 1;
            opt->backPrev = 0;
            opt->prev1IsChar = true;
            opt->prev2 = false;
          }
        }
      }
    }

    startLen = 2; /* speed optimization */
    {
    uint32_t repIndex;
    for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++)
    {
      uint32_t lenTest;
      uint32_t lenTestTemp;
      uint32_t price;
      const uint8_t *data2 = data - (reps[repIndex] + 1);
      if (data[0] != data2[0] || data[1] != data2[1])
        continue;
      for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++);
      while (lenEnd < cur + lenTest)
        p->opt[++lenEnd].price = kInfinityPrice;
      lenTestTemp = lenTest;
      price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState);
      do
      {
        uint32_t curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2];
        COptimal *opt = &p->opt[cur + lenTest];
        if (curAndLenPrice < opt->price)
        {
          opt->price = curAndLenPrice;
          opt->posPrev = cur;
          opt->backPrev = repIndex;
          opt->prev1IsChar = false;
        }
      }
      while (--lenTest >= 2);
      lenTest = lenTestTemp;

      if (repIndex == 0)
        startLen = lenTest + 1;

      /* if (_maxMode) */
        {
          uint32_t lenTest2 = lenTest + 1;
          uint32_t limit = lenTest2 + p->numFastBytes;
          uint32_t nextRepMatchPrice;
          if (limit > numAvailFull)
            limit = numAvailFull;
          for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++);
          lenTest2 -= lenTest + 1;
          if (lenTest2 >= 2)
          {
            uint32_t state2 = kRepNextStates[state];
            uint32_t posStateNext = (position + lenTest) & p->pbMask;
            uint32_t curAndLenCharPrice =
                price + p->repLenEnc.prices[posState][lenTest - 2] +
                GET_PRICE_0(p->isMatch[state2][posStateNext]) +
                LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]),
                    data[lenTest], data2[lenTest], p->ProbPrices);
            state2 = kLiteralNextStates[state2];
            posStateNext = (position + lenTest + 1) & p->pbMask;
            nextRepMatchPrice = curAndLenCharPrice +
                GET_PRICE_1(p->isMatch[state2][posStateNext]) +
                GET_PRICE_1(p->isRep[state2]);

            /* for (; lenTest2 >= 2; lenTest2--) */
            {
              uint32_t curAndLenPrice;
              COptimal *opt;
              uint32_t offset = cur + lenTest + 1 + lenTest2;
              while (lenEnd < offset)
                p->opt[++lenEnd].price = kInfinityPrice;
              curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext);
              opt = &p->opt[offset];
              if (curAndLenPrice < opt->price)
              {
                opt->price = curAndLenPrice;
                opt->posPrev = cur + lenTest + 1;
                opt->backPrev = 0;
                opt->prev1IsChar = true;
                opt->prev2 = true;
                opt->posPrev2 = cur;
                opt->backPrev2 = repIndex;
              }
            }
          }
        }
    }
    }
    /* for (uint32_t lenTest = 2; lenTest <= newLen; lenTest++) */
    if (newLen > numAvail)
    {
      newLen = numAvail;
      for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2);
      matches[numPairs] = newLen;
      numPairs += 2;
    }
    if (newLen >= startLen)
    {
      uint32_t normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]);
      uint32_t offs, curBack, posSlot;
      uint32_t lenTest;
      while (lenEnd < cur + newLen)
        p->opt[++lenEnd].price = kInfinityPrice;

      offs = 0;
      while (startLen > matches[offs])
        offs += 2;
      curBack = matches[offs + 1];
      GetPosSlot2(curBack, posSlot);
      for (lenTest = /*2*/ startLen; ; lenTest++)
      {
        uint32_t curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN];
        uint32_t lenToPosState = GetLenToPosState(lenTest);
        COptimal *opt;
        if (curBack < kNumFullDistances)
          curAndLenPrice += p->distancesPrices[lenToPosState][curBack];
        else
          curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask];

        opt = &p->opt[cur + lenTest];
        if (curAndLenPrice < opt->price)
        {
          opt->price = curAndLenPrice;
          opt->posPrev = cur;
          opt->backPrev = curBack + LZMA_NUM_REPS;
          opt->prev1IsChar = false;
        }

        if (/*_maxMode && */lenTest == matches[offs])
        {
          /* Try Match + Literal + Rep0 */
          const uint8_t *data2 = data - (curBack + 1);
          uint32_t lenTest2 = lenTest + 1;
          uint32_t limit = lenTest2 + p->numFastBytes;
          uint32_t nextRepMatchPrice;
          if (limit > numAvailFull)
            limit = numAvailFull;
          for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++);
          lenTest2 -= lenTest + 1;
          if (lenTest2 >= 2)
          {
            uint32_t state2 = kMatchNextStates[state];
            uint32_t posStateNext = (position + lenTest) & p->pbMask;
            uint32_t curAndLenCharPrice = curAndLenPrice +
                GET_PRICE_0(p->isMatch[state2][posStateNext]) +
                LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]),
                    data[lenTest], data2[lenTest], p->ProbPrices);
            state2 = kLiteralNextStates[state2];
            posStateNext = (posStateNext + 1) & p->pbMask;
            nextRepMatchPrice = curAndLenCharPrice +
                GET_PRICE_1(p->isMatch[state2][posStateNext]) +
                GET_PRICE_1(p->isRep[state2]);

            /* for (; lenTest2 >= 2; lenTest2--) */
            {
              uint32_t offset = cur + lenTest + 1 + lenTest2;
              uint32_t curAndLenPrice;
              COptimal *opt;
              while (lenEnd < offset)
                p->opt[++lenEnd].price = kInfinityPrice;
              curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext);
              opt = &p->opt[offset];
              if (curAndLenPrice < opt->price)
              {
                opt->price = curAndLenPrice;
                opt->posPrev = cur + lenTest + 1;
                opt->backPrev = 0;
                opt->prev1IsChar = true;
                opt->prev2 = true;
                opt->posPrev2 = cur;
                opt->backPrev2 = curBack + LZMA_NUM_REPS;
              }
            }
          }
          offs += 2;
          if (offs == numPairs)
            break;
          curBack = matches[offs + 1];
          if (curBack >= kNumFullDistances)
            GetPosSlot2(curBack, posSlot);
        }
      }
    }
  }
}

#define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist))

static uint32_t GetOptimumFast(CLzmaEnc *p, uint32_t *backRes)
{
  uint32_t numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i;
  const uint8_t *data;
  const uint32_t *matches;

  if (p->additionalOffset == 0)
    mainLen = ReadMatchDistances(p, &numPairs);
  else
  {
    mainLen = p->longestMatchLength;
    numPairs = p->numPairs;
  }

  numAvail = p->numAvail;
  *backRes = (uint32_t)-1;
  if (numAvail < 2)
    return 1;
  if (numAvail > LZMA_MATCH_LEN_MAX)
    numAvail = LZMA_MATCH_LEN_MAX;
  data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1;

  repLen = repIndex = 0;
  for (i = 0; i < LZMA_NUM_REPS; i++)
  {
    uint32_t len;
    const uint8_t *data2 = data - (p->reps[i] + 1);
    if (data[0] != data2[0] || data[1] != data2[1])
      continue;
    for (len = 2; len < numAvail && data[len] == data2[len]; len++);
    if (len >= p->numFastBytes)
    {
      *backRes = i;
      MovePos(p, len - 1);
      return len;
    }
    if (len > repLen)
    {
      repIndex = i;
      repLen = len;
    }
  }

  matches = p->matches;
  if (mainLen >= p->numFastBytes)
  {
    *backRes = matches[numPairs - 1] + LZMA_NUM_REPS;
    MovePos(p, mainLen - 1);
    return mainLen;
  }

  mainDist = 0; /* for GCC */
  if (mainLen >= 2)
  {
    mainDist = matches[numPairs - 1];
    while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1)
    {
      if (!ChangePair(matches[numPairs - 3], mainDist))
        break;
      numPairs -= 2;
      mainLen = matches[numPairs - 2];
      mainDist = matches[numPairs - 1];
    }
    if (mainLen == 2 && mainDist >= 0x80)
      mainLen = 1;
  }

  if (repLen >= 2 && (
        (repLen + 1 >= mainLen) ||
        (repLen + 2 >= mainLen && mainDist >= (1 << 9)) ||
        (repLen + 3 >= mainLen && mainDist >= (1 << 15))))
  {
    *backRes = repIndex;
    MovePos(p, repLen - 1);
    return repLen;
  }

  if (mainLen < 2 || numAvail <= 2)
    return 1;

  p->longestMatchLength = ReadMatchDistances(p, &p->numPairs);
  if (p->longestMatchLength >= 2)
  {
    uint32_t newDistance = matches[p->numPairs - 1];
    if ((p->longestMatchLength >= mainLen && newDistance < mainDist) ||
        (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) ||
        (p->longestMatchLength > mainLen + 1) ||
        (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist)))
      return 1;
  }

  data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1;
  for (i = 0; i < LZMA_NUM_REPS; i++)
  {
    uint32_t len, limit;
    const uint8_t *data2 = data - (p->reps[i] + 1);
    if (data[0] != data2[0] || data[1] != data2[1])
      continue;
    limit = mainLen - 1;
    for (len = 2; len < limit && data[len] == data2[len]; len++);
    if (len >= limit)
      return 1;
  }
  *backRes = mainDist + LZMA_NUM_REPS;
  MovePos(p, mainLen - 2);
  return mainLen;
}

static void WriteEndMarker(CLzmaEnc *p, uint32_t posState)
{
  uint32_t len;
  RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1);
  RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0);
  p->state = kMatchNextStates[p->state];
  len = LZMA_MATCH_LEN_MIN;
  LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices);
  RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1);
  RangeEnc_EncodeDirectBits(&p->rc, (((uint32_t)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits);
  RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask);
}

static SRes CheckErrors(CLzmaEnc *p)
{
  if (p->result != SZ_OK)
    return p->result;
  if (p->rc.res != SZ_OK)
    p->result = SZ_ERROR_WRITE;
  if (p->matchFinderBase.result != SZ_OK)
    p->result = SZ_ERROR_READ;
  if (p->result != SZ_OK)
    p->finished = true;
  return p->result;
}

static SRes Flush(CLzmaEnc *p, uint32_t nowPos)
{
  /* ReleaseMFStream(); */
  p->finished = true;
  if (p->writeEndMark)
    WriteEndMarker(p, nowPos & p->pbMask);
  RangeEnc_FlushData(&p->rc);
  RangeEnc_FlushStream(&p->rc);
  return CheckErrors(p);
}

static void FillAlignPrices(CLzmaEnc *p)
{
  uint32_t i;
  for (i = 0; i < kAlignTableSize; i++)
    p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices);
  p->alignPriceCount = 0;
}

static void FillDistancesPrices(CLzmaEnc *p)
{
  uint32_t tempPrices[kNumFullDistances];
  uint32_t i, lenToPosState;
  for (i = kStartPosModelIndex; i < kNumFullDistances; i++)
  {
    uint32_t posSlot = GetPosSlot1(i);
    uint32_t footerBits = ((posSlot >> 1) - 1);
    uint32_t base = ((2 | (posSlot & 1)) << footerBits);
    tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices);
  }

  for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++)
  {
    uint32_t posSlot;
    const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState];
    uint32_t *posSlotPrices = p->posSlotPrices[lenToPosState];
    for (posSlot = 0; posSlot < p->distTableSize; posSlot++)
      posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices);
    for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++)
      posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits);

    {
      uint32_t *distancesPrices = p->distancesPrices[lenToPosState];
      uint32_t i;
      for (i = 0; i < kStartPosModelIndex; i++)
        distancesPrices[i] = posSlotPrices[i];
      for (; i < kNumFullDistances; i++)
        distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i];
    }
  }
  p->matchPriceCount = 0;
}

void LzmaEnc_Construct(CLzmaEnc *p)
{
  RangeEnc_Construct(&p->rc);
  MatchFinder_Construct(&p->matchFinderBase);

  {
    CLzmaEncProps props;
    LzmaEncProps_Init(&props);
    LzmaEnc_SetProps(p, &props);
  }

  #ifndef LZMA_LOG_BSR
  LzmaEnc_FastPosInit(p->g_FastPos);
  #endif

  LzmaEnc_InitPriceTables(p->ProbPrices);
  p->litProbs = 0;
  p->saveState.litProbs = 0;
}

CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc)
{
  void *p;
  p = alloc->Alloc(alloc, sizeof(CLzmaEnc));
  if (p != 0)
    LzmaEnc_Construct((CLzmaEnc *)p);
  return p;
}

void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc)
{
  alloc->Free(alloc, p->litProbs);
  alloc->Free(alloc, p->saveState.litProbs);
  p->litProbs = 0;
  p->saveState.litProbs = 0;
}

void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  MatchFinder_Free(&p->matchFinderBase, allocBig);
  LzmaEnc_FreeLits(p, alloc);
  RangeEnc_Free(&p->rc, alloc);
}

void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig);
  alloc->Free(alloc, p);
}

static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, bool useLimits, uint32_t maxPackSize, uint32_t maxUnpackSize)
{
  uint32_t nowPos32, startPos32;
  if (p->needInit)
  {
    p->matchFinder.Init(p->matchFinderObj);
    p->needInit = 0;
  }

  if (p->finished)
    return p->result;
  RINOK(CheckErrors(p));

  nowPos32 = (uint32_t)p->nowPos64;
  startPos32 = nowPos32;

  if (p->nowPos64 == 0)
  {
    uint32_t numPairs;
    uint8_t curByte;
    if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0)
      return Flush(p, nowPos32);
    ReadMatchDistances(p, &numPairs);
    RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0);
    p->state = kLiteralNextStates[p->state];
    curByte = p->matchFinder.GetIndexByte(p->matchFinderObj, 0 - p->additionalOffset);
    LitEnc_Encode(&p->rc, p->litProbs, curByte);
    p->additionalOffset--;
    nowPos32++;
  }

  if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0)
  for (;;)
  {
    uint32_t pos, len, posState;

    if (p->fastMode)
      len = GetOptimumFast(p, &pos);
    else
      len = GetOptimum(p, nowPos32, &pos);

    posState = nowPos32 & p->pbMask;
    if (len == 1 && pos == (uint32_t)-1)
    {
      uint8_t curByte;
      CLzmaProb *probs;
      const uint8_t *data;

      RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0);
      data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset;
      curByte = *data;
      probs = LIT_PROBS(nowPos32, *(data - 1));
      if (IsCharState(p->state))
        LitEnc_Encode(&p->rc, probs, curByte);
      else
        LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1));
      p->state = kLiteralNextStates[p->state];
    }
    else
    {
      RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1);
      if (pos < LZMA_NUM_REPS)
      {
        RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1);
        if (pos == 0)
        {
          RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0);
          RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1));
        }
        else
        {
          uint32_t distance = p->reps[pos];
          RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1);
          if (pos == 1)
            RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0);
          else
          {
            RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1);
            RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2);
            if (pos == 3)
              p->reps[3] = p->reps[2];
            p->reps[2] = p->reps[1];
          }
          p->reps[1] = p->reps[0];
          p->reps[0] = distance;
        }
        if (len == 1)
          p->state = kShortRepNextStates[p->state];
        else
        {
          LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices);
          p->state = kRepNextStates[p->state];
        }
      }
      else
      {
        uint32_t posSlot;
        RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0);
        p->state = kMatchNextStates[p->state];
        LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices);
        pos -= LZMA_NUM_REPS;
        GetPosSlot(pos, posSlot);
        RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot);

        if (posSlot >= kStartPosModelIndex)
        {
          uint32_t footerBits = ((posSlot >> 1) - 1);
          uint32_t base = ((2 | (posSlot & 1)) << footerBits);
          uint32_t posReduced = pos - base;

          if (posSlot < kEndPosModelIndex)
            RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced);
          else
          {
            RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits);
            RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask);
            p->alignPriceCount++;
          }
        }
        p->reps[3] = p->reps[2];
        p->reps[2] = p->reps[1];
        p->reps[1] = p->reps[0];
        p->reps[0] = pos;
        p->matchPriceCount++;
      }
    }
    p->additionalOffset -= len;
    nowPos32 += len;
    if (p->additionalOffset == 0)
    {
      uint32_t processed;
      if (!p->fastMode)
      {
        if (p->matchPriceCount >= (1 << 7))
          FillDistancesPrices(p);
        if (p->alignPriceCount >= kAlignTableSize)
          FillAlignPrices(p);
      }
      if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0)
        break;
      processed = nowPos32 - startPos32;
      if (useLimits)
      {
        if (processed + kNumOpts + 300 >= maxUnpackSize ||
            RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize)
          break;
      }
      else if (processed >= (1 << 15))
      {
        p->nowPos64 += nowPos32 - startPos32;
        return CheckErrors(p);
      }
    }
  }
  p->nowPos64 += nowPos32 - startPos32;
  return Flush(p, nowPos32);
}

#define kBigHashDicLimit ((uint32_t)1 << 24)

static SRes LzmaEnc_Alloc(CLzmaEnc *p, uint32_t keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  uint32_t beforeSize = kNumOpts;
  if (!RangeEnc_Alloc(&p->rc, alloc))
    return SZ_ERROR_MEM;

  {
    unsigned lclp = p->lc + p->lp;
    if (p->litProbs == 0 || p->saveState.litProbs == 0 || p->lclp != lclp)
    {
      LzmaEnc_FreeLits(p, alloc);
      p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb));
      p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb));
      if (p->litProbs == 0 || p->saveState.litProbs == 0)
      {
        LzmaEnc_FreeLits(p, alloc);
        return SZ_ERROR_MEM;
      }
      p->lclp = lclp;
    }
  }

  p->matchFinderBase.bigHash = (p->dictSize > kBigHashDicLimit);

  if (beforeSize + p->dictSize < keepWindowSize)
    beforeSize = keepWindowSize - p->dictSize;

  {
    if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig))
      return SZ_ERROR_MEM;
    p->matchFinderObj = &p->matchFinderBase;
    MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder);
  }
  return SZ_OK;
}

void LzmaEnc_Init(CLzmaEnc *p)
{
  uint32_t i;
  p->state = 0;
  for (i = 0 ; i < LZMA_NUM_REPS; i++)
    p->reps[i] = 0;

  RangeEnc_Init(&p->rc);

  for (i = 0; i < kNumStates; i++)
  {
    uint32_t j;
    for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++)
    {
      p->isMatch[i][j] = kProbInitValue;
      p->isRep0Long[i][j] = kProbInitValue;
    }
    p->isRep[i] = kProbInitValue;
    p->isRepG0[i] = kProbInitValue;
    p->isRepG1[i] = kProbInitValue;
    p->isRepG2[i] = kProbInitValue;
  }

  {
    uint32_t num = 0x300 << (p->lp + p->lc);
    for (i = 0; i < num; i++)
      p->litProbs[i] = kProbInitValue;
  }

  {
    for (i = 0; i < kNumLenToPosStates; i++)
    {
      CLzmaProb *probs = p->posSlotEncoder[i];
      uint32_t j;
      for (j = 0; j < (1 << kNumPosSlotBits); j++)
        probs[j] = kProbInitValue;
    }
  }
  {
    for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++)
      p->posEncoders[i] = kProbInitValue;
  }

  LenEnc_Init(&p->lenEnc.p);
  LenEnc_Init(&p->repLenEnc.p);

  for (i = 0; i < (1 << kNumAlignBits); i++)
    p->posAlignEncoder[i] = kProbInitValue;

  p->optimumEndIndex = 0;
  p->optimumCurrentIndex = 0;
  p->additionalOffset = 0;

  p->pbMask = (1 << p->pb) - 1;
  p->lpMask = (1 << p->lp) - 1;
}

void LzmaEnc_InitPrices(CLzmaEnc *p)
{
  if (!p->fastMode)
  {
    FillDistancesPrices(p);
    FillAlignPrices(p);
  }

  p->lenEnc.tableSize =
  p->repLenEnc.tableSize =
      p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN;
  LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices);
  LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices);
}

static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, uint32_t keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  uint32_t i;
  for (i = 0; i < (uint32_t)kDicLogSizeMaxCompress; i++)
    if (p->dictSize <= ((uint32_t)1 << i))
      break;
  p->distTableSize = i * 2;

  p->finished = false;
  p->result = SZ_OK;
  RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig));
  LzmaEnc_Init(p);
  LzmaEnc_InitPrices(p);
  p->nowPos64 = 0;
  return SZ_OK;
}

static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream,
    ISzAlloc *alloc, ISzAlloc *allocBig)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  p->matchFinderBase.stream = inStream;
  p->needInit = 1;
  p->rc.outStream = outStream;
  return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig);
}

SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp,
    ISeqInStream *inStream, uint32_t keepWindowSize,
    ISzAlloc *alloc, ISzAlloc *allocBig)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  p->matchFinderBase.stream = inStream;
  p->needInit = 1;
  return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig);
}

static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const uint8_t *src, size_t srcLen)
{
  p->matchFinderBase.directInput = 1;
  p->matchFinderBase.bufferBase = (uint8_t *)src;
  p->matchFinderBase.directInputRem = srcLen;
}

SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const uint8_t *src, size_t srcLen,
    uint32_t keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  LzmaEnc_SetInputBuf(p, src, srcLen);
  p->needInit = 1;

  return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig);
}

void LzmaEnc_Finish(CLzmaEncHandle /*pp*/)
{
}

struct CSeqOutStreamBuf
{
  ISeqOutStream funcTable;
  uint8_t *data;
  size_t rem;
  bool overflow;
};

static size_t MyWrite(void *pp, const void *data, size_t size)
{
  CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp;
  if (p->rem < size)
  {
    size = p->rem;
    p->overflow = true;
  }
  memcpy(p->data, data, size);
  p->rem -= size;
  p->data += size;
  return size;
}

uint32_t LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp)
{
  const CLzmaEnc *p = (CLzmaEnc *)pp;
  return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj);
}

const uint8_t *LzmaEnc_GetCurBuf(CLzmaEncHandle pp)
{
  const CLzmaEnc *p = (CLzmaEnc *)pp;
  return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset;
}

SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, bool reInit,
    uint8_t *dest, size_t *destLen, uint32_t desiredPackSize, uint32_t *unpackSize)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  uint64_t nowPos64;
  SRes res;
  CSeqOutStreamBuf outStream;

  outStream.funcTable.Write = MyWrite;
  outStream.data = dest;
  outStream.rem = *destLen;
  outStream.overflow = false;

  p->writeEndMark = false;
  p->finished = false;
  p->result = SZ_OK;

  if (reInit)
    LzmaEnc_Init(p);
  LzmaEnc_InitPrices(p);
  nowPos64 = p->nowPos64;
  RangeEnc_Init(&p->rc);
  p->rc.outStream = &outStream.funcTable;

  res = LzmaEnc_CodeOneBlock(p, true, desiredPackSize, *unpackSize);

  *unpackSize = (uint32_t)(p->nowPos64 - nowPos64);
  *destLen -= outStream.rem;
  if (outStream.overflow)
    return SZ_ERROR_OUTPUT_EOF;

  return res;
}

static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress)
{
  SRes res = SZ_OK;

  for (;;)
  {
    res = LzmaEnc_CodeOneBlock(p, false, 0, 0);
    if (res != SZ_OK || p->finished != 0)
      break;
    if (progress != 0)
    {
      res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc));
      if (res != SZ_OK)
      {
        res = SZ_ERROR_PROGRESS;
        break;
      }
    }
  }
  LzmaEnc_Finish(p);
  return res;
}

SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress,
    ISzAlloc *alloc, ISzAlloc *allocBig)
{
  RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig));
  return LzmaEnc_Encode2((CLzmaEnc *)pp, progress);
}

SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, uint8_t *props, size_t *size)
{
  CLzmaEnc *p = (CLzmaEnc *)pp;
  int i;
  uint32_t dictSize = p->dictSize;
  if (*size < LZMA_PROPS_SIZE)
    return SZ_ERROR_PARAM;
  *size = LZMA_PROPS_SIZE;
  props[0] = (uint8_t)((p->pb * 5 + p->lp) * 9 + p->lc);

  for (i = 11; i <= 30; i++)
  {
    if (dictSize <= ((uint32_t)2 << i))
    {
      dictSize = (2 << i);
      break;
    }
    if (dictSize <= ((uint32_t)3 << i))
    {
      dictSize = (3 << i);
      break;
    }
  }

  for (i = 0; i < 4; i++)
    props[1 + i] = (uint8_t)(dictSize >> (8 * i));
  return SZ_OK;
}

SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, uint8_t *dest, size_t *destLen, const uint8_t *src, size_t srcLen,
    int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  SRes res;
  CLzmaEnc *p = (CLzmaEnc *)pp;

  CSeqOutStreamBuf outStream;

  LzmaEnc_SetInputBuf(p, src, srcLen);

  outStream.funcTable.Write = MyWrite;
  outStream.data = dest;
  outStream.rem = *destLen;
  outStream.overflow = false;

  p->writeEndMark = writeEndMark;

  p->rc.outStream = &outStream.funcTable;
  res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig);
  if (res == SZ_OK)
    res = LzmaEnc_Encode2(p, progress);

  *destLen -= outStream.rem;
  if (outStream.overflow)
    return SZ_ERROR_OUTPUT_EOF;
  return res;
}

SRes LzmaEncode(uint8_t *dest, size_t *destLen, const uint8_t *src, size_t srcLen,
    const CLzmaEncProps *props, uint8_t *propsEncoded, size_t *propsSize, int writeEndMark,
    ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig)
{
  CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc);
  SRes res;
  if (p == 0)
    return SZ_ERROR_MEM;

  res = LzmaEnc_SetProps(p, props);
  if (res == SZ_OK)
  {
    res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize);
    if (res == SZ_OK)
      res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen,
          writeEndMark, progress, alloc, allocBig);
  }

  LzmaEnc_Destroy(p, alloc, allocBig);
  return res;
}

} // namespace lzma
// End of LZMA compression library by Igor Pavlov

#ifndef UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H
#define UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H
static void *LzmaAlloc(void* /*p*/, size_t size) { return new char[size]; }
static void LzmaFree(void* /*p*/, void *address) { delete[] (char*) address; }
static lzma::ISzAlloc lzmaAllocator = { LzmaAlloc, LzmaFree };
#endif // UFAL_CPPUTILS_COMPRESSOR_LZMA_ALLOCATOR_H

bool compressor::save(ostream& os, const binary_encoder& enc) {
  size_t uncompressed_size = enc.data.size(), compressed_size = 2 * enc.data.size() + 100;
  vector<unsigned char> compressed(compressed_size);

  lzma::CLzmaEncProps props;
  lzma::LzmaEncProps_Init(&props);
  unsigned char props_encoded[LZMA_PROPS_SIZE];
  size_t props_encoded_size = LZMA_PROPS_SIZE;

  auto res = lzma::LzmaEncode(compressed.data(), &compressed_size, enc.data.data(), uncompressed_size, &props, props_encoded, &props_encoded_size, 0, nullptr, &lzmaAllocator, &lzmaAllocator);
  if (res != SZ_OK) return false;

  uint32_t poor_crc = uncompressed_size * 19991 + compressed_size * 199999991 + 1234567890;
  if (uint32_t(uncompressed_size) != uncompressed_size || uint32_t(compressed_size) != compressed_size) return false;
  if (!os.write((const char*) &uncompressed_size, sizeof(uint32_t))) return false;
  if (!os.write((const char*) &compressed_size, sizeof(uint32_t))) return false;
  if (!os.write((const char*) &poor_crc, sizeof(uint32_t))) return false;
  if (!os.write((const char*) props_encoded, sizeof(props_encoded))) return false;
  if (!os.write((const char*) compressed.data(), compressed_size)) return false;

  return true;
}

} // namespace utils

/////////
// File: version/version.cpp
/////////

// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Returns current version.
version version::current() {
  return {1, 2, 0, ""};
}

// Returns multi-line formated version and copyright string.
string version::version_and_copyright(const string& other_libraries) {
  ostringstream info;

  auto udpipe = version::current();
  auto unilib = unilib::version::current();
  auto morphodita = morphodita::version::current();
  auto parsito = parsito::version::current();

  info << "UDPipe version " << udpipe.major << '.' << udpipe.minor << '.' << udpipe.patch
       << (udpipe.prerelease.empty() ? "" : "-") << udpipe.prerelease
       << " (using UniLib " << unilib.major << '.' << unilib.minor << '.' << unilib.patch
       << (unilib.prerelease.empty() ? "" : "-") << unilib.prerelease
       << ",\nMorphoDiTa " << morphodita.major << '.' << morphodita.minor << '.' << unilib.patch
       << (morphodita.prerelease.empty() ? "" : "-") << morphodita.prerelease
       << ", Parsito " << parsito.major << '.' << parsito.minor << '.' << unilib.patch
       << (parsito.prerelease.empty() ? "" : "-") << parsito.prerelease
       << (other_libraries.empty() ? "" : " and ") << other_libraries << ")\n"
          "Copyright 2016 by Institute of Formal and Applied Linguistics, Faculty of\n"
          "Mathematics and Physics, Charles University in Prague, Czech Republic.";

  return info.str();
}

} // namespace udpipe
} // namespace ufal