dictionary/system/codec_test.cc

// Copyright 2010-2018, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "dictionary/system/codec.h"

#include <memory>
#include <string>
#include <vector>

#include "base/logging.h"
#include "base/util.h"
#include "dictionary/dictionary_token.h"
#include "dictionary/system/codec_interface.h"
#include "dictionary/system/words_info.h"
#include "testing/base/public/googletest.h"
#include "testing/base/public/gunit.h"

namespace mozc {
namespace dictionary {
namespace {

using std::unique_ptr;

::testing::AssertionResult MakeAssertResult(
    bool success, char32 c, const char *message) {
  if (success) {
    return ::testing::AssertionSuccess();
  }
  return ::testing::AssertionFailure()
      << message << " c = " << Util::StringPrintf("U+%05X", c);
}

::testing::AssertionResult IsExpectedEncodedSize(
    char32 c, const string &encoded) {
  const string::size_type size = encoded.size();
  if (c == 0x00) {
    return ::testing::AssertionFailure() << "NUL is not supported.";
  }
  if (c <= 0xff) {
    return MakeAssertResult(size == 2, c,
                            "U+00?? (ASCII) should be encoded into 2 bytes.");
  }
  if (0x10000 <= c && c <= 0x10ffff) {
    if ((c & 0xffff) == 0) {
      return MakeAssertResult(
          size == 2, c, "U+?0000 and U+100000 are encoded into 2 bytes.");
    }
    if ((c & 0xff) == 0) {
      return MakeAssertResult(
          size == 3, c, "U+???00 and U+10??00 are encoded into 3 bytes.");
    }
    if (((c & 0xff00) >> 8) == 0) {
      return MakeAssertResult(
          size == 3, c, "U+?00?? and U+1000?? are encoded into 3 bytes.");
    }
    return MakeAssertResult(
        size == 4, c,
        "[U+10000, U+10FFFF] except for U+???00, U+?00??, U+10??00 and "
        "U+1000?? should be encoded into 4 bytes.");
  }
  if (0x10ffff < c) {
    return MakeAssertResult(
        false, c, "U+110000 and greater are not supported.");
  }
  if (0xffff < c) {
    return MakeAssertResult(false, c, "Should not reach here.");
  }

  // Hereafter, |c| should be represented as 0x????
  const uint16 s = static_cast<uint16>(c);
  if ((s & 0xff) == 0) {
    return MakeAssertResult(size == 2, c, "U+??00 are encoded into 2 bytes.");
  }
  if (0x3041 <= s && s < 0x3095) {
    return MakeAssertResult(
        size == 1, c, "Hiragana(85 characters) are encoded into 1 byte.");
  }
  if (0x30a1 <= s && s < 0x30fd) {
    return MakeAssertResult(
        size == 1, c, "Katakana (92 characters) are encoded into 1 byte.");
  }
  if (0x4e00 <= s && s < 0x9800) {
    return MakeAssertResult(size == 2, c,
                            "Frequent Kanji and others (74*256 characters) "
                            "are encoded into 2 bytes.");
  }
  return MakeAssertResult(size == 3, c,
                          "Other charaters should be encoded into 3bytes.");
}


}  // namespace

class SystemDictionaryCodecTest : public ::testing::Test {
 protected:
  virtual void SetUp() {
    SystemDictionaryCodecFactory::SetCodec(NULL);
    ResetAllTokens();
  }

  virtual void TearDown() {
    SystemDictionaryCodecFactory::SetCodec(NULL);
    ResetAllTokens();
  }

  void ResetAllTokens() {
    ClearTokens(&source_tokens_);
    ClearTokens(&decoded_tokens_);
  }

  void ClearTokens(std::vector<TokenInfo> *tokens) const {
    for (size_t i = 0; i < tokens->size(); ++i) {
      delete tokens->at(i).token;
    }
    tokens->clear();
  }

  void InitTokens(int size) {
    for (size_t i = 0; i < size; ++i) {
      Token *t = new Token();
      TokenInfo token_info(t);
      token_info.id_in_value_trie = 0;
      source_tokens_.push_back(token_info);
    }
  }

  void SetDefaultPos(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->pos_type = TokenInfo::DEFAULT_POS;
    // set id randomly
    const int id = Util::Random(50);
    token_info->token->lid = id;
    token_info->token->rid = (Util::Random(2) == 0) ? id : id + 1;
  }

  void SetFrequentPos(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->pos_type = TokenInfo::FREQUENT_POS;
    // set id randomly
    const int id = Util::Random(256);
    token_info->id_in_frequent_pos_map = id;
  }

  void SetSamePos(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->pos_type = TokenInfo::SAME_AS_PREV_POS;
  }

  void SetRandPos() {
    for (size_t i = 0; i < source_tokens_.size(); ++i) {
      CHECK(source_tokens_[i].token);
      int n = Util::Random(TokenInfo::POS_TYPE_SIZE);
      CHECK_GE(n, 0);
      CHECK_LT(n, TokenInfo::POS_TYPE_SIZE);
      if (i == 0 && n == 2) {
        // First token cannot be the same pos.
        n = 0;
      }

      if (n == 0) {
        SetDefaultPos(&source_tokens_[i]);
      } else if (n == 1) {
        SetFrequentPos(&source_tokens_[i]);
      } else if (n == 2) {
        SetSamePos(&source_tokens_[i]);
      } else {
        FAIL();
      }
    }
  }

  void SetDefaultCost(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->cost_type = TokenInfo::DEFAULT_COST;
    // set cost randomly
    const int cost = Util::Random(8000);
    token_info->token->cost = cost;
  }

  void SetSmallCost(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->cost_type = TokenInfo::CAN_USE_SMALL_ENCODING;
    // set cost randomly
    const int cost = Util::Random(8000);
    token_info->token->cost = cost;
  }

  void SetRandCost() {
    for (size_t i = 0; i < source_tokens_.size(); ++i) {
      CHECK(source_tokens_[i].token);
      int n = Util::Random(TokenInfo::COST_TYPE_SIZE);
      CHECK_GE(n, 0);
      CHECK_LT(n, TokenInfo::POS_TYPE_SIZE);
      if (n == 0) {
        SetDefaultCost(&source_tokens_[i]);
      } else if (n == 1) {
        SetSmallCost(&source_tokens_[i]);
      }
    }
  }

  void SetDefaultValue(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->value_type = TokenInfo::DEFAULT_VALUE;
    // set id randomly
    const int id = Util::Random(50000);
    token_info->id_in_value_trie = id;
  }

  void SetSameValue(TokenInfo *token_info) const {
    CHECK(token_info);
    token_info->value_type = TokenInfo::SAME_AS_PREV_VALUE;
  }

  void SetRandValue() {
    for (size_t i = 0; i < source_tokens_.size(); ++i) {
      CHECK(source_tokens_[i].token);
      int n = Util::Random(TokenInfo::VALUE_TYPE_SIZE);
      CHECK_GE(n, 0);
      CHECK_LT(n, TokenInfo::VALUE_TYPE_SIZE);
      if (i == 0 && n == 1) {
        // first token cannot be the same as before.
        n = 0;
      }
      if (n == 0) {
        SetDefaultValue(&source_tokens_[i]);
      } else if (n == 1) {
        SetSameValue(&source_tokens_[i]);
      } else if (n == 2) {
        source_tokens_[i].value_type = TokenInfo::AS_IS_HIRAGANA;
      } else if (n == 3) {
        source_tokens_[i].value_type = TokenInfo::AS_IS_KATAKANA;
      }
    }
  }

  void SetRandLabel() {
    for (size_t i = 0; i < source_tokens_.size(); ++i) {
      CHECK(source_tokens_[i].token);
      int n = Util::Random(Token::LABEL_SIZE);
      CHECK_GE(n, 0);
      CHECK_LT(n, Token::LABEL_SIZE);
      if (n == 0) {
        source_tokens_[i].token->attributes = Token::NONE;
      } else if (n == 1) {
        source_tokens_[i].token->attributes = Token::SPELLING_CORRECTION;
      }
    }
  }

  void CheckDecoded() const {
    EXPECT_EQ(source_tokens_.size(), decoded_tokens_.size());
    for (size_t i = 0; i < source_tokens_.size(); ++i) {
      EXPECT_TRUE(source_tokens_[i].token != NULL);
      EXPECT_TRUE(decoded_tokens_[i].token != NULL);

      EXPECT_EQ(source_tokens_[i].token->attributes,
                decoded_tokens_[i].token->attributes);

      EXPECT_EQ(source_tokens_[i].pos_type, decoded_tokens_[i].pos_type);
      if (source_tokens_[i].pos_type == TokenInfo::DEFAULT_POS) {
        EXPECT_EQ(source_tokens_[i].token->lid, decoded_tokens_[i].token->lid);
        EXPECT_EQ(source_tokens_[i].token->rid, decoded_tokens_[i].token->rid);
      } else if (source_tokens_[i].pos_type == TokenInfo::FREQUENT_POS) {
        EXPECT_EQ(source_tokens_[i].id_in_frequent_pos_map,
                  decoded_tokens_[i].id_in_frequent_pos_map);
      }

      if (source_tokens_[i].cost_type == TokenInfo::DEFAULT_COST) {
        EXPECT_EQ(source_tokens_[i].token->cost,
                  decoded_tokens_[i].token->cost);
      } else {  // small cost
        EXPECT_NEAR(source_tokens_[i].token->cost,
                    decoded_tokens_[i].token->cost,
                    256);
      }

      EXPECT_EQ(source_tokens_[i].value_type, decoded_tokens_[i].value_type);
      if (source_tokens_[i].value_type == TokenInfo::DEFAULT_VALUE) {
        EXPECT_EQ(source_tokens_[i].id_in_value_trie,
                  decoded_tokens_[i].id_in_value_trie);
      }
    }
  }

  std::vector<TokenInfo> source_tokens_;
  std::vector<TokenInfo> decoded_tokens_;
};

class SystemDictionaryCodecMock : public SystemDictionaryCodecInterface {
 public:
  const string GetSectionNameForKey() const { return "Mock"; }
  const string GetSectionNameForValue() const { return "Mock"; }
  const string GetSectionNameForTokens() const { return "Mock"; }
  const string GetSectionNameForPos() const { return "Mock"; }
  virtual void EncodeKey(const StringPiece src, string *dst) const {}
  virtual void DecodeKey(const StringPiece src, string *dst) const {}
  virtual size_t GetEncodedKeyLength(const StringPiece src) const { return 0; }
  virtual size_t GetDecodedKeyLength(const StringPiece src) const { return 0; }
  virtual void EncodeValue(const StringPiece src, string *dst) const {}
  virtual void DecodeValue(const StringPiece src, string *dst) const {}
  virtual void EncodeTokens(
      const std::vector<TokenInfo> &tokens, string *output) const {}
  virtual void DecodeTokens(
      const uint8 *ptr, std::vector<TokenInfo> *tokens) const {}
  virtual bool DecodeToken(
      const uint8 *ptr, TokenInfo *token_info, int *read_bytes) const {
    *read_bytes = 0;
    return false;
  }
  virtual bool ReadTokenForReverseLookup(
      const uint8 *ptr, int *value_id, int *read_bytes) const { return false; }
  virtual uint8 GetTokensTerminationFlag() const { return 0xff; }
};

TEST_F(SystemDictionaryCodecTest, FactoryTest) {
  unique_ptr<SystemDictionaryCodecMock> mock(new SystemDictionaryCodecMock);
  SystemDictionaryCodecFactory::SetCodec(mock.get());
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  EXPECT_EQ("Mock", codec->GetSectionNameForKey());
}

TEST_F(SystemDictionaryCodecTest, KeyCodecKanaTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  const string original = "よみ";
  string encoded;
  codec->EncodeKey(original, &encoded);
  // hiragana should be encoded in 1 byte
  EXPECT_EQ(2, encoded.size());
  EXPECT_EQ(encoded.size(), codec->GetEncodedKeyLength(original));
  string decoded;
  codec->DecodeKey(encoded, &decoded);
  EXPECT_EQ(original, decoded);
  EXPECT_EQ(decoded.size(), codec->GetDecodedKeyLength(encoded));
}


TEST_F(SystemDictionaryCodecTest, KeyCodecSymbolTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  const string original = "・ー";
  string encoded;
  codec->EncodeKey(original, &encoded);
  // middle dot and prolonged sound should be encoded in 1 byte
  EXPECT_EQ(2, encoded.size());
  EXPECT_EQ(encoded.size(), codec->GetEncodedKeyLength(original));
  string decoded;
  codec->DecodeKey(encoded, &decoded);
  EXPECT_EQ(original, decoded);
  EXPECT_EQ(decoded.size(), codec->GetDecodedKeyLength(encoded));
}

TEST_F(SystemDictionaryCodecTest, ValueCodecTest) {
  unique_ptr<SystemDictionaryCodec> codec(new SystemDictionaryCodec);
  // TODO(toshiyuki): Use 0x10ffff instead when UCS4 is supported.
  const char32 kMaxUniChar = 0x10ffff;
  for (char32 c = 0x01; c <= kMaxUniChar; ++c) {
    string original;
    Util::UCS4ToUTF8(c, &original);
    string encoded;
    codec->EncodeValue(original, &encoded);
    EXPECT_TRUE(IsExpectedEncodedSize(c, encoded));
    string decoded;
    codec->DecodeValue(encoded, &decoded);
    EXPECT_EQ(original, decoded)
        << "failed at: " << static_cast<uint32>(c);
  }
}

TEST_F(SystemDictionaryCodecTest, ValueCodecKanaTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  const string original = "もジ";
  string encoded;
  codec->EncodeValue(original, &encoded);
  // kana should be encoded in 1 byte
  EXPECT_EQ(2, encoded.size());
  string decoded;
  codec->DecodeValue(encoded, &decoded);
  EXPECT_EQ(original, decoded);
}

TEST_F(SystemDictionaryCodecTest, ValueCodecAsciiTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  const string original = "word";
  string encoded;
  codec->EncodeValue(original, &encoded);
  // ascii should be encoded in 2 bytes
  EXPECT_EQ(8, encoded.size());
  string decoded;
  codec->DecodeValue(encoded, &decoded);
  EXPECT_EQ(original, decoded);
}

TEST_F(SystemDictionaryCodecTest, TokenDefaultPosTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(1);
  SetDefaultPos(&source_tokens_[0]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenFrequentPosTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(1);
  SetFrequentPos(&source_tokens_[0]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenSamePosTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  {
    InitTokens(2);
    SetDefaultPos(&source_tokens_[0]);
    SetSamePos(&source_tokens_[1]);
    string encoded;
    codec->EncodeTokens(source_tokens_, &encoded);
    EXPECT_GT(encoded.size(), 0);
    codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                        &decoded_tokens_);
    CheckDecoded();
  }
  ResetAllTokens();
  {
    InitTokens(2);
    SetFrequentPos(&source_tokens_[0]);
    SetSamePos(&source_tokens_[1]);
    string encoded;
    codec->EncodeTokens(source_tokens_, &encoded);
    EXPECT_GT(encoded.size(), 0);
    codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                        &decoded_tokens_);
    CheckDecoded();
  }
}

TEST_F(SystemDictionaryCodecTest, TokenRandomPosTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandPos();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenDefaultCostTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(1);
  SetDefaultCost(&source_tokens_[0]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenSmallCostTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(1);
  SetSmallCost(&source_tokens_[0]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenRandomCostTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandCost();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenDefaultValueTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(1);
  SetDefaultValue(&source_tokens_[0]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, UCS4CharactersTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  const string ucs4_including =
      // "������������������������������������������������������"
      "\xf0\xa0\x80\x8b\xf0\xa1\x88\xbd\xf0\xa1\x8c\x9b\xf0\xa1\x91\xae\xf0"
      "\xa1\xa2\xbd\xf0\xa0\xae\x9f\xf0\xa1\x9a\xb4\xf0\xa1\xb8\xb4\xf0\xa3"
      "\x87\x84\xf0\xa3\x97\x84\xf0\xa3\x9c\xbf\xf0\xa3\x9d\xa3\xf0\xa3\xb3"
      "\xbe\xf0\xa4\x9f\xb1\xf0\xa5\x92\x8e\xf0\xa5\x94\x8e\xf0\xa5\x9d\xb1"
      "\xf0\xa5\xa7\x84\xf0\xa5\xb6\xa1\xf0\xa6\xab\xbf\xf0\xa6\xb9\x80\xf0"
      "\xa7\x83\xb4\xf0\xa7\x9a\x84\xf0\xa8\x89\xb7\xf0\xa8\x8f\x8d\xf0\xaa"
      "\x86\x90\xf0\xa0\x82\x89"
      // "������������������������������������������������������"
      "\xf0\xa0\x82\xa2\xf0\xa0\x82\xa4\xf0\xa0\x86\xa2\xf0\xa0\x88\x93\xf0"
      "\xa0\x8c\xab\xf0\xa0\x8e\x81\xf0\xa0\x8d\xb1\xf0\xa0\x8f\xb9\xf0\xa0"
      "\x91\x8a\xf0\xa0\x94\x89\xf0\xa0\x97\x96\xf0\xa0\x98\xa8\xf0\xa0\x9d"
      "\x8f\xf0\xa0\xa0\x87\xf0\xa0\xa0\xba\xf0\xa0\xa2\xb9\xf0\xa0\xa5\xbc"
      "\xf0\xa0\xa6\x9d\xf0\xa0\xab\x93\xf0\xa0\xac\x9d\xf0\xa0\xb5\x85\xf0"
      "\xa0\xb7\xa1\xf0\xa0\xba\x95\xf0\xa0\xb9\xad\xf0\xa0\xb9\xa4\xf0\xa0"
      "\xbd\x9f\xf0\xa1\x88\x81"
      // "������������������������������������������������������"
      "\xf0\xa1\x89\x95\xf0\xa1\x89\xbb\xf0\xa1\x89\xb4\xf0\xa1\x8b\xa4\xf0"
      "\xa1\x8b\x97\xf0\xa1\x8b\xbd\xf0\xa1\x8c\xb6\xf0\xa1\x8d\x84\xf0\xa1"
      "\x8f\x84\xf0\xa1\x91\xad\xf0\xa1\x97\x97\xf0\xa6\xb0\xa9\xf0\xa1\x99"
      "\x87\xf0\xa1\x9c\x86\xf0\xa1\x9d\x82\xf0\xa1\xa7\x83\xf0\xa1\xb1\x96"
      "\xf0\xa1\xb4\xad\xf0\xa1\xb5\x85\xf0\xa1\xb5\xb8\xf0\xa1\xb5\xa2\xf0"
      "\xa1\xb6\xa1\xf0\xa1\xb6\x9c\xf0\xa1\xb6\x92\xf0\xa1\xb6\xb7\xf0\xa1"
      "\xb7\xa0\xf0\xa1\xb8\xb3"
      // "������������������������������������������������������"
      "\xf0\xa1\xbc\x9e\xf0\xa1\xbd\xb6\xf0\xa1\xbf\xba\xf0\xa2\x85\xbb\xf0"
      "\xa2\x8c\x9e\xf0\xa2\x8e\xad\xf0\xa2\x9b\xb3\xf0\xa2\xa1\x9b\xf0\xa2"
      "\xa2\xab\xf0\xa2\xa6\x8f\xf0\xa2\xaa\xb8\xf0\xa2\xad\x8f\xf0\xa2\xad"
      "\x90\xf0\xa2\xad\x86\xf0\xa2\xb0\x9d\xf0\xa2\xae\xa6\xf0\xa2\xb0\xa4"
      "\xf0\xa2\xb7\xa1\xf0\xa3\x87\x83\xf0\xa3\x87\xb5\xf0\xa3\x86\xb6\xf0"
      "\xa3\x8d\xb2\xf0\xa3\x8f\x93\xf0\xa3\x8f\x92\xf0\xa3\x8f\x90\xf0\xa3\x8f"
      "\xa4\xf0\xa3\x8f\x95"
      // "������������������������������������������������������"
      "\xf0\xa3\x8f\x9a\xf0\xa3\x8f\x9f\xf0\xa3\x91\x8a\xf0\xa3\x91\x91\xf0"
      "\xa3\x91\x8b\xf0\xa3\x91\xa5\xf0\xa3\x93\xa4\xf0\xa3\x95\x9a\xf0\xa3"
      "\x96\x94\xf0\xa3\x98\xb9\xf0\xa3\x99\x87\xf0\xa3\x98\xb8\xf0\xa3\x98"
      "\xba\xf0\xa3\x9c\x9c\xf0\xa3\x9c\x8c\xf0\xa3\x9d\xa4\xf0\xa3\x9f\xbf"
      "\xf0\xa3\x9f\xa7\xf0\xa3\xa0\xa4\xf0\xa3\xa0\xbd\xf0\xa3\xaa\x98\xf0"
      "\xa3\xb1\xbf\xf0\xa3\xb4\x80\xf0\xa3\xb5\x80\xf0\xa3\xb7\xba\xf0\xa3"
      "\xb7\xb9\xf0\xa3\xb7\x93"
      // "������������������������������������������������������"
      "\xf0\xa3\xbd\xbe\xf0\xa4\x82\x96\xf0\xa4\x84\x83\xf0\xa4\x87\x86\xf0"
      "\xa4\x87\xbe\xf0\xa4\x8e\xbc\xf0\xa4\x98\xa9\xf0\xa4\x9a\xa5\xf0\xa4"
      "\xa2\x96\xf0\xa4\xa9\x8d\xf0\xa4\xad\x96\xf0\xa4\xad\xaf\xf0\xa4\xb0"
      "\x96\xf0\xa4\xb4\x94\xf0\xa4\xb8\x8e\xf0\xa4\xb8\xb7\xf0\xa4\xb9\xaa"
      "\xf0\xa4\xba\x8b\xf0\xa5\x81\x8a\xf0\xa5\x81\x95\xf0\xa5\x84\xa2\xf0"
      "\xa5\x86\xa9\xf0\xa5\x87\xa5\xf0\xa5\x87\x8d\xf0\xa5\x88\x9e\xf0\xa5"
      "\x89\x8c\xf0\xa5\x90\xae"
      // "������������������������������������������������������"
      "\xf0\xa5\x93\x99\xf0\xa5\x96\xa7\xf0\xa5\x9e\xa9\xf0\xa5\x9e\xb4\xf0"
      "\xa5\xa7\x94\xf0\xa5\xab\xa4\xf0\xa5\xab\xa3\xf0\xa5\xab\xb1\xf0\xa5"
      "\xae\xb2\xf0\xa5\xb1\x8b\xf0\xa5\xb1\xa4\xf0\xa5\xb8\xae\xf0\xa5\xb9"
      "\x96\xf0\xa5\xb9\xa5\xf0\xa5\xb9\xa2\xf0\xa5\xbb\x98\xf0\xa5\xbb\x82"
      "\xf0\xa5\xbb\xa8\xf0\xa5\xbc\xa3\xf0\xa5\xbd\x9c\xf0\xa5\xbf\xa0\xf0"
      "\xa5\xbf\x94\xf0\xa6\x80\x8c\xf0\xa5\xbf\xbb\xf0\xa6\x80\x97\xf0\xa6"
      "\x81\xa0\xf0\xa6\x83\xad"
      // "������������������������������������������������������"
      "\xf0\xa6\x89\xb0\xf0\xa6\x8a\x86\xf0\xa6\x8d\x8c\xf0\xa3\xb4\x8e\xf0"
      "\xa6\x90\x82\xf0\xa6\x99\xbe\xf0\xa6\x9a\xb0\xf0\xa6\x9c\x9d\xf0\xa6"
      "\xa3\x9d\xf0\xa6\xa3\xaa\xf0\xa6\xa5\x91\xf0\xa6\xa5\xaf\xf0\xa6\xa7"
      "\x9d\xf0\xa6\xa8\x9e\xf0\xa6\xa9\x98\xf0\xa6\xaa\x8c\xf0\xa6\xaa\xb7"
      "\xf0\xa6\xb1\xb3\xf0\xa6\xb3\x9d\xf0\xa6\xb9\xa5\xf0\xa6\xbe\x94\xf0"
      "\xa6\xbf\xb8\xf0\xa6\xbf\xb6\xf0\xa6\xbf\xb7\xf0\xa7\x84\x8d\xf0\xa7"
      "\x84\xb9\xf0\xa7\x8f\x9b"
      // "������������������������������������������������������"
      "\xf0\xa7\x8f\x9a\xf0\xa7\x8f\xbe\xf0\xa7\x90\x90\xf0\xa7\x91\x89\xf0"
      "\xa7\x98\x95\xf0\xa7\x98\x94\xf0\xa7\x98\xb1\xf0\xa7\x9a\x93\xf0\xa7"
      "\x9c\x8e\xf0\xa7\x9c\xa3\xf0\xa7\x9d\x92\xf0\xa7\xa6\x85\xf0\xa7\xaa"
      "\x84\xf0\xa7\xae\xb3\xf0\xa7\xae\xbe\xf0\xa7\xaf\x87\xf0\xa7\xb2\xb8"
      "\xf0\xa7\xb6\xa0\xf0\xa7\xb8\x90\xf0\xa7\xbe\xb7\xf0\xa8\x82\x8a\xf0"
      "\xa8\x82\xbb\xf0\xa8\x8a\x82\xf0\xa8\x8b\xb3\xf0\xa8\x90\x8c\xf0\xa8"
      "\x91\x95\xf0\xa8\x95\xab"
      // "������������������������������������������������������"
      "\xf0\xa8\x97\x88\xf0\xa8\x97\x89\xf0\xa8\x9b\x97\xf0\xa8\x9b\xba\xf0"
      "\xa8\xa5\x89\xf0\xa8\xa5\x86\xf0\xa8\xa5\xab\xf0\xa8\xa6\x87\xf0\xa8"
      "\xa6\x88\xf0\xa8\xa6\xba\xf0\xa8\xa6\xbb\xf0\xa8\xa8\x9e\xf0\xa8\xa8"
      "\xa9\xf0\xa8\xa9\xb1\xf0\xa8\xa9\x83\xf0\xa8\xaa\x99\xf0\xa8\xab\x8d"
      "\xf0\xa8\xab\xa4\xf0\xa8\xab\x9d\xf0\xa8\xaf\x81\xf0\xa8\xaf\xaf\xf0\xa8"
      "\xb4\x90\xf0\xa8\xb5\xb1\xf0\xa8\xb7\xbb\xf0\xa8\xb8\x9f\xf0\xa8\xb8"
      "\xb6\xf0\xa8\xba\x89"
      // "������������������������������������������������������"
      "\xf0\xa8\xbb\xab\xf0\xa8\xbc\xb2\xf0\xa8\xbf\xb8\xf0\xa9\x8a\xa0\xf0"
      "\xa9\x8a\xb1\xf0\xa9\x92\x90\xf0\xa9\x97\x8f\xf0\xa9\x99\xbf\xf0\xa9"
      "\x9b\xb0\xf0\xa9\x9c\x99\xf0\xa9\x9d\x90\xf0\xa9\xa3\x86\xf0\xa9\xa9"
      "\xb2\xf0\xa9\xb7\x9b\xf0\xa9\xb8\xbd\xf0\xa9\xb8\x95\xf0\xa9\xba\x8a"
      "\xf0\xa9\xb9\x89\xf0\xa9\xbb\x84\xf0\xa9\xbb\xa9\xf0\xa9\xbb\x9b\xf0"
      "\xa9\xbf\x8e\xf0\xaa\x80\xaf\xf0\xaa\x80\x9a\xf0\xaa\x83\xb9\xf0\xaa"
      "\x82\x82\xf0\xa2\x88\x98"
      // "������������"
      "\xf0\xaa\x8e\x8c\xf0\xaa\x90\xb7\xf0\xaa\x97\xb1\xf0\xaa\x98\x82\xf0"
      "\xaa\x98\x9a\xf0\xaa\x9a\xb2";
  string encoded;
  codec->EncodeValue(ucs4_including, &encoded);
  EXPECT_GT(encoded.size(), 0);
  string decoded;
  codec->DecodeValue(encoded, &decoded);
  EXPECT_EQ(ucs4_including, decoded);
}

TEST_F(SystemDictionaryCodecTest, TokenSameValueTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(2);
  SetDefaultValue(&source_tokens_[0]);
  SetSameValue(&source_tokens_[1]);
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenRandomValueTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandValue();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenRandomLabelTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandLabel();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, TokenRandomTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandPos();
  SetRandCost();
  SetRandValue();
  SetRandLabel();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                      &decoded_tokens_);
  CheckDecoded();
}

TEST_F(SystemDictionaryCodecTest, ReadTokenRandomTest) {
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  InitTokens(50);
  Util::SetRandomSeed(0);
  SetRandPos();
  SetRandCost();
  SetRandValue();
  SetRandLabel();
  string encoded;
  codec->EncodeTokens(source_tokens_, &encoded);
  EXPECT_GT(encoded.size(), 0);
  int read_num = 0;
  int offset = 0;
  while (true) {
    int read_byte = 0;
    int value_id = -1;
    const bool is_last_token = !(codec->ReadTokenForReverseLookup(
        reinterpret_cast<const unsigned char *>(encoded.data()) + offset,
        &value_id,
        &read_byte));
    if (source_tokens_[read_num].value_type == TokenInfo::DEFAULT_VALUE) {
      EXPECT_EQ(source_tokens_[read_num].id_in_value_trie, value_id);
    } else {
      EXPECT_EQ(-1, value_id);
    }
    offset += read_byte;
    ++read_num;
    if (is_last_token) {
      break;
    }
  }
  EXPECT_EQ(source_tokens_.size(), read_num);
}

TEST_F(SystemDictionaryCodecTest, CodecTest) {
  unique_ptr<SystemDictionaryCodec> impl(new SystemDictionaryCodec);
  SystemDictionaryCodecFactory::SetCodec(impl.get());
  SystemDictionaryCodecInterface *codec =
      SystemDictionaryCodecFactory::GetCodec();
  {  // Token
    InitTokens(50);
    Util::SetRandomSeed(0);
    SetRandPos();
    SetRandCost();
    SetRandValue();
    SetRandLabel();
    string encoded;
    codec->EncodeTokens(source_tokens_, &encoded);
    EXPECT_GT(encoded.size(), 0);
    codec->DecodeTokens(reinterpret_cast<const unsigned char *>(encoded.data()),
                        &decoded_tokens_);
    CheckDecoded();

    // ReadTokens
    int read_num = 0;
    int offset = 0;
    while (true) {
      int read_byte = 0;
      int value_id = -1;
      const bool is_last_token = !(codec->ReadTokenForReverseLookup(
          reinterpret_cast<const unsigned char *>(encoded.data()) + offset,
          &value_id,
          &read_byte));
      if (source_tokens_[read_num].value_type == TokenInfo::DEFAULT_VALUE) {
        EXPECT_EQ(source_tokens_[read_num].id_in_value_trie, value_id);
      } else {
        EXPECT_EQ(-1, value_id);
      }
      offset += read_byte;
      ++read_num;
      if (is_last_token) {
        break;
      }
    }
    EXPECT_EQ(source_tokens_.size(), read_num);
  }
  {  // Value
    string original;
    {
      char32 a_ucs4 = '!';
      Util::SetRandomSeed(0);
      for (size_t i = 0; i < 10000; ++i) {
        // U+4E00-9FFF CJK Unified Ideographs
        const char32 c = a_ucs4 + static_cast<uint16>(Util::Random(0x9f00));
        Util::UCS4ToUTF8Append(c, &original);
      }
    }
    string encoded;
    codec->EncodeValue(original, &encoded);
    string decoded;
    codec->DecodeValue(encoded, &decoded);
    EXPECT_EQ(original, decoded);
  }
  {  // Key
    string original;
    {
      char32 a_ucs4 = 0x3041;  // "ぁ"
      Util::SetRandomSeed(0);
      for (size_t i = 0; i < 1000; ++i) {
        const char32 c = a_ucs4 + static_cast<uint16>(Util::Random(1000));
        Util::UCS4ToUTF8Append(c, &original);
      }
    }
    string encoded;
    codec->EncodeKey(original, &encoded);
    EXPECT_EQ(encoded.size(), codec->GetEncodedKeyLength(original));
    string decoded;
    codec->DecodeKey(encoded, &decoded);
    EXPECT_EQ(original, decoded);
    EXPECT_EQ(decoded.size(), codec->GetDecodedKeyLength(encoded));
  }
}


}  // namespace dictionary
}  // namespace mozc