1 // Copyright 2010-2018, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 // * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 #include <algorithm>
32 #include <map>
33 #include <memory>
34 #include <string>
35 #include <utility>
36 #include <vector>
37
38 #include "base/logging.h"
39 #include "base/port.h"
40 #include "base/system_util.h"
41 #include "config/config_handler.h"
42 #include "converter/quality_regression_util.h"
43 #include "data_manager/data_manager.h"
44 #include "engine/engine.h"
45 #include "protocol/commands.pb.h"
46 #include "protocol/config.pb.h"
47 #include "session/request_test_util.h"
48 #include "testing/base/public/gunit.h"
49 #include "testing/base/public/mozctest.h"
50
51 namespace mozc {
52
53 // Test data is provided in external file.
54 struct TestCase {
55 const bool enabled;
56 const char *line;
57 };
58 extern TestCase kTestData[];
59
60 namespace {
61
62 using quality_regression::QualityRegressionUtil;
63
64 class QualityRegressionTest : public ::testing::Test {
65 protected:
RunTestForPlatform(uint32 platform,QualityRegressionUtil * util)66 static void RunTestForPlatform(uint32 platform, QualityRegressionUtil *util) {
67 CHECK(util);
68 std::map<string, std::vector<std::pair<float, string>>> results,
69 disabled_results;
70
71 int num_executed_cases = 0, num_disabled_cases = 0;
72 for (size_t i = 0; kTestData[i].line; ++i) {
73 const string &tsv_line = kTestData[i].line;
74 QualityRegressionUtil::TestItem item;
75 CHECK(item.ParseFromTSV(tsv_line));
76 if (!(item.platform & platform)) {
77 continue;
78 }
79 string actual_value;
80 const bool test_result = util->ConvertAndTest(item, &actual_value);
81
82 std::map<string, std::vector<std::pair<float, string>>> *table = nullptr;
83 if (kTestData[i].enabled) {
84 ++num_executed_cases;
85 table = &results;
86 } else {
87 LOG(INFO) << "DISABLED: " << kTestData[i].line;
88 ++num_disabled_cases;
89 table = &disabled_results;
90 }
91
92 const string &label = item.label;
93 string line = tsv_line;
94 line.append("\tActual: ").append(actual_value);
95 if (test_result) {
96 // use "-1.0" as a dummy expected ratio
97 (*table)[label].push_back(std::make_pair(-1.0, line));
98 } else {
99 (*table)[label].push_back(std::make_pair(item.accuracy, line));
100 }
101 }
102
103 ExamineResults(true, platform, &results);
104 ExamineResults(false, platform, &disabled_results);
105
106 const int total_cases = num_executed_cases + num_disabled_cases;
107 LOG(INFO) << "Tested " << num_executed_cases << " / "
108 << total_cases << " entries.";
109 }
110
111 // If |enabled| parameter is true, then actual conversion results are tested
112 // and any failure is reported as test failure. If false, actual conversion
113 // results don't affect test results but closable issues are reported.
ExamineResults(const bool enabled,uint32 platform,std::map<string,std::vector<std::pair<float,string>>> * results)114 static void ExamineResults(
115 const bool enabled, uint32 platform,
116 std::map<string, std::vector<std::pair<float, string>>> *results) {
117 for (auto it = results->begin(); it != results->end(); ++it) {
118 std::vector<std::pair<float, string>> *values = &it->second;
119 std::sort(values->begin(), values->end());
120 size_t correct = 0;
121 bool all_passed = true;
122 for (const auto &value : *values) {
123 const float accuracy = value.first;
124 if (accuracy < 0) {
125 ++correct;
126 continue;
127 }
128 // Print failed example for failed label
129 const float actual_ratio = 1.0 * correct / values->size();
130 if (enabled) {
131 EXPECT_LT(accuracy, actual_ratio) << value.second
132 << " " << accuracy
133 << " " << actual_ratio;
134 } else {
135 if (accuracy < actual_ratio) {
136 LOG(INFO) << "PASSED (DISABLED): "
137 << it->first << ": " << value.second;
138 } else {
139 LOG(INFO) << "FAILED (DISABLED): "
140 << it->first << ": " << value.second;
141 all_passed = false;
142 }
143 }
144 }
145 LOG(INFO) << "Accuracy: " << it->first << " "
146 << 1.0 * correct / values->size();
147 if (!enabled && all_passed) {
148 LOG(INFO) << "CLOSED ISSUE [platform = "
149 << QualityRegressionUtil::GetPlatformString(platform)
150 << "]: " << it->first << " with "
151 << it->second.size() << " cases";
152 }
153 }
154 }
155
156 private:
157 const testing::ScopedTmpUserProfileDirectory scoped_profile_dir_;
158 };
159
CreateEngine(const string & data_file_path,const string & magic_number,const string & engine_type)160 std::unique_ptr<EngineInterface> CreateEngine(const string &data_file_path,
161 const string &magic_number,
162 const string &engine_type) {
163 std::unique_ptr<DataManager> data_manager(new DataManager);
164 const auto status = data_manager->InitFromFile(data_file_path, magic_number);
165 if (status != DataManager::Status::OK) {
166 LOG(ERROR) << "Failed to load " << data_file_path
167 << ": " << DataManager::StatusCodeToString(status);
168 return nullptr;
169 }
170 if (engine_type == "desktop") {
171 return Engine::CreateDesktopEngine(std::move(data_manager));
172 }
173 if (engine_type == "mobile") {
174 return Engine::CreateMobileEngine(std::move(data_manager));
175 }
176 LOG(ERROR) << "Invalid engine type: " << engine_type;
177 return nullptr;
178 }
179
180
181 } // namespace
182 } // namespace mozc
183