1 // Copyright 2010-2018, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 // * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 #include "converter/converter.h"
31
32 #include <memory>
33 #include <string>
34 #include <vector>
35
36 #include "base/logging.h"
37 #include "base/port.h"
38 #include "base/system_util.h"
39 #include "base/util.h"
40 #include "composer/composer.h"
41 #include "composer/table.h"
42 #include "config/config_handler.h"
43 #include "converter/connector.h"
44 #include "converter/converter_interface.h"
45 #include "converter/immutable_converter.h"
46 #include "converter/immutable_converter_interface.h"
47 #include "converter/node.h"
48 #include "converter/segmenter.h"
49 #include "converter/segments.h"
50 #include "data_manager/data_manager_interface.h"
51 #include "data_manager/testing/mock_data_manager.h"
52 #include "dictionary/dictionary_impl.h"
53 #include "dictionary/dictionary_interface.h"
54 #include "dictionary/dictionary_mock.h"
55 #include "dictionary/pos_group.h"
56 #include "dictionary/pos_matcher.h"
57 #include "dictionary/suffix_dictionary.h"
58 #include "dictionary/suppression_dictionary.h"
59 #include "dictionary/system/system_dictionary.h"
60 #include "dictionary/system/value_dictionary.h"
61 #include "dictionary/user_dictionary.h"
62 #include "dictionary/user_dictionary_stub.h"
63 #include "dictionary/user_pos.h"
64 #include "engine/engine.h"
65 #include "engine/engine_interface.h"
66 #include "engine/mock_data_engine_factory.h"
67 #include "prediction/dictionary_predictor.h"
68 #include "prediction/predictor.h"
69 #include "prediction/predictor_interface.h"
70 #include "prediction/suggestion_filter.h"
71 #include "prediction/user_history_predictor.h"
72 #include "protocol/commands.pb.h"
73 #include "protocol/config.pb.h"
74 #include "request/conversion_request.h"
75 #include "rewriter/rewriter.h"
76 #include "rewriter/rewriter_interface.h"
77 #include "testing/base/public/googletest.h"
78 #include "testing/base/public/gunit.h"
79 #include "testing/base/public/mozctest.h"
80 #include "transliteration/transliteration.h"
81 #include "usage_stats/usage_stats.h"
82 #include "usage_stats/usage_stats_testing_util.h"
83
84 namespace mozc {
85 namespace {
86
87 using dictionary::DictionaryImpl;
88 using dictionary::DictionaryInterface;
89 using dictionary::DictionaryMock;
90 using dictionary::PosGroup;
91 using dictionary::POSMatcher;
92 using dictionary::SuffixDictionary;
93 using dictionary::SuppressionDictionary;
94 using dictionary::SystemDictionary;
95 using dictionary::Token;
96 using dictionary::UserDictionaryStub;
97 using dictionary::ValueDictionary;
98 using usage_stats::UsageStats;
99
100 class StubPredictor : public PredictorInterface {
101 public:
StubPredictor()102 StubPredictor() : predictor_name_("StubPredictor") {}
103
PredictForRequest(const ConversionRequest & request,Segments * segments) const104 bool PredictForRequest(const ConversionRequest &request, Segments *segments)
105 const override {
106 return true;
107 }
108
GetPredictorName() const109 const string &GetPredictorName() const override {
110 return predictor_name_;
111 }
112
113 private:
114 const string predictor_name_;
115 };
116
117 class StubRewriter : public RewriterInterface {
Rewrite(const ConversionRequest & request,Segments * segments) const118 bool Rewrite(const ConversionRequest &request,
119 Segments *segments) const override {
120 return true;
121 }
122 };
123
CreateSuffixDictionaryFromDataManager(const DataManagerInterface & data_manager)124 SuffixDictionary *CreateSuffixDictionaryFromDataManager(
125 const DataManagerInterface &data_manager) {
126 StringPiece suffix_key_array_data, suffix_value_array_data;
127 const uint32 *token_array;
128 data_manager.GetSuffixDictionaryData(&suffix_key_array_data,
129 &suffix_value_array_data,
130 &token_array);
131 return new SuffixDictionary(suffix_key_array_data,
132 suffix_value_array_data,
133 token_array);
134 }
135
136 class InsertDummyWordsRewriter : public RewriterInterface {
Rewrite(const ConversionRequest &,Segments * segments) const137 bool Rewrite(const ConversionRequest &, Segments *segments) const override {
138 for (size_t i = 0; i < segments->conversion_segments_size(); ++i) {
139 Segment *seg = segments->mutable_conversion_segment(i);
140 {
141 Segment::Candidate *cand = seg->add_candidate();
142 cand->Init();
143 cand->key = "tobefiltered";
144 cand->value = "ToBeFiltered";
145 }
146 {
147 Segment::Candidate *cand = seg->add_candidate();
148 cand->Init();
149 cand->key = "nottobefiltered";
150 cand->value = "NotToBeFiltered";
151 }
152 }
153 return true;
154 }
155 };
156
157 } // namespace
158
159 class ConverterTest : public ::testing::Test {
160 protected:
161 enum PredictorType {
162 STUB_PREDICTOR,
163 DEFAULT_PREDICTOR,
164 MOBILE_PREDICTOR,
165 };
166
167 struct UserDefinedEntry {
168 const string key;
169 const string value;
170 const user_dictionary::UserDictionary::PosType pos;
UserDefinedEntrymozc::ConverterTest::UserDefinedEntry171 UserDefinedEntry(const string &k, const string &v,
172 user_dictionary::UserDictionary::PosType p)
173 : key(k), value(v), pos(p) {}
174 };
175
176 // Workaround for C2512 error (no default appropriate constructor) on MSVS.
ConverterTest()177 ConverterTest() {}
~ConverterTest()178 ~ConverterTest() override {}
179
SetUp()180 void SetUp() override {
181 UsageStats::ClearAllStatsForTest();
182 }
183
TearDown()184 void TearDown() override {
185 UsageStats::ClearAllStatsForTest();
186 }
187
188 // This struct holds resources used by converter.
189 struct ConverterAndData {
190 std::unique_ptr<testing::MockDataManager> data_manager;
191 std::unique_ptr<DictionaryInterface> user_dictionary;
192 std::unique_ptr<SuppressionDictionary> suppression_dictionary;
193 std::unique_ptr<DictionaryInterface> suffix_dictionary;
194 std::unique_ptr<const Connector> connector;
195 std::unique_ptr<const Segmenter> segmenter;
196 std::unique_ptr<DictionaryInterface> dictionary;
197 std::unique_ptr<const PosGroup> pos_group;
198 std::unique_ptr<const SuggestionFilter> suggestion_filter;
199 std::unique_ptr<ImmutableConverterInterface> immutable_converter;
200 std::unique_ptr<ConverterImpl> converter;
201 dictionary::POSMatcher pos_matcher;
202 };
203
204 // Returns initialized predictor for the given type.
205 // Note that all fields of |converter_and_data| should be filled including
206 // |converter_and_data.converter|. |converter| will be initialized using
207 // predictor pointer, but predictor need the pointer for |converter| for
208 // initializing. Prease see mozc/engine/engine.cc for details.
209 // Caller should manage the ownership.
CreatePredictor(PredictorType predictor_type,const POSMatcher * pos_matcher,const ConverterAndData & converter_and_data)210 PredictorInterface *CreatePredictor(
211 PredictorType predictor_type,
212 const POSMatcher *pos_matcher,
213 const ConverterAndData &converter_and_data) {
214 if (predictor_type == STUB_PREDICTOR) {
215 return new StubPredictor;
216 }
217
218 PredictorInterface *(*predictor_factory)(
219 PredictorInterface *, PredictorInterface *) = nullptr;
220 bool enable_content_word_learning = false;
221
222 switch (predictor_type) {
223 case DEFAULT_PREDICTOR:
224 predictor_factory = DefaultPredictor::CreateDefaultPredictor;
225 enable_content_word_learning = false;
226 break;
227 case MOBILE_PREDICTOR:
228 predictor_factory = MobilePredictor::CreateMobilePredictor;
229 enable_content_word_learning = true;
230 break;
231 default:
232 LOG(ERROR) << "Should not come here: Invalid predictor type.";
233 predictor_factory = DefaultPredictor::CreateDefaultPredictor;
234 enable_content_word_learning = false;
235 break;
236 }
237
238 CHECK(converter_and_data.converter.get()) << "converter should be filled.";
239
240 // Create a predictor with three sub-predictors, dictionary predictor, user
241 // history predictor, and extra predictor.
242 PredictorInterface *dictionary_predictor =
243 new DictionaryPredictor(*converter_and_data.data_manager,
244 converter_and_data.converter.get(),
245 converter_and_data.immutable_converter.get(),
246 converter_and_data.dictionary.get(),
247 converter_and_data.suffix_dictionary.get(),
248 converter_and_data.connector.get(),
249 converter_and_data.segmenter.get(),
250 pos_matcher,
251 converter_and_data.suggestion_filter.get());
252 CHECK(dictionary_predictor);
253
254 PredictorInterface *user_history_predictor =
255 new UserHistoryPredictor(
256 converter_and_data.dictionary.get(),
257 pos_matcher,
258 converter_and_data.suppression_dictionary.get(),
259 enable_content_word_learning);
260 CHECK(user_history_predictor);
261
262 PredictorInterface *ret_predictor =
263 (*predictor_factory)(dictionary_predictor, user_history_predictor);
264 CHECK(ret_predictor);
265 return ret_predictor;
266 }
267
268 // Initializes ConverterAndData with mock data set using given
269 // |user_dictionary| and |suppression_dictionary|.
270 // ConverterAndData takes ownership of |user_dicationary| and
271 // |suppression_dictionary|
InitConverterAndData(DictionaryInterface * user_dictionary,SuppressionDictionary * suppression_dictionary,RewriterInterface * rewriter,PredictorType predictor_type,ConverterAndData * converter_and_data)272 void InitConverterAndData(DictionaryInterface *user_dictionary,
273 SuppressionDictionary *suppression_dictionary,
274 RewriterInterface *rewriter,
275 PredictorType predictor_type,
276 ConverterAndData *converter_and_data) {
277 converter_and_data->data_manager.reset(new testing::MockDataManager());
278 const auto &data_manager = *converter_and_data->data_manager;
279
280 const char *dictionary_data = nullptr;
281 int dictionary_size = 0;
282 data_manager.GetSystemDictionaryData(&dictionary_data, &dictionary_size);
283
284 converter_and_data->pos_matcher.Set(data_manager.GetPOSMatcherData());
285
286 SystemDictionary *sysdic =
287 SystemDictionary::Builder(dictionary_data, dictionary_size).Build();
288 converter_and_data->user_dictionary.reset(user_dictionary);
289 converter_and_data->suppression_dictionary.reset(suppression_dictionary);
290 converter_and_data->dictionary.reset(new DictionaryImpl(
291 sysdic, // DictionaryImpl takes the ownership
292 new ValueDictionary(converter_and_data->pos_matcher,
293 &sysdic->value_trie()),
294 converter_and_data->user_dictionary.get(),
295 converter_and_data->suppression_dictionary.get(),
296 &converter_and_data->pos_matcher));
297 converter_and_data->pos_group.reset(
298 new PosGroup(data_manager.GetPosGroupData()));
299 converter_and_data->suggestion_filter.reset(
300 CreateSuggestionFilter(data_manager));
301 converter_and_data->suffix_dictionary.reset(
302 CreateSuffixDictionaryFromDataManager(data_manager));
303 converter_and_data->connector.reset(
304 Connector::CreateFromDataManager(data_manager));
305 converter_and_data->segmenter.reset(
306 Segmenter::CreateFromDataManager(data_manager));
307 converter_and_data->immutable_converter.reset(
308 new ImmutableConverterImpl(
309 converter_and_data->dictionary.get(),
310 converter_and_data->suffix_dictionary.get(),
311 converter_and_data->suppression_dictionary.get(),
312 converter_and_data->connector.get(),
313 converter_and_data->segmenter.get(),
314 &converter_and_data->pos_matcher,
315 converter_and_data->pos_group.get(),
316 converter_and_data->suggestion_filter.get()));
317 converter_and_data->converter.reset(new ConverterImpl);
318
319 PredictorInterface *predictor = CreatePredictor(
320 predictor_type, &converter_and_data->pos_matcher, *converter_and_data);
321 converter_and_data->converter->Init(
322 &converter_and_data->pos_matcher,
323 converter_and_data->suppression_dictionary.get(),
324 predictor,
325 rewriter,
326 converter_and_data->immutable_converter.get());
327 }
328
CreateConverterAndData(RewriterInterface * rewriter,PredictorType predictor_type)329 ConverterAndData *CreateConverterAndData(RewriterInterface *rewriter,
330 PredictorType predictor_type) {
331 ConverterAndData *ret = new ConverterAndData;
332 InitConverterAndData(new UserDictionaryStub, new SuppressionDictionary,
333 rewriter, predictor_type, ret);
334 return ret;
335 }
336
CreateStubbedConverterAndData()337 ConverterAndData *CreateStubbedConverterAndData() {
338 return CreateConverterAndData(new StubRewriter, STUB_PREDICTOR);
339 }
340
CreateConverterAndDataWithInsertDummyWordsRewriter()341 ConverterAndData *CreateConverterAndDataWithInsertDummyWordsRewriter() {
342 return CreateConverterAndData(new InsertDummyWordsRewriter, STUB_PREDICTOR);
343 }
344
CreateConverterAndDataWithUserDefinedEntries(const std::vector<UserDefinedEntry> & user_defined_entries,RewriterInterface * rewriter,PredictorType predictor_type)345 ConverterAndData *CreateConverterAndDataWithUserDefinedEntries(
346 const std::vector<UserDefinedEntry> &user_defined_entries,
347 RewriterInterface *rewriter, PredictorType predictor_type) {
348 ConverterAndData *ret = new ConverterAndData;
349
350 ret->pos_matcher.Set(mock_data_manager_.GetPOSMatcherData());
351
352 SuppressionDictionary *suppression_dictionary = new SuppressionDictionary;
353 dictionary::UserDictionary *user_dictionary =
354 new dictionary::UserDictionary(
355 dictionary::UserPOS::CreateFromDataManager(mock_data_manager_),
356 ret->pos_matcher,
357 suppression_dictionary);
358 InitConverterAndData(
359 user_dictionary, suppression_dictionary, rewriter, predictor_type, ret);
360
361 {
362 user_dictionary::UserDictionaryStorage storage;
363 typedef user_dictionary::UserDictionary::Entry UserEntry;
364 user_dictionary::UserDictionary *dictionary = storage.add_dictionaries();
365 for (const auto &user_entry : user_defined_entries) {
366 UserEntry *entry = dictionary->add_entries();
367 entry->set_key(user_entry.key);
368 entry->set_value(user_entry.value);
369 entry->set_pos(user_entry.pos);
370 }
371
372 user_dictionary->Load(storage);
373 }
374 return ret;
375 }
376
CreateEngineWithMobilePredictor()377 EngineInterface *CreateEngineWithMobilePredictor() {
378 return Engine::CreateMobileEngineHelper<testing::MockDataManager>()
379 .release();
380 }
381
FindCandidateByValue(const string & value,const Segment & segment) const382 bool FindCandidateByValue(const string &value, const Segment &segment) const {
383 for (size_t i = 0; i < segment.candidates_size(); ++i) {
384 if (segment.candidate(i).value == value) {
385 return true;
386 }
387 }
388 return false;
389 }
390
default_request() const391 const commands::Request &default_request() const {
392 return default_request_;
393 }
394
CreateSuggestionFilter(const DataManagerInterface & data_manager)395 static SuggestionFilter *CreateSuggestionFilter(
396 const DataManagerInterface &data_manager) {
397 const char *data = nullptr;
398 size_t size = 0;
399 data_manager.GetSuggestionFilterData(&data, &size);
400 return new SuggestionFilter(data, size);
401 }
402
403 private:
404 const testing::ScopedTmpUserProfileDirectory scoped_profile_dir_;
405 const testing::MockDataManager mock_data_manager_;
406 const commands::Request default_request_;
407 mozc::usage_stats::scoped_usage_stats_enabler usage_stats_enabler_;
408 };
409
410 // test for issue:2209644
411 // just checking whether this causes segmentation fault or not.
412 // TODO(toshiyuki): make dictionary mock and test strictly.
TEST_F(ConverterTest,CanConvertTest)413 TEST_F(ConverterTest, CanConvertTest) {
414 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
415 ConverterInterface *converter = engine->GetConverter();
416 CHECK(converter);
417 {
418 Segments segments;
419 EXPECT_TRUE(converter->StartConversion(&segments, "-"));
420 }
421 {
422 Segments segments;
423 EXPECT_TRUE(converter->StartConversion(&segments, "おきておきて"));
424 }
425 }
426
427 namespace {
ContextAwareConvert(const string & first_key,const string & first_value,const string & second_key)428 string ContextAwareConvert(const string &first_key,
429 const string &first_value,
430 const string &second_key) {
431 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
432 ConverterInterface *converter = engine->GetConverter();
433 CHECK(converter);
434
435 Segments segments;
436 EXPECT_TRUE(converter->StartConversion(&segments, first_key));
437
438 string converted;
439 int segment_num = 0;
440 while (1) {
441 int position = -1;
442 for (size_t i = 0; i < segments.segment(segment_num).candidates_size();
443 ++i) {
444 const string &value = segments.segment(segment_num).candidate(i).value;
445 if (first_value.substr(converted.size(), value.size()) == value) {
446 position = static_cast<int>(i);
447 converted += value;
448 break;
449 }
450 }
451
452 if (position < 0) {
453 break;
454 }
455
456 EXPECT_TRUE(converter->CommitSegmentValue(&segments, 0, position))
457 << first_value;
458
459 ++segment_num;
460
461 if (first_value == converted) {
462 break;
463 }
464 }
465 EXPECT_EQ(first_value, converted) << first_value;
466 // TODO(team): Use StartConversionForRequest instead of StartConversion.
467 const ConversionRequest default_request;
468 EXPECT_TRUE(converter->FinishConversion(default_request, &segments));
469 EXPECT_TRUE(converter->StartConversion(&segments, second_key));
470 EXPECT_EQ(segment_num + 1, segments.segments_size());
471
472 return segments.segment(segment_num).candidate(0).value;
473 }
474 } // namespace
475
TEST_F(ConverterTest,ContextAwareConversionTest)476 TEST_F(ConverterTest, ContextAwareConversionTest) {
477 // Desirable context aware conversions
478 EXPECT_EQ("一髪", ContextAwareConvert("きき", "危機", "いっぱつ"));
479 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 2000, 1, 2000, 2000);
480 EXPECT_TIMING_STATS("SubmittedLengthx1000", 2000, 1, 2000, 2000);
481 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 1000, 1, 1000, 1000);
482 EXPECT_COUNT_STATS("SubmittedTotalLength", 2);
483
484 EXPECT_EQ("大", ContextAwareConvert("きょうと", "京都", "だい"));
485 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 4000, 2, 2000, 2000);
486 EXPECT_TIMING_STATS("SubmittedLengthx1000", 4000, 2, 2000, 2000);
487 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 2000, 2, 1000, 1000);
488 EXPECT_COUNT_STATS("SubmittedTotalLength", 4);
489
490 EXPECT_EQ("点", ContextAwareConvert("もんだい", "問題", "てん"));
491 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 6000, 3, 2000, 2000);
492 EXPECT_TIMING_STATS("SubmittedLengthx1000", 6000, 3, 2000, 2000);
493 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 3000, 3, 1000, 1000);
494 EXPECT_COUNT_STATS("SubmittedTotalLength", 6);
495
496 EXPECT_EQ("陽水", ContextAwareConvert("いのうえ", "井上", "ようすい"));
497 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 8000, 4, 2000, 2000);
498 EXPECT_TIMING_STATS("SubmittedLengthx1000", 8000, 4, 2000, 2000);
499 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 4000, 4, 1000, 1000);
500 EXPECT_COUNT_STATS("SubmittedTotalLength", 8);
501
502 // Undesirable context aware conversions
503 EXPECT_NE("宗号", ContextAwareConvert("19じ", "19時", "しゅうごう"));
504 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 11000, 6, 1000, 2000);
505 EXPECT_TIMING_STATS("SubmittedLengthx1000", 11000, 5, 2000, 3000);
506 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 6000, 5, 1000, 2000);
507 EXPECT_COUNT_STATS("SubmittedTotalLength", 11);
508
509 EXPECT_NE("な前", ContextAwareConvert("の", "の", "なまえ"));
510 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 12000, 7, 1000, 2000);
511 EXPECT_TIMING_STATS("SubmittedLengthx1000", 12000, 6, 1000, 3000);
512 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 7000, 6, 1000, 2000);
513 EXPECT_COUNT_STATS("SubmittedTotalLength", 12);
514
515 EXPECT_NE("し料", ContextAwareConvert("の", "の", "しりょう"));
516 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 13000, 8, 1000, 2000);
517 EXPECT_TIMING_STATS("SubmittedLengthx1000", 13000, 7, 1000, 3000);
518 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 8000, 7, 1000, 2000);
519 EXPECT_COUNT_STATS("SubmittedTotalLength", 13);
520
521 EXPECT_NE("し礼賛", ContextAwareConvert("ぼくと", "僕と", "しらいさん"));
522 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 15000, 9, 1000, 2000);
523 EXPECT_TIMING_STATS("SubmittedLengthx1000", 15000, 8, 1000, 3000);
524 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 9000, 8, 1000, 2000);
525 EXPECT_COUNT_STATS("SubmittedTotalLength", 15);
526 }
527
TEST_F(ConverterTest,CommitSegmentValue)528 TEST_F(ConverterTest, CommitSegmentValue) {
529 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
530 ConverterInterface *converter = engine->GetConverter();
531 CHECK(converter);
532 Segments segments;
533
534 {
535 // Prepare a segment, with candidates "1" and "2";
536 Segment *segment = segments.add_segment();
537 segment->add_candidate()->value = "1";
538 segment->add_candidate()->value = "2";
539 }
540 {
541 // Prepare a segment, with candidates "3" and "4";
542 Segment *segment = segments.add_segment();
543 segment->add_candidate()->value = "3";
544 segment->add_candidate()->value = "4";
545 }
546 {
547 // Commit the candidate whose value is "2".
548 EXPECT_TRUE(converter->CommitSegmentValue(&segments, 0, 1));
549 EXPECT_EQ(2, segments.segments_size());
550 EXPECT_EQ(0, segments.history_segments_size());
551 EXPECT_EQ(2, segments.conversion_segments_size());
552 const Segment &segment = segments.conversion_segment(0);
553 EXPECT_EQ(Segment::FIXED_VALUE, segment.segment_type());
554 EXPECT_EQ("2", segment.candidate(0).value);
555 EXPECT_NE(0,
556 segment.candidate(0).attributes & Segment::Candidate::RERANKED);
557 }
558 {
559 // Make the segment SUBMITTED
560 segments.mutable_conversion_segment(0)->set_segment_type(
561 Segment::SUBMITTED);
562 EXPECT_EQ(2, segments.segments_size());
563 EXPECT_EQ(1, segments.history_segments_size());
564 EXPECT_EQ(1, segments.conversion_segments_size());
565 }
566 {
567 // Commit the candidate whose value is "3".
568 EXPECT_TRUE(converter->CommitSegmentValue(&segments, 0, 0));
569 EXPECT_EQ(2, segments.segments_size());
570 EXPECT_EQ(1, segments.history_segments_size());
571 EXPECT_EQ(1, segments.conversion_segments_size());
572 const Segment &segment = segments.conversion_segment(0);
573 EXPECT_EQ(Segment::FIXED_VALUE, segment.segment_type());
574 EXPECT_EQ("3", segment.candidate(0).value);
575 EXPECT_EQ(0,
576 segment.candidate(0).attributes & Segment::Candidate::RERANKED);
577 }
578 }
579
TEST_F(ConverterTest,CommitSegments)580 TEST_F(ConverterTest, CommitSegments) {
581 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
582 ConverterInterface *converter = engine->GetConverter();
583 CHECK(converter);
584 Segments segments;
585
586 // History segment.
587 {
588 Segment *segment = segments.add_segment();
589 segment->set_key("あした");
590 segment->set_segment_type(Segment::HISTORY);
591 Segment::Candidate *candidate = segment->add_candidate();
592 candidate->key = "あした";
593 candidate->value = "今日";
594 }
595
596 {
597 Segment *segment = segments.add_segment();
598 segment->set_key("かつこうに");
599 Segment::Candidate *candidate = segment->add_candidate();
600 candidate->value = "学校に";
601 candidate->key = "がっこうに";
602 }
603
604 {
605 Segment *segment = segments.add_segment();
606 segment->set_key("いく");
607 Segment::Candidate *candidate = segment->add_candidate();
608 candidate->value = "行く";
609 candidate->key = "いく";
610 }
611
612 // Test "CommitFirstSegment" feature.
613 {
614 // Commit 1st segment.
615 std::vector<size_t> index_list;
616 index_list.push_back(0);
617 converter->CommitSegments(&segments, index_list);
618
619 EXPECT_EQ(2, segments.history_segments_size());
620 EXPECT_EQ(1, segments.conversion_segments_size());
621 EXPECT_EQ(Segment::HISTORY, segments.history_segment(0).segment_type());
622 EXPECT_EQ(Segment::SUBMITTED, segments.history_segment(1).segment_type());
623
624 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 3000, 1, 3000, 3000);
625 EXPECT_TIMING_STATS("SubmittedLengthx1000", 3000, 1, 3000, 3000);
626 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 1000, 1, 1000, 1000);
627 EXPECT_COUNT_STATS("SubmittedTotalLength", 3);
628 }
629
630 // Reset the test data.
631 segments.mutable_history_segment(1)->set_segment_type(Segment::FREE);
632
633 // Test "CommitHeadToFocusedSegment" feature.
634 {
635 // Commit 1st and 2nd segments.
636 std::vector<size_t> index_list;
637 index_list.push_back(0);
638 index_list.push_back(0);
639 converter->CommitSegments(&segments, index_list);
640
641 EXPECT_EQ(3, segments.history_segments_size());
642 EXPECT_EQ(0, segments.conversion_segments_size());
643 EXPECT_EQ(Segment::HISTORY, segments.history_segment(0).segment_type());
644 EXPECT_EQ(Segment::SUBMITTED, segments.history_segment(1).segment_type());
645 EXPECT_EQ(Segment::SUBMITTED, segments.history_segment(2).segment_type());
646
647 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 8000, 3, 2000, 3000);
648 EXPECT_TIMING_STATS("SubmittedLengthx1000", 8000, 2, 3000, 5000);
649 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 3000, 2, 1000, 2000);
650 EXPECT_COUNT_STATS("SubmittedTotalLength", 8);
651 }
652 }
653
TEST_F(ConverterTest,CommitPartialSuggestionSegmentValue)654 TEST_F(ConverterTest, CommitPartialSuggestionSegmentValue) {
655 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
656 ConverterInterface *converter = engine->GetConverter();
657 CHECK(converter);
658 Segments segments;
659
660 {
661 // Prepare a segment, with candidates "1" and "2";
662 Segment *segment = segments.add_segment();
663 segment->add_candidate()->value = "1";
664 segment->add_candidate()->value = "2";
665 }
666 {
667 // Prepare a segment, with candidates "3" and "4";
668 Segment *segment = segments.add_segment();
669 segment->add_candidate()->value = "3";
670 segment->add_candidate()->value = "4";
671 }
672 {
673 // Commit the candidate whose value is "2".
674 EXPECT_TRUE(converter->CommitPartialSuggestionSegmentValue(
675 &segments, 0, 1, "left2", "right2"));
676 EXPECT_EQ(3, segments.segments_size());
677 EXPECT_EQ(1, segments.history_segments_size());
678 EXPECT_EQ(2, segments.conversion_segments_size());
679 {
680 // The tail segment of the history segments uses
681 // CommitPartialSuggestionSegmentValue's |current_segment_key| parameter
682 // and contains original value.
683 const Segment &segment =
684 segments.history_segment(segments.history_segments_size() - 1);
685 EXPECT_EQ(Segment::SUBMITTED, segment.segment_type());
686 EXPECT_EQ("2", segment.candidate(0).value);
687 EXPECT_EQ("left2", segment.key());
688 EXPECT_NE(0,
689 segment.candidate(0).attributes & Segment::Candidate::RERANKED);
690 }
691 {
692 // The head segment of the conversion segments uses |new_segment_key|.
693 const Segment &segment = segments.conversion_segment(0);
694 EXPECT_EQ(Segment::FREE, segment.segment_type());
695 EXPECT_EQ("right2", segment.key());
696 }
697 }
698 }
699
TEST_F(ConverterTest,CommitPartialSuggestionUsageStats)700 TEST_F(ConverterTest, CommitPartialSuggestionUsageStats) {
701 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
702 ConverterInterface *converter = engine->GetConverter();
703 CHECK(converter);
704 Segments segments;
705
706 // History segment.
707 {
708 Segment *segment = segments.add_segment();
709 segment->set_key("あした");
710 segment->set_segment_type(Segment::HISTORY);
711
712 Segment::Candidate *candidate = segment->add_candidate();
713 candidate->key = "あした";
714 candidate->value = "今日";
715 }
716
717 {
718 Segment *segment = segments.add_segment();
719 segment->set_key("かつこうに");
720
721 Segment::Candidate *candidate = segment->add_candidate();
722 candidate->value = "学校に";
723 candidate->key = "がっこうに";
724
725 candidate = segment->add_candidate();
726 candidate->value = "格好に";
727 candidate->key = "かっこうに";
728
729 candidate = segment->add_candidate();
730 candidate->value = "かつこうに";
731 candidate->key = "かつこうに";
732 }
733
734 EXPECT_STATS_NOT_EXIST("CommitPartialSuggestion");
735 EXPECT_TRUE(converter->CommitPartialSuggestionSegmentValue(
736 &segments, 0, 1, "かつこうに", "いく"));
737 EXPECT_EQ(2, segments.history_segments_size());
738 EXPECT_EQ(1, segments.conversion_segments_size());
739 EXPECT_EQ(Segment::HISTORY, segments.history_segment(0).segment_type());
740 EXPECT_EQ(Segment::SUBMITTED, segments.history_segment(1).segment_type());
741 {
742 // The tail segment of the history segments uses
743 // CommitPartialSuggestionSegmentValue's |current_segment_key| parameter
744 // and contains original value.
745 const Segment &segment =
746 segments.history_segment(segments.history_segments_size() - 1);
747 EXPECT_EQ(Segment::SUBMITTED, segment.segment_type());
748 EXPECT_EQ("格好に", segment.candidate(0).value);
749 EXPECT_EQ("かっこうに", segment.candidate(0).key);
750 EXPECT_EQ("かつこうに", segment.key());
751 EXPECT_NE(0,
752 segment.candidate(0).attributes & Segment::Candidate::RERANKED);
753 }
754 {
755 // The head segment of the conversion segments uses |new_segment_key|.
756 const Segment &segment = segments.conversion_segment(0);
757 EXPECT_EQ(Segment::FREE, segment.segment_type());
758 EXPECT_EQ("いく", segment.key());
759 }
760
761 EXPECT_COUNT_STATS("CommitPartialSuggestion", 1);
762 EXPECT_TIMING_STATS("SubmittedSegmentLengthx1000", 3000, 1, 3000, 3000);
763 EXPECT_TIMING_STATS("SubmittedLengthx1000", 3000, 1, 3000, 3000);
764 EXPECT_TIMING_STATS("SubmittedSegmentNumberx1000", 1000, 1, 1000, 1000);
765 EXPECT_COUNT_STATS("SubmittedTotalLength", 3);
766 }
767
TEST_F(ConverterTest,CommitAutoPartialSuggestionUsageStats)768 TEST_F(ConverterTest, CommitAutoPartialSuggestionUsageStats) {
769 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
770 ConverterInterface *converter = engine->GetConverter();
771 CHECK(converter);
772 Segments segments;
773
774 {
775 Segment *segment = segments.add_segment();
776 segment->set_key("かつこうにいく");
777
778 Segment::Candidate *candidate = segment->add_candidate();
779 candidate->value = "学校にいく";
780 candidate->key = "がっこうにいく";
781
782 candidate = segment->add_candidate();
783 candidate->value = "学校に行く";
784 candidate->key = "がっこうにいく";
785
786 candidate = segment->add_candidate();
787 candidate->value = "学校に";
788 candidate->key = "がっこうに";
789 }
790
791 EXPECT_STATS_NOT_EXIST("CommitPartialSuggestion");
792 EXPECT_TRUE(converter->CommitPartialSuggestionSegmentValue(
793 &segments, 0, 2, "かつこうに", "いく"));
794 EXPECT_EQ(2, segments.segments_size());
795 EXPECT_EQ(1, segments.history_segments_size());
796 EXPECT_EQ(1, segments.conversion_segments_size());
797 {
798 // The tail segment of the history segments uses
799 // CommitPartialSuggestionSegmentValue's |current_segment_key| parameter
800 // and contains original value.
801 const Segment &segment =
802 segments.history_segment(segments.history_segments_size() - 1);
803 EXPECT_EQ(Segment::SUBMITTED, segment.segment_type());
804 EXPECT_EQ("学校に", segment.candidate(0).value);
805 EXPECT_EQ("がっこうに", segment.candidate(0).key);
806 EXPECT_EQ("かつこうに", segment.key());
807 EXPECT_NE(0,
808 segment.candidate(0).attributes & Segment::Candidate::RERANKED);
809 }
810 {
811 // The head segment of the conversion segments uses |new_segment_key|.
812 const Segment &segment = segments.conversion_segment(0);
813 EXPECT_EQ(Segment::FREE, segment.segment_type());
814 EXPECT_EQ("いく", segment.key());
815 }
816
817 EXPECT_COUNT_STATS("CommitAutoPartialSuggestion", 1);
818 }
819
TEST_F(ConverterTest,CandidateKeyTest)820 TEST_F(ConverterTest, CandidateKeyTest) {
821 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
822 ConverterInterface *converter = engine->GetConverter();
823 CHECK(converter);
824 Segments segments;
825 EXPECT_TRUE(converter->StartConversion(&segments, "わたしは"));
826 EXPECT_EQ(1, segments.segments_size());
827 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).key);
828 EXPECT_EQ("わたし", segments.segment(0).candidate(0).content_key);
829 }
830
TEST_F(ConverterTest,Regression3437022)831 TEST_F(ConverterTest, Regression3437022) {
832 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
833 ConverterInterface *converter = engine->GetConverter();
834 Segments segments;
835
836 const string kKey1 = "けいたい";
837 const string kKey2 = "でんわ";
838
839 const string kValue1 = "携帯";
840 const string kValue2 = "電話";
841
842 {
843 // Make sure converte result is one segment
844 EXPECT_TRUE(converter->StartConversion(&segments, kKey1 + kKey2));
845 EXPECT_EQ(1, segments.conversion_segments_size());
846 EXPECT_EQ(kValue1 + kValue2,
847 segments.conversion_segment(0).candidate(0).value);
848 }
849 {
850 // Make sure we can convert first part
851 segments.Clear();
852 EXPECT_TRUE(converter->StartConversion(&segments, kKey1));
853 EXPECT_EQ(1, segments.conversion_segments_size());
854 EXPECT_EQ(kValue1, segments.conversion_segment(0).candidate(0).value);
855 }
856 {
857 // Make sure we can convert last part
858 segments.Clear();
859 EXPECT_TRUE(converter->StartConversion(&segments, kKey2));
860 EXPECT_EQ(1, segments.conversion_segments_size());
861 EXPECT_EQ(kValue2, segments.conversion_segment(0).candidate(0).value);
862 }
863
864 // Add compound entry to suppression dictionary
865 segments.Clear();
866
867 SuppressionDictionary *dic = engine->GetSuppressionDictionary();
868 dic->Lock();
869 dic->AddEntry(kKey1 + kKey2, kValue1 + kValue2);
870 dic->UnLock();
871
872 EXPECT_TRUE(converter->StartConversion(&segments, kKey1 + kKey2));
873
874 int rest_size = 0;
875 for (size_t i = 1; i < segments.conversion_segments_size(); ++i) {
876 rest_size += Util::CharsLen(
877 segments.conversion_segment(i).candidate(0).key);
878 }
879
880 // Expand segment so that the entire part will become one segment
881 if (rest_size > 0) {
882 const ConversionRequest default_request;
883 EXPECT_TRUE(converter->ResizeSegment(
884 &segments, default_request, 0, rest_size));
885 }
886
887 EXPECT_EQ(1, segments.conversion_segments_size());
888 EXPECT_NE(kValue1 + kValue2,
889 segments.conversion_segment(0).candidate(0).value);
890
891 dic->Lock();
892 dic->Clear();
893 dic->UnLock();
894 }
895
TEST_F(ConverterTest,CompletePOSIds)896 TEST_F(ConverterTest, CompletePOSIds) {
897 const char *kTestKeys[] = {
898 "きょうと",
899 "いきます",
900 "うつくしい",
901 "おおきな",
902 "いっちゃわないね",
903 "わたしのなまえはなかのです",
904 };
905
906 std::unique_ptr<ConverterAndData> converter_and_data(
907 CreateStubbedConverterAndData());
908 ConverterImpl *converter = converter_and_data->converter.get();
909 for (size_t i = 0; i < arraysize(kTestKeys); ++i) {
910 Segments segments;
911 segments.set_request_type(Segments::PREDICTION);
912 Segment *seg = segments.add_segment();
913 seg->set_key(kTestKeys[i]);
914 seg->set_segment_type(Segment::FREE);
915 segments.set_max_prediction_candidates_size(20);
916 CHECK(converter_and_data->immutable_converter->Convert(&segments));
917 const int lid = segments.segment(0).candidate(0).lid;
918 const int rid = segments.segment(0).candidate(0).rid;
919 Segment::Candidate candidate;
920 candidate.value = segments.segment(0).candidate(0).value;
921 candidate.key = segments.segment(0).candidate(0).key;
922 candidate.lid = 0;
923 candidate.rid = 0;
924 converter->CompletePOSIds(&candidate);
925 EXPECT_EQ(lid, candidate.lid);
926 EXPECT_EQ(rid, candidate.rid);
927 EXPECT_NE(candidate.lid, 0);
928 EXPECT_NE(candidate.rid, 0);
929 }
930
931 {
932 Segment::Candidate candidate;
933 candidate.key = "test";
934 candidate.value = "test";
935 candidate.lid = 10;
936 candidate.rid = 11;
937 converter->CompletePOSIds(&candidate);
938 EXPECT_EQ(10, candidate.lid);
939 EXPECT_EQ(11, candidate.rid);
940 }
941 }
942
TEST_F(ConverterTest,Regression3046266)943 TEST_F(ConverterTest, Regression3046266) {
944 // Shouldn't correct nodes at the beginning of a sentence.
945 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
946 ConverterInterface *converter = engine->GetConverter();
947 Segments segments;
948
949 // Can be any string that has "ん" at the end
950 const char kKey1[] = "かん";
951
952 // Can be any string that has a vowel at the beginning
953 const char kKey2[] = "あか";
954
955 const char kValueNotExpected[] = "中";
956
957 EXPECT_TRUE(converter->StartConversion(&segments, kKey1));
958 EXPECT_EQ(1, segments.conversion_segments_size());
959 EXPECT_TRUE(converter->CommitSegmentValue(&segments, 0, 0));
960
961 // TODO(team): Use StartConversionForRequest instead of StartConversion.
962 const ConversionRequest default_request;
963 EXPECT_TRUE(converter->FinishConversion(default_request, &segments));
964
965 EXPECT_TRUE(converter->StartConversion(&segments, kKey2));
966 EXPECT_EQ(1, segments.conversion_segments_size());
967 const Segment &segment = segments.conversion_segment(0);
968 for (size_t i = 0; i < segment.candidates_size(); ++i) {
969 EXPECT_NE(segment.candidate(i).value, kValueNotExpected);
970 }
971 }
972
TEST_F(ConverterTest,Regression5502496)973 TEST_F(ConverterTest, Regression5502496) {
974 // Make sure key correction works for the first word of a sentence.
975 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
976 ConverterInterface *converter = engine->GetConverter();
977 Segments segments;
978
979 const char kKey[] = "みんあ";
980 const char kValueExpected[] = "みんな";
981
982 EXPECT_TRUE(converter->StartConversion(&segments, kKey));
983 EXPECT_EQ(1, segments.conversion_segments_size());
984 const Segment &segment = segments.conversion_segment(0);
985 bool found = false;
986 for (size_t i = 0; i < segment.candidates_size(); ++i) {
987 if (segment.candidate(i).value == kValueExpected) {
988 found = true;
989 }
990 }
991 EXPECT_TRUE(found);
992 }
993
TEST_F(ConverterTest,StartSuggestionForRequest)994 TEST_F(ConverterTest, StartSuggestionForRequest) {
995 commands::Request client_request;
996 client_request.set_mixed_conversion(true);
997
998 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
999 ConverterInterface *converter = engine->GetConverter();
1000 CHECK(converter);
1001
1002 const string kShi = "し";
1003
1004 composer::Table table;
1005 table.AddRule("si", kShi, "");
1006 table.AddRule("shi", kShi, "");
1007 config::Config config;
1008
1009 {
1010 composer::Composer composer(&table, &client_request, &config);
1011
1012 composer.InsertCharacter("shi");
1013
1014 Segments segments;
1015 const ConversionRequest request(&composer, &client_request, &config);
1016 EXPECT_TRUE(converter->StartSuggestionForRequest(request, &segments));
1017 EXPECT_EQ(1, segments.segments_size());
1018 ASSERT_TRUE(segments.segment(0).meta_candidates_size() >=
1019 transliteration::HALF_ASCII);
1020 EXPECT_EQ(
1021 "shi",
1022 segments.segment(0).meta_candidate(transliteration::HALF_ASCII).value);
1023 }
1024
1025 {
1026 composer::Composer composer(&table, &client_request, &config);
1027
1028 composer.InsertCharacter("si");
1029
1030 Segments segments;
1031 const ConversionRequest request(&composer, &client_request, &config);
1032 EXPECT_TRUE(converter->StartSuggestionForRequest(request, &segments));
1033 EXPECT_EQ(1, segments.segments_size());
1034 ASSERT_TRUE(segments.segment(0).meta_candidates_size() >=
1035 transliteration::HALF_ASCII);
1036 EXPECT_EQ(
1037 "si",
1038 segments.segment(0).meta_candidate(transliteration::HALF_ASCII).value);
1039 }
1040 }
1041
TEST_F(ConverterTest,StartPartialPrediction)1042 TEST_F(ConverterTest, StartPartialPrediction) {
1043 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1044 ConverterInterface *converter = engine->GetConverter();
1045 CHECK(converter);
1046 Segments segments;
1047 EXPECT_TRUE(converter->StartPartialPrediction(&segments, "わたしは"));
1048 EXPECT_EQ(1, segments.segments_size());
1049 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).key);
1050 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).content_key);
1051 }
1052
TEST_F(ConverterTest,StartPartialSuggestion)1053 TEST_F(ConverterTest, StartPartialSuggestion) {
1054 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1055 ConverterInterface *converter = engine->GetConverter();
1056 CHECK(converter);
1057 Segments segments;
1058 EXPECT_TRUE(converter->StartPartialSuggestion(&segments, "わたしは"));
1059 EXPECT_EQ(1, segments.segments_size());
1060 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).key);
1061 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).content_key);
1062 }
1063
TEST_F(ConverterTest,StartPartialPrediction_mobile)1064 TEST_F(ConverterTest, StartPartialPrediction_mobile) {
1065 std::unique_ptr<EngineInterface> engine(CreateEngineWithMobilePredictor());
1066 ConverterInterface *converter = engine->GetConverter();
1067 CHECK(converter);
1068 Segments segments;
1069 EXPECT_TRUE(converter->StartPartialPrediction(&segments, "わたしは"));
1070 EXPECT_EQ(1, segments.segments_size());
1071 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).key);
1072 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).content_key);
1073 }
1074
TEST_F(ConverterTest,StartPartialSuggestion_mobile)1075 TEST_F(ConverterTest, StartPartialSuggestion_mobile) {
1076 std::unique_ptr<EngineInterface> engine(CreateEngineWithMobilePredictor());
1077 ConverterInterface *converter = engine->GetConverter();
1078 CHECK(converter);
1079 Segments segments;
1080 EXPECT_TRUE(converter->StartPartialSuggestion(&segments, "わたしは"));
1081 EXPECT_EQ(1, segments.segments_size());
1082 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).key);
1083 EXPECT_EQ("わたしは", segments.segment(0).candidate(0).content_key);
1084 }
1085
TEST_F(ConverterTest,MaybeSetConsumedKeySizeToSegment)1086 TEST_F(ConverterTest, MaybeSetConsumedKeySizeToSegment) {
1087 const size_t consumed_key_size = 5;
1088 const size_t original_consumed_key_size = 10;
1089
1090 Segment segment;
1091 // 1st candidate without PARTIALLY_KEY_CONSUMED
1092 segment.push_back_candidate();
1093 // 2nd candidate with PARTIALLY_KEY_CONSUMED
1094 Segment::Candidate *candidate2 = segment.push_back_candidate();
1095 candidate2->attributes |= Segment::Candidate::PARTIALLY_KEY_CONSUMED;
1096 candidate2->consumed_key_size = original_consumed_key_size;
1097 // 1st meta candidate without PARTIALLY_KEY_CONSUMED
1098 segment.add_meta_candidate();
1099 // 2nd meta candidate with PARTIALLY_KEY_CONSUMED
1100 Segment::Candidate *meta_candidate2 = segment.add_meta_candidate();
1101 meta_candidate2->attributes |= Segment::Candidate::PARTIALLY_KEY_CONSUMED;
1102 meta_candidate2->consumed_key_size = original_consumed_key_size;
1103
1104 ConverterImpl::MaybeSetConsumedKeySizeToSegment(consumed_key_size, &segment);
1105 EXPECT_NE(0, (segment.candidate(0).attributes &
1106 Segment::Candidate::PARTIALLY_KEY_CONSUMED));
1107 EXPECT_EQ(consumed_key_size, segment.candidate(0).consumed_key_size);
1108 EXPECT_NE(0, (segment.candidate(1).attributes &
1109 Segment::Candidate::PARTIALLY_KEY_CONSUMED));
1110 EXPECT_EQ(original_consumed_key_size,
1111 segment.candidate(1).consumed_key_size);
1112 EXPECT_NE(0, (segment.meta_candidate(0).attributes &
1113 Segment::Candidate::PARTIALLY_KEY_CONSUMED));
1114 EXPECT_EQ(consumed_key_size, segment.meta_candidate(0).consumed_key_size);
1115 EXPECT_NE(0, (segment.meta_candidate(1).attributes &
1116 Segment::Candidate::PARTIALLY_KEY_CONSUMED));
1117 EXPECT_EQ(original_consumed_key_size,
1118 segment.meta_candidate(1).consumed_key_size);
1119 }
1120
TEST_F(ConverterTest,Predict_SetKey)1121 TEST_F(ConverterTest, Predict_SetKey) {
1122 const char kPredictionKey[] = "prediction key";
1123 const char kPredictionKey2[] = "prediction key2";
1124 // Tests whether SetKey method is called or not.
1125 struct TestData {
1126 const Segments::RequestType request_type;
1127 const char *key;
1128 const bool expect_reset;
1129 };
1130 const TestData test_data_list[] = {
1131 {Segments::CONVERSION, nullptr, true},
1132 {Segments::CONVERSION, kPredictionKey, true},
1133 {Segments::CONVERSION, kPredictionKey2, true},
1134 {Segments::REVERSE_CONVERSION, nullptr, true},
1135 {Segments::REVERSE_CONVERSION, kPredictionKey, true},
1136 {Segments::REVERSE_CONVERSION, kPredictionKey2, true},
1137 {Segments::PREDICTION, nullptr, true},
1138 {Segments::PREDICTION, kPredictionKey2, true},
1139 {Segments::SUGGESTION, nullptr, true},
1140 {Segments::SUGGESTION, kPredictionKey2, true},
1141 {Segments::PARTIAL_PREDICTION, nullptr, true},
1142 {Segments::PARTIAL_PREDICTION, kPredictionKey2, true},
1143 {Segments::PARTIAL_SUGGESTION, nullptr, true},
1144 {Segments::PARTIAL_SUGGESTION, kPredictionKey2, true},
1145 // If we are predicting, and one or more segment exists,
1146 // and the segments's key equals to the input key, then do not reset.
1147 {Segments::PREDICTION, kPredictionKey, false},
1148 {Segments::SUGGESTION, kPredictionKey, true},
1149 {Segments::PARTIAL_PREDICTION, kPredictionKey, false},
1150 {Segments::PARTIAL_SUGGESTION, kPredictionKey, true},
1151 };
1152
1153 std::unique_ptr<ConverterAndData> converter_and_data(
1154 CreateStubbedConverterAndData());
1155 ConverterImpl *converter = converter_and_data->converter.get();
1156 ASSERT_NE(nullptr, converter);
1157
1158 // Note that TearDown method will reset above stubs.
1159
1160 for (size_t i = 0; i < arraysize(test_data_list); ++i) {
1161 const TestData &test_data = test_data_list[i];
1162 Segments segments;
1163 segments.set_request_type(test_data.request_type);
1164
1165 if (test_data.key) {
1166 Segment *seg = segments.add_segment();
1167 seg->Clear();
1168 seg->set_key(test_data.key);
1169 // The segment has a candidate.
1170 seg->add_candidate();
1171 }
1172 const ConversionRequest request;
1173 converter->Predict(request, kPredictionKey,
1174 Segments::PREDICTION, &segments);
1175
1176 EXPECT_EQ(1, segments.conversion_segments_size());
1177 EXPECT_EQ(kPredictionKey, segments.conversion_segment(0).key());
1178 EXPECT_EQ(test_data.expect_reset ? 0 : 1,
1179 segments.conversion_segment(0).candidates_size());
1180 }
1181 }
1182
TEST_F(ConverterTest,VariantExpansionForSuggestion)1183 TEST_F(ConverterTest, VariantExpansionForSuggestion) {
1184 // Create Converter with mock user dictionary
1185 testing::MockDataManager data_manager;
1186 std::unique_ptr<DictionaryMock> mock_user_dictionary(new DictionaryMock);
1187
1188 mock_user_dictionary->AddLookupPredictive(
1189 "てすと",
1190 "てすと",
1191 "<>!?",
1192 Token::USER_DICTIONARY);
1193 mock_user_dictionary->AddLookupPrefix(
1194 "てすと",
1195 "てすと",
1196 "<>!?",
1197 Token::USER_DICTIONARY);
1198 std::unique_ptr<SuppressionDictionary> suppression_dictionary(
1199 new SuppressionDictionary);
1200
1201 const char *dictionary_data = nullptr;
1202 int dictionary_size = 0;
1203 data_manager.GetSystemDictionaryData(&dictionary_data, &dictionary_size);
1204
1205 const dictionary::POSMatcher pos_matcher(
1206 data_manager.GetPOSMatcherData());
1207
1208 SystemDictionary *sysdic =
1209 SystemDictionary::Builder(dictionary_data, dictionary_size).Build();
1210 std::unique_ptr<DictionaryInterface> dictionary(new DictionaryImpl(
1211 sysdic, // DictionaryImpl takes the ownership
1212 new ValueDictionary(pos_matcher, &sysdic->value_trie()),
1213 mock_user_dictionary.get(),
1214 suppression_dictionary.get(),
1215 &pos_matcher));
1216 std::unique_ptr<const PosGroup> pos_group(
1217 new PosGroup(data_manager.GetPosGroupData()));
1218 std::unique_ptr<const DictionaryInterface> suffix_dictionary(
1219 CreateSuffixDictionaryFromDataManager(data_manager));
1220 std::unique_ptr<const Connector> connector(
1221 Connector::CreateFromDataManager(data_manager));
1222 std::unique_ptr<const Segmenter> segmenter(
1223 Segmenter::CreateFromDataManager(data_manager));
1224 std::unique_ptr<const SuggestionFilter> suggestion_filter(
1225 CreateSuggestionFilter(data_manager));
1226 std::unique_ptr<ImmutableConverterInterface> immutable_converter(
1227 new ImmutableConverterImpl(dictionary.get(),
1228 suffix_dictionary.get(),
1229 suppression_dictionary.get(),
1230 connector.get(),
1231 segmenter.get(),
1232 &pos_matcher,
1233 pos_group.get(),
1234 suggestion_filter.get()));
1235 std::unique_ptr<const SuggestionFilter> suggegstion_filter(
1236 CreateSuggestionFilter(data_manager));
1237 std::unique_ptr<ConverterImpl> converter(new ConverterImpl);
1238 const DictionaryInterface *kNullDictionary = nullptr;
1239 converter->Init(&pos_matcher,
1240 suppression_dictionary.get(),
1241 DefaultPredictor::CreateDefaultPredictor(
1242 new DictionaryPredictor(
1243 data_manager,
1244 converter.get(),
1245 immutable_converter.get(),
1246 dictionary.get(),
1247 suffix_dictionary.get(),
1248 connector.get(),
1249 segmenter.get(),
1250 &pos_matcher,
1251 suggegstion_filter.get()),
1252 new UserHistoryPredictor(dictionary.get(),
1253 &pos_matcher,
1254 suppression_dictionary.get(),
1255 false)),
1256 new RewriterImpl(converter.get(),
1257 &data_manager,
1258 pos_group.get(),
1259 kNullDictionary),
1260 immutable_converter.get());
1261
1262 Segments segments;
1263 {
1264 // Dictionary suggestion
1265 EXPECT_TRUE(converter->StartSuggestion(&segments, "てすと"));
1266 EXPECT_EQ(1, segments.conversion_segments_size());
1267 EXPECT_LE(1, segments.conversion_segment(0).candidates_size());
1268 EXPECT_TRUE(FindCandidateByValue("<>!?", segments.conversion_segment(0)));
1269 EXPECT_FALSE(
1270 FindCandidateByValue("<>!?", segments.conversion_segment(0)));
1271 }
1272 {
1273 // Realtime conversion
1274 segments.Clear();
1275 EXPECT_TRUE(converter->StartSuggestion(&segments, "てすとの"));
1276 EXPECT_EQ(1, segments.conversion_segments_size());
1277 EXPECT_LE(1, segments.conversion_segment(0).candidates_size());
1278 EXPECT_TRUE(FindCandidateByValue("<>!?の", segments.conversion_segment(0)));
1279 EXPECT_FALSE(
1280 FindCandidateByValue("<>!?の", segments.conversion_segment(0)));
1281 }
1282 }
1283
TEST_F(ConverterTest,ComposerKeySelection)1284 TEST_F(ConverterTest, ComposerKeySelection) {
1285 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1286 ConverterInterface *converter = engine->GetConverter();
1287 composer::Table table;
1288 config::Config config;
1289 {
1290 Segments segments;
1291 composer::Composer composer(&table, &default_request(), &config);
1292 composer.InsertCharacterPreedit("わたしh");
1293 ConversionRequest request(&composer, &default_request(), &config);
1294 request.set_composer_key_selection(ConversionRequest::CONVERSION_KEY);
1295 converter->StartConversionForRequest(request, &segments);
1296 EXPECT_EQ(2, segments.conversion_segments_size());
1297 EXPECT_EQ("私", segments.conversion_segment(0).candidate(0).value);
1298 EXPECT_EQ("h", segments.conversion_segment(1).candidate(0).value);
1299 }
1300 {
1301 Segments segments;
1302 composer::Composer composer(&table, &default_request(), &config);
1303 composer.InsertCharacterPreedit("わたしh");
1304 ConversionRequest request(&composer, &default_request(), &config);
1305 request.set_composer_key_selection(ConversionRequest::PREDICTION_KEY);
1306 converter->StartConversionForRequest(request, &segments);
1307 EXPECT_EQ(1, segments.conversion_segments_size());
1308 EXPECT_EQ("私", segments.conversion_segment(0).candidate(0).value);
1309 }
1310 }
1311
TEST_F(ConverterTest,SuppressionDictionaryForRewriter)1312 TEST_F(ConverterTest, SuppressionDictionaryForRewriter) {
1313 std::unique_ptr<ConverterAndData> ret(
1314 CreateConverterAndDataWithInsertDummyWordsRewriter());
1315
1316 // Set up suppression dictionary
1317 ret->suppression_dictionary->Lock();
1318 ret->suppression_dictionary->AddEntry("tobefiltered", "ToBeFiltered");
1319 ret->suppression_dictionary->UnLock();
1320 EXPECT_FALSE(ret->suppression_dictionary->IsEmpty());
1321
1322 // Convert
1323 composer::Table table;
1324 config::Config config;
1325 composer::Composer composer(&table, &default_request(), &config);
1326 composer.InsertCharacter("dummy");
1327 const ConversionRequest request(&composer, &default_request(), &config);
1328 Segments segments;
1329 EXPECT_TRUE(ret->converter->StartConversionForRequest(request, &segments));
1330
1331 // Verify that words inserted by the rewriter is suppressed if its in the
1332 // suppression_dictionary.
1333 for (size_t i = 0; i < segments.conversion_segments_size(); ++i) {
1334 const Segment &seg = segments.conversion_segment(i);
1335 EXPECT_FALSE(FindCandidateByValue("ToBeFiltered", seg));
1336 EXPECT_TRUE(FindCandidateByValue("NotToBeFiltered", seg));
1337 }
1338 }
1339
TEST_F(ConverterTest,EmptyConvertReverse_Issue8661091)1340 TEST_F(ConverterTest, EmptyConvertReverse_Issue8661091) {
1341 // This is a test case against b/8661091.
1342 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1343 ConverterInterface *converter = engine->GetConverter();
1344
1345 Segments segments;
1346 EXPECT_FALSE(converter->StartReverseConversion(&segments, ""));
1347 }
1348
TEST_F(ConverterTest,StartReverseConversion)1349 TEST_F(ConverterTest, StartReverseConversion) {
1350 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1351 const ConverterInterface *converter = engine->GetConverter();
1352
1353 const string kHonKanji = "本";
1354 const string kHonHiragana = "ほん";
1355 const string kMuryouKanji = "無料";
1356 const string kMuryouHiragana = "むりょう";
1357 const string kFullWidthSpace = " "; // full-width space
1358 {
1359 // Test for single Kanji character.
1360 const string &kInput = kHonKanji;
1361 Segments segments;
1362 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1363 ASSERT_EQ(1, segments.segments_size());
1364 ASSERT_LE(1, segments.conversion_segment(0).candidates_size());
1365 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1366 }
1367 {
1368 // Test for multi-Kanji character.
1369 const string &kInput = kMuryouKanji;
1370 Segments segments;
1371 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1372 ASSERT_EQ(1, segments.segments_size());
1373 ASSERT_LE(1, segments.conversion_segment(0).candidates_size());
1374 EXPECT_EQ(kMuryouHiragana,
1375 segments.conversion_segment(0).candidate(0).value);
1376 }
1377 {
1378 // Test for multi terms separated by a space.
1379 const string &kInput = kHonKanji + " " + kMuryouKanji;
1380 Segments segments;
1381 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1382 ASSERT_EQ(3, segments.segments_size());
1383 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1384 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1385 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1386 EXPECT_EQ(" ", segments.conversion_segment(1).candidate(0).value);
1387 ASSERT_LT(0, segments.conversion_segment(2).candidates_size());
1388 EXPECT_EQ(kMuryouHiragana,
1389 segments.conversion_segment(2).candidate(0).value);
1390 }
1391 {
1392 // Test for multi terms separated by multiple spaces.
1393 const string &kInput = kHonKanji + " " + kMuryouKanji;
1394 Segments segments;
1395 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1396 ASSERT_EQ(3, segments.segments_size());
1397 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1398 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1399 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1400 EXPECT_EQ(" ", segments.conversion_segment(1).candidate(0).value);
1401 ASSERT_LT(0, segments.conversion_segment(2).candidates_size());
1402 EXPECT_EQ(kMuryouHiragana,
1403 segments.conversion_segment(2).candidate(0).value);
1404 }
1405 {
1406 // Test for leading white spaces.
1407 const string &kInput = " " + kHonKanji;
1408 Segments segments;
1409 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1410 ASSERT_EQ(2, segments.segments_size());
1411 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1412 EXPECT_EQ(" ", segments.conversion_segment(0).candidate(0).value);
1413 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1414 EXPECT_EQ(kHonHiragana, segments.conversion_segment(1).candidate(0).value);
1415 }
1416 {
1417 // Test for trailing white spaces.
1418 const string &kInput = kMuryouKanji + " ";
1419 Segments segments;
1420 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1421 ASSERT_EQ(2, segments.segments_size());
1422 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1423 EXPECT_EQ(kMuryouHiragana,
1424 segments.conversion_segment(0).candidate(0).value);
1425 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1426 EXPECT_EQ(" ", segments.conversion_segment(1).candidate(0).value);
1427 }
1428 {
1429 // Test for multi terms separated by a full-width space.
1430 const string &kInput = kHonKanji + kFullWidthSpace + kMuryouKanji;
1431 Segments segments;
1432 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1433 ASSERT_EQ(3, segments.segments_size());
1434 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1435 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1436 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1437 EXPECT_EQ(kFullWidthSpace,
1438 segments.conversion_segment(1).candidate(0).value);
1439 ASSERT_LT(0, segments.conversion_segment(2).candidates_size());
1440 EXPECT_EQ(kMuryouHiragana,
1441 segments.conversion_segment(2).candidate(0).value);
1442 }
1443 {
1444 // Test for multi terms separated by two full-width spaces.
1445 const string &kFullWidthSpace2 = kFullWidthSpace + kFullWidthSpace;
1446 const string &kInput = kHonKanji + kFullWidthSpace2 + kMuryouKanji;
1447 Segments segments;
1448 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1449 ASSERT_EQ(3, segments.segments_size());
1450 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1451 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1452 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1453 EXPECT_EQ(kFullWidthSpace2,
1454 segments.conversion_segment(1).candidate(0).value);
1455 ASSERT_LT(0, segments.conversion_segment(2).candidates_size());
1456 EXPECT_EQ(kMuryouHiragana,
1457 segments.conversion_segment(2).candidate(0).value);
1458 }
1459 {
1460 // Test for multi terms separated by the mix of full- and half-width spaces.
1461 const string &kFullWidthSpace2 = kFullWidthSpace + " ";
1462 const string &kInput = kHonKanji + kFullWidthSpace2 + kMuryouKanji;
1463 Segments segments;
1464 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInput));
1465 ASSERT_EQ(3, segments.segments_size());
1466 ASSERT_LT(0, segments.conversion_segment(0).candidates_size());
1467 EXPECT_EQ(kHonHiragana, segments.conversion_segment(0).candidate(0).value);
1468 ASSERT_LT(0, segments.conversion_segment(1).candidates_size());
1469 EXPECT_EQ(kFullWidthSpace2,
1470 segments.conversion_segment(1).candidate(0).value);
1471 ASSERT_LT(0, segments.conversion_segment(2).candidates_size());
1472 EXPECT_EQ(kMuryouHiragana,
1473 segments.conversion_segment(2).candidate(0).value);
1474 }
1475 {
1476 // Test for math expressions; see b/9398304.
1477 const string &kInputHalf = "365*24*60*60*1000=";
1478 Segments segments;
1479 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInputHalf));
1480 ASSERT_EQ(1, segments.segments_size());
1481 ASSERT_EQ(1, segments.conversion_segment(0).candidates_size());
1482 EXPECT_EQ(kInputHalf, segments.conversion_segment(0).candidate(0).value);
1483
1484 // Test for full-width characters.
1485 segments.Clear();
1486 const string &kInputFull = "365*24*60*60*1000=";
1487 EXPECT_TRUE(converter->StartReverseConversion(&segments, kInputFull));
1488 ASSERT_EQ(1, segments.segments_size());
1489 ASSERT_EQ(1, segments.conversion_segment(0).candidates_size());
1490 EXPECT_EQ(kInputHalf, segments.conversion_segment(0).candidate(0).value);
1491 }
1492 }
1493
TEST_F(ConverterTest,GetLastConnectivePart)1494 TEST_F(ConverterTest, GetLastConnectivePart) {
1495 std::unique_ptr<ConverterAndData> converter_and_data(
1496 CreateStubbedConverterAndData());
1497 ConverterImpl *converter = converter_and_data->converter.get();
1498
1499 {
1500 string key;
1501 string value;
1502 uint16 id = 0;
1503 EXPECT_FALSE(converter->GetLastConnectivePart("", &key, &value, &id));
1504 EXPECT_FALSE(converter->GetLastConnectivePart(" ", &key, &value, &id));
1505 EXPECT_FALSE(converter->GetLastConnectivePart(" ", &key, &value, &id));
1506 }
1507
1508 {
1509 string key;
1510 string value;
1511 uint16 id = 0;
1512 EXPECT_TRUE(converter->GetLastConnectivePart("a", &key, &value, &id));
1513 EXPECT_EQ("a", key);
1514 EXPECT_EQ("a", value);
1515 EXPECT_EQ(converter_and_data->converter->pos_matcher_->GetUniqueNounId(),
1516 id);
1517
1518 EXPECT_TRUE(converter->GetLastConnectivePart("a ", &key, &value, &id));
1519 EXPECT_EQ("a", key);
1520 EXPECT_EQ("a", value);
1521
1522 EXPECT_FALSE(converter->GetLastConnectivePart("a ", &key, &value, &id));
1523
1524 EXPECT_TRUE(converter->GetLastConnectivePart("a ", &key, &value, &id));
1525 EXPECT_EQ("a", key);
1526 EXPECT_EQ("a", value);
1527
1528 EXPECT_TRUE(converter->GetLastConnectivePart("a10a", &key, &value, &id));
1529 EXPECT_EQ("a", key);
1530 EXPECT_EQ("a", value);
1531
1532 EXPECT_TRUE(converter->GetLastConnectivePart("a", &key, &value, &id));
1533 EXPECT_EQ("a", key);
1534 EXPECT_EQ("a", value);
1535 }
1536
1537 {
1538 string key;
1539 string value;
1540 uint16 id = 0;
1541 EXPECT_TRUE(converter->GetLastConnectivePart("10", &key, &value, &id));
1542 EXPECT_EQ("10", key);
1543 EXPECT_EQ("10", value);
1544 EXPECT_EQ(converter_and_data->converter->pos_matcher_->GetNumberId(), id);
1545
1546 EXPECT_TRUE(converter->GetLastConnectivePart("10a10", &key, &value, &id));
1547 EXPECT_EQ("10", key);
1548 EXPECT_EQ("10", value);
1549
1550 EXPECT_TRUE(converter->GetLastConnectivePart("10", &key, &value, &id));
1551 EXPECT_EQ("10", key);
1552 EXPECT_EQ("10", value);
1553 }
1554
1555 {
1556 string key;
1557 string value;
1558 uint16 id = 0;
1559 EXPECT_FALSE(converter->GetLastConnectivePart("あ", &key, &value, &id));
1560 }
1561 }
1562
TEST_F(ConverterTest,ReconstructHistory)1563 TEST_F(ConverterTest, ReconstructHistory) {
1564 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1565 ConverterInterface *converter = engine->GetConverter();
1566
1567 const char kTen[] = "10";
1568
1569 Segments segments;
1570 EXPECT_TRUE(converter->ReconstructHistory(&segments, kTen));
1571 EXPECT_EQ(1, segments.segments_size());
1572 const Segment &segment = segments.segment(0);
1573 EXPECT_EQ(Segment::HISTORY, segment.segment_type());
1574 EXPECT_EQ("10", segment.key());
1575 EXPECT_EQ(1, segment.candidates_size());
1576 const Segment::Candidate &candidate = segment.candidate(0);
1577 EXPECT_EQ(Segment::Candidate::NO_LEARNING, candidate.attributes);
1578 EXPECT_EQ("10", candidate.content_key);
1579 EXPECT_EQ("10", candidate.key);
1580 EXPECT_EQ(kTen, candidate.content_value);
1581 EXPECT_EQ(kTen, candidate.value);
1582 EXPECT_NE(0, candidate.lid);
1583 EXPECT_NE(0, candidate.rid);
1584 }
1585
TEST_F(ConverterTest,LimitCandidatesSize)1586 TEST_F(ConverterTest, LimitCandidatesSize) {
1587 std::unique_ptr<EngineInterface> engine(MockDataEngineFactory::Create());
1588 ConverterInterface *converter = engine->GetConverter();
1589
1590 composer::Table table;
1591 const config::Config &config = config::ConfigHandler::DefaultConfig();
1592 mozc::commands::Request request_proto;
1593 mozc::composer::Composer composer(&table, &request_proto, &config);
1594 composer.InsertCharacterPreedit("あ");
1595 ConversionRequest request(&composer, &request_proto, &config);
1596
1597 Segments segments;
1598 ASSERT_TRUE(converter->StartConversionForRequest(request, &segments));
1599 ASSERT_EQ(1, segments.conversion_segments_size());
1600 const int original_candidates_size =
1601 segments.segment(0).candidates_size();
1602 const int original_meta_candidates_size =
1603 segments.segment(0).meta_candidates_size();
1604 EXPECT_LT(0, original_candidates_size - 1 - original_meta_candidates_size)
1605 << "original candidates size: " << original_candidates_size
1606 << ", original meta candidates size: " << original_meta_candidates_size;
1607
1608 segments.Clear();
1609 request_proto.set_candidates_size_limit(original_candidates_size - 1);
1610 ASSERT_TRUE(converter->StartConversionForRequest(request, &segments));
1611 ASSERT_EQ(1, segments.conversion_segments_size());
1612 EXPECT_GE(original_candidates_size - 1,
1613 segments.segment(0).candidates_size());
1614 EXPECT_LE(original_candidates_size - 1 - original_meta_candidates_size,
1615 segments.segment(0).candidates_size());
1616 EXPECT_EQ(original_meta_candidates_size,
1617 segments.segment(0).meta_candidates_size());
1618
1619 segments.Clear();
1620 request_proto.set_candidates_size_limit(0);
1621 ASSERT_TRUE(converter->StartConversionForRequest(request, &segments));
1622 ASSERT_EQ(1, segments.conversion_segments_size());
1623 EXPECT_EQ(1, segments.segment(0).candidates_size());
1624 EXPECT_EQ(original_meta_candidates_size,
1625 segments.segment(0).meta_candidates_size());
1626 }
1627
TEST_F(ConverterTest,UserEntryShouldBePromoted)1628 TEST_F(ConverterTest, UserEntryShouldBePromoted) {
1629 using user_dictionary::UserDictionary;
1630 std::vector<UserDefinedEntry> user_defined_entries;
1631 // "哀" is not in the test dictionary
1632 user_defined_entries.push_back(
1633 UserDefinedEntry("あい", "哀", UserDictionary::NOUN));
1634
1635 std::unique_ptr<ConverterAndData> ret(
1636 CreateConverterAndDataWithUserDefinedEntries(
1637 user_defined_entries, new StubRewriter, STUB_PREDICTOR));
1638
1639 ConverterInterface *converter = ret->converter.get();
1640 CHECK(converter);
1641 {
1642 Segments segments;
1643 EXPECT_TRUE(converter->StartConversion(&segments, "あい"));
1644 ASSERT_EQ(1, segments.conversion_segments_size());
1645 ASSERT_LT(1, segments.conversion_segment(0).candidates_size());
1646 EXPECT_EQ("哀", segments.conversion_segment(0).candidate(0).value);
1647 }
1648 }
1649
TEST_F(ConverterTest,UserEntryShouldBePromoted_MobilePrediction)1650 TEST_F(ConverterTest, UserEntryShouldBePromoted_MobilePrediction) {
1651 using user_dictionary::UserDictionary;
1652 std::vector<UserDefinedEntry> user_defined_entries;
1653 // "哀" is not in the test dictionary
1654 user_defined_entries.push_back(
1655 UserDefinedEntry("あい", "哀", UserDictionary::NOUN));
1656
1657 std::unique_ptr<ConverterAndData> ret(
1658 CreateConverterAndDataWithUserDefinedEntries(
1659 user_defined_entries, new StubRewriter, MOBILE_PREDICTOR));
1660
1661 ConverterInterface *converter = ret->converter.get();
1662 CHECK(converter);
1663 {
1664 Segments segments;
1665 EXPECT_TRUE(converter->StartPrediction(&segments, "あい"));
1666 ASSERT_EQ(1, segments.conversion_segments_size());
1667 ASSERT_LT(1, segments.conversion_segment(0).candidates_size());
1668
1669 // "哀" should be the top result for the key "あい".
1670 int first_ai_index = -1;
1671 for (int i = 0; i < segments.conversion_segment(0).candidates_size(); ++i) {
1672 if (segments.conversion_segment(0).candidate(i).key == "あい") {
1673 first_ai_index = i;
1674 break;
1675 }
1676 }
1677 ASSERT_NE(-1, first_ai_index);
1678 EXPECT_EQ("哀",
1679 segments.conversion_segment(0).candidate(first_ai_index).value);
1680 }
1681 }
1682
TEST_F(ConverterTest,SuppressionEntryShouldBePrioritized)1683 TEST_F(ConverterTest, SuppressionEntryShouldBePrioritized) {
1684 using user_dictionary::UserDictionary;
1685 std::vector<UserDefinedEntry> user_defined_entries;
1686 // "哀" is not in the test dictionary
1687 user_defined_entries.push_back(
1688 UserDefinedEntry("あい", "哀", UserDictionary::NOUN));
1689 user_defined_entries.push_back(
1690 UserDefinedEntry("あい", "哀", UserDictionary::SUPPRESSION_WORD));
1691
1692 std::unique_ptr<ConverterAndData> ret(
1693 CreateConverterAndDataWithUserDefinedEntries(
1694 user_defined_entries, new StubRewriter, STUB_PREDICTOR));
1695
1696 ConverterInterface *converter = ret->converter.get();
1697 CHECK(converter);
1698 {
1699 Segments segments;
1700 EXPECT_TRUE(converter->StartConversion(&segments, "あい"));
1701 ASSERT_EQ(1, segments.conversion_segments_size());
1702 ASSERT_LT(1, segments.conversion_segment(0).candidates_size());
1703 EXPECT_FALSE(FindCandidateByValue("哀", segments.conversion_segment(0)));
1704 }
1705 }
1706
TEST_F(ConverterTest,SuppressionEntryShouldBePrioritized_Prediction)1707 TEST_F(ConverterTest, SuppressionEntryShouldBePrioritized_Prediction) {
1708 using user_dictionary::UserDictionary;
1709 std::vector<UserDefinedEntry> user_defined_entries;
1710 // "哀" is not in the test dictionary
1711 user_defined_entries.push_back(
1712 UserDefinedEntry("あい", "哀", UserDictionary::NOUN));
1713 user_defined_entries.push_back(
1714 UserDefinedEntry("あい", "哀", UserDictionary::SUPPRESSION_WORD));
1715
1716 PredictorType types[] = {DEFAULT_PREDICTOR, MOBILE_PREDICTOR};
1717 for (int i = 0; i < arraysize(types); ++i) {
1718 std::unique_ptr<ConverterAndData> ret(
1719 CreateConverterAndDataWithUserDefinedEntries(
1720 user_defined_entries, new StubRewriter, types[i]));
1721 ConverterInterface *converter = ret->converter.get();
1722 CHECK(converter);
1723 {
1724 Segments segments;
1725 EXPECT_TRUE(converter->StartPrediction(&segments, "あい"));
1726 ASSERT_EQ(1, segments.conversion_segments_size());
1727 ASSERT_LT(1, segments.conversion_segment(0).candidates_size());
1728 EXPECT_FALSE(FindCandidateByValue("哀", segments.conversion_segment(0)));
1729 }
1730 }
1731 }
1732
TEST_F(ConverterTest,AbbreviationShouldBeIndependent)1733 TEST_F(ConverterTest, AbbreviationShouldBeIndependent) {
1734 using user_dictionary::UserDictionary;
1735 std::vector<UserDefinedEntry> user_defined_entries;
1736 user_defined_entries.push_back(
1737 UserDefinedEntry("じゅ", "Google+", UserDictionary::ABBREVIATION));
1738
1739 std::unique_ptr<ConverterAndData> ret(
1740 CreateConverterAndDataWithUserDefinedEntries(
1741 user_defined_entries, new StubRewriter, STUB_PREDICTOR));
1742
1743 ConverterInterface *converter = ret->converter.get();
1744 CHECK(converter);
1745 {
1746 Segments segments;
1747 EXPECT_TRUE(converter->StartConversion(&segments, "じゅうじか"));
1748 ASSERT_EQ(1, segments.conversion_segments_size());
1749 EXPECT_FALSE(
1750 FindCandidateByValue("Google+うじか", segments.conversion_segment(0)));
1751 }
1752 }
1753
TEST_F(ConverterTest,AbbreviationShouldBeIndependent_Prediction)1754 TEST_F(ConverterTest, AbbreviationShouldBeIndependent_Prediction) {
1755 using user_dictionary::UserDictionary;
1756 std::vector<UserDefinedEntry> user_defined_entries;
1757 user_defined_entries.push_back(
1758 UserDefinedEntry("じゅ", "Google+", UserDictionary::ABBREVIATION));
1759
1760 PredictorType types[] = {DEFAULT_PREDICTOR, MOBILE_PREDICTOR};
1761 for (int i = 0; i < arraysize(types); ++i) {
1762 std::unique_ptr<ConverterAndData> ret(
1763 CreateConverterAndDataWithUserDefinedEntries(
1764 user_defined_entries, new StubRewriter, types[i]));
1765
1766 ConverterInterface *converter = ret->converter.get();
1767 CHECK(converter);
1768
1769 {
1770 Segments segments;
1771 EXPECT_TRUE(converter->StartPrediction(&segments, "じゅうじか"));
1772 ASSERT_EQ(1, segments.conversion_segments_size());
1773 EXPECT_FALSE(FindCandidateByValue("Google+うじか",
1774 segments.conversion_segment(0)));
1775 }
1776 }
1777 }
1778
TEST_F(ConverterTest,SuggestionOnlyShouldBeIndependent)1779 TEST_F(ConverterTest, SuggestionOnlyShouldBeIndependent) {
1780 using user_dictionary::UserDictionary;
1781 std::vector<UserDefinedEntry> user_defined_entries;
1782 user_defined_entries.push_back(
1783 UserDefinedEntry("じゅ", "Google+", UserDictionary::SUGGESTION_ONLY));
1784
1785 std::unique_ptr<ConverterAndData> ret(
1786 CreateConverterAndDataWithUserDefinedEntries(
1787 user_defined_entries, new StubRewriter, STUB_PREDICTOR));
1788
1789 ConverterInterface *converter = ret->converter.get();
1790 CHECK(converter);
1791 {
1792 Segments segments;
1793 EXPECT_TRUE(converter->StartConversion(&segments, "じゅうじか"));
1794 ASSERT_EQ(1, segments.conversion_segments_size());
1795 EXPECT_FALSE(
1796 FindCandidateByValue("Google+うじか", segments.conversion_segment(0)));
1797 }
1798 }
1799
TEST_F(ConverterTest,SuggestionOnlyShouldBeIndependent_Prediction)1800 TEST_F(ConverterTest, SuggestionOnlyShouldBeIndependent_Prediction) {
1801 using user_dictionary::UserDictionary;
1802 std::vector<UserDefinedEntry> user_defined_entries;
1803 user_defined_entries.push_back(
1804 UserDefinedEntry("じゅ", "Google+", UserDictionary::SUGGESTION_ONLY));
1805
1806 PredictorType types[] = {DEFAULT_PREDICTOR, MOBILE_PREDICTOR};
1807 for (int i = 0; i < arraysize(types); ++i) {
1808 std::unique_ptr<ConverterAndData> ret(
1809 CreateConverterAndDataWithUserDefinedEntries(
1810 user_defined_entries, new StubRewriter, types[i]));
1811
1812 ConverterInterface *converter = ret->converter.get();
1813 CHECK(converter);
1814 {
1815 Segments segments;
1816 EXPECT_TRUE(converter->StartConversion(&segments, "じゅうじか"));
1817 ASSERT_EQ(1, segments.conversion_segments_size());
1818 EXPECT_FALSE(FindCandidateByValue("Google+うじか",
1819 segments.conversion_segment(0)));
1820 }
1821 }
1822 }
1823
1824 } // namespace mozc
1825