1 // Copyright 2010-2018, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 // * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 #include "rewriter/user_segment_history_rewriter.h"
31
32 #include <memory>
33 #include <string>
34
35 #include "base/clock.h"
36 #include "base/clock_mock.h"
37 #include "base/file_util.h"
38 #include "base/logging.h"
39 #include "base/number_util.h"
40 #include "base/system_util.h"
41 #include "base/util.h"
42 #include "config/character_form_manager.h"
43 #include "config/config_handler.h"
44 #include "converter/segments.h"
45 #include "data_manager/testing/mock_data_manager.h"
46 #include "dictionary/pos_group.h"
47 #include "dictionary/pos_matcher.h"
48 #include "protocol/config.pb.h"
49 #include "request/conversion_request.h"
50 #include "rewriter/number_rewriter.h"
51 #include "rewriter/variants_rewriter.h"
52 #include "testing/base/public/googletest.h"
53 #include "testing/base/public/gunit.h"
54
55 namespace mozc {
56 namespace {
57
58 using config::CharacterFormManager;
59 using config::Config;
60 using config::ConfigHandler;
61 using dictionary::POSMatcher;
62 using dictionary::PosGroup;
63
64 const size_t kCandidatesSize = 20;
65
InitSegments(Segments * segments,size_t size,size_t candidate_size)66 void InitSegments(Segments *segments, size_t size,
67 size_t candidate_size) {
68 segments->Clear();
69 for (size_t i = 0; i < size; ++i) {
70 Segment *segment = segments->add_segment();
71 CHECK(segment);
72 segment->set_key(string("segment") +
73 std::to_string(static_cast<uint32>(i)));
74 for (size_t j = 0; j < candidate_size; ++j) {
75 Segment::Candidate *c = segment->add_candidate();
76 c->content_key = segment->key();
77 c->content_value =
78 string("candidate") + std::to_string(static_cast<uint32>(j));
79 c->value = c->content_value;
80 if (j == 0) {
81 c->attributes |= Segment::Candidate::BEST_CANDIDATE;
82 }
83 }
84 CHECK_EQ(segment->candidates_size(), candidate_size);
85 }
86 CHECK_EQ(segments->segments_size(), size);
87 }
88
InitSegments(Segments * segments,size_t size)89 void InitSegments(Segments *segments, size_t size) {
90 InitSegments(segments, size, kCandidatesSize);
91 }
92
AppendCandidateSuffix(Segment * segment,size_t index,const string & suffix,uint16 lid,uint16 rid)93 void AppendCandidateSuffix(Segment *segment, size_t index,
94 const string &suffix,
95 uint16 lid, uint16 rid) {
96 segment->set_key(segment->key() + suffix);
97 segment->mutable_candidate(index)->value += suffix;
98 segment->mutable_candidate(index)->lid = lid;
99 segment->mutable_candidate(index)->rid = rid;
100 }
101
AppendCandidateSuffixWithLid(Segment * segment,size_t index,const string & suffix,uint16 lid)102 void AppendCandidateSuffixWithLid(Segment *segment, size_t index,
103 const string &suffix,
104 uint16 lid) {
105 // if lid == 0 and rid == 0, we assume that candidate is t13n.
106 // we set 1 as rid to avoid this.
107 AppendCandidateSuffix(segment, index, suffix, lid, 1);
108 }
109
110 } // namespace
111
112 class UserSegmentHistoryRewriterTest : public ::testing::Test {
113 protected:
UserSegmentHistoryRewriterTest()114 UserSegmentHistoryRewriterTest() {
115 request_.set_config(&config_);
116 }
117
SetUp()118 void SetUp() override {
119 SystemUtil::SetUserProfileDirectory(FLAGS_test_tmpdir);
120
121 ConfigHandler::GetDefaultConfig(&config_);
122 for (int i = 0; i < config_.character_form_rules_size(); ++i) {
123 Config::CharacterFormRule *rule =
124 config_.mutable_character_form_rules(i);
125 if (rule->group() == "0" || rule->group() == "A" ||
126 rule->group() == "(){}[]") {
127 rule->set_preedit_character_form(Config::HALF_WIDTH);
128 rule->set_conversion_character_form(Config::HALF_WIDTH);
129 }
130 }
131 CharacterFormManager::GetCharacterFormManager()->ReloadConfig(config_);
132
133 Clock::SetClockForUnitTest(NULL);
134
135 pos_matcher_.Set(mock_data_manager_.GetPOSMatcherData());
136 pos_group_.reset(new PosGroup(mock_data_manager_.GetPosGroupData()));
137 ASSERT_TRUE(pos_group_.get() != NULL);
138 }
139
TearDown()140 void TearDown() override {
141 Clock::SetClockForUnitTest(NULL);
142
143 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
144 CreateUserSegmentHistoryRewriter());
145 rewriter->Clear();
146 // reset config
147 ConfigHandler::GetDefaultConfig(&config_);
148 CharacterFormManager::GetCharacterFormManager()->SetDefaultRule();
149 }
150
pos_matcher() const151 const POSMatcher &pos_matcher() const {
152 return pos_matcher_;
153 }
154
CreateNumberRewriter() const155 NumberRewriter *CreateNumberRewriter() const {
156 return new NumberRewriter(&mock_data_manager_);
157 }
158
CreateUserSegmentHistoryRewriter() const159 UserSegmentHistoryRewriter *CreateUserSegmentHistoryRewriter() const {
160 return new UserSegmentHistoryRewriter(&pos_matcher_, pos_group_.get());
161 }
162
SetNumberForm(Config::CharacterForm form)163 void SetNumberForm(Config::CharacterForm form) {
164 for (size_t i = 0; i < config_.character_form_rules_size(); ++i) {
165 Config::CharacterFormRule *rule = config_.mutable_character_form_rules(i);
166 if (rule->group() == "0") {
167 rule->set_conversion_character_form(form);
168 }
169 }
170 CharacterFormManager::GetCharacterFormManager()->ReloadConfig(config_);
171 EXPECT_EQ(form,
172 CharacterFormManager::GetCharacterFormManager()->
173 GetConversionCharacterForm("0"));
174 }
175
176 ConversionRequest request_;
177 Config config_;
178
179 private:
180 const testing::MockDataManager mock_data_manager_;
181 POSMatcher pos_matcher_;
182 std::unique_ptr<const PosGroup> pos_group_;
183 DISALLOW_COPY_AND_ASSIGN(UserSegmentHistoryRewriterTest);
184 };
185
TEST_F(UserSegmentHistoryRewriterTest,CreateFile)186 TEST_F(UserSegmentHistoryRewriterTest, CreateFile) {
187 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
188 CreateUserSegmentHistoryRewriter());
189 const string history_file = FileUtil::JoinPath(FLAGS_test_tmpdir,
190 "/segment.db");
191 EXPECT_TRUE(FileUtil::FileExists(history_file));
192 }
193
TEST_F(UserSegmentHistoryRewriterTest,InvalidInputsTest)194 TEST_F(UserSegmentHistoryRewriterTest, InvalidInputsTest) {
195 Segments segments;
196 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
197 CreateUserSegmentHistoryRewriter());
198 segments.Clear();
199 EXPECT_FALSE(rewriter->Rewrite(request_, &segments));
200 rewriter->Finish(request_, &segments);
201 }
202
TEST_F(UserSegmentHistoryRewriterTest,IncognitoModeTest)203 TEST_F(UserSegmentHistoryRewriterTest, IncognitoModeTest) {
204 Segments segments;
205 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
206 CreateUserSegmentHistoryRewriter());
207
208 {
209 config_.set_incognito_mode(false);
210 InitSegments(&segments, 1);
211 segments.mutable_segment(0)->move_candidate(2, 0);
212 segments.mutable_segment(0)->mutable_candidate(0)->attributes
213 |= Segment::Candidate::RERANKED;
214 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
215 rewriter->Finish(request_, &segments);
216 InitSegments(&segments, 1);
217 rewriter->Rewrite(request_, &segments);
218 EXPECT_EQ("candidate2",
219 segments.segment(0).candidate(0).value);
220
221 config_.set_incognito_mode(true);
222 InitSegments(&segments, 1);
223 rewriter->Rewrite(request_, &segments);
224 EXPECT_EQ("candidate0",
225 segments.segment(0).candidate(0).value);
226 }
227
228 {
229 rewriter->Clear(); // clear history
230 config_.set_incognito_mode(true);
231 InitSegments(&segments, 1);
232 segments.mutable_segment(0)->move_candidate(2, 0);
233 segments.mutable_segment(0)->mutable_candidate(0)->attributes
234 |= Segment::Candidate::RERANKED;
235 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
236 rewriter->Finish(request_, &segments);
237 InitSegments(&segments, 1);
238 rewriter->Rewrite(request_, &segments);
239 EXPECT_EQ("candidate0",
240 segments.segment(0).candidate(0).value);
241 }
242 }
243
TEST_F(UserSegmentHistoryRewriterTest,ConfigTest)244 TEST_F(UserSegmentHistoryRewriterTest, ConfigTest) {
245 Segments segments;
246 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
247 CreateUserSegmentHistoryRewriter());
248
249 {
250 config_.set_history_learning_level(Config::DEFAULT_HISTORY);
251 InitSegments(&segments, 1);
252 segments.mutable_segment(0)->move_candidate(2, 0);
253 segments.mutable_segment(0)->mutable_candidate(0)->attributes
254 |= Segment::Candidate::RERANKED;
255 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
256 rewriter->Finish(request_, &segments);
257 InitSegments(&segments, 1);
258 rewriter->Rewrite(request_, &segments);
259 EXPECT_EQ("candidate2",
260 segments.segment(0).candidate(0).value);
261
262 config_.set_history_learning_level(Config::NO_HISTORY);
263 InitSegments(&segments, 1);
264 rewriter->Rewrite(request_, &segments);
265 EXPECT_EQ("candidate0",
266 segments.segment(0).candidate(0).value);
267
268 config_.set_history_learning_level(Config::READ_ONLY);
269 InitSegments(&segments, 1);
270 rewriter->Rewrite(request_, &segments);
271 EXPECT_EQ("candidate2",
272 segments.segment(0).candidate(0).value);
273 }
274
275 {
276 config_.set_history_learning_level(Config::NO_HISTORY);
277 InitSegments(&segments, 1);
278 segments.mutable_segment(0)->move_candidate(2, 0);
279 segments.mutable_segment(0)->mutable_candidate(0)->attributes
280 |= Segment::Candidate::RERANKED;
281 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
282 rewriter->Finish(request_, &segments);
283 InitSegments(&segments, 1);
284 rewriter->Rewrite(request_, &segments);
285 EXPECT_EQ("candidate0",
286 segments.segment(0).candidate(0).value);
287 }
288 }
289
TEST_F(UserSegmentHistoryRewriterTest,DisableTest)290 TEST_F(UserSegmentHistoryRewriterTest, DisableTest) {
291 Segments segments;
292 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
293 CreateUserSegmentHistoryRewriter());
294
295 {
296 InitSegments(&segments, 1);
297 segments.set_user_history_enabled(true);
298 segments.mutable_segment(0)->move_candidate(2, 0);
299 segments.mutable_segment(0)->mutable_candidate(0)->attributes
300 |= Segment::Candidate::RERANKED;
301 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
302 rewriter->Finish(request_, &segments);
303 InitSegments(&segments, 1);
304 rewriter->Rewrite(request_, &segments);
305 EXPECT_EQ("candidate2",
306 segments.segment(0).candidate(0).value);
307
308 InitSegments(&segments, 1);
309 segments.set_user_history_enabled(false);
310 rewriter->Rewrite(request_, &segments);
311 EXPECT_EQ("candidate0",
312 segments.segment(0).candidate(0).value);
313
314 InitSegments(&segments, 1);
315 segments.set_user_history_enabled(true);
316 rewriter->Rewrite(request_, &segments);
317 EXPECT_EQ("candidate2",
318 segments.segment(0).candidate(0).value);
319 }
320
321 {
322 InitSegments(&segments, 1);
323 segments.set_user_history_enabled(false);
324 segments.mutable_segment(0)->move_candidate(2, 0);
325 segments.mutable_segment(0)->mutable_candidate(0)->attributes
326 |= Segment::Candidate::RERANKED;
327 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
328 rewriter->Finish(request_, &segments);
329 InitSegments(&segments, 1);
330 rewriter->Rewrite(request_, &segments);
331 segments.set_user_history_enabled(true);
332 EXPECT_EQ("candidate0",
333 segments.segment(0).candidate(0).value);
334 }
335 }
336
TEST_F(UserSegmentHistoryRewriterTest,BasicTest)337 TEST_F(UserSegmentHistoryRewriterTest, BasicTest) {
338 Segments segments;
339 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
340 CreateUserSegmentHistoryRewriter());
341
342 rewriter->Clear();
343
344 {
345 InitSegments(&segments, 2);
346
347 segments.mutable_segment(0)->move_candidate(2, 0);
348 segments.mutable_segment(0)->mutable_candidate(0)->attributes
349 |= Segment::Candidate::RERANKED;
350 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
351 rewriter->Finish(request_, &segments);
352 InitSegments(&segments, 2);
353 rewriter->Rewrite(request_, &segments);
354
355 EXPECT_EQ("candidate2",
356 segments.segment(0).candidate(0).value);
357 EXPECT_EQ("candidate0",
358 segments.segment(1).candidate(0).value);
359
360 InitSegments(&segments, 1);
361 rewriter->Rewrite(request_, &segments);
362 EXPECT_EQ("candidate2",
363 segments.segment(0).candidate(0).value);
364 segments.mutable_segment(0)->move_candidate(1, 0);
365 segments.mutable_segment(0)->mutable_candidate(0)->attributes
366 |= Segment::Candidate::RERANKED;
367 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
368 rewriter->Finish(request_, &segments);
369
370 InitSegments(&segments, 1);
371 rewriter->Rewrite(request_, &segments);
372
373 EXPECT_EQ("candidate0",
374 segments.segment(0).candidate(0).value);
375
376 InitSegments(&segments, 2);
377 rewriter->Rewrite(request_, &segments);
378 EXPECT_EQ("candidate2",
379 segments.segment(0).candidate(0).value);
380 EXPECT_EQ("candidate0",
381 segments.segment(1).candidate(0).value);
382 segments.mutable_segment(1)->move_candidate(3, 0);
383 segments.mutable_segment(1)->mutable_candidate(0)->attributes
384 |= Segment::Candidate::RERANKED;
385 segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
386 rewriter->Finish(request_, &segments);
387
388 InitSegments(&segments, 2);
389 rewriter->Rewrite(request_, &segments);
390 EXPECT_EQ("candidate2",
391 segments.segment(0).candidate(0).value);
392 EXPECT_EQ("candidate3",
393 segments.segment(1).candidate(0).value);
394 }
395
396 rewriter->Clear();
397 {
398 InitSegments(&segments, 2);
399
400 segments.mutable_segment(0)->move_candidate(2, 0);
401 segments.mutable_segment(0)->mutable_candidate(0)->attributes
402 |= Segment::Candidate::RERANKED;
403 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
404 rewriter->Finish(request_, &segments);
405 InitSegments(&segments, 2);
406 rewriter->Rewrite(request_, &segments);
407
408 EXPECT_EQ("candidate2",
409 segments.segment(0).candidate(0).value);
410 EXPECT_EQ("candidate0",
411 segments.segment(1).candidate(0).value);
412
413 InitSegments(&segments, 1);
414 rewriter->Rewrite(request_, &segments);
415 EXPECT_EQ("candidate2",
416 segments.segment(0).candidate(0).value);
417
418 // back to the original
419 segments.mutable_segment(0)->move_candidate(1, 0);
420 segments.mutable_segment(0)->mutable_candidate(0)->attributes
421 |= Segment::Candidate::RERANKED;
422 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
423 rewriter->Finish(request_, &segments);
424
425 InitSegments(&segments, 1);
426 rewriter->Rewrite(request_, &segments);
427 EXPECT_EQ("candidate0",
428 segments.segment(0).candidate(0).value);
429 }
430 }
431
432 // Test for Issue 2155278
TEST_F(UserSegmentHistoryRewriterTest,SequenceTest)433 TEST_F(UserSegmentHistoryRewriterTest, SequenceTest) {
434 Segments segments;
435 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
436 CreateUserSegmentHistoryRewriter());
437
438 rewriter->Clear();
439
440 const uint64 kSeconds = 0;
441 const uint32 kMicroSeconds = 0;
442 ClockMock clock(kSeconds, kMicroSeconds);
443 Clock::SetClockForUnitTest(&clock);
444
445 {
446 InitSegments(&segments, 1);
447
448 segments.mutable_segment(0)->move_candidate(2, 0);
449 segments.mutable_segment(0)->mutable_candidate(0)->attributes
450 |= Segment::Candidate::RERANKED;
451 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
452 EXPECT_EQ("candidate2",
453 segments.segment(0).candidate(0).value);
454 rewriter->Finish(request_, &segments); // learn "candidate2"
455
456 // Next timestamp of learning should be newer than previous learning.
457 clock.PutClockForward(1, 0);
458
459 InitSegments(&segments, 2);
460 segments.mutable_segment(0)->move_candidate(2, 0);
461 segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
462 segments.mutable_segment(1)->set_key(segments.segment(0).key());
463 EXPECT_EQ(1, segments.history_segments_size());
464 rewriter->Rewrite(request_, &segments);
465
466 EXPECT_EQ("candidate2",
467 segments.segment(0).candidate(0).value);
468 EXPECT_EQ("candidate2",
469 segments.segment(1).candidate(0).value);
470 // 2 0 1 3 4 ..
471
472 segments.mutable_segment(1)->move_candidate(3, 0);
473 segments.mutable_segment(1)->mutable_candidate(0)->attributes
474 |= Segment::Candidate::RERANKED;
475 segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
476 EXPECT_EQ("candidate3",
477 segments.segment(1).candidate(0).value);
478 rewriter->Finish(request_, &segments); // learn "candidate3"
479
480 clock.PutClockForward(1, 0);
481
482 InitSegments(&segments, 3);
483 segments.mutable_segment(0)->move_candidate(2, 0);
484 segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
485 segments.mutable_segment(1)->move_candidate(3, 0);
486 segments.mutable_segment(1)->set_segment_type(Segment::HISTORY);
487 segments.mutable_segment(1)->set_key(segments.segment(0).key());
488 segments.mutable_segment(2)->set_key(segments.segment(0).key());
489 EXPECT_EQ(2, segments.history_segments_size());
490 rewriter->Rewrite(request_, &segments);
491
492 EXPECT_EQ("candidate2",
493 segments.segment(0).candidate(0).value);
494 EXPECT_EQ("candidate3",
495 segments.segment(1).candidate(0).value);
496 EXPECT_EQ("candidate3",
497 segments.segment(2).candidate(0).value);
498 // 3 2 0 1 4 ..
499
500 segments.mutable_segment(2)->move_candidate(1, 0);
501 segments.mutable_segment(2)->mutable_candidate(0)->attributes
502 |= Segment::Candidate::RERANKED;
503 segments.mutable_segment(2)->set_segment_type(Segment::FIXED_VALUE);
504 EXPECT_EQ("candidate2",
505 segments.segment(2).candidate(0).value);
506 rewriter->Finish(request_, &segments); // learn "candidate2"
507
508 clock.PutClockForward(1, 0);
509
510 InitSegments(&segments, 4);
511 segments.mutable_segment(0)->move_candidate(2, 0);
512 segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
513 segments.mutable_segment(1)->move_candidate(3, 0);
514 segments.mutable_segment(1)->set_segment_type(Segment::HISTORY);
515 segments.mutable_segment(1)->set_key(segments.segment(0).key());
516 segments.mutable_segment(2)->move_candidate(2, 0);
517 segments.mutable_segment(2)->set_segment_type(Segment::HISTORY);
518 segments.mutable_segment(2)->set_key(segments.segment(0).key());
519 segments.mutable_segment(3)->set_key(segments.segment(0).key());
520 EXPECT_EQ(3, segments.history_segments_size());
521 rewriter->Rewrite(request_, &segments);
522
523 EXPECT_EQ("candidate2",
524 segments.segment(0).candidate(0).value);
525 EXPECT_EQ("candidate3",
526 segments.segment(1).candidate(0).value);
527 EXPECT_EQ("candidate2",
528 segments.segment(2).candidate(0).value);
529 EXPECT_EQ("candidate2",
530 segments.segment(3).candidate(0).value);
531 // 2 3 0 1 4 ..
532 }
533 }
534
TEST_F(UserSegmentHistoryRewriterTest,DupTest)535 TEST_F(UserSegmentHistoryRewriterTest, DupTest) {
536 Segments segments;
537 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
538 CreateUserSegmentHistoryRewriter());
539
540 rewriter->Clear();
541
542 const uint64 kSeconds = 0;
543 const uint32 kMicroSeconds = 0;
544 ClockMock clock(kSeconds, kMicroSeconds);
545 Clock::SetClockForUnitTest(&clock);
546
547 {
548 InitSegments(&segments, 1);
549 segments.mutable_segment(0)->move_candidate(4, 0);
550 segments.mutable_segment(0)->mutable_candidate(0)->attributes
551 |= Segment::Candidate::RERANKED;
552 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
553 rewriter->Finish(request_, &segments);
554 InitSegments(&segments, 1);
555 rewriter->Rewrite(request_, &segments);
556
557 // restored
558 // 4,0,1,2,3,5,...
559 EXPECT_EQ("candidate4",
560 segments.segment(0).candidate(0).value);
561 segments.mutable_segment(0)->move_candidate(4, 0);
562 segments.mutable_segment(0)->mutable_candidate(0)->attributes
563 |= Segment::Candidate::RERANKED;
564 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
565 clock.PutClockForward(1, 0);
566 rewriter->Finish(request_, &segments);
567 InitSegments(&segments, 1);
568 rewriter->Rewrite(request_, &segments);
569
570 // 3,4,0,1,2,5
571 EXPECT_EQ("candidate3",
572 segments.segment(0).candidate(0).value);
573 EXPECT_EQ("candidate4",
574 segments.segment(0).candidate(1).value);
575 segments.mutable_segment(0)->move_candidate(4, 0);
576 segments.mutable_segment(0)->mutable_candidate(0)->attributes
577 |= Segment::Candidate::RERANKED;
578 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
579 clock.PutClockForward(1, 0);
580 rewriter->Finish(request_, &segments);
581 InitSegments(&segments, 1);
582 rewriter->Rewrite(request_, &segments);
583
584 EXPECT_EQ("candidate2",
585 segments.segment(0).candidate(0).value);
586 EXPECT_EQ("candidate3",
587 segments.segment(0).candidate(1).value);
588 EXPECT_EQ("candidate4",
589 segments.segment(0).candidate(2).value);
590 }
591 }
592
TEST_F(UserSegmentHistoryRewriterTest,LearningType)593 TEST_F(UserSegmentHistoryRewriterTest, LearningType) {
594 Segments segments;
595 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
596 CreateUserSegmentHistoryRewriter());
597
598 {
599 rewriter->Clear();
600 InitSegments(&segments, 1);
601 segments.mutable_segment(0)->move_candidate(2, 0);
602 segments.mutable_segment(0)->mutable_candidate(0)->attributes
603 |= Segment::Candidate::RERANKED;
604 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
605 segments.mutable_segment(0)->mutable_candidate(0)->attributes
606 |= Segment::Candidate::NO_LEARNING;
607 rewriter->Finish(request_, &segments);
608 InitSegments(&segments, 1);
609 rewriter->Rewrite(request_, &segments);
610 EXPECT_EQ("candidate0",
611 segments.segment(0).candidate(0).value);
612 }
613
614 {
615 rewriter->Clear();
616 InitSegments(&segments, 1);
617 segments.mutable_segment(0)->move_candidate(2, 0);
618 segments.mutable_segment(0)->mutable_candidate(0)->attributes
619 |= Segment::Candidate::RERANKED;
620 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
621 segments.mutable_segment(0)->mutable_candidate(0)->attributes |=
622 Segment::Candidate::NO_HISTORY_LEARNING;
623 rewriter->Finish(request_, &segments);
624 InitSegments(&segments, 1);
625 rewriter->Rewrite(request_, &segments);
626 EXPECT_EQ("candidate0",
627 segments.segment(0).candidate(0).value);
628 }
629
630 {
631 rewriter->Clear();
632 InitSegments(&segments, 1);
633 segments.mutable_segment(0)->move_candidate(2, 0);
634 segments.mutable_segment(0)->mutable_candidate(0)->attributes
635 |= Segment::Candidate::RERANKED;
636 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
637 segments.mutable_segment(0)->mutable_candidate(0)->attributes |=
638 Segment::Candidate::NO_SUGGEST_LEARNING;
639 rewriter->Finish(request_, &segments);
640 InitSegments(&segments, 1);
641 rewriter->Rewrite(request_, &segments);
642 EXPECT_EQ("candidate2",
643 segments.segment(0).candidate(0).value);
644 }
645 }
646
TEST_F(UserSegmentHistoryRewriterTest,ContextSensitive)647 TEST_F(UserSegmentHistoryRewriterTest, ContextSensitive) {
648 Segments segments;
649 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
650 CreateUserSegmentHistoryRewriter());
651
652 rewriter->Clear();
653 {
654 InitSegments(&segments, 2);
655 segments.mutable_segment(0)->move_candidate(2, 0);
656 segments.mutable_segment(0)->mutable_candidate(0)->attributes
657 |= Segment::Candidate::RERANKED;
658 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
659 segments.mutable_segment(0)->mutable_candidate(0)->attributes
660 |= Segment::Candidate::CONTEXT_SENSITIVE;
661 rewriter->Finish(request_, &segments);
662 InitSegments(&segments, 2);
663 rewriter->Rewrite(request_, &segments);
664
665 // fire if two segments
666 EXPECT_EQ("candidate2",
667 segments.segment(0).candidate(0).value);
668 // not fire if single segment
669 InitSegments(&segments, 1);
670 rewriter->Rewrite(request_, &segments);
671 EXPECT_EQ("candidate0",
672 segments.segment(0).candidate(0).value);
673 }
674
675 rewriter->Clear();
676 {
677 InitSegments(&segments, 1);
678 segments.mutable_segment(0)->move_candidate(2, 0);
679 segments.mutable_segment(0)->mutable_candidate(0)->attributes
680 |= Segment::Candidate::RERANKED;
681 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
682 segments.mutable_segment(0)->mutable_candidate(0)->attributes
683 |= Segment::Candidate::CONTEXT_SENSITIVE;
684 rewriter->Finish(request_, &segments);
685
686 // fire if even in single segment
687 InitSegments(&segments, 1);
688 rewriter->Rewrite(request_, &segments);
689 EXPECT_EQ("candidate2",
690 segments.segment(0).candidate(0).value);
691
692 // not fire if two segments
693 InitSegments(&segments, 2);
694 rewriter->Rewrite(request_, &segments);
695 EXPECT_EQ("candidate0",
696 segments.segment(0).candidate(0).value);
697 }
698 }
699
TEST_F(UserSegmentHistoryRewriterTest,ContentValueLearning)700 TEST_F(UserSegmentHistoryRewriterTest, ContentValueLearning) {
701 Segments segments;
702 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
703 CreateUserSegmentHistoryRewriter());
704
705 rewriter->Clear();
706 {
707 InitSegments(&segments, 1);
708 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
709 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
710 segments.mutable_segment(0)->move_candidate(2, 0);
711 segments.mutable_segment(0)->mutable_candidate(0)->attributes
712 |= Segment::Candidate::RERANKED;
713 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
714
715 rewriter->Finish(request_, &segments);
716
717 InitSegments(&segments, 1);
718 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
719 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
720
721 rewriter->Rewrite(request_, &segments);
722
723 // exact match
724 EXPECT_EQ("candidate2:all",
725 segments.segment(0).candidate(0).value);
726
727 InitSegments(&segments, 1);
728 rewriter->Rewrite(request_, &segments);
729
730 // content value only:
731 // in both learning/applying phase, lid and suffix are the same
732 // as those of top candidates.
733 EXPECT_EQ("candidate2",
734 segments.segment(0).candidate(0).value);
735
736 InitSegments(&segments, 1);
737 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":other", 0);
738 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 0);
739
740 rewriter->Rewrite(request_, &segments);
741
742 EXPECT_EQ("candidate2:other",
743 segments.segment(0).candidate(0).value);
744 }
745
746 // In learning phase, lid is different
747 rewriter->Clear();
748 {
749 InitSegments(&segments, 1);
750 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
751 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
752 segments.mutable_segment(0)->move_candidate(2, 0);
753 segments.mutable_segment(0)->mutable_candidate(0)->attributes
754 |= Segment::Candidate::RERANKED;
755 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
756 rewriter->Finish(request_, &segments);
757 InitSegments(&segments, 1);
758 rewriter->Rewrite(request_, &segments);
759 EXPECT_EQ("candidate0",
760 segments.segment(0).candidate(0).value);
761 }
762
763 // In learning phase, suffix (functional value) is different
764 rewriter->Clear();
765 {
766 InitSegments(&segments, 1);
767 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, "", 0);
768 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 1);
769 segments.mutable_segment(0)->move_candidate(2, 0);
770 segments.mutable_segment(0)->mutable_candidate(0)->attributes
771 |= Segment::Candidate::RERANKED;
772 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
773 rewriter->Finish(request_, &segments);
774 InitSegments(&segments, 1);
775 rewriter->Rewrite(request_, &segments);
776 EXPECT_EQ("candidate0",
777 segments.segment(0).candidate(0).value);
778 }
779
780 // In apply phase, lid is different
781 rewriter->Clear();
782 {
783 InitSegments(&segments, 1);
784 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
785 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
786 segments.mutable_segment(0)->move_candidate(2, 0);
787 segments.mutable_segment(0)->mutable_candidate(0)->attributes
788 |= Segment::Candidate::RERANKED;
789 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
790 rewriter->Finish(request_, &segments);
791 InitSegments(&segments, 1);
792 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":other", 0);
793 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 1);
794 rewriter->Rewrite(request_, &segments);
795 EXPECT_EQ("candidate0:other",
796 segments.segment(0).candidate(0).value);
797 }
798
799 // In apply phase, suffix (functional value) is different
800 rewriter->Clear();
801 {
802 InitSegments(&segments, 1);
803 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
804 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
805 segments.mutable_segment(0)->move_candidate(2, 0);
806 segments.mutable_segment(0)->mutable_candidate(0)->attributes
807 |= Segment::Candidate::RERANKED;
808 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
809 rewriter->Finish(request_, &segments);
810 InitSegments(&segments, 1);
811 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, "", 0);
812 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 0);
813 rewriter->Rewrite(request_, &segments);
814 EXPECT_EQ("candidate0",
815 segments.segment(0).candidate(0).value);
816 }
817 }
818
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableTest)819 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableTest) {
820 Segments segments;
821 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
822 CreateUserSegmentHistoryRewriter());
823
824 rewriter->Clear();
825 {
826 InitSegments(&segments, 2);
827 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
828 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
829 segments.mutable_segment(0)->move_candidate(2, 0);
830 segments.mutable_segment(0)->mutable_candidate(0)->attributes
831 |= Segment::Candidate::RERANKED;
832 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
833
834 rewriter->Finish(request_, &segments);
835
836 InitSegments(&segments, 1);
837 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
838 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
839
840 rewriter->Rewrite(request_, &segments);
841
842 EXPECT_EQ("candidate2:all",
843 segments.segment(0).candidate(0).value);
844
845 InitSegments(&segments, 1);
846 rewriter->Rewrite(request_, &segments);
847 }
848
849 rewriter->Clear();
850 {
851 InitSegments(&segments, 1);
852 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
853 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
854 segments.mutable_segment(0)->move_candidate(2, 0);
855 segments.mutable_segment(0)->mutable_candidate(0)->attributes
856 |= Segment::Candidate::RERANKED;
857 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
858
859 rewriter->Finish(request_, &segments);
860
861 InitSegments(&segments, 2);
862 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
863 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
864
865 rewriter->Rewrite(request_, &segments);
866
867 EXPECT_EQ("candidate2:all",
868 segments.segment(0).candidate(0).value);
869
870 InitSegments(&segments, 1);
871 rewriter->Rewrite(request_, &segments);
872 }
873
874 rewriter->Clear();
875 {
876 InitSegments(&segments, 2);
877 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
878 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
879 segments.mutable_segment(0)->move_candidate(2, 0);
880 segments.mutable_segment(0)->mutable_candidate(0)->attributes
881 |= Segment::Candidate::RERANKED;
882 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
883 rewriter->Finish(request_, &segments);
884 InitSegments(&segments, 1);
885 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
886 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
887 rewriter->Rewrite(request_, &segments);
888 EXPECT_EQ("candidate0:all",
889 segments.segment(0).candidate(0).value);
890 }
891
892 rewriter->Clear();
893 {
894 InitSegments(&segments, 2);
895 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
896 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
897 segments.mutable_segment(0)->move_candidate(2, 0);
898 segments.mutable_segment(0)->mutable_candidate(0)->attributes
899 |= Segment::Candidate::RERANKED;
900 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
901 rewriter->Finish(request_, &segments);
902 InitSegments(&segments, 1);
903 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
904 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
905 rewriter->Rewrite(request_, &segments);
906 EXPECT_EQ("candidate0:all",
907 segments.segment(0).candidate(0).value);
908 }
909
910 rewriter->Clear();
911 {
912 InitSegments(&segments, 1);
913 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
914 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
915 segments.mutable_segment(0)->move_candidate(2, 0);
916 segments.mutable_segment(0)->mutable_candidate(0)->attributes
917 |= Segment::Candidate::RERANKED;
918 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
919 rewriter->Finish(request_, &segments);
920 InitSegments(&segments, 2);
921 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
922 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
923 rewriter->Rewrite(request_, &segments);
924 EXPECT_EQ("candidate0:all",
925 segments.segment(0).candidate(0).value);
926 }
927
928 rewriter->Clear();
929 {
930 InitSegments(&segments, 1);
931 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
932 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
933 segments.mutable_segment(0)->move_candidate(2, 0);
934 segments.mutable_segment(0)->mutable_candidate(0)->attributes
935 |= Segment::Candidate::RERANKED;
936 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
937 rewriter->Finish(request_, &segments);
938 InitSegments(&segments, 2);
939 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
940 AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
941 rewriter->Rewrite(request_, &segments);
942 EXPECT_EQ("candidate0:all",
943 segments.segment(0).candidate(0).value);
944 }
945 }
946
TEST_F(UserSegmentHistoryRewriterTest,NotReplaceableForDifferentId)947 TEST_F(UserSegmentHistoryRewriterTest, NotReplaceableForDifferentId) {
948 Segments segments;
949 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
950 CreateUserSegmentHistoryRewriter());
951
952 rewriter->Clear();
953 {
954 InitSegments(&segments, 2);
955 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
956 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 200, 300);
957 segments.mutable_segment(0)->move_candidate(2, 0);
958 segments.mutable_segment(0)->mutable_candidate(0)->attributes
959 |= Segment::Candidate::RERANKED;
960 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
961
962 rewriter->Finish(request_, &segments);
963
964 InitSegments(&segments, 2);
965 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
966 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 200, 300);
967 segments.mutable_segment(1)->mutable_candidate(0)->value =
968 "not_same_as_before";
969
970 rewriter->Rewrite(request_, &segments);
971
972 EXPECT_NE("candidate2:all",
973 segments.segment(0).candidate(0).value);
974 }
975 }
976
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableForSameId)977 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableForSameId) {
978 Segments segments;
979 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
980 CreateUserSegmentHistoryRewriter());
981
982 rewriter->Clear();
983 {
984 InitSegments(&segments, 2);
985 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
986 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 1, 1);
987 segments.mutable_segment(0)->move_candidate(2, 0);
988 segments.mutable_segment(0)->mutable_candidate(0)->attributes
989 |= Segment::Candidate::RERANKED;
990 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
991
992 rewriter->Finish(request_, &segments);
993
994 InitSegments(&segments, 2);
995 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
996 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 1, 1);
997 segments.mutable_segment(1)->mutable_candidate(0)->value =
998 "not_same_as_before";
999
1000 rewriter->Rewrite(request_, &segments);
1001
1002 EXPECT_EQ("candidate2:all",
1003 segments.segment(0).candidate(0).value);
1004 }
1005 }
1006
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableT13NTest)1007 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableT13NTest) {
1008 Segments segments;
1009 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1010 CreateUserSegmentHistoryRewriter());
1011
1012 rewriter->Clear();
1013 {
1014 InitSegments(&segments, 2);
1015 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
1016 // Prepare candidate2 as T13N candidate.
1017 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 0, 0);
1018 segments.mutable_segment(0)->move_candidate(2, 0);
1019 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1020 |= Segment::Candidate::RERANKED;
1021 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1022
1023 rewriter->Finish(request_, &segments);
1024
1025 InitSegments(&segments, 2);
1026 AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
1027 AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 0, 0);
1028 segments.mutable_segment(1)->mutable_candidate(0)->value =
1029 "not_same_as_before";
1030
1031 rewriter->Rewrite(request_, &segments);
1032
1033 EXPECT_EQ("candidate2:all",
1034 segments.segment(0).candidate(0).value);
1035 }
1036 }
1037
TEST_F(UserSegmentHistoryRewriterTest,LeftRightNumber)1038 TEST_F(UserSegmentHistoryRewriterTest, LeftRightNumber) {
1039 Segments segments;
1040 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1041 CreateUserSegmentHistoryRewriter());
1042
1043 rewriter->Clear();
1044
1045 {
1046 InitSegments(&segments, 2);
1047
1048 segments.mutable_segment(0)->mutable_candidate(0)->value = "1234";
1049 segments.mutable_segment(1)->move_candidate(2, 0);
1050 segments.mutable_segment(1)->mutable_candidate(0)->attributes
1051 |= Segment::Candidate::RERANKED;
1052 segments.mutable_segment(1)->mutable_candidate(0)->attributes
1053 |= Segment::Candidate::CONTEXT_SENSITIVE;
1054 segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1055 rewriter->Finish(request_, &segments);
1056 EXPECT_EQ("1234",
1057 segments.segment(0).candidate(0).value);
1058 EXPECT_EQ("candidate2",
1059 segments.segment(1).candidate(0).value);
1060
1061 InitSegments(&segments, 2);
1062 // different num.
1063 segments.mutable_segment(0)->mutable_candidate(0)->value = "5678";
1064 rewriter->Rewrite(request_, &segments);
1065 EXPECT_EQ("5678",
1066 segments.segment(0).candidate(0).value);
1067 EXPECT_EQ("candidate2",
1068 segments.segment(1).candidate(0).value);
1069 }
1070
1071 {
1072 InitSegments(&segments, 2);
1073
1074 segments.mutable_segment(1)->mutable_candidate(0)->value = "1234";
1075 segments.mutable_segment(0)->move_candidate(2, 0);
1076 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1077 |= Segment::Candidate::RERANKED;
1078 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1079 |= Segment::Candidate::CONTEXT_SENSITIVE;
1080 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1081 rewriter->Finish(request_, &segments);
1082 EXPECT_EQ("candidate2",
1083 segments.segment(0).candidate(0).value);
1084 EXPECT_EQ("1234",
1085 segments.segment(1).candidate(0).value);
1086
1087 InitSegments(&segments, 2);
1088 // different num.
1089 segments.mutable_segment(1)->mutable_candidate(0)->value = "5678";
1090 rewriter->Rewrite(request_, &segments);
1091 EXPECT_EQ("candidate2",
1092 segments.segment(0).candidate(0).value);
1093 EXPECT_EQ("5678",
1094 segments.segment(1).candidate(0).value);
1095 }
1096 }
1097
TEST_F(UserSegmentHistoryRewriterTest,BacketMatching)1098 TEST_F(UserSegmentHistoryRewriterTest, BacketMatching) {
1099 Segments segments;
1100 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1101 CreateUserSegmentHistoryRewriter());
1102
1103 rewriter->Clear();
1104
1105 {
1106 InitSegments(&segments, 1);
1107 Segment::Candidate *candidate =
1108 segments.mutable_segment(0)->insert_candidate(2);
1109 candidate->value = "(";
1110 candidate->content_value = "(";
1111 candidate->content_key = "(";
1112 segments.mutable_segment(0)->move_candidate(2, 0);
1113 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1114 |= Segment::Candidate::RERANKED;
1115 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1116 rewriter->Finish(request_, &segments);
1117 }
1118
1119 {
1120 InitSegments(&segments, 1);
1121 Segment::Candidate *candidate =
1122 segments.mutable_segment(0)->insert_candidate(2);
1123 candidate->value = ")";
1124 candidate->content_value = ")";
1125 candidate->content_key = ")";
1126
1127 rewriter->Rewrite(request_, &segments);
1128
1129 EXPECT_EQ(")",
1130 segments.segment(0).candidate(0).value);
1131 }
1132 }
1133
1134 // issue 2262691
TEST_F(UserSegmentHistoryRewriterTest,MultipleLearning)1135 TEST_F(UserSegmentHistoryRewriterTest, MultipleLearning) {
1136 Segments segments;
1137 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1138 CreateUserSegmentHistoryRewriter());
1139
1140 rewriter->Clear();
1141
1142 {
1143 InitSegments(&segments, 1);
1144 segments.mutable_segment(0)->set_key("key1");
1145 Segment::Candidate *candidate =
1146 segments.mutable_segment(0)->insert_candidate(2);
1147 candidate->value = "value1";
1148 candidate->content_value = "value1";
1149 candidate->content_key = "key1";
1150 segments.mutable_segment(0)->move_candidate(2, 0);
1151 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1152 |= Segment::Candidate::RERANKED;
1153 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1154 rewriter->Finish(request_, &segments);
1155 }
1156
1157 {
1158 InitSegments(&segments, 1);
1159 segments.mutable_segment(0)->set_key("key2");
1160 Segment::Candidate *candidate =
1161 segments.mutable_segment(0)->insert_candidate(2);
1162 candidate->value = "value2";
1163 candidate->content_value = "value2";
1164 candidate->content_key = "key2";
1165 segments.mutable_segment(0)->move_candidate(2, 0);
1166 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1167 |= Segment::Candidate::RERANKED;
1168 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1169 rewriter->Finish(request_, &segments);
1170 }
1171
1172 {
1173 InitSegments(&segments, 1);
1174 segments.mutable_segment(0)->set_key("key1");
1175 Segment::Candidate *candidate =
1176 segments.mutable_segment(0)->insert_candidate(2);
1177 candidate->value = "value2";
1178 candidate->content_value = "value2";
1179 candidate->content_key = "key2";
1180 candidate = segments.mutable_segment(0)->insert_candidate(3);
1181 candidate->value = "value1";
1182 candidate->content_value = "value1";
1183 candidate->content_key = "key1";
1184
1185 rewriter->Rewrite(request_, &segments);
1186
1187 EXPECT_EQ("value1",
1188 segments.segment(0).candidate(0).value);
1189 }
1190 }
1191
TEST_F(UserSegmentHistoryRewriterTest,NumberSpecial)1192 TEST_F(UserSegmentHistoryRewriterTest, NumberSpecial) {
1193 Segments segments;
1194 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1195 CreateUserSegmentHistoryRewriter());
1196 std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1197
1198 rewriter->Clear();
1199
1200 {
1201 segments.Clear();
1202 segments.add_segment();
1203 segments.mutable_segment(0)->set_key("12");
1204 Segment::Candidate *candidate =
1205 segments.mutable_segment(0)->insert_candidate(0);
1206 candidate->value = "⑫";
1207 candidate->content_value = "⑫";
1208 candidate->content_key = "12";
1209 candidate->lid = pos_matcher().GetNumberId();
1210 candidate->rid = pos_matcher().GetNumberId();
1211 candidate->style = NumberUtil::NumberString::NUMBER_CIRCLED;
1212 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1213 |= Segment::Candidate::RERANKED;
1214 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1215 rewriter->Finish(request_, &segments);
1216 }
1217
1218 {
1219 segments.Clear();
1220 segments.add_segment();
1221 segments.mutable_segment(0)->set_key("14");
1222 {
1223 Segment::Candidate *candidate =
1224 segments.mutable_segment(0)->insert_candidate(0);
1225 candidate->value = "14";
1226 candidate->content_value = "14";
1227 candidate->content_key = "14";
1228 candidate->lid = pos_matcher().GetNumberId();
1229 candidate->rid = pos_matcher().GetNumberId();
1230 }
1231 EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1232 rewriter->Rewrite(request_, &segments);
1233
1234 EXPECT_EQ("⑭", segments.segment(0).candidate(0).value);
1235 }
1236 }
1237
TEST_F(UserSegmentHistoryRewriterTest,NumberHalfWidth)1238 TEST_F(UserSegmentHistoryRewriterTest, NumberHalfWidth) {
1239 SetNumberForm(Config::HALF_WIDTH);
1240 Segments segments;
1241 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1242 CreateUserSegmentHistoryRewriter());
1243 std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1244
1245 rewriter->Clear();
1246
1247 {
1248 segments.Clear();
1249 segments.add_segment();
1250 segments.mutable_segment(0)->set_key("1234");
1251 Segment::Candidate *candidate =
1252 segments.mutable_segment(0)->insert_candidate(0);
1253 candidate->value = "1,234";
1254 candidate->content_value = "1,234";
1255 candidate->content_key = "1234";
1256 candidate->lid = pos_matcher().GetNumberId();
1257 candidate->rid = pos_matcher().GetNumberId();
1258 candidate->style =
1259 NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_FULLWIDTH;
1260 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1261 rewriter->Finish(request_, &segments); // full-width for separated number
1262 }
1263
1264 {
1265 segments.Clear();
1266 segments.add_segment();
1267 segments.mutable_segment(0)->set_key("1234");
1268 {
1269 Segment::Candidate *candidate =
1270 segments.mutable_segment(0)->insert_candidate(0);
1271 candidate->value = "1234";
1272 candidate->content_value = "1234";
1273 candidate->content_key = "1234";
1274 candidate->lid = pos_matcher().GetNumberId();
1275 candidate->rid = pos_matcher().GetNumberId();
1276 }
1277
1278 EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1279 rewriter->Rewrite(request_, &segments);
1280
1281 EXPECT_EQ("1,234", segments.segment(0).candidate(0).value);
1282 }
1283 }
1284
TEST_F(UserSegmentHistoryRewriterTest,NumberFullWidth)1285 TEST_F(UserSegmentHistoryRewriterTest, NumberFullWidth) {
1286 SetNumberForm(Config::FULL_WIDTH);
1287 Segments segments;
1288 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1289 CreateUserSegmentHistoryRewriter());
1290 std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1291
1292 rewriter->Clear();
1293
1294 {
1295 segments.Clear();
1296 segments.add_segment();
1297 segments.mutable_segment(0)->set_key("1234");
1298 Segment::Candidate *candidate =
1299 segments.mutable_segment(0)->insert_candidate(0);
1300 candidate->value = "1,234";
1301 candidate->content_value = "1,2344";
1302 candidate->content_key = "1234";
1303 candidate->lid = pos_matcher().GetNumberId();
1304 candidate->rid = pos_matcher().GetNumberId();
1305 candidate->style =
1306 NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_HALFWIDTH;
1307 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1308 rewriter->Finish(request_, &segments); // half-width for separated number
1309 }
1310
1311 {
1312 segments.Clear();
1313 segments.add_segment();
1314 {
1315 segments.mutable_segment(0)->set_key("1234");
1316 Segment::Candidate *candidate =
1317 segments.mutable_segment(0)->insert_candidate(0);
1318 candidate->value = "1234";
1319 candidate->content_value = "1234";
1320 candidate->content_key = "1234";
1321 candidate->lid = pos_matcher().GetNumberId();
1322 candidate->rid = pos_matcher().GetNumberId();
1323 }
1324 EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1325 rewriter->Rewrite(request_, &segments);
1326
1327 EXPECT_EQ("1,234", segments.segment(0).candidate(0).value);
1328 }
1329 }
1330
TEST_F(UserSegmentHistoryRewriterTest,NumberNoSeparated)1331 TEST_F(UserSegmentHistoryRewriterTest, NumberNoSeparated) {
1332 SetNumberForm(Config::HALF_WIDTH);
1333 Segments segments;
1334 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1335 CreateUserSegmentHistoryRewriter());
1336 std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1337
1338 rewriter->Clear();
1339
1340 {
1341 segments.Clear();
1342 segments.add_segment();
1343 segments.mutable_segment(0)->set_key("10");
1344 Segment::Candidate *candidate =
1345 segments.mutable_segment(0)->insert_candidate(0);
1346 candidate->value = "十";
1347 candidate->content_value = "十";
1348 candidate->content_key = "10";
1349 candidate->lid = pos_matcher().GetNumberId();
1350 candidate->rid = pos_matcher().GetNumberId();
1351 candidate->style = NumberUtil::NumberString::NUMBER_KANJI;
1352 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1353 rewriter->Finish(request_, &segments); // learn kanji
1354 }
1355 {
1356 segments.Clear();
1357 segments.add_segment();
1358 segments.mutable_segment(0)->set_key("1234");
1359 Segment::Candidate *candidate =
1360 segments.mutable_segment(0)->insert_candidate(0);
1361 candidate->value = "1,234";
1362 candidate->content_value = "1,234";
1363 candidate->content_key = "1234";
1364 candidate->lid = pos_matcher().GetNumberId();
1365 candidate->rid = pos_matcher().GetNumberId();
1366 candidate->style =
1367 NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_HALFWIDTH;
1368 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1369 rewriter->Finish(request_, &segments); // learn kanji
1370 }
1371
1372 {
1373 InitSegments(&segments, 1);
1374 segments.mutable_segment(0)->set_key("9");
1375 {
1376 Segment::Candidate *candidate =
1377 segments.mutable_segment(0)->insert_candidate(0);
1378 candidate->value = "9";
1379 candidate->content_value = "9";
1380 candidate->content_key = "9";
1381 candidate->lid = pos_matcher().GetNumberId();
1382 candidate->rid = pos_matcher().GetNumberId();
1383 }
1384 EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1385 rewriter->Rewrite(request_, &segments);
1386
1387 // 9, not "九"
1388 EXPECT_EQ("9", segments.segment(0).candidate(0).value);
1389 }
1390 }
1391
TEST_F(UserSegmentHistoryRewriterTest,Regression2459519)1392 TEST_F(UserSegmentHistoryRewriterTest, Regression2459519) {
1393 Segments segments;
1394 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1395 CreateUserSegmentHistoryRewriter());
1396
1397 rewriter->Clear();
1398
1399 const uint64 kSeconds = 0;
1400 const uint32 kMicroSeconds = 0;
1401 ClockMock clock(kSeconds, kMicroSeconds);
1402 Clock::SetClockForUnitTest(&clock);
1403
1404 InitSegments(&segments, 1);
1405 segments.mutable_segment(0)->move_candidate(2, 0);
1406 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1407 |= Segment::Candidate::RERANKED;
1408 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1409 rewriter->Finish(request_, &segments);
1410
1411 InitSegments(&segments, 1);
1412 rewriter->Rewrite(request_, &segments);
1413 EXPECT_EQ("candidate2",
1414 segments.segment(0).candidate(0).value);
1415 EXPECT_EQ("candidate0",
1416 segments.segment(0).candidate(1).value);
1417
1418 segments.mutable_segment(0)->move_candidate(1, 0);
1419 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1420 |= Segment::Candidate::RERANKED;
1421 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1422 clock.PutClockForward(1, 0);
1423 rewriter->Finish(request_, &segments);
1424
1425 InitSegments(&segments, 1);
1426 rewriter->Rewrite(request_, &segments);
1427 EXPECT_EQ("candidate0",
1428 segments.segment(0).candidate(0).value);
1429 EXPECT_EQ("candidate2",
1430 segments.segment(0).candidate(1).value);
1431
1432 segments.mutable_segment(0)->move_candidate(1, 0);
1433 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1434 |= Segment::Candidate::RERANKED;
1435 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1436 clock.PutClockForward(1, 0);
1437 rewriter->Finish(request_, &segments);
1438
1439 InitSegments(&segments, 1);
1440 rewriter->Rewrite(request_, &segments);
1441 EXPECT_EQ("candidate2",
1442 segments.segment(0).candidate(0).value);
1443 EXPECT_EQ("candidate0",
1444 segments.segment(0).candidate(1).value);
1445 }
1446
TEST_F(UserSegmentHistoryRewriterTest,Regression2459520)1447 TEST_F(UserSegmentHistoryRewriterTest, Regression2459520) {
1448 Segments segments;
1449 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1450 CreateUserSegmentHistoryRewriter());
1451
1452 rewriter->Clear();
1453
1454 InitSegments(&segments, 2);
1455 segments.mutable_segment(0)->move_candidate(2, 0);
1456 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1457 |= Segment::Candidate::RERANKED;
1458 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1459
1460 segments.mutable_segment(1)->move_candidate(3, 0);
1461 segments.mutable_segment(1)->mutable_candidate(0)->attributes
1462 |= Segment::Candidate::RERANKED;
1463 segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1464 rewriter->Finish(request_, &segments);
1465
1466 InitSegments(&segments, 2);
1467 rewriter->Rewrite(request_, &segments);
1468 EXPECT_EQ("candidate2",
1469 segments.segment(0).candidate(0).value);
1470 EXPECT_EQ("candidate3",
1471 segments.segment(1).candidate(0).value);
1472 }
1473
TEST_F(UserSegmentHistoryRewriterTest,PuntuationsTest)1474 TEST_F(UserSegmentHistoryRewriterTest, PuntuationsTest) {
1475 Segments segments;
1476 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1477 CreateUserSegmentHistoryRewriter());
1478
1479 const uint16 id = pos_matcher().GetJapanesePunctuationsId();
1480
1481 rewriter->Clear();
1482
1483 InitSegments(&segments, 2);
1484 segments.mutable_segment(1)->set_key(".");
1485 for (int i = 1; i < kCandidatesSize; ++i) {
1486 segments.mutable_segment(1)->mutable_candidate(i)->lid = id;
1487 segments.mutable_segment(1)->mutable_candidate(i)->rid = id;
1488 segments.mutable_segment(1)->mutable_candidate(i)->value = ".";
1489 }
1490 segments.mutable_segment(1)->move_candidate(2, 0);
1491 segments.mutable_segment(1)->mutable_candidate(0)->attributes
1492 |= Segment::Candidate::RERANKED;
1493 segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1494 rewriter->Finish(request_, &segments);
1495
1496 InitSegments(&segments, 2);
1497 segments.mutable_segment(1)->set_key(".");
1498 for (int i = 1; i < kCandidatesSize; ++i) {
1499 segments.mutable_segment(1)->mutable_candidate(i)->lid = id;
1500 segments.mutable_segment(1)->mutable_candidate(i)->rid = id;
1501 segments.mutable_segment(1)->mutable_candidate(i)->value = ".";
1502 }
1503
1504 // Punctuation is not remembered
1505 rewriter->Rewrite(request_, &segments);
1506 EXPECT_EQ("candidate0", segments.segment(1).candidate(0).value);
1507 }
1508
TEST_F(UserSegmentHistoryRewriterTest,Regression3264619)1509 TEST_F(UserSegmentHistoryRewriterTest, Regression3264619) {
1510 Segments segments;
1511 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1512 CreateUserSegmentHistoryRewriter());
1513
1514 // Too big candidates
1515 InitSegments(&segments, 2, 1024);
1516 segments.mutable_segment(0)->move_candidate(512, 0);
1517 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1518 |= Segment::Candidate::RERANKED;
1519 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1520 rewriter->Finish(request_, &segments);
1521 InitSegments(&segments, 2, 1024);
1522 rewriter->Rewrite(request_, &segments);
1523
1524 EXPECT_EQ("candidate512",
1525 segments.segment(0).candidate(0).value);
1526 EXPECT_EQ("candidate0",
1527 segments.segment(1).candidate(0).value);
1528 }
1529
TEST_F(UserSegmentHistoryRewriterTest,RandomTest)1530 TEST_F(UserSegmentHistoryRewriterTest, RandomTest) {
1531 Segments segments;
1532 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1533 CreateUserSegmentHistoryRewriter());
1534
1535 const uint64 kSeconds = 0;
1536 const uint32 kMicroSeconds = 0;
1537 ClockMock clock(kSeconds, kMicroSeconds);
1538 Clock::SetClockForUnitTest(&clock);
1539
1540 rewriter->Clear();
1541 for (int i = 0; i < 5; ++i) {
1542 InitSegments(&segments, 1);
1543 const int n = Util::Random(10);
1544 const string expected = segments.segment(0).candidate(n).value;
1545 segments.mutable_segment(0)->move_candidate(n, 0);
1546 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1547 |= Segment::Candidate::RERANKED;
1548 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1549 EXPECT_EQ(expected,
1550 segments.segment(0).candidate(0).value);
1551 rewriter->Finish(request_, &segments);
1552 InitSegments(&segments, 1);
1553 rewriter->Rewrite(request_, &segments);
1554 EXPECT_EQ(expected,
1555 segments.segment(0).candidate(0).value);
1556 clock.PutClockForward(1, 0); // update LRU timer
1557 }
1558 }
1559
TEST_F(UserSegmentHistoryRewriterTest,AnnotationAfterLearning)1560 TEST_F(UserSegmentHistoryRewriterTest, AnnotationAfterLearning) {
1561 Segments segments;
1562 std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1563 CreateUserSegmentHistoryRewriter());
1564
1565 {
1566 segments.Clear();
1567 InitSegments(&segments, 1, 2);
1568 segments.mutable_segment(0)->set_key("abc");
1569 Segment::Candidate *candidate =
1570 segments.mutable_segment(0)->mutable_candidate(1);
1571 candidate->value = "abc";
1572 candidate->content_value = "abc";
1573 candidate->content_key = "abc";
1574 candidate->description = "[全] アルファベット";
1575 segments.mutable_segment(0)->move_candidate(1, 0);
1576 segments.mutable_segment(0)->mutable_candidate(0)->attributes
1577 |= Segment::Candidate::RERANKED;
1578 segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1579 rewriter->Finish(request_, &segments);
1580 }
1581
1582 {
1583 segments.Clear();
1584 InitSegments(&segments, 1, 2);
1585 segments.mutable_segment(0)->set_key("abc");
1586 Segment::Candidate *candidate =
1587 segments.mutable_segment(0)->mutable_candidate(1);
1588 candidate->value = "abc";
1589 candidate->content_value = "abc";
1590 candidate->content_key = "abc";
1591 candidate->description = "[全]アルファベット";
1592 candidate->content_key = "abc";
1593 rewriter->Rewrite(request_, &segments);
1594 EXPECT_EQ("abc", segments.segment(0).candidate(0).content_value);
1595 // "[半] アルファベット"
1596 string expectation = VariantsRewriter::kHalfWidth;
1597 const string alphabet = VariantsRewriter::kAlphabet;
1598 if (alphabet.size() != 0) {
1599 expectation += ' ' + alphabet;
1600 }
1601 EXPECT_EQ(expectation,
1602 segments.segment(0).candidate(0).description);
1603 rewriter->Finish(request_, &segments);
1604 }
1605 }
1606
1607 } // namespace mozc
1608