1 // Copyright 2010-2018, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 //     * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 //     * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 //     * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 
30 #include "rewriter/user_segment_history_rewriter.h"
31 
32 #include <memory>
33 #include <string>
34 
35 #include "base/clock.h"
36 #include "base/clock_mock.h"
37 #include "base/file_util.h"
38 #include "base/logging.h"
39 #include "base/number_util.h"
40 #include "base/system_util.h"
41 #include "base/util.h"
42 #include "config/character_form_manager.h"
43 #include "config/config_handler.h"
44 #include "converter/segments.h"
45 #include "data_manager/testing/mock_data_manager.h"
46 #include "dictionary/pos_group.h"
47 #include "dictionary/pos_matcher.h"
48 #include "protocol/config.pb.h"
49 #include "request/conversion_request.h"
50 #include "rewriter/number_rewriter.h"
51 #include "rewriter/variants_rewriter.h"
52 #include "testing/base/public/googletest.h"
53 #include "testing/base/public/gunit.h"
54 
55 namespace mozc {
56 namespace {
57 
58 using config::CharacterFormManager;
59 using config::Config;
60 using config::ConfigHandler;
61 using dictionary::POSMatcher;
62 using dictionary::PosGroup;
63 
64 const size_t kCandidatesSize = 20;
65 
InitSegments(Segments * segments,size_t size,size_t candidate_size)66 void InitSegments(Segments *segments, size_t size,
67                   size_t candidate_size) {
68   segments->Clear();
69   for (size_t i = 0; i < size; ++i) {
70     Segment *segment = segments->add_segment();
71     CHECK(segment);
72     segment->set_key(string("segment") +
73                      std::to_string(static_cast<uint32>(i)));
74     for (size_t j = 0; j < candidate_size; ++j) {
75       Segment::Candidate *c = segment->add_candidate();
76       c->content_key = segment->key();
77       c->content_value =
78           string("candidate") + std::to_string(static_cast<uint32>(j));
79       c->value = c->content_value;
80       if (j == 0) {
81         c->attributes |= Segment::Candidate::BEST_CANDIDATE;
82       }
83     }
84     CHECK_EQ(segment->candidates_size(), candidate_size);
85   }
86   CHECK_EQ(segments->segments_size(), size);
87 }
88 
InitSegments(Segments * segments,size_t size)89 void InitSegments(Segments *segments, size_t size) {
90   InitSegments(segments, size, kCandidatesSize);
91 }
92 
AppendCandidateSuffix(Segment * segment,size_t index,const string & suffix,uint16 lid,uint16 rid)93 void AppendCandidateSuffix(Segment *segment, size_t index,
94                            const string &suffix,
95                            uint16 lid, uint16 rid) {
96   segment->set_key(segment->key() + suffix);
97   segment->mutable_candidate(index)->value += suffix;
98   segment->mutable_candidate(index)->lid = lid;
99   segment->mutable_candidate(index)->rid = rid;
100 }
101 
AppendCandidateSuffixWithLid(Segment * segment,size_t index,const string & suffix,uint16 lid)102 void AppendCandidateSuffixWithLid(Segment *segment, size_t index,
103                                   const string &suffix,
104                                   uint16 lid) {
105   // if lid == 0 and rid == 0, we assume that candidate is t13n.
106   // we set 1 as rid to avoid this.
107   AppendCandidateSuffix(segment, index, suffix, lid, 1);
108 }
109 
110 }  // namespace
111 
112 class UserSegmentHistoryRewriterTest : public ::testing::Test {
113  protected:
UserSegmentHistoryRewriterTest()114   UserSegmentHistoryRewriterTest() {
115     request_.set_config(&config_);
116   }
117 
SetUp()118   void SetUp() override {
119     SystemUtil::SetUserProfileDirectory(FLAGS_test_tmpdir);
120 
121     ConfigHandler::GetDefaultConfig(&config_);
122     for (int i = 0; i < config_.character_form_rules_size(); ++i) {
123       Config::CharacterFormRule *rule =
124           config_.mutable_character_form_rules(i);
125       if (rule->group() == "0" || rule->group() == "A" ||
126           rule->group() == "(){}[]") {
127         rule->set_preedit_character_form(Config::HALF_WIDTH);
128         rule->set_conversion_character_form(Config::HALF_WIDTH);
129       }
130     }
131     CharacterFormManager::GetCharacterFormManager()->ReloadConfig(config_);
132 
133     Clock::SetClockForUnitTest(NULL);
134 
135     pos_matcher_.Set(mock_data_manager_.GetPOSMatcherData());
136     pos_group_.reset(new PosGroup(mock_data_manager_.GetPosGroupData()));
137     ASSERT_TRUE(pos_group_.get() != NULL);
138   }
139 
TearDown()140   void TearDown() override {
141     Clock::SetClockForUnitTest(NULL);
142 
143     std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
144         CreateUserSegmentHistoryRewriter());
145     rewriter->Clear();
146     // reset config
147     ConfigHandler::GetDefaultConfig(&config_);
148     CharacterFormManager::GetCharacterFormManager()->SetDefaultRule();
149   }
150 
pos_matcher() const151   const POSMatcher &pos_matcher() const {
152     return pos_matcher_;
153   }
154 
CreateNumberRewriter() const155   NumberRewriter *CreateNumberRewriter() const {
156     return new NumberRewriter(&mock_data_manager_);
157   }
158 
CreateUserSegmentHistoryRewriter() const159   UserSegmentHistoryRewriter *CreateUserSegmentHistoryRewriter() const {
160     return new UserSegmentHistoryRewriter(&pos_matcher_, pos_group_.get());
161   }
162 
SetNumberForm(Config::CharacterForm form)163   void SetNumberForm(Config::CharacterForm form) {
164     for (size_t i = 0; i < config_.character_form_rules_size(); ++i) {
165       Config::CharacterFormRule *rule = config_.mutable_character_form_rules(i);
166       if (rule->group() == "0") {
167         rule->set_conversion_character_form(form);
168       }
169     }
170     CharacterFormManager::GetCharacterFormManager()->ReloadConfig(config_);
171     EXPECT_EQ(form,
172               CharacterFormManager::GetCharacterFormManager()->
173               GetConversionCharacterForm("0"));
174   }
175 
176   ConversionRequest request_;
177   Config config_;
178 
179  private:
180   const testing::MockDataManager mock_data_manager_;
181   POSMatcher pos_matcher_;
182   std::unique_ptr<const PosGroup> pos_group_;
183   DISALLOW_COPY_AND_ASSIGN(UserSegmentHistoryRewriterTest);
184 };
185 
TEST_F(UserSegmentHistoryRewriterTest,CreateFile)186 TEST_F(UserSegmentHistoryRewriterTest, CreateFile) {
187   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
188       CreateUserSegmentHistoryRewriter());
189   const string history_file = FileUtil::JoinPath(FLAGS_test_tmpdir,
190                                                  "/segment.db");
191   EXPECT_TRUE(FileUtil::FileExists(history_file));
192 }
193 
TEST_F(UserSegmentHistoryRewriterTest,InvalidInputsTest)194 TEST_F(UserSegmentHistoryRewriterTest, InvalidInputsTest) {
195   Segments segments;
196   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
197       CreateUserSegmentHistoryRewriter());
198   segments.Clear();
199   EXPECT_FALSE(rewriter->Rewrite(request_, &segments));
200   rewriter->Finish(request_, &segments);
201 }
202 
TEST_F(UserSegmentHistoryRewriterTest,IncognitoModeTest)203 TEST_F(UserSegmentHistoryRewriterTest, IncognitoModeTest) {
204   Segments segments;
205   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
206       CreateUserSegmentHistoryRewriter());
207 
208   {
209     config_.set_incognito_mode(false);
210     InitSegments(&segments, 1);
211     segments.mutable_segment(0)->move_candidate(2, 0);
212     segments.mutable_segment(0)->mutable_candidate(0)->attributes
213         |= Segment::Candidate::RERANKED;
214     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
215     rewriter->Finish(request_, &segments);
216     InitSegments(&segments, 1);
217     rewriter->Rewrite(request_, &segments);
218     EXPECT_EQ("candidate2",
219               segments.segment(0).candidate(0).value);
220 
221     config_.set_incognito_mode(true);
222     InitSegments(&segments, 1);
223     rewriter->Rewrite(request_, &segments);
224     EXPECT_EQ("candidate0",
225               segments.segment(0).candidate(0).value);
226   }
227 
228   {
229     rewriter->Clear();   // clear history
230     config_.set_incognito_mode(true);
231     InitSegments(&segments, 1);
232     segments.mutable_segment(0)->move_candidate(2, 0);
233     segments.mutable_segment(0)->mutable_candidate(0)->attributes
234         |= Segment::Candidate::RERANKED;
235     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
236     rewriter->Finish(request_, &segments);
237     InitSegments(&segments, 1);
238     rewriter->Rewrite(request_, &segments);
239     EXPECT_EQ("candidate0",
240               segments.segment(0).candidate(0).value);
241   }
242 }
243 
TEST_F(UserSegmentHistoryRewriterTest,ConfigTest)244 TEST_F(UserSegmentHistoryRewriterTest, ConfigTest) {
245   Segments segments;
246   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
247       CreateUserSegmentHistoryRewriter());
248 
249   {
250     config_.set_history_learning_level(Config::DEFAULT_HISTORY);
251     InitSegments(&segments, 1);
252     segments.mutable_segment(0)->move_candidate(2, 0);
253     segments.mutable_segment(0)->mutable_candidate(0)->attributes
254         |= Segment::Candidate::RERANKED;
255     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
256     rewriter->Finish(request_, &segments);
257     InitSegments(&segments, 1);
258     rewriter->Rewrite(request_, &segments);
259     EXPECT_EQ("candidate2",
260               segments.segment(0).candidate(0).value);
261 
262     config_.set_history_learning_level(Config::NO_HISTORY);
263     InitSegments(&segments, 1);
264     rewriter->Rewrite(request_, &segments);
265     EXPECT_EQ("candidate0",
266               segments.segment(0).candidate(0).value);
267 
268     config_.set_history_learning_level(Config::READ_ONLY);
269     InitSegments(&segments, 1);
270     rewriter->Rewrite(request_, &segments);
271     EXPECT_EQ("candidate2",
272               segments.segment(0).candidate(0).value);
273   }
274 
275   {
276     config_.set_history_learning_level(Config::NO_HISTORY);
277     InitSegments(&segments, 1);
278     segments.mutable_segment(0)->move_candidate(2, 0);
279     segments.mutable_segment(0)->mutable_candidate(0)->attributes
280         |= Segment::Candidate::RERANKED;
281     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
282     rewriter->Finish(request_, &segments);
283     InitSegments(&segments, 1);
284     rewriter->Rewrite(request_, &segments);
285     EXPECT_EQ("candidate0",
286               segments.segment(0).candidate(0).value);
287   }
288 }
289 
TEST_F(UserSegmentHistoryRewriterTest,DisableTest)290 TEST_F(UserSegmentHistoryRewriterTest, DisableTest) {
291   Segments segments;
292   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
293       CreateUserSegmentHistoryRewriter());
294 
295   {
296     InitSegments(&segments, 1);
297     segments.set_user_history_enabled(true);
298     segments.mutable_segment(0)->move_candidate(2, 0);
299     segments.mutable_segment(0)->mutable_candidate(0)->attributes
300         |= Segment::Candidate::RERANKED;
301     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
302     rewriter->Finish(request_, &segments);
303     InitSegments(&segments, 1);
304     rewriter->Rewrite(request_, &segments);
305     EXPECT_EQ("candidate2",
306               segments.segment(0).candidate(0).value);
307 
308     InitSegments(&segments, 1);
309     segments.set_user_history_enabled(false);
310     rewriter->Rewrite(request_, &segments);
311     EXPECT_EQ("candidate0",
312               segments.segment(0).candidate(0).value);
313 
314     InitSegments(&segments, 1);
315     segments.set_user_history_enabled(true);
316     rewriter->Rewrite(request_, &segments);
317     EXPECT_EQ("candidate2",
318               segments.segment(0).candidate(0).value);
319   }
320 
321   {
322     InitSegments(&segments, 1);
323     segments.set_user_history_enabled(false);
324     segments.mutable_segment(0)->move_candidate(2, 0);
325     segments.mutable_segment(0)->mutable_candidate(0)->attributes
326         |= Segment::Candidate::RERANKED;
327     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
328     rewriter->Finish(request_, &segments);
329     InitSegments(&segments, 1);
330     rewriter->Rewrite(request_, &segments);
331     segments.set_user_history_enabled(true);
332     EXPECT_EQ("candidate0",
333               segments.segment(0).candidate(0).value);
334   }
335 }
336 
TEST_F(UserSegmentHistoryRewriterTest,BasicTest)337 TEST_F(UserSegmentHistoryRewriterTest, BasicTest) {
338   Segments segments;
339   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
340       CreateUserSegmentHistoryRewriter());
341 
342   rewriter->Clear();
343 
344   {
345     InitSegments(&segments, 2);
346 
347     segments.mutable_segment(0)->move_candidate(2, 0);
348     segments.mutable_segment(0)->mutable_candidate(0)->attributes
349         |= Segment::Candidate::RERANKED;
350     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
351     rewriter->Finish(request_, &segments);
352     InitSegments(&segments, 2);
353     rewriter->Rewrite(request_, &segments);
354 
355     EXPECT_EQ("candidate2",
356               segments.segment(0).candidate(0).value);
357     EXPECT_EQ("candidate0",
358               segments.segment(1).candidate(0).value);
359 
360     InitSegments(&segments, 1);
361     rewriter->Rewrite(request_, &segments);
362     EXPECT_EQ("candidate2",
363               segments.segment(0).candidate(0).value);
364     segments.mutable_segment(0)->move_candidate(1, 0);
365     segments.mutable_segment(0)->mutable_candidate(0)->attributes
366         |= Segment::Candidate::RERANKED;
367     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
368     rewriter->Finish(request_, &segments);
369 
370     InitSegments(&segments, 1);
371     rewriter->Rewrite(request_, &segments);
372 
373     EXPECT_EQ("candidate0",
374               segments.segment(0).candidate(0).value);
375 
376     InitSegments(&segments, 2);
377     rewriter->Rewrite(request_, &segments);
378     EXPECT_EQ("candidate2",
379               segments.segment(0).candidate(0).value);
380     EXPECT_EQ("candidate0",
381               segments.segment(1).candidate(0).value);
382     segments.mutable_segment(1)->move_candidate(3, 0);
383     segments.mutable_segment(1)->mutable_candidate(0)->attributes
384         |= Segment::Candidate::RERANKED;
385     segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
386     rewriter->Finish(request_, &segments);
387 
388     InitSegments(&segments, 2);
389     rewriter->Rewrite(request_, &segments);
390     EXPECT_EQ("candidate2",
391               segments.segment(0).candidate(0).value);
392     EXPECT_EQ("candidate3",
393               segments.segment(1).candidate(0).value);
394   }
395 
396   rewriter->Clear();
397   {
398     InitSegments(&segments, 2);
399 
400     segments.mutable_segment(0)->move_candidate(2, 0);
401     segments.mutable_segment(0)->mutable_candidate(0)->attributes
402         |= Segment::Candidate::RERANKED;
403     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
404     rewriter->Finish(request_, &segments);
405     InitSegments(&segments, 2);
406     rewriter->Rewrite(request_, &segments);
407 
408     EXPECT_EQ("candidate2",
409               segments.segment(0).candidate(0).value);
410     EXPECT_EQ("candidate0",
411               segments.segment(1).candidate(0).value);
412 
413     InitSegments(&segments, 1);
414     rewriter->Rewrite(request_, &segments);
415     EXPECT_EQ("candidate2",
416               segments.segment(0).candidate(0).value);
417 
418     // back to the original
419     segments.mutable_segment(0)->move_candidate(1, 0);
420     segments.mutable_segment(0)->mutable_candidate(0)->attributes
421         |= Segment::Candidate::RERANKED;
422     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
423     rewriter->Finish(request_, &segments);
424 
425     InitSegments(&segments, 1);
426     rewriter->Rewrite(request_, &segments);
427     EXPECT_EQ("candidate0",
428               segments.segment(0).candidate(0).value);
429   }
430 }
431 
432 // Test for Issue 2155278
TEST_F(UserSegmentHistoryRewriterTest,SequenceTest)433 TEST_F(UserSegmentHistoryRewriterTest, SequenceTest) {
434   Segments segments;
435   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
436       CreateUserSegmentHistoryRewriter());
437 
438   rewriter->Clear();
439 
440   const uint64 kSeconds = 0;
441   const uint32 kMicroSeconds = 0;
442   ClockMock clock(kSeconds, kMicroSeconds);
443   Clock::SetClockForUnitTest(&clock);
444 
445   {
446     InitSegments(&segments, 1);
447 
448     segments.mutable_segment(0)->move_candidate(2, 0);
449     segments.mutable_segment(0)->mutable_candidate(0)->attributes
450         |= Segment::Candidate::RERANKED;
451     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
452     EXPECT_EQ("candidate2",
453               segments.segment(0).candidate(0).value);
454     rewriter->Finish(request_, &segments);  // learn "candidate2"
455 
456     // Next timestamp of learning should be newer than previous learning.
457     clock.PutClockForward(1, 0);
458 
459     InitSegments(&segments, 2);
460     segments.mutable_segment(0)->move_candidate(2, 0);
461     segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
462     segments.mutable_segment(1)->set_key(segments.segment(0).key());
463     EXPECT_EQ(1, segments.history_segments_size());
464     rewriter->Rewrite(request_, &segments);
465 
466     EXPECT_EQ("candidate2",
467               segments.segment(0).candidate(0).value);
468     EXPECT_EQ("candidate2",
469               segments.segment(1).candidate(0).value);
470     // 2 0 1 3 4 ..
471 
472     segments.mutable_segment(1)->move_candidate(3, 0);
473     segments.mutable_segment(1)->mutable_candidate(0)->attributes
474         |= Segment::Candidate::RERANKED;
475     segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
476     EXPECT_EQ("candidate3",
477               segments.segment(1).candidate(0).value);
478     rewriter->Finish(request_, &segments);  // learn "candidate3"
479 
480     clock.PutClockForward(1, 0);
481 
482     InitSegments(&segments, 3);
483     segments.mutable_segment(0)->move_candidate(2, 0);
484     segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
485     segments.mutable_segment(1)->move_candidate(3, 0);
486     segments.mutable_segment(1)->set_segment_type(Segment::HISTORY);
487     segments.mutable_segment(1)->set_key(segments.segment(0).key());
488     segments.mutable_segment(2)->set_key(segments.segment(0).key());
489     EXPECT_EQ(2, segments.history_segments_size());
490     rewriter->Rewrite(request_, &segments);
491 
492     EXPECT_EQ("candidate2",
493               segments.segment(0).candidate(0).value);
494     EXPECT_EQ("candidate3",
495               segments.segment(1).candidate(0).value);
496     EXPECT_EQ("candidate3",
497               segments.segment(2).candidate(0).value);
498     // 3 2 0 1 4 ..
499 
500     segments.mutable_segment(2)->move_candidate(1, 0);
501     segments.mutable_segment(2)->mutable_candidate(0)->attributes
502         |= Segment::Candidate::RERANKED;
503     segments.mutable_segment(2)->set_segment_type(Segment::FIXED_VALUE);
504     EXPECT_EQ("candidate2",
505               segments.segment(2).candidate(0).value);
506     rewriter->Finish(request_, &segments);  // learn "candidate2"
507 
508     clock.PutClockForward(1, 0);
509 
510     InitSegments(&segments, 4);
511     segments.mutable_segment(0)->move_candidate(2, 0);
512     segments.mutable_segment(0)->set_segment_type(Segment::HISTORY);
513     segments.mutable_segment(1)->move_candidate(3, 0);
514     segments.mutable_segment(1)->set_segment_type(Segment::HISTORY);
515     segments.mutable_segment(1)->set_key(segments.segment(0).key());
516     segments.mutable_segment(2)->move_candidate(2, 0);
517     segments.mutable_segment(2)->set_segment_type(Segment::HISTORY);
518     segments.mutable_segment(2)->set_key(segments.segment(0).key());
519     segments.mutable_segment(3)->set_key(segments.segment(0).key());
520     EXPECT_EQ(3, segments.history_segments_size());
521     rewriter->Rewrite(request_, &segments);
522 
523     EXPECT_EQ("candidate2",
524               segments.segment(0).candidate(0).value);
525     EXPECT_EQ("candidate3",
526               segments.segment(1).candidate(0).value);
527     EXPECT_EQ("candidate2",
528               segments.segment(2).candidate(0).value);
529     EXPECT_EQ("candidate2",
530               segments.segment(3).candidate(0).value);
531     // 2 3 0 1 4 ..
532   }
533 }
534 
TEST_F(UserSegmentHistoryRewriterTest,DupTest)535 TEST_F(UserSegmentHistoryRewriterTest, DupTest) {
536   Segments segments;
537   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
538       CreateUserSegmentHistoryRewriter());
539 
540   rewriter->Clear();
541 
542   const uint64 kSeconds = 0;
543   const uint32 kMicroSeconds = 0;
544   ClockMock clock(kSeconds, kMicroSeconds);
545   Clock::SetClockForUnitTest(&clock);
546 
547   {
548     InitSegments(&segments, 1);
549     segments.mutable_segment(0)->move_candidate(4, 0);
550     segments.mutable_segment(0)->mutable_candidate(0)->attributes
551         |= Segment::Candidate::RERANKED;
552     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
553     rewriter->Finish(request_, &segments);
554     InitSegments(&segments, 1);
555     rewriter->Rewrite(request_, &segments);
556 
557     // restored
558     // 4,0,1,2,3,5,...
559     EXPECT_EQ("candidate4",
560               segments.segment(0).candidate(0).value);
561     segments.mutable_segment(0)->move_candidate(4, 0);
562     segments.mutable_segment(0)->mutable_candidate(0)->attributes
563         |= Segment::Candidate::RERANKED;
564     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
565     clock.PutClockForward(1, 0);
566     rewriter->Finish(request_, &segments);
567     InitSegments(&segments, 1);
568     rewriter->Rewrite(request_, &segments);
569 
570     // 3,4,0,1,2,5
571     EXPECT_EQ("candidate3",
572               segments.segment(0).candidate(0).value);
573     EXPECT_EQ("candidate4",
574               segments.segment(0).candidate(1).value);
575     segments.mutable_segment(0)->move_candidate(4, 0);
576     segments.mutable_segment(0)->mutable_candidate(0)->attributes
577         |= Segment::Candidate::RERANKED;
578     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
579     clock.PutClockForward(1, 0);
580     rewriter->Finish(request_, &segments);
581     InitSegments(&segments, 1);
582     rewriter->Rewrite(request_, &segments);
583 
584     EXPECT_EQ("candidate2",
585               segments.segment(0).candidate(0).value);
586     EXPECT_EQ("candidate3",
587               segments.segment(0).candidate(1).value);
588     EXPECT_EQ("candidate4",
589               segments.segment(0).candidate(2).value);
590   }
591 }
592 
TEST_F(UserSegmentHistoryRewriterTest,LearningType)593 TEST_F(UserSegmentHistoryRewriterTest, LearningType) {
594   Segments segments;
595   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
596       CreateUserSegmentHistoryRewriter());
597 
598   {
599     rewriter->Clear();
600     InitSegments(&segments, 1);
601     segments.mutable_segment(0)->move_candidate(2, 0);
602     segments.mutable_segment(0)->mutable_candidate(0)->attributes
603         |= Segment::Candidate::RERANKED;
604     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
605     segments.mutable_segment(0)->mutable_candidate(0)->attributes
606         |= Segment::Candidate::NO_LEARNING;
607     rewriter->Finish(request_, &segments);
608     InitSegments(&segments, 1);
609     rewriter->Rewrite(request_, &segments);
610     EXPECT_EQ("candidate0",
611               segments.segment(0).candidate(0).value);
612   }
613 
614   {
615     rewriter->Clear();
616     InitSegments(&segments, 1);
617     segments.mutable_segment(0)->move_candidate(2, 0);
618     segments.mutable_segment(0)->mutable_candidate(0)->attributes
619         |= Segment::Candidate::RERANKED;
620     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
621     segments.mutable_segment(0)->mutable_candidate(0)->attributes |=
622         Segment::Candidate::NO_HISTORY_LEARNING;
623     rewriter->Finish(request_, &segments);
624     InitSegments(&segments, 1);
625     rewriter->Rewrite(request_, &segments);
626     EXPECT_EQ("candidate0",
627               segments.segment(0).candidate(0).value);
628   }
629 
630   {
631     rewriter->Clear();
632     InitSegments(&segments, 1);
633     segments.mutable_segment(0)->move_candidate(2, 0);
634     segments.mutable_segment(0)->mutable_candidate(0)->attributes
635         |= Segment::Candidate::RERANKED;
636     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
637     segments.mutable_segment(0)->mutable_candidate(0)->attributes |=
638         Segment::Candidate::NO_SUGGEST_LEARNING;
639     rewriter->Finish(request_, &segments);
640     InitSegments(&segments, 1);
641     rewriter->Rewrite(request_, &segments);
642     EXPECT_EQ("candidate2",
643               segments.segment(0).candidate(0).value);
644   }
645 }
646 
TEST_F(UserSegmentHistoryRewriterTest,ContextSensitive)647 TEST_F(UserSegmentHistoryRewriterTest, ContextSensitive) {
648   Segments segments;
649   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
650       CreateUserSegmentHistoryRewriter());
651 
652   rewriter->Clear();
653   {
654     InitSegments(&segments, 2);
655     segments.mutable_segment(0)->move_candidate(2, 0);
656     segments.mutable_segment(0)->mutable_candidate(0)->attributes
657         |= Segment::Candidate::RERANKED;
658     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
659     segments.mutable_segment(0)->mutable_candidate(0)->attributes
660         |= Segment::Candidate::CONTEXT_SENSITIVE;
661     rewriter->Finish(request_, &segments);
662     InitSegments(&segments, 2);
663     rewriter->Rewrite(request_, &segments);
664 
665     // fire if two segments
666     EXPECT_EQ("candidate2",
667               segments.segment(0).candidate(0).value);
668     // not fire if single segment
669     InitSegments(&segments, 1);
670     rewriter->Rewrite(request_, &segments);
671     EXPECT_EQ("candidate0",
672               segments.segment(0).candidate(0).value);
673   }
674 
675   rewriter->Clear();
676   {
677     InitSegments(&segments, 1);
678     segments.mutable_segment(0)->move_candidate(2, 0);
679     segments.mutable_segment(0)->mutable_candidate(0)->attributes
680         |= Segment::Candidate::RERANKED;
681     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
682     segments.mutable_segment(0)->mutable_candidate(0)->attributes
683         |= Segment::Candidate::CONTEXT_SENSITIVE;
684     rewriter->Finish(request_, &segments);
685 
686     // fire if even in single segment
687     InitSegments(&segments, 1);
688     rewriter->Rewrite(request_, &segments);
689     EXPECT_EQ("candidate2",
690               segments.segment(0).candidate(0).value);
691 
692     // not fire if two segments
693     InitSegments(&segments, 2);
694     rewriter->Rewrite(request_, &segments);
695     EXPECT_EQ("candidate0",
696               segments.segment(0).candidate(0).value);
697   }
698 }
699 
TEST_F(UserSegmentHistoryRewriterTest,ContentValueLearning)700 TEST_F(UserSegmentHistoryRewriterTest, ContentValueLearning) {
701   Segments segments;
702   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
703       CreateUserSegmentHistoryRewriter());
704 
705   rewriter->Clear();
706   {
707     InitSegments(&segments, 1);
708     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
709     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
710     segments.mutable_segment(0)->move_candidate(2, 0);
711     segments.mutable_segment(0)->mutable_candidate(0)->attributes
712         |= Segment::Candidate::RERANKED;
713     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
714 
715     rewriter->Finish(request_, &segments);
716 
717     InitSegments(&segments, 1);
718     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
719     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
720 
721     rewriter->Rewrite(request_, &segments);
722 
723     // exact match
724     EXPECT_EQ("candidate2:all",
725               segments.segment(0).candidate(0).value);
726 
727     InitSegments(&segments, 1);
728     rewriter->Rewrite(request_, &segments);
729 
730     // content value only:
731     // in both learning/applying phase, lid and suffix are the same
732     // as those of top candidates.
733     EXPECT_EQ("candidate2",
734               segments.segment(0).candidate(0).value);
735 
736     InitSegments(&segments, 1);
737     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":other", 0);
738     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 0);
739 
740     rewriter->Rewrite(request_, &segments);
741 
742     EXPECT_EQ("candidate2:other",
743               segments.segment(0).candidate(0).value);
744   }
745 
746   // In learning phase, lid is different
747   rewriter->Clear();
748   {
749     InitSegments(&segments, 1);
750     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
751     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
752     segments.mutable_segment(0)->move_candidate(2, 0);
753     segments.mutable_segment(0)->mutable_candidate(0)->attributes
754         |= Segment::Candidate::RERANKED;
755     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
756     rewriter->Finish(request_, &segments);
757     InitSegments(&segments, 1);
758     rewriter->Rewrite(request_, &segments);
759     EXPECT_EQ("candidate0",
760               segments.segment(0).candidate(0).value);
761   }
762 
763   // In learning phase, suffix (functional value) is different
764   rewriter->Clear();
765   {
766     InitSegments(&segments, 1);
767     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, "", 0);
768     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 1);
769     segments.mutable_segment(0)->move_candidate(2, 0);
770     segments.mutable_segment(0)->mutable_candidate(0)->attributes
771         |= Segment::Candidate::RERANKED;
772     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
773     rewriter->Finish(request_, &segments);
774     InitSegments(&segments, 1);
775     rewriter->Rewrite(request_, &segments);
776     EXPECT_EQ("candidate0",
777               segments.segment(0).candidate(0).value);
778   }
779 
780   // In apply phase, lid is different
781   rewriter->Clear();
782   {
783     InitSegments(&segments, 1);
784     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
785     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
786     segments.mutable_segment(0)->move_candidate(2, 0);
787     segments.mutable_segment(0)->mutable_candidate(0)->attributes
788         |= Segment::Candidate::RERANKED;
789     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
790     rewriter->Finish(request_, &segments);
791     InitSegments(&segments, 1);
792     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":other", 0);
793     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 1);
794     rewriter->Rewrite(request_, &segments);
795     EXPECT_EQ("candidate0:other",
796               segments.segment(0).candidate(0).value);
797   }
798 
799   // In apply phase, suffix (functional value) is different
800   rewriter->Clear();
801   {
802     InitSegments(&segments, 1);
803     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
804     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
805     segments.mutable_segment(0)->move_candidate(2, 0);
806     segments.mutable_segment(0)->mutable_candidate(0)->attributes
807         |= Segment::Candidate::RERANKED;
808     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
809     rewriter->Finish(request_, &segments);
810     InitSegments(&segments, 1);
811     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, "", 0);
812     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":other", 0);
813     rewriter->Rewrite(request_, &segments);
814     EXPECT_EQ("candidate0",
815               segments.segment(0).candidate(0).value);
816   }
817 }
818 
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableTest)819 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableTest) {
820   Segments segments;
821   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
822       CreateUserSegmentHistoryRewriter());
823 
824   rewriter->Clear();
825   {
826     InitSegments(&segments, 2);
827     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
828     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
829     segments.mutable_segment(0)->move_candidate(2, 0);
830     segments.mutable_segment(0)->mutable_candidate(0)->attributes
831         |= Segment::Candidate::RERANKED;
832     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
833 
834     rewriter->Finish(request_, &segments);
835 
836     InitSegments(&segments, 1);
837     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
838     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
839 
840     rewriter->Rewrite(request_, &segments);
841 
842     EXPECT_EQ("candidate2:all",
843               segments.segment(0).candidate(0).value);
844 
845     InitSegments(&segments, 1);
846     rewriter->Rewrite(request_, &segments);
847   }
848 
849   rewriter->Clear();
850   {
851     InitSegments(&segments, 1);
852     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
853     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
854     segments.mutable_segment(0)->move_candidate(2, 0);
855     segments.mutable_segment(0)->mutable_candidate(0)->attributes
856         |= Segment::Candidate::RERANKED;
857     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
858 
859     rewriter->Finish(request_, &segments);
860 
861     InitSegments(&segments, 2);
862     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
863     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
864 
865     rewriter->Rewrite(request_, &segments);
866 
867     EXPECT_EQ("candidate2:all",
868               segments.segment(0).candidate(0).value);
869 
870     InitSegments(&segments, 1);
871     rewriter->Rewrite(request_, &segments);
872   }
873 
874   rewriter->Clear();
875   {
876     InitSegments(&segments, 2);
877     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
878     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
879     segments.mutable_segment(0)->move_candidate(2, 0);
880     segments.mutable_segment(0)->mutable_candidate(0)->attributes
881         |= Segment::Candidate::RERANKED;
882     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
883     rewriter->Finish(request_, &segments);
884     InitSegments(&segments, 1);
885     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
886     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
887     rewriter->Rewrite(request_, &segments);
888     EXPECT_EQ("candidate0:all",
889               segments.segment(0).candidate(0).value);
890   }
891 
892   rewriter->Clear();
893   {
894     InitSegments(&segments, 2);
895     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
896     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
897     segments.mutable_segment(0)->move_candidate(2, 0);
898     segments.mutable_segment(0)->mutable_candidate(0)->attributes
899         |= Segment::Candidate::RERANKED;
900     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
901     rewriter->Finish(request_, &segments);
902     InitSegments(&segments, 1);
903     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
904     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
905     rewriter->Rewrite(request_, &segments);
906     EXPECT_EQ("candidate0:all",
907               segments.segment(0).candidate(0).value);
908   }
909 
910   rewriter->Clear();
911   {
912     InitSegments(&segments, 1);
913     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
914     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
915     segments.mutable_segment(0)->move_candidate(2, 0);
916     segments.mutable_segment(0)->mutable_candidate(0)->attributes
917         |= Segment::Candidate::RERANKED;
918     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
919     rewriter->Finish(request_, &segments);
920     InitSegments(&segments, 2);
921     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
922     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
923     rewriter->Rewrite(request_, &segments);
924     EXPECT_EQ("candidate0:all",
925               segments.segment(0).candidate(0).value);
926   }
927 
928   rewriter->Clear();
929   {
930     InitSegments(&segments, 1);
931     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
932     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 0);
933     segments.mutable_segment(0)->move_candidate(2, 0);
934     segments.mutable_segment(0)->mutable_candidate(0)->attributes
935         |= Segment::Candidate::RERANKED;
936     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
937     rewriter->Finish(request_, &segments);
938     InitSegments(&segments, 2);
939     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 0, ":all", 0);
940     AppendCandidateSuffixWithLid(segments.mutable_segment(0), 2, ":all", 1);
941     rewriter->Rewrite(request_, &segments);
942     EXPECT_EQ("candidate0:all",
943               segments.segment(0).candidate(0).value);
944   }
945 }
946 
TEST_F(UserSegmentHistoryRewriterTest,NotReplaceableForDifferentId)947 TEST_F(UserSegmentHistoryRewriterTest, NotReplaceableForDifferentId) {
948   Segments segments;
949   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
950       CreateUserSegmentHistoryRewriter());
951 
952   rewriter->Clear();
953   {
954     InitSegments(&segments, 2);
955     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
956     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 200, 300);
957     segments.mutable_segment(0)->move_candidate(2, 0);
958     segments.mutable_segment(0)->mutable_candidate(0)->attributes
959         |= Segment::Candidate::RERANKED;
960     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
961 
962     rewriter->Finish(request_, &segments);
963 
964     InitSegments(&segments, 2);
965     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
966     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 200, 300);
967     segments.mutable_segment(1)->mutable_candidate(0)->value =
968         "not_same_as_before";
969 
970     rewriter->Rewrite(request_, &segments);
971 
972     EXPECT_NE("candidate2:all",
973               segments.segment(0).candidate(0).value);
974   }
975 }
976 
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableForSameId)977 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableForSameId) {
978   Segments segments;
979   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
980       CreateUserSegmentHistoryRewriter());
981 
982   rewriter->Clear();
983   {
984     InitSegments(&segments, 2);
985     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
986     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 1, 1);
987     segments.mutable_segment(0)->move_candidate(2, 0);
988     segments.mutable_segment(0)->mutable_candidate(0)->attributes
989         |= Segment::Candidate::RERANKED;
990     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
991 
992     rewriter->Finish(request_, &segments);
993 
994     InitSegments(&segments, 2);
995     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
996     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 1, 1);
997     segments.mutable_segment(1)->mutable_candidate(0)->value =
998         "not_same_as_before";
999 
1000     rewriter->Rewrite(request_, &segments);
1001 
1002     EXPECT_EQ("candidate2:all",
1003               segments.segment(0).candidate(0).value);
1004   }
1005 }
1006 
TEST_F(UserSegmentHistoryRewriterTest,ReplaceableT13NTest)1007 TEST_F(UserSegmentHistoryRewriterTest, ReplaceableT13NTest) {
1008   Segments segments;
1009   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1010       CreateUserSegmentHistoryRewriter());
1011 
1012   rewriter->Clear();
1013   {
1014     InitSegments(&segments, 2);
1015     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
1016     // Prepare candidate2 as T13N candidate.
1017     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 0, 0);
1018     segments.mutable_segment(0)->move_candidate(2, 0);
1019     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1020         |= Segment::Candidate::RERANKED;
1021     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1022 
1023     rewriter->Finish(request_, &segments);
1024 
1025     InitSegments(&segments, 2);
1026     AppendCandidateSuffix(segments.mutable_segment(0), 0, ":all", 1, 1);
1027     AppendCandidateSuffix(segments.mutable_segment(0), 2, ":all", 0, 0);
1028     segments.mutable_segment(1)->mutable_candidate(0)->value =
1029         "not_same_as_before";
1030 
1031     rewriter->Rewrite(request_, &segments);
1032 
1033     EXPECT_EQ("candidate2:all",
1034               segments.segment(0).candidate(0).value);
1035   }
1036 }
1037 
TEST_F(UserSegmentHistoryRewriterTest,LeftRightNumber)1038 TEST_F(UserSegmentHistoryRewriterTest, LeftRightNumber) {
1039   Segments segments;
1040   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1041       CreateUserSegmentHistoryRewriter());
1042 
1043   rewriter->Clear();
1044 
1045   {
1046     InitSegments(&segments, 2);
1047 
1048     segments.mutable_segment(0)->mutable_candidate(0)->value = "1234";
1049     segments.mutable_segment(1)->move_candidate(2, 0);
1050     segments.mutable_segment(1)->mutable_candidate(0)->attributes
1051         |= Segment::Candidate::RERANKED;
1052     segments.mutable_segment(1)->mutable_candidate(0)->attributes
1053         |= Segment::Candidate::CONTEXT_SENSITIVE;
1054     segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1055     rewriter->Finish(request_, &segments);
1056     EXPECT_EQ("1234",
1057               segments.segment(0).candidate(0).value);
1058     EXPECT_EQ("candidate2",
1059               segments.segment(1).candidate(0).value);
1060 
1061     InitSegments(&segments, 2);
1062     // different num.
1063     segments.mutable_segment(0)->mutable_candidate(0)->value = "5678";
1064     rewriter->Rewrite(request_, &segments);
1065     EXPECT_EQ("5678",
1066               segments.segment(0).candidate(0).value);
1067     EXPECT_EQ("candidate2",
1068               segments.segment(1).candidate(0).value);
1069   }
1070 
1071   {
1072     InitSegments(&segments, 2);
1073 
1074     segments.mutable_segment(1)->mutable_candidate(0)->value = "1234";
1075     segments.mutable_segment(0)->move_candidate(2, 0);
1076     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1077         |= Segment::Candidate::RERANKED;
1078     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1079         |= Segment::Candidate::CONTEXT_SENSITIVE;
1080     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1081     rewriter->Finish(request_, &segments);
1082     EXPECT_EQ("candidate2",
1083               segments.segment(0).candidate(0).value);
1084     EXPECT_EQ("1234",
1085               segments.segment(1).candidate(0).value);
1086 
1087     InitSegments(&segments, 2);
1088     // different num.
1089     segments.mutable_segment(1)->mutable_candidate(0)->value = "5678";
1090     rewriter->Rewrite(request_, &segments);
1091     EXPECT_EQ("candidate2",
1092               segments.segment(0).candidate(0).value);
1093     EXPECT_EQ("5678",
1094               segments.segment(1).candidate(0).value);
1095   }
1096 }
1097 
TEST_F(UserSegmentHistoryRewriterTest,BacketMatching)1098 TEST_F(UserSegmentHistoryRewriterTest, BacketMatching) {
1099   Segments segments;
1100   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1101       CreateUserSegmentHistoryRewriter());
1102 
1103   rewriter->Clear();
1104 
1105   {
1106     InitSegments(&segments, 1);
1107     Segment::Candidate *candidate =
1108         segments.mutable_segment(0)->insert_candidate(2);
1109     candidate->value = "(";
1110     candidate->content_value = "(";
1111     candidate->content_key = "(";
1112     segments.mutable_segment(0)->move_candidate(2, 0);
1113     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1114         |= Segment::Candidate::RERANKED;
1115     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1116     rewriter->Finish(request_, &segments);
1117   }
1118 
1119   {
1120     InitSegments(&segments, 1);
1121     Segment::Candidate *candidate =
1122         segments.mutable_segment(0)->insert_candidate(2);
1123     candidate->value = ")";
1124     candidate->content_value = ")";
1125     candidate->content_key = ")";
1126 
1127     rewriter->Rewrite(request_, &segments);
1128 
1129     EXPECT_EQ(")",
1130               segments.segment(0).candidate(0).value);
1131   }
1132 }
1133 
1134 // issue 2262691
TEST_F(UserSegmentHistoryRewriterTest,MultipleLearning)1135 TEST_F(UserSegmentHistoryRewriterTest, MultipleLearning) {
1136   Segments segments;
1137   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1138       CreateUserSegmentHistoryRewriter());
1139 
1140   rewriter->Clear();
1141 
1142   {
1143     InitSegments(&segments, 1);
1144     segments.mutable_segment(0)->set_key("key1");
1145     Segment::Candidate *candidate =
1146         segments.mutable_segment(0)->insert_candidate(2);
1147     candidate->value = "value1";
1148     candidate->content_value = "value1";
1149     candidate->content_key = "key1";
1150     segments.mutable_segment(0)->move_candidate(2, 0);
1151     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1152         |= Segment::Candidate::RERANKED;
1153     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1154     rewriter->Finish(request_, &segments);
1155   }
1156 
1157   {
1158     InitSegments(&segments, 1);
1159     segments.mutable_segment(0)->set_key("key2");
1160     Segment::Candidate *candidate =
1161         segments.mutable_segment(0)->insert_candidate(2);
1162     candidate->value = "value2";
1163     candidate->content_value = "value2";
1164     candidate->content_key = "key2";
1165     segments.mutable_segment(0)->move_candidate(2, 0);
1166     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1167         |= Segment::Candidate::RERANKED;
1168     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1169     rewriter->Finish(request_, &segments);
1170   }
1171 
1172   {
1173     InitSegments(&segments, 1);
1174     segments.mutable_segment(0)->set_key("key1");
1175     Segment::Candidate *candidate =
1176         segments.mutable_segment(0)->insert_candidate(2);
1177     candidate->value = "value2";
1178     candidate->content_value = "value2";
1179     candidate->content_key = "key2";
1180     candidate = segments.mutable_segment(0)->insert_candidate(3);
1181     candidate->value = "value1";
1182     candidate->content_value = "value1";
1183     candidate->content_key = "key1";
1184 
1185     rewriter->Rewrite(request_, &segments);
1186 
1187     EXPECT_EQ("value1",
1188               segments.segment(0).candidate(0).value);
1189   }
1190 }
1191 
TEST_F(UserSegmentHistoryRewriterTest,NumberSpecial)1192 TEST_F(UserSegmentHistoryRewriterTest, NumberSpecial) {
1193   Segments segments;
1194   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1195       CreateUserSegmentHistoryRewriter());
1196   std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1197 
1198   rewriter->Clear();
1199 
1200   {
1201     segments.Clear();
1202     segments.add_segment();
1203     segments.mutable_segment(0)->set_key("12");
1204     Segment::Candidate *candidate =
1205         segments.mutable_segment(0)->insert_candidate(0);
1206     candidate->value = "⑫";
1207     candidate->content_value = "⑫";
1208     candidate->content_key = "12";
1209     candidate->lid = pos_matcher().GetNumberId();
1210     candidate->rid = pos_matcher().GetNumberId();
1211     candidate->style = NumberUtil::NumberString::NUMBER_CIRCLED;
1212     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1213         |= Segment::Candidate::RERANKED;
1214     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1215     rewriter->Finish(request_, &segments);
1216   }
1217 
1218   {
1219     segments.Clear();
1220     segments.add_segment();
1221     segments.mutable_segment(0)->set_key("14");
1222     {
1223       Segment::Candidate *candidate =
1224           segments.mutable_segment(0)->insert_candidate(0);
1225       candidate->value = "14";
1226       candidate->content_value = "14";
1227       candidate->content_key = "14";
1228       candidate->lid = pos_matcher().GetNumberId();
1229       candidate->rid = pos_matcher().GetNumberId();
1230     }
1231     EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1232     rewriter->Rewrite(request_, &segments);
1233 
1234     EXPECT_EQ("⑭", segments.segment(0).candidate(0).value);
1235   }
1236 }
1237 
TEST_F(UserSegmentHistoryRewriterTest,NumberHalfWidth)1238 TEST_F(UserSegmentHistoryRewriterTest, NumberHalfWidth) {
1239   SetNumberForm(Config::HALF_WIDTH);
1240   Segments segments;
1241   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1242       CreateUserSegmentHistoryRewriter());
1243   std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1244 
1245   rewriter->Clear();
1246 
1247   {
1248     segments.Clear();
1249     segments.add_segment();
1250     segments.mutable_segment(0)->set_key("1234");
1251     Segment::Candidate *candidate =
1252         segments.mutable_segment(0)->insert_candidate(0);
1253     candidate->value = "１，２３４";
1254     candidate->content_value = "１，２３４";
1255     candidate->content_key = "1234";
1256     candidate->lid = pos_matcher().GetNumberId();
1257     candidate->rid = pos_matcher().GetNumberId();
1258     candidate->style =
1259         NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_FULLWIDTH;
1260     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1261     rewriter->Finish(request_, &segments);  // full-width for separated number
1262   }
1263 
1264   {
1265     segments.Clear();
1266     segments.add_segment();
1267     segments.mutable_segment(0)->set_key("1234");
1268     {
1269       Segment::Candidate *candidate =
1270           segments.mutable_segment(0)->insert_candidate(0);
1271       candidate->value = "1234";
1272       candidate->content_value = "1234";
1273       candidate->content_key = "1234";
1274       candidate->lid = pos_matcher().GetNumberId();
1275       candidate->rid = pos_matcher().GetNumberId();
1276     }
1277 
1278     EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1279     rewriter->Rewrite(request_, &segments);
1280 
1281     EXPECT_EQ("1,234", segments.segment(0).candidate(0).value);
1282   }
1283 }
1284 
TEST_F(UserSegmentHistoryRewriterTest,NumberFullWidth)1285 TEST_F(UserSegmentHistoryRewriterTest, NumberFullWidth) {
1286   SetNumberForm(Config::FULL_WIDTH);
1287   Segments segments;
1288   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1289       CreateUserSegmentHistoryRewriter());
1290   std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1291 
1292   rewriter->Clear();
1293 
1294   {
1295     segments.Clear();
1296     segments.add_segment();
1297     segments.mutable_segment(0)->set_key("1234");
1298     Segment::Candidate *candidate =
1299         segments.mutable_segment(0)->insert_candidate(0);
1300     candidate->value = "1,234";
1301     candidate->content_value = "1,2344";
1302     candidate->content_key = "1234";
1303     candidate->lid = pos_matcher().GetNumberId();
1304     candidate->rid = pos_matcher().GetNumberId();
1305     candidate->style =
1306         NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_HALFWIDTH;
1307     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1308     rewriter->Finish(request_, &segments);  // half-width for separated number
1309   }
1310 
1311   {
1312     segments.Clear();
1313     segments.add_segment();
1314     {
1315       segments.mutable_segment(0)->set_key("1234");
1316       Segment::Candidate *candidate =
1317           segments.mutable_segment(0)->insert_candidate(0);
1318       candidate->value = "1234";
1319       candidate->content_value = "1234";
1320       candidate->content_key = "1234";
1321       candidate->lid = pos_matcher().GetNumberId();
1322       candidate->rid = pos_matcher().GetNumberId();
1323     }
1324     EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1325     rewriter->Rewrite(request_, &segments);
1326 
1327     EXPECT_EQ("１，２３４", segments.segment(0).candidate(0).value);
1328   }
1329 }
1330 
TEST_F(UserSegmentHistoryRewriterTest,NumberNoSeparated)1331 TEST_F(UserSegmentHistoryRewriterTest, NumberNoSeparated) {
1332   SetNumberForm(Config::HALF_WIDTH);
1333   Segments segments;
1334   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1335       CreateUserSegmentHistoryRewriter());
1336   std::unique_ptr<NumberRewriter> number_rewriter(CreateNumberRewriter());
1337 
1338   rewriter->Clear();
1339 
1340   {
1341     segments.Clear();
1342     segments.add_segment();
1343     segments.mutable_segment(0)->set_key("10");
1344     Segment::Candidate *candidate =
1345         segments.mutable_segment(0)->insert_candidate(0);
1346     candidate->value = "十";
1347     candidate->content_value = "十";
1348     candidate->content_key = "10";
1349     candidate->lid = pos_matcher().GetNumberId();
1350     candidate->rid = pos_matcher().GetNumberId();
1351     candidate->style = NumberUtil::NumberString::NUMBER_KANJI;
1352     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1353     rewriter->Finish(request_, &segments);  // learn kanji
1354   }
1355   {
1356     segments.Clear();
1357     segments.add_segment();
1358     segments.mutable_segment(0)->set_key("1234");
1359     Segment::Candidate *candidate =
1360         segments.mutable_segment(0)->insert_candidate(0);
1361     candidate->value = "1,234";
1362     candidate->content_value = "1,234";
1363     candidate->content_key = "1234";
1364     candidate->lid = pos_matcher().GetNumberId();
1365     candidate->rid = pos_matcher().GetNumberId();
1366     candidate->style =
1367         NumberUtil::NumberString::NUMBER_SEPARATED_ARABIC_HALFWIDTH;
1368     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1369     rewriter->Finish(request_, &segments);  // learn kanji
1370   }
1371 
1372   {
1373     InitSegments(&segments, 1);
1374     segments.mutable_segment(0)->set_key("9");
1375     {
1376       Segment::Candidate *candidate =
1377           segments.mutable_segment(0)->insert_candidate(0);
1378       candidate->value = "9";
1379       candidate->content_value = "9";
1380       candidate->content_key = "9";
1381       candidate->lid = pos_matcher().GetNumberId();
1382       candidate->rid = pos_matcher().GetNumberId();
1383     }
1384     EXPECT_TRUE(number_rewriter->Rewrite(request_, &segments));
1385     rewriter->Rewrite(request_, &segments);
1386 
1387     // 9, not "九"
1388     EXPECT_EQ("9", segments.segment(0).candidate(0).value);
1389   }
1390 }
1391 
TEST_F(UserSegmentHistoryRewriterTest,Regression2459519)1392 TEST_F(UserSegmentHistoryRewriterTest, Regression2459519) {
1393   Segments segments;
1394   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1395       CreateUserSegmentHistoryRewriter());
1396 
1397   rewriter->Clear();
1398 
1399   const uint64 kSeconds = 0;
1400   const uint32 kMicroSeconds = 0;
1401   ClockMock clock(kSeconds, kMicroSeconds);
1402   Clock::SetClockForUnitTest(&clock);
1403 
1404   InitSegments(&segments, 1);
1405   segments.mutable_segment(0)->move_candidate(2, 0);
1406   segments.mutable_segment(0)->mutable_candidate(0)->attributes
1407       |= Segment::Candidate::RERANKED;
1408   segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1409   rewriter->Finish(request_, &segments);
1410 
1411   InitSegments(&segments, 1);
1412   rewriter->Rewrite(request_, &segments);
1413   EXPECT_EQ("candidate2",
1414             segments.segment(0).candidate(0).value);
1415   EXPECT_EQ("candidate0",
1416             segments.segment(0).candidate(1).value);
1417 
1418   segments.mutable_segment(0)->move_candidate(1, 0);
1419   segments.mutable_segment(0)->mutable_candidate(0)->attributes
1420       |= Segment::Candidate::RERANKED;
1421   segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1422   clock.PutClockForward(1, 0);
1423   rewriter->Finish(request_, &segments);
1424 
1425   InitSegments(&segments, 1);
1426   rewriter->Rewrite(request_, &segments);
1427   EXPECT_EQ("candidate0",
1428             segments.segment(0).candidate(0).value);
1429   EXPECT_EQ("candidate2",
1430             segments.segment(0).candidate(1).value);
1431 
1432   segments.mutable_segment(0)->move_candidate(1, 0);
1433   segments.mutable_segment(0)->mutable_candidate(0)->attributes
1434       |= Segment::Candidate::RERANKED;
1435   segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1436   clock.PutClockForward(1, 0);
1437   rewriter->Finish(request_, &segments);
1438 
1439   InitSegments(&segments, 1);
1440   rewriter->Rewrite(request_, &segments);
1441   EXPECT_EQ("candidate2",
1442             segments.segment(0).candidate(0).value);
1443   EXPECT_EQ("candidate0",
1444             segments.segment(0).candidate(1).value);
1445 }
1446 
TEST_F(UserSegmentHistoryRewriterTest,Regression2459520)1447 TEST_F(UserSegmentHistoryRewriterTest, Regression2459520) {
1448   Segments segments;
1449   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1450       CreateUserSegmentHistoryRewriter());
1451 
1452   rewriter->Clear();
1453 
1454   InitSegments(&segments, 2);
1455   segments.mutable_segment(0)->move_candidate(2, 0);
1456   segments.mutable_segment(0)->mutable_candidate(0)->attributes
1457       |= Segment::Candidate::RERANKED;
1458   segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1459 
1460   segments.mutable_segment(1)->move_candidate(3, 0);
1461   segments.mutable_segment(1)->mutable_candidate(0)->attributes
1462       |= Segment::Candidate::RERANKED;
1463   segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1464   rewriter->Finish(request_, &segments);
1465 
1466   InitSegments(&segments, 2);
1467   rewriter->Rewrite(request_, &segments);
1468   EXPECT_EQ("candidate2",
1469             segments.segment(0).candidate(0).value);
1470   EXPECT_EQ("candidate3",
1471             segments.segment(1).candidate(0).value);
1472 }
1473 
TEST_F(UserSegmentHistoryRewriterTest,PuntuationsTest)1474 TEST_F(UserSegmentHistoryRewriterTest, PuntuationsTest) {
1475   Segments segments;
1476   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1477       CreateUserSegmentHistoryRewriter());
1478 
1479   const uint16 id = pos_matcher().GetJapanesePunctuationsId();
1480 
1481   rewriter->Clear();
1482 
1483   InitSegments(&segments, 2);
1484   segments.mutable_segment(1)->set_key(".");
1485   for (int i = 1; i < kCandidatesSize; ++i) {
1486     segments.mutable_segment(1)->mutable_candidate(i)->lid = id;
1487     segments.mutable_segment(1)->mutable_candidate(i)->rid = id;
1488     segments.mutable_segment(1)->mutable_candidate(i)->value = ".";
1489   }
1490   segments.mutable_segment(1)->move_candidate(2, 0);
1491   segments.mutable_segment(1)->mutable_candidate(0)->attributes
1492       |= Segment::Candidate::RERANKED;
1493   segments.mutable_segment(1)->set_segment_type(Segment::FIXED_VALUE);
1494   rewriter->Finish(request_, &segments);
1495 
1496   InitSegments(&segments, 2);
1497   segments.mutable_segment(1)->set_key(".");
1498   for (int i = 1; i < kCandidatesSize; ++i) {
1499     segments.mutable_segment(1)->mutable_candidate(i)->lid = id;
1500     segments.mutable_segment(1)->mutable_candidate(i)->rid = id;
1501     segments.mutable_segment(1)->mutable_candidate(i)->value = ".";
1502   }
1503 
1504   // Punctuation is not remembered
1505   rewriter->Rewrite(request_, &segments);
1506   EXPECT_EQ("candidate0", segments.segment(1).candidate(0).value);
1507 }
1508 
TEST_F(UserSegmentHistoryRewriterTest,Regression3264619)1509 TEST_F(UserSegmentHistoryRewriterTest, Regression3264619) {
1510   Segments segments;
1511   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1512       CreateUserSegmentHistoryRewriter());
1513 
1514   // Too big candidates
1515   InitSegments(&segments, 2, 1024);
1516   segments.mutable_segment(0)->move_candidate(512, 0);
1517   segments.mutable_segment(0)->mutable_candidate(0)->attributes
1518       |= Segment::Candidate::RERANKED;
1519   segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1520   rewriter->Finish(request_, &segments);
1521   InitSegments(&segments, 2, 1024);
1522   rewriter->Rewrite(request_, &segments);
1523 
1524   EXPECT_EQ("candidate512",
1525             segments.segment(0).candidate(0).value);
1526   EXPECT_EQ("candidate0",
1527             segments.segment(1).candidate(0).value);
1528 }
1529 
TEST_F(UserSegmentHistoryRewriterTest,RandomTest)1530 TEST_F(UserSegmentHistoryRewriterTest, RandomTest) {
1531   Segments segments;
1532   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1533       CreateUserSegmentHistoryRewriter());
1534 
1535   const uint64 kSeconds = 0;
1536   const uint32 kMicroSeconds = 0;
1537   ClockMock clock(kSeconds, kMicroSeconds);
1538   Clock::SetClockForUnitTest(&clock);
1539 
1540   rewriter->Clear();
1541   for (int i = 0; i < 5; ++i) {
1542     InitSegments(&segments, 1);
1543     const int n = Util::Random(10);
1544     const string expected = segments.segment(0).candidate(n).value;
1545     segments.mutable_segment(0)->move_candidate(n, 0);
1546     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1547         |= Segment::Candidate::RERANKED;
1548     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1549     EXPECT_EQ(expected,
1550               segments.segment(0).candidate(0).value);
1551     rewriter->Finish(request_, &segments);
1552     InitSegments(&segments, 1);
1553     rewriter->Rewrite(request_, &segments);
1554     EXPECT_EQ(expected,
1555               segments.segment(0).candidate(0).value);
1556     clock.PutClockForward(1, 0);  // update LRU timer
1557   }
1558 }
1559 
TEST_F(UserSegmentHistoryRewriterTest,AnnotationAfterLearning)1560 TEST_F(UserSegmentHistoryRewriterTest, AnnotationAfterLearning) {
1561   Segments segments;
1562   std::unique_ptr<UserSegmentHistoryRewriter> rewriter(
1563       CreateUserSegmentHistoryRewriter());
1564 
1565   {
1566     segments.Clear();
1567     InitSegments(&segments, 1, 2);
1568     segments.mutable_segment(0)->set_key("abc");
1569     Segment::Candidate *candidate =
1570         segments.mutable_segment(0)->mutable_candidate(1);
1571     candidate->value = "ａｂｃ";
1572     candidate->content_value = "ａｂｃ";
1573     candidate->content_key = "abc";
1574     candidate->description = "[全] アルファベット";
1575     segments.mutable_segment(0)->move_candidate(1, 0);
1576     segments.mutable_segment(0)->mutable_candidate(0)->attributes
1577         |= Segment::Candidate::RERANKED;
1578     segments.mutable_segment(0)->set_segment_type(Segment::FIXED_VALUE);
1579     rewriter->Finish(request_, &segments);
1580   }
1581 
1582   {
1583     segments.Clear();
1584     InitSegments(&segments, 1, 2);
1585     segments.mutable_segment(0)->set_key("abc");
1586     Segment::Candidate *candidate =
1587         segments.mutable_segment(0)->mutable_candidate(1);
1588     candidate->value = "ａｂｃ";
1589     candidate->content_value = "ａｂｃ";
1590     candidate->content_key = "abc";
1591     candidate->description = "[全]アルファベット";
1592     candidate->content_key = "abc";
1593     rewriter->Rewrite(request_, &segments);
1594     EXPECT_EQ("abc", segments.segment(0).candidate(0).content_value);
1595     // "[半] アルファベット"
1596     string expectation = VariantsRewriter::kHalfWidth;
1597     const string alphabet = VariantsRewriter::kAlphabet;
1598     if (alphabet.size() != 0) {
1599       expectation += ' ' + alphabet;
1600     }
1601     EXPECT_EQ(expectation,
1602               segments.segment(0).candidate(0).description);
1603     rewriter->Finish(request_, &segments);
1604   }
1605 }
1606 
1607 }  // namespace mozc
1608