1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <stddef.h>
6
7 #include <algorithm>
8
9 #include "base/logging.h"
10 #include "base/stl_util.h"
11 #include "base/strings/string_piece.h"
12 #include "base/strings/utf_offset_string_conversions.h"
13 #include "testing/gtest/include/gtest/gtest.h"
14
15 namespace base {
16
17 namespace {
18
19 static const size_t kNpos = string16::npos;
20
21 } // namespace
22
TEST(UTFOffsetStringConversionsTest,AdjustOffset)23 TEST(UTFOffsetStringConversionsTest, AdjustOffset) {
24 struct UTF8ToUTF16Case {
25 const char* utf8;
26 size_t input_offset;
27 size_t output_offset;
28 } utf8_to_utf16_cases[] = {
29 {"", 0, 0},
30 {"", kNpos, kNpos},
31 {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, kNpos},
32 {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1},
33 {"\xed\xb0\x80z", 3, 3},
34 {"A\xF0\x90\x8C\x80z", 1, 1},
35 {"A\xF0\x90\x8C\x80z", 2, kNpos},
36 {"A\xF0\x90\x8C\x80z", 5, 3},
37 {"A\xF0\x90\x8C\x80z", 6, 4},
38 {"A\xF0\x90\x8C\x80z", kNpos, kNpos},
39 };
40 for (const auto& i : utf8_to_utf16_cases) {
41 const size_t offset = i.input_offset;
42 std::vector<size_t> offsets;
43 offsets.push_back(offset);
44 UTF8ToUTF16AndAdjustOffsets(i.utf8, &offsets);
45 EXPECT_EQ(i.output_offset, offsets[0]);
46 }
47
48 struct UTF16ToUTF8Case {
49 char16 utf16[10];
50 size_t input_offset;
51 size_t output_offset;
52 } utf16_to_utf8_cases[] = {
53 {{}, 0, 0},
54 // Converted to 3-byte utf-8 sequences
55 {{0x5909, 0x63DB}, 3, kNpos},
56 {{0x5909, 0x63DB}, 2, 6},
57 {{0x5909, 0x63DB}, 1, 3},
58 {{0x5909, 0x63DB}, 0, 0},
59 // Converted to 2-byte utf-8 sequences
60 {{'A', 0x00bc, 0x00be, 'z'}, 1, 1},
61 {{'A', 0x00bc, 0x00be, 'z'}, 2, 3},
62 {{'A', 0x00bc, 0x00be, 'z'}, 3, 5},
63 {{'A', 0x00bc, 0x00be, 'z'}, 4, 6},
64 // Surrogate pair
65 {{'A', 0xd800, 0xdf00, 'z'}, 1, 1},
66 {{'A', 0xd800, 0xdf00, 'z'}, 2, kNpos},
67 {{'A', 0xd800, 0xdf00, 'z'}, 3, 5},
68 {{'A', 0xd800, 0xdf00, 'z'}, 4, 6},
69 };
70 for (size_t i = 0; i < base::size(utf16_to_utf8_cases); ++i) {
71 size_t offset = utf16_to_utf8_cases[i].input_offset;
72 std::vector<size_t> offsets;
73 offsets.push_back(offset);
74 UTF16ToUTF8AndAdjustOffsets(utf16_to_utf8_cases[i].utf16, &offsets);
75 EXPECT_EQ(utf16_to_utf8_cases[i].output_offset, offsets[0]) << i;
76 }
77 }
78
TEST(UTFOffsetStringConversionsTest,LimitOffsets)79 TEST(UTFOffsetStringConversionsTest, LimitOffsets) {
80 const OffsetAdjuster::Adjustments kNoAdjustments;
81 const size_t kLimit = 10;
82 const size_t kItems = 20;
83 std::vector<size_t> size_ts;
84 for (size_t t = 0; t < kItems; ++t) {
85 size_ts.push_back(t);
86 OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
87 }
88 size_t unlimited_count = 0;
89 for (auto ti : size_ts) {
90 if (ti != kNpos)
91 ++unlimited_count;
92 }
93 EXPECT_EQ(11U, unlimited_count);
94
95 // Reverse the values in the vector and try again.
96 size_ts.clear();
97 for (size_t t = kItems; t > 0; --t) {
98 size_ts.push_back(t - 1);
99 OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
100 }
101 unlimited_count = 0;
102 for (auto ti : size_ts) {
103 if (ti != kNpos)
104 ++unlimited_count;
105 }
106 EXPECT_EQ(11U, unlimited_count);
107 }
108
TEST(UTFOffsetStringConversionsTest,AdjustOffsets)109 TEST(UTFOffsetStringConversionsTest, AdjustOffsets) {
110 // Imagine we have strings as shown in the following cases where the
111 // X's represent encoded characters.
112 // 1: abcXXXdef ==> abcXdef
113 {
114 std::vector<size_t> offsets;
115 for (size_t t = 0; t <= 9; ++t)
116 offsets.push_back(t);
117 OffsetAdjuster::Adjustments adjustments;
118 adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
119 OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
120 size_t expected_1[] = {0, 1, 2, 3, kNpos, kNpos, 4, 5, 6, 7};
121 EXPECT_EQ(offsets.size(), base::size(expected_1));
122 for (size_t i = 0; i < base::size(expected_1); ++i)
123 EXPECT_EQ(expected_1[i], offsets[i]);
124 }
125
126 // 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
127 {
128 std::vector<size_t> offsets;
129 for (size_t t = 0; t <= 23; ++t)
130 offsets.push_back(t);
131 OffsetAdjuster::Adjustments adjustments;
132 adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
133 adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
134 adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
135 adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
136 OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
137 size_t expected_2[] = {
138 0, kNpos, kNpos, 1, 2, kNpos, kNpos, kNpos, 4, 5, 6, kNpos, kNpos, kNpos,
139 kNpos, kNpos, kNpos, 10, 11, 12, 13, kNpos, kNpos, 14
140 };
141 EXPECT_EQ(offsets.size(), base::size(expected_2));
142 for (size_t i = 0; i < base::size(expected_2); ++i)
143 EXPECT_EQ(expected_2[i], offsets[i]);
144 }
145
146 // 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
147 {
148 std::vector<size_t> offsets;
149 for (size_t t = 0; t <= 17; ++t)
150 offsets.push_back(t);
151 OffsetAdjuster::Adjustments adjustments;
152 adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
153 adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
154 adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
155 adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
156 OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
157 size_t expected_3[] = {
158 0, kNpos, kNpos, 0, 1, kNpos, kNpos, kNpos, 5, 6, 7, 8, kNpos, kNpos, 11,
159 12, kNpos, 12
160 };
161 EXPECT_EQ(offsets.size(), base::size(expected_3));
162 for (size_t i = 0; i < base::size(expected_3); ++i)
163 EXPECT_EQ(expected_3[i], offsets[i]);
164 }
165 }
166
TEST(UTFOffsetStringConversionsTest,UnadjustOffsets)167 TEST(UTFOffsetStringConversionsTest, UnadjustOffsets) {
168 // Imagine we have strings as shown in the following cases where the
169 // X's represent encoded characters.
170 // 1: abcXXXdef ==> abcXdef
171 {
172 std::vector<size_t> offsets;
173 for (size_t t = 0; t <= 7; ++t)
174 offsets.push_back(t);
175 OffsetAdjuster::Adjustments adjustments;
176 adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
177 OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
178 size_t expected_1[] = {0, 1, 2, 3, 6, 7, 8, 9};
179 EXPECT_EQ(offsets.size(), base::size(expected_1));
180 for (size_t i = 0; i < base::size(expected_1); ++i)
181 EXPECT_EQ(expected_1[i], offsets[i]);
182 }
183
184 // 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
185 {
186 std::vector<size_t> offsets;
187 for (size_t t = 0; t <= 14; ++t)
188 offsets.push_back(t);
189 OffsetAdjuster::Adjustments adjustments;
190 adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
191 adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
192 adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
193 adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
194 OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
195 size_t expected_2[] = {
196 0, 3, 4, kNpos, 8, 9, 10, kNpos, kNpos, kNpos, 17, 18, 19, 20, 23
197 };
198 EXPECT_EQ(offsets.size(), base::size(expected_2));
199 for (size_t i = 0; i < base::size(expected_2); ++i)
200 EXPECT_EQ(expected_2[i], offsets[i]);
201 }
202
203 // 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
204 {
205 std::vector<size_t> offsets;
206 for (size_t t = 0; t <= 12; ++t)
207 offsets.push_back(t);
208 OffsetAdjuster::Adjustments adjustments;
209 adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
210 adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
211 adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
212 adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
213 OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
214 size_t expected_3[] = {
215 0, // this could just as easily be 3
216 4, kNpos, kNpos, kNpos, 8, 9, 10, 11, kNpos, kNpos, 14,
217 15 // this could just as easily be 17
218 };
219 EXPECT_EQ(offsets.size(), base::size(expected_3));
220 for (size_t i = 0; i < base::size(expected_3); ++i)
221 EXPECT_EQ(expected_3[i], offsets[i]);
222 }
223 }
224
225 // MergeSequentialAdjustments is used by net/base/escape.{h,cc} and
226 // net/base/net_util.{h,cc}. The two tests EscapeTest.AdjustOffset and
227 // NetUtilTest.FormatUrlWithOffsets test its behavior extensively. This
228 // is simply a short, additional test.
TEST(UTFOffsetStringConversionsTest,MergeSequentialAdjustments)229 TEST(UTFOffsetStringConversionsTest, MergeSequentialAdjustments) {
230 // Pretend the input string is "abcdefghijklmnopqrstuvwxyz".
231
232 // Set up |first_adjustments| to
233 // - remove the leading "a"
234 // - combine the "bc" into one character (call it ".")
235 // - remove the "f"
236 // - remove the "tuv"
237 // The resulting string should be ".deghijklmnopqrswxyz".
238 OffsetAdjuster::Adjustments first_adjustments;
239 first_adjustments.push_back(OffsetAdjuster::Adjustment(0, 1, 0));
240 first_adjustments.push_back(OffsetAdjuster::Adjustment(1, 2, 1));
241 first_adjustments.push_back(OffsetAdjuster::Adjustment(5, 1, 0));
242 first_adjustments.push_back(OffsetAdjuster::Adjustment(19, 3, 0));
243
244 // Set up |adjustments_on_adjusted_string| to
245 // - combine the "." character that replaced "bc" with "d" into one character
246 // (call it "?")
247 // - remove the "egh"
248 // - expand the "i" into two characters (call them "12")
249 // - combine the "jkl" into one character (call it "@")
250 // - expand the "z" into two characters (call it "34")
251 // The resulting string should be "?12@mnopqrswxy34".
252 OffsetAdjuster::Adjustments adjustments_on_adjusted_string;
253 adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
254 0, 2, 1));
255 adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
256 2, 3, 0));
257 adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
258 5, 1, 2));
259 adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
260 6, 3, 1));
261 adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
262 19, 1, 2));
263
264 // Now merge the adjustments and check the results.
265 OffsetAdjuster::MergeSequentialAdjustments(first_adjustments,
266 &adjustments_on_adjusted_string);
267 // The merged adjustments should look like
268 // - combine abcd into "?"
269 // - note: it's also reasonable for the Merge function to instead produce
270 // two adjustments instead of this, one to remove a and another to
271 // combine bcd into "?". This test verifies the current behavior.
272 // - remove efgh
273 // - expand i into "12"
274 // - combine jkl into "@"
275 // - remove tuv
276 // - expand z into "34"
277 ASSERT_EQ(6u, adjustments_on_adjusted_string.size());
278 EXPECT_EQ(0u, adjustments_on_adjusted_string[0].original_offset);
279 EXPECT_EQ(4u, adjustments_on_adjusted_string[0].original_length);
280 EXPECT_EQ(1u, adjustments_on_adjusted_string[0].output_length);
281 EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_offset);
282 EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_length);
283 EXPECT_EQ(0u, adjustments_on_adjusted_string[1].output_length);
284 EXPECT_EQ(8u, adjustments_on_adjusted_string[2].original_offset);
285 EXPECT_EQ(1u, adjustments_on_adjusted_string[2].original_length);
286 EXPECT_EQ(2u, adjustments_on_adjusted_string[2].output_length);
287 EXPECT_EQ(9u, adjustments_on_adjusted_string[3].original_offset);
288 EXPECT_EQ(3u, adjustments_on_adjusted_string[3].original_length);
289 EXPECT_EQ(1u, adjustments_on_adjusted_string[3].output_length);
290 EXPECT_EQ(19u, adjustments_on_adjusted_string[4].original_offset);
291 EXPECT_EQ(3u, adjustments_on_adjusted_string[4].original_length);
292 EXPECT_EQ(0u, adjustments_on_adjusted_string[4].output_length);
293 EXPECT_EQ(25u, adjustments_on_adjusted_string[5].original_offset);
294 EXPECT_EQ(1u, adjustments_on_adjusted_string[5].original_length);
295 EXPECT_EQ(2u, adjustments_on_adjusted_string[5].output_length);
296 }
297
298 } // namespace base
299