1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/url_formatter/spoof_checks/idn_spoof_checker.h"
6
7 #include "base/check_op.h"
8 #include "base/logging.h"
9 #include "base/no_destructor.h"
10 #include "base/numerics/safe_conversions.h"
11 #include "base/strings/string_piece.h"
12 #include "base/strings/string_split.h"
13 #include "base/strings/string_util.h"
14 #include "base/strings/utf_string_conversions.h"
15 #include "base/threading/thread_local_storage.h"
16 #include "build/build_config.h"
17 #include "net/base/lookup_string_in_fixed_set.h"
18 #include "third_party/icu/source/common/unicode/schriter.h"
19 #include "third_party/icu/source/common/unicode/unistr.h"
20 #include "third_party/icu/source/i18n/unicode/regex.h"
21 #include "third_party/icu/source/i18n/unicode/translit.h"
22 #include "third_party/icu/source/i18n/unicode/uspoof.h"
23
24 namespace url_formatter {
25
26 namespace {
27
BitLength(uint32_t input)28 uint8_t BitLength(uint32_t input) {
29 uint8_t number_of_bits = 0;
30 while (input != 0) {
31 number_of_bits++;
32 input >>= 1;
33 }
34 return number_of_bits;
35 }
36
37 class TopDomainPreloadDecoder : public net::extras::PreloadDecoder {
38 public:
39 using net::extras::PreloadDecoder::PreloadDecoder;
~TopDomainPreloadDecoder()40 ~TopDomainPreloadDecoder() override {}
41
ReadEntry(net::extras::PreloadDecoder::BitReader * reader,const std::string & search,size_t current_search_offset,bool * out_found)42 bool ReadEntry(net::extras::PreloadDecoder::BitReader* reader,
43 const std::string& search,
44 size_t current_search_offset,
45 bool* out_found) override {
46 // Make sure the assigned bit length is enough to encode all SkeletonType
47 // values.
48 DCHECK_EQ(kSkeletonTypeBitLength,
49 BitLength(url_formatter::SkeletonType::kMaxValue));
50
51 bool is_same_skeleton;
52
53 if (!reader->Next(&is_same_skeleton))
54 return false;
55
56 TopDomainEntry top_domain;
57 if (!reader->Next(&top_domain.is_top_500))
58 return false;
59 uint32_t skeletontype_value;
60 if (!reader->Read(kSkeletonTypeBitLength, &skeletontype_value))
61 return false;
62 top_domain.skeleton_type =
63 static_cast<url_formatter::SkeletonType>(skeletontype_value);
64 if (is_same_skeleton) {
65 top_domain.domain = search;
66 } else {
67 bool has_com_suffix = false;
68 if (!reader->Next(&has_com_suffix))
69 return false;
70
71 for (char c;; top_domain.domain += c) {
72 huffman_decoder().Decode(reader, &c);
73 if (c == net::extras::PreloadDecoder::kEndOfTable)
74 break;
75 }
76 if (has_com_suffix)
77 top_domain.domain += ".com";
78 }
79 if (current_search_offset == 0) {
80 *out_found = true;
81 DCHECK(!top_domain.domain.empty());
82 result_ = top_domain;
83 }
84 return true;
85 }
86
matching_top_domain() const87 TopDomainEntry matching_top_domain() const { return result_; }
88
89 private:
90 TopDomainEntry result_;
91 };
92
93 // Stores whole-script-confusable information about a written script.
94 // Used to populate a list of WholeScriptConfusable structs.
95 struct WholeScriptConfusableData {
96 const char* const script_regex;
97 const char* const latin_lookalike_letters;
98 const std::vector<std::string> allowed_tlds;
99 };
100
OnThreadTermination(void * regex_matcher)101 void OnThreadTermination(void* regex_matcher) {
102 delete reinterpret_cast<icu::RegexMatcher*>(regex_matcher);
103 }
104
DangerousPatternTLS()105 base::ThreadLocalStorage::Slot& DangerousPatternTLS() {
106 static base::NoDestructor<base::ThreadLocalStorage::Slot>
107 dangerous_pattern_tls(&OnThreadTermination);
108 return *dangerous_pattern_tls;
109 }
110
111 // Allow middle dot (U+00B7) only on Catalan domains when between two 'l's, to
112 // permit the Catalan character ela geminada to be expressed.
113 // See https://tools.ietf.org/html/rfc5892#appendix-A.3 for details.
HasUnsafeMiddleDot(const icu::UnicodeString & label_string,base::StringPiece top_level_domain)114 bool HasUnsafeMiddleDot(const icu::UnicodeString& label_string,
115 base::StringPiece top_level_domain) {
116 int last_index = 0;
117 while (true) {
118 int index = label_string.indexOf("·", last_index);
119 if (index < 0) {
120 break;
121 }
122 DCHECK_LT(index, label_string.length());
123 if (top_level_domain != "cat") {
124 // Non-Catalan domains cannot contain middle dot.
125 return true;
126 }
127 // Middle dot at the beginning or end.
128 if (index == 0 || index == label_string.length() - 1) {
129 return true;
130 }
131 // Middle dot not surrounded by an 'l'.
132 if (label_string[index - 1] != 'l' || label_string[index + 1] != 'l') {
133 return true;
134 }
135 last_index = index + 1;
136 }
137 return false;
138 }
139
IsSubdomainOf(base::StringPiece16 hostname,const base::string16 & top_domain)140 bool IsSubdomainOf(base::StringPiece16 hostname,
141 const base::string16& top_domain) {
142 DCHECK_NE(hostname, top_domain);
143 DCHECK(!hostname.empty());
144 DCHECK(!top_domain.empty());
145 return base::EndsWith(hostname, base::ASCIIToUTF16(".") + top_domain,
146 base::CompareCase::INSENSITIVE_ASCII);
147 }
148
149 #include "components/url_formatter/spoof_checks/top_domains/domains-trie-inc.cc"
150
151 // All the domains in the above file have 4 or fewer labels.
152 const size_t kNumberOfLabelsToCheck = 4;
153
154 IDNSpoofChecker::HuffmanTrieParams g_trie_params{
155 kTopDomainsHuffmanTree, sizeof(kTopDomainsHuffmanTree), kTopDomainsTrie,
156 kTopDomainsTrieBits, kTopDomainsRootPosition};
157
158 } // namespace
159
WholeScriptConfusable(std::unique_ptr<icu::UnicodeSet> arg_all_letters,std::unique_ptr<icu::UnicodeSet> arg_latin_lookalike_letters,const std::vector<std::string> & arg_allowed_tlds)160 IDNSpoofChecker::WholeScriptConfusable::WholeScriptConfusable(
161 std::unique_ptr<icu::UnicodeSet> arg_all_letters,
162 std::unique_ptr<icu::UnicodeSet> arg_latin_lookalike_letters,
163 const std::vector<std::string>& arg_allowed_tlds)
164 : all_letters(std::move(arg_all_letters)),
165 latin_lookalike_letters(std::move(arg_latin_lookalike_letters)),
166 allowed_tlds(arg_allowed_tlds) {}
167
168 IDNSpoofChecker::WholeScriptConfusable::~WholeScriptConfusable() = default;
169
IDNSpoofChecker()170 IDNSpoofChecker::IDNSpoofChecker() {
171 UErrorCode status = U_ZERO_ERROR;
172 checker_ = uspoof_open(&status);
173 if (U_FAILURE(status)) {
174 checker_ = nullptr;
175 return;
176 }
177
178 // At this point, USpoofChecker has all the checks enabled except
179 // for USPOOF_CHAR_LIMIT (USPOOF_{RESTRICTION_LEVEL, INVISIBLE,
180 // MIXED_SCRIPT_CONFUSABLE, WHOLE_SCRIPT_CONFUSABLE, MIXED_NUMBERS, ANY_CASE})
181 // This default configuration is adjusted below as necessary.
182
183 // Set the restriction level to high. It allows mixing Latin with one logical
184 // CJK script (+ COMMON and INHERITED), but does not allow any other script
185 // mixing (e.g. Latin + Cyrillic, Latin + Armenian, Cyrillic + Greek). Note
186 // that each of {Han + Bopomofo} for Chinese, {Hiragana, Katakana, Han} for
187 // Japanese, and {Hangul, Han} for Korean is treated as a single logical
188 // script.
189 // See http://www.unicode.org/reports/tr39/#Restriction_Level_Detection
190 uspoof_setRestrictionLevel(checker_, USPOOF_HIGHLY_RESTRICTIVE);
191
192 // Sets allowed characters in IDN labels and turns on USPOOF_CHAR_LIMIT.
193 SetAllowedUnicodeSet(&status);
194
195 // Enable the return of auxillary (non-error) information.
196 // We used to disable WHOLE_SCRIPT_CONFUSABLE check explicitly, but as of
197 // ICU 58.1, WSC is a no-op in a single string check API.
198 int32_t checks = uspoof_getChecks(checker_, &status) | USPOOF_AUX_INFO;
199 uspoof_setChecks(checker_, checks, &status);
200
201 // Four characters handled differently by IDNA 2003 and IDNA 2008. UTS46
202 // transitional processing treats them as IDNA 2003 does; maps U+00DF and
203 // U+03C2 and drops U+200[CD].
204 deviation_characters_ = icu::UnicodeSet(
205 UNICODE_STRING_SIMPLE("[\\u00df\\u03c2\\u200c\\u200d]"), status);
206 deviation_characters_.freeze();
207
208 // Latin letters outside ASCII. 'Script_Extensions=Latin' is not necessary
209 // because additional characters pulled in with scx=Latn are not included in
210 // the allowed set.
211 non_ascii_latin_letters_ =
212 icu::UnicodeSet(UNICODE_STRING_SIMPLE("[[:Latin:] - [a-zA-Z]]"), status);
213 non_ascii_latin_letters_.freeze();
214
215 // The following two sets are parts of |dangerous_patterns_|.
216 kana_letters_exceptions_ = icu::UnicodeSet(
217 UNICODE_STRING_SIMPLE("[\\u3078-\\u307a\\u30d8-\\u30da\\u30fb-\\u30fe]"),
218 status);
219 kana_letters_exceptions_.freeze();
220 combining_diacritics_exceptions_ =
221 icu::UnicodeSet(UNICODE_STRING_SIMPLE("[\\u0300-\\u0339]"), status);
222 combining_diacritics_exceptions_.freeze();
223
224 const WholeScriptConfusableData kWholeScriptConfusables[] = {
225 {// Armenian
226 "[[:Armn:]]",
227 "[ագզէլհյոսւօՙ]",
228 {"am"}},
229 {// Cyrillic
230 "[[:Cyrl:]]",
231 "[аысԁеԍһіюјӏорԗԛѕԝхуъЬҽпгѵѡ]",
232 // TLDs containing most of the Cyrillic domains.
233 {"bg", "by", "kz", "pyc", "ru", "su", "ua", "uz"}},
234 {// Ethiopic (Ge'ez). Variants of these characters such as ሁ and ሡ could
235 // arguably be added to this list. However, we are only restricting
236 // the more obvious characters to keep the list short and to reduce the
237 // probability of false positives.
238 // Potential set: [ሀሁሃሠሡሰሱሲስበቡቢተቱቲታነከኩኪካኬክዐዑዕዖዘዙዚዛዝዞጠጡጢጣጦፐፒꬁꬂꬅ]
239 "[[:Ethi:]]",
240 "[ሀሠሰስበነተከዐዕዘጠፐꬅ]",
241 {"er", "et"}},
242 {// Georgian
243 "[[:Geor:]]",
244 "[იოყძხჽჿ]",
245 {"ge"}},
246 {// Greek
247 "[[:Grek:]]",
248 // This ignores variants such as ά, έ, ή, ί.
249 "[αικνρυωηοτ]",
250 {"gr"}},
251 {// Hebrew
252 "[[:Hebr:]]",
253 "[דוחיןסװײ׳ﬦ]",
254 // TLDs containing most of the Hebrew domains.
255 {"il"}},
256 // Indic scripts in the recommended set. No ccTLDs are allowlisted.
257 {// Bengali
258 "[[:Beng:]]", "[০৭]"},
259 {// Devanagari
260 "[[:Deva:]]", "[ऽ०ॱ]"},
261 {// Gujarati
262 "[[:Gujr:]]", "[ડટ૦૧]"},
263 {// Gurmukhi
264 "[[:Guru:]]", "[੦੧]"},
265 {// Kannada
266 "[[:Knda:]]", "[ಽ೦೧]"},
267 {// Malayalam
268 "[[:Mlym:]]", "[ടഠധനറ൦]"},
269 {// Oriya
270 "[[:Orya:]]", "[ଠ୦୮]"},
271 {// Tamil
272 "[[:Taml:]]", "[டப௦]"},
273 {// Telugu
274 "[[:Telu:]]", "[౦౧]"},
275 {// Myanmar. Shan digits (႐႑႕႖႗) are already blocked from mixing with
276 // other Myanmar characters. However, they can still be used to form
277 // WSC spoofs, so they are included here (they are encoded because macOS
278 // doesn't display them properly).
279 // U+104A (၊) and U+U+104A(။) are excluded as they are signs and are
280 // blocked.
281 "[[:Mymr:]]",
282 "[ခဂငထပဝ၀၂ၔၜ\u1090\u1091\u1095\u1096\u1097]",
283 {"mm"}},
284 {// Thai
285 "[[:Thai:]]",
286 // Some of the Thai characters are only confusable on Linux, so the Linux
287 // set is larger than other platforms. Ideally we don't want to have any
288 // differences between platforms, but doing so is unavoidable here as
289 // these characters look significantly different between Linux and other
290 // platforms.
291 // The ideal fix would be to change the omnibox font used for Thai. In
292 // that case, the Linux-only list should be revisited and potentially
293 // removed.
294 #if defined(OS_LINUX) || defined(OS_CHROMEOS) || defined(OS_BSD)
295 "[ทนบพรหเแ๐ดลปฟม]",
296 #else
297 "[บพเแ๐]",
298 #endif
299 {"th"}},
300 };
301 for (const WholeScriptConfusableData& data : kWholeScriptConfusables) {
302 auto all_letters = std::make_unique<icu::UnicodeSet>(
303 icu::UnicodeString::fromUTF8(data.script_regex), status);
304 DCHECK(U_SUCCESS(status));
305 auto latin_lookalikes = std::make_unique<icu::UnicodeSet>(
306 icu::UnicodeString::fromUTF8(data.latin_lookalike_letters), status);
307 DCHECK(U_SUCCESS(status));
308 auto script = std::make_unique<WholeScriptConfusable>(
309 std::move(all_letters), std::move(latin_lookalikes), data.allowed_tlds);
310 wholescriptconfusables_.push_back(std::move(script));
311 }
312
313 // These characters are, or look like, digits. A domain label entirely made of
314 // digit-lookalikes or digits is blocked.
315 digits_ = icu::UnicodeSet(UNICODE_STRING_SIMPLE("[0-9]"), status);
316 digits_.freeze();
317 digit_lookalikes_ = icu::UnicodeSet(
318 icu::UnicodeString::fromUTF8("[θ२২੨੨૨೩೭շзҙӡउওਤ੩૩౩ဒვპੜ੫丩ㄐճ৪੪୫૭୨౨]"),
319 status);
320 digit_lookalikes_.freeze();
321
322 DCHECK(U_SUCCESS(status));
323 // This set is used to determine whether or not to apply a slow
324 // transliteration to remove diacritics to a given hostname before the
325 // confusable skeleton calculation for comparison with top domain names. If
326 // it has any character outside the set, the expensive step will be skipped
327 // because it cannot match any of top domain names.
328 // The last ([\u0300-\u0339] is a shorthand for "[:Identifier_Status=Allowed:]
329 // & [:Script_Extensions=Inherited:] - [\\u200C\\u200D]". The latter is a
330 // subset of the former but it does not matter because hostnames with
331 // characters outside the latter set would be rejected in an earlier step.
332 lgc_letters_n_ascii_ = icu::UnicodeSet(
333 UNICODE_STRING_SIMPLE("[[:Latin:][:Greek:][:Cyrillic:][0-9\\u002e_"
334 "\\u002d][\\u0300-\\u0339]]"),
335 status);
336 lgc_letters_n_ascii_.freeze();
337
338 // Latin small letter thorn ("þ", U+00FE) can be used to spoof both b and p.
339 // It's used in modern Icelandic orthography, so allow it for the Icelandic
340 // ccTLD (.is) but block in any other TLD. Also block Latin small letter eth
341 // ("ð", U+00F0) which can be used to spoof the letter o.
342 icelandic_characters_ =
343 icu::UnicodeSet(UNICODE_STRING_SIMPLE("[\\u00fe\\u00f0]"), status);
344 icelandic_characters_.freeze();
345
346 DCHECK(U_SUCCESS(status))
347 << "Spoofchecker initalization failed due to an error: "
348 << u_errorName(status);
349
350 skeleton_generator_ = std::make_unique<SkeletonGenerator>(checker_);
351 }
352
~IDNSpoofChecker()353 IDNSpoofChecker::~IDNSpoofChecker() {
354 uspoof_close(checker_);
355 }
356
SafeToDisplayAsUnicode(base::StringPiece16 label,base::StringPiece top_level_domain,base::StringPiece16 top_level_domain_unicode)357 IDNSpoofChecker::Result IDNSpoofChecker::SafeToDisplayAsUnicode(
358 base::StringPiece16 label,
359 base::StringPiece top_level_domain,
360 base::StringPiece16 top_level_domain_unicode) {
361 UErrorCode status = U_ZERO_ERROR;
362 int32_t result =
363 uspoof_check(checker_, label.data(),
364 base::checked_cast<int32_t>(label.size()), nullptr, &status);
365 // If uspoof_check fails (due to library failure), or if any of the checks
366 // fail, treat the IDN as unsafe.
367 if (U_FAILURE(status) || (result & USPOOF_ALL_CHECKS)) {
368 return Result::kICUSpoofChecks;
369 }
370
371 icu::UnicodeString label_string(false /* isTerminated */, label.data(),
372 base::checked_cast<int32_t>(label.size()));
373
374 // A punycode label with 'xn--' prefix is not subject to the URL
375 // canonicalization and is stored as it is in GURL. If it encodes a deviation
376 // character (UTS 46; e.g. U+00DF/sharp-s), it should be still shown in
377 // punycode instead of Unicode. Without this check, xn--fu-hia for
378 // 'fu<sharp-s>' would be converted to 'fu<sharp-s>' for display because
379 // "UTS 46 section 4 Processing step 4" applies validity criteria for
380 // non-transitional processing (i.e. do not map deviation characters) to any
381 // punycode labels regardless of whether transitional or non-transitional is
382 // chosen. On the other hand, 'fu<sharp-s>' typed or copy and pasted
383 // as Unicode would be canonicalized to 'fuss' by GURL and is displayed as
384 // such. See http://crbug.com/595263 .
385 if (deviation_characters_.containsSome(label_string))
386 return Result::kDeviationCharacters;
387
388 // Disallow Icelandic confusables for domains outside Iceland's ccTLD (.is).
389 if (label_string.length() > 1 && top_level_domain != "is" &&
390 icelandic_characters_.containsSome(label_string))
391 return Result::kTLDSpecificCharacters;
392
393 // Disallow Latin Schwa (U+0259) for domains outside Azerbaijan's ccTLD (.az).
394 if (label_string.length() > 1 && top_level_domain != "az" &&
395 label_string.indexOf("ə") != -1)
396 return Result::kTLDSpecificCharacters;
397
398 // Disallow middle dot (U+00B7) when unsafe.
399 if (HasUnsafeMiddleDot(label_string, top_level_domain)) {
400 return Result::kUnsafeMiddleDot;
401 }
402
403 // If there's no script mixing, the input is regarded as safe without any
404 // extra check unless it falls into one of three categories:
405 // - contains Kana letter exceptions
406 // - the TLD is ASCII and the input is made entirely of whole script
407 // characters confusable that look like Latin letters.
408 // - it has combining diacritic marks.
409 // Note that the following combinations of scripts are treated as a 'logical'
410 // single script.
411 // - Chinese: Han, Bopomofo, Common
412 // - Japanese: Han, Hiragana, Katakana, Common
413 // - Korean: Hangul, Han, Common
414 result &= USPOOF_RESTRICTION_LEVEL_MASK;
415 if (result == USPOOF_ASCII)
416 return Result::kSafe;
417
418 if (result == USPOOF_SINGLE_SCRIPT_RESTRICTIVE &&
419 kana_letters_exceptions_.containsNone(label_string) &&
420 combining_diacritics_exceptions_.containsNone(label_string)) {
421 for (auto const& script : wholescriptconfusables_) {
422 if (IsLabelWholeScriptConfusableForScript(*script, label_string) &&
423 !IsWholeScriptConfusableAllowedForTLD(*script, top_level_domain,
424 top_level_domain_unicode)) {
425 return Result::kWholeScriptConfusable;
426 }
427 }
428 // Disallow domains that contain only numbers and number-spoofs.
429 if (IsDigitLookalike(label_string))
430 return Result::kDigitLookalikes;
431
432 return Result::kSafe;
433 }
434
435 // Disallow domains that contain only numbers and number-spoofs.
436 if (IsDigitLookalike(label_string))
437 return Result::kDigitLookalikes;
438
439 // Additional checks for |label| with multiple scripts, one of which is Latin.
440 // Disallow non-ASCII Latin letters to mix with a non-Latin script.
441 // Note that the non-ASCII Latin check should not be applied when the entire
442 // label is made of Latin. Checking with lgc_letters set here should be fine
443 // because script mixing of LGC is already rejected.
444 if (non_ascii_latin_letters_.containsSome(label_string) &&
445 !lgc_letters_n_ascii_.containsAll(label_string))
446 return Result::kNonAsciiLatinCharMixedWithNonLatin;
447
448 icu::RegexMatcher* dangerous_pattern =
449 reinterpret_cast<icu::RegexMatcher*>(DangerousPatternTLS().Get());
450 if (!dangerous_pattern) {
451 // The parentheses in the below strings belong to the raw string sequence
452 // R"(...)". They are NOT part of the regular expression. Each sub
453 // regex is OR'ed with the | operator.
454 dangerous_pattern = new icu::RegexMatcher(
455 icu::UnicodeString(
456 // Disallow the following as they may be mistaken for slashes when
457 // they're surrounded by non-Japanese scripts (i.e. has non-Katakana
458 // Hiragana or Han scripts on both sides):
459 // "ノ" (Katakana no, U+30ce), "ソ" (Katakana so, U+30bd),
460 // "ゾ" (Katakana zo, U+30be), "ン" (Katakana n, U+30f3),
461 // "丶" (CJK unified ideograph, U+4E36),
462 // "乀" (CJK unified ideograph, U+4E40),
463 // "乁" (CJK unified ideograph, U+4E41),
464 // "丿" (CJK unified ideograph, U+4E3F).
465 // If {no, so, zo, n} next to a
466 // non-Japanese script on either side is disallowed, legitimate
467 // cases like '{vitamin in Katakana}b6' are blocked. Note that
468 // trying to block those characters when used alone as a label is
469 // futile because those cases would not reach here. Also disallow
470 // what used to be blocked by mixed-script-confusable (MSC)
471 // detection. ICU 58 does not detect MSC any more for a single input
472 // string. See http://bugs.icu-project.org/trac/ticket/12823 .
473 // TODO(jshin): adjust the pattern once the above ICU bug is fixed.
474 R"([^\p{scx=kana}\p{scx=hira}\p{scx=hani}])"
475 R"([\u30ce\u30f3\u30bd\u30be\u4e36\u4e40\u4e41\u4e3f])"
476 R"([^\p{scx=kana}\p{scx=hira}\p{scx=hani}]|)"
477
478 // Disallow U+30FD (Katakana iteration mark) and U+30FE (Katakana
479 // voiced iteration mark) unless they're preceded by a Katakana.
480 R"([^\p{scx=kana}][\u30fd\u30fe]|^[\u30fd\u30fe]|)"
481
482 // Disallow three Hiragana letters (U+307[8-A]) or Katakana letters
483 // (U+30D[8-A]) that look exactly like each other when they're used
484 // in a label otherwise entirely in Katakana or Hiragana.
485 R"(^[\p{scx=kana}]+[\u3078-\u307a][\p{scx=kana}]+$|)"
486 R"(^[\p{scx=hira}]+[\u30d8-\u30da][\p{scx=hira}]+$|)"
487
488 // Disallow U+30FB (Katakana Middle Dot) and U+30FC (Hiragana-
489 // Katakana Prolonged Sound) used out-of-context.
490 R"([^\p{scx=kana}\p{scx=hira}]\u30fc|^\u30fc|)"
491 R"([a-z]\u30fb|\u30fb[a-z]|)"
492
493 // Disallow these CJK ideographs if they are next to non-CJK
494 // characters. These characters can be used to spoof Latin
495 // characters or punctuation marks:
496 // U+4E00 (一), U+3127 (ㄧ), U+4E28 (丨), U+4E5B (乛), U+4E03 (七),
497 // U+4E05 (丅), U+5341 (十), U+3007 (〇), U+3112 (ㄒ), U+311A (ㄚ),
498 // U+311F (ㄟ), U+3128 (ㄨ), U+3129 (ㄩ), U+3108 (ㄈ), U+31BA (ㆺ),
499 // U+31B3 (ㆳ), U+5DE5 (工), U+31B2 (ㆲ), U+8BA0 (讠), U+4E01 (丁)
500 // These characters are already blocked:
501 // U+2F00 (⼀) (normalized to U+4E00), U+3192 (㆒), U+2F02 (⼂),
502 // U+2F17 (⼗) and U+3038 (〸) (both normalized to U+5341 (十)).
503 // Check if there is non-{Hiragana, Katagana, Han, Bopomofo} on the
504 // left.
505 R"([^\p{scx=kana}\p{scx=hira}\p{scx=hani}\p{scx=bopo}])"
506 R"([\u4e00\u3127\u4e28\u4e5b\u4e03\u4e05\u5341\u3007\u3112)"
507 R"(\u311a\u311f\u3128\u3129\u3108\u31ba\u31b3\u5dE5)"
508 R"(\u31b2\u8ba0\u4e01]|)"
509 // Check if there is non-{Hiragana, Katagana, Han, Bopomofo} on the
510 // right.
511 R"([\u4e00\u3127\u4e28\u4e5b\u4e03\u4e05\u5341\u3007\u3112)"
512 R"(\u311a\u311f\u3128\u3129\u3108\u31ba\u31b3\u5de5)"
513 R"(\u31b2\u8ba0\u4e01])"
514 R"([^\p{scx=kana}\p{scx=hira}\p{scx=hani}\p{scx=bopo}]|)"
515
516 // Disallow combining diacritical mark (U+0300-U+0339) after a
517 // non-LGC character. Other combining diacritical marks are not in
518 // the allowed character set.
519 R"([^\p{scx=latn}\p{scx=grek}\p{scx=cyrl}][\u0300-\u0339]|)"
520
521 // Disallow dotless i (U+0131) followed by a combining mark.
522 R"(\u0131[\u0300-\u0339]|)"
523
524 // Disallow combining Kana voiced sound marks.
525 R"(\u3099|\u309A|)"
526
527 // Disallow U+0307 (dot above) after 'i', 'j', 'l' or dotless i
528 // (U+0131). Dotless j (U+0237) is not in the allowed set to begin
529 // with.
530 R"([ijl]\u0307)",
531 -1, US_INV),
532 0, status);
533 DangerousPatternTLS().Set(dangerous_pattern);
534 }
535 dangerous_pattern->reset(label_string);
536 if (dangerous_pattern->find()) {
537 return Result::kDangerousPattern;
538 }
539 return Result::kSafe;
540 }
541
GetSimilarTopDomain(base::StringPiece16 hostname)542 TopDomainEntry IDNSpoofChecker::GetSimilarTopDomain(
543 base::StringPiece16 hostname) {
544 DCHECK(!hostname.empty());
545 for (const std::string& skeleton : GetSkeletons(hostname)) {
546 DCHECK(!skeleton.empty());
547 TopDomainEntry matching_top_domain = LookupSkeletonInTopDomains(skeleton);
548 if (!matching_top_domain.domain.empty()) {
549 const base::string16 top_domain =
550 base::UTF8ToUTF16(matching_top_domain.domain);
551 // Return an empty result if hostname is a top domain itself, or a
552 // subdomain of top domain. This prevents subdomains of top domains from
553 // being marked as spoofs. Without this check, éxample.blogspot.com
554 // would return blogspot.com and treated as a top domain lookalike.
555 if (hostname == top_domain || IsSubdomainOf(hostname, top_domain)) {
556 return TopDomainEntry();
557 }
558 return matching_top_domain;
559 }
560 }
561 return TopDomainEntry();
562 }
563
GetSkeletons(base::StringPiece16 hostname) const564 Skeletons IDNSpoofChecker::GetSkeletons(base::StringPiece16 hostname) const {
565 return skeleton_generator_->GetSkeletons(hostname);
566 }
567
LookupSkeletonInTopDomains(const std::string & skeleton,SkeletonType skeleton_type)568 TopDomainEntry IDNSpoofChecker::LookupSkeletonInTopDomains(
569 const std::string& skeleton,
570 SkeletonType skeleton_type) {
571 DCHECK(!skeleton.empty());
572 // There are no other guarantees about a skeleton string such as not including
573 // a dot. Skeleton of certain characters are dots (e.g. "۰" (U+06F0)).
574 TopDomainPreloadDecoder preload_decoder(
575 g_trie_params.huffman_tree, g_trie_params.huffman_tree_size,
576 g_trie_params.trie, g_trie_params.trie_bits,
577 g_trie_params.trie_root_position);
578 auto labels = base::SplitStringPiece(skeleton, ".", base::KEEP_WHITESPACE,
579 base::SPLIT_WANT_ALL);
580
581 if (labels.size() > kNumberOfLabelsToCheck) {
582 labels.erase(labels.begin(),
583 labels.begin() + labels.size() - kNumberOfLabelsToCheck);
584 }
585
586 while (labels.size() > 0) {
587 // A full skeleton needs at least two labels to match.
588 if (labels.size() == 1 && skeleton_type == SkeletonType::kFull) {
589 break;
590 }
591 std::string partial_skeleton = base::JoinString(labels, ".");
592 bool match = false;
593 bool decoded = preload_decoder.Decode(partial_skeleton, &match);
594 DCHECK(decoded);
595 if (!decoded)
596 return TopDomainEntry();
597
598 if (match)
599 return preload_decoder.matching_top_domain();
600
601 labels.erase(labels.begin());
602 }
603 return TopDomainEntry();
604 }
605
SetAllowedUnicodeSet(UErrorCode * status)606 void IDNSpoofChecker::SetAllowedUnicodeSet(UErrorCode* status) {
607 if (U_FAILURE(*status))
608 return;
609
610 // The recommended set is a set of characters for identifiers in a
611 // security-sensitive environment taken from UTR 39
612 // (http://unicode.org/reports/tr39/) and
613 // http://www.unicode.org/Public/security/latest/xidmodifications.txt .
614 // The inclusion set comes from "Candidate Characters for Inclusion
615 // in idenfiers" of UTR 31 (http://www.unicode.org/reports/tr31). The list
616 // may change over the time and will be updated whenever the version of ICU
617 // used in Chromium is updated.
618 const icu::UnicodeSet* recommended_set =
619 uspoof_getRecommendedUnicodeSet(status);
620 icu::UnicodeSet allowed_set;
621 allowed_set.addAll(*recommended_set);
622 const icu::UnicodeSet* inclusion_set = uspoof_getInclusionUnicodeSet(status);
623 allowed_set.addAll(*inclusion_set);
624
625 // The sections below refer to Mozilla's IDN blacklist:
626 // http://kb.mozillazine.org/Network.IDN.blacklist_chars
627 //
628 // U+0338 (Combining Long Solidus Overlay) is included in the recommended set,
629 // but is blacklisted by Mozilla. It is dropped because it can look like a
630 // slash when rendered with a broken font.
631 allowed_set.remove(0x338u);
632 // U+05F4 (Hebrew Punctuation Gershayim) is in the inclusion set, but is
633 // blacklisted by Mozilla. We keep it, even though it can look like a double
634 // quotation mark. Using it in Hebrew should be safe. When used with a
635 // non-Hebrew script, it'd be filtered by other checks in place.
636
637 // The following 5 characters are disallowed because they're in NV8 (invalid
638 // in IDNA 2008).
639 allowed_set.remove(0x58au); // Armenian Hyphen
640 // U+2010 (Hyphen) is in the inclusion set, but we drop it because it can be
641 // confused with an ASCII U+002D (Hyphen-Minus).
642 allowed_set.remove(0x2010u);
643 // U+2019 is hard to notice when sitting next to a regular character.
644 allowed_set.remove(0x2019u); // Right Single Quotation Mark
645 // U+2027 (Hyphenation Point) is in the inclusion set, but is blacklisted by
646 // Mozilla. It is dropped, as it can be confused with U+30FB (Katakana Middle
647 // Dot).
648 allowed_set.remove(0x2027u);
649 allowed_set.remove(0x30a0u); // Katakana-Hiragana Double Hyphen
650
651 // Block {Single,double}-quotation-mark look-alikes.
652 allowed_set.remove(0x2bbu); // Modifier Letter Turned Comma
653 allowed_set.remove(0x2bcu); // Modifier Letter Apostrophe
654
655 // Block modifier letter voicing.
656 allowed_set.remove(0x2ecu);
657
658 // Block historic character Latin Kra (also blocked by Mozilla).
659 allowed_set.remove(0x0138);
660
661 // No need to block U+144A (Canadian Syllabics West-Cree P) separately
662 // because it's blocked from mixing with other scripts including Latin.
663
664 #if defined(OS_APPLE)
665 // The following characters are reported as present in the default macOS
666 // system UI font, but they render as blank. Remove them from the allowed
667 // set to prevent spoofing until the font issue is resolved.
668
669 // Arabic letter KASHMIRI YEH. Not used in Arabic and Persian.
670 allowed_set.remove(0x0620u);
671
672 // Tibetan characters used for transliteration of ancient texts:
673 allowed_set.remove(0x0F8Cu);
674 allowed_set.remove(0x0F8Du);
675 allowed_set.remove(0x0F8Eu);
676 allowed_set.remove(0x0F8Fu);
677 #endif
678
679 // Disallow extremely rarely used LGC character blocks.
680 // Cyllic Ext A is not in the allowed set. Neither are Latin Ext-{C,E}.
681 allowed_set.remove(0x01CDu, 0x01DCu); // Latin Ext B; Pinyin
682 allowed_set.remove(0x1C80u, 0x1C8Fu); // Cyrillic Extended-C
683 allowed_set.remove(0x1E00u, 0x1E9Bu); // Latin Extended Additional
684 allowed_set.remove(0x1F00u, 0x1FFFu); // Greek Extended
685 allowed_set.remove(0xA640u, 0xA69Fu); // Cyrillic Extended-B
686 allowed_set.remove(0xA720u, 0xA7FFu); // Latin Extended-D
687
688 uspoof_setAllowedUnicodeSet(checker_, &allowed_set, status);
689 }
690
IsDigitLookalike(const icu::UnicodeString & label)691 bool IDNSpoofChecker::IsDigitLookalike(const icu::UnicodeString& label) {
692 bool has_lookalike_char = false;
693 icu::StringCharacterIterator it(label);
694 for (it.setToStart(); it.hasNext();) {
695 const UChar32 c = it.next32PostInc();
696 if (digits_.contains(c)) {
697 continue;
698 }
699 if (digit_lookalikes_.contains(c)) {
700 has_lookalike_char = true;
701 continue;
702 }
703 return false;
704 }
705 return has_lookalike_char;
706 }
707
708 // static
IsWholeScriptConfusableAllowedForTLD(const WholeScriptConfusable & script,base::StringPiece tld,base::StringPiece16 tld_unicode)709 bool IDNSpoofChecker::IsWholeScriptConfusableAllowedForTLD(
710 const WholeScriptConfusable& script,
711 base::StringPiece tld,
712 base::StringPiece16 tld_unicode) {
713 icu::UnicodeString tld_string(
714 false /* isTerminated */, tld_unicode.data(),
715 base::checked_cast<int32_t>(tld_unicode.size()));
716 // Allow if the TLD contains any letter from the script, in which case it's
717 // likely to be a TLD in that script.
718 if (script.all_letters->containsSome(tld_string)) {
719 return true;
720 }
721 return base::Contains(script.allowed_tlds, tld);
722 }
723
724 // static
IsLabelWholeScriptConfusableForScript(const WholeScriptConfusable & script,const icu::UnicodeString & label)725 bool IDNSpoofChecker::IsLabelWholeScriptConfusableForScript(
726 const WholeScriptConfusable& script,
727 const icu::UnicodeString& label) {
728 // Collect all the letters of |label| using |script.all_letters| and see if
729 // they're a subset of |script.latin_lookalike_letters|.
730 // An alternative approach is to include [0-9] and [_-] in script.all_letters
731 // and checking if it contains all letters of |label|. However, this would not
732 // work if a label has non-letters outside ASCII.
733 icu::UnicodeSet label_characters_belonging_to_script;
734 icu::StringCharacterIterator it(label);
735 for (it.setToStart(); it.hasNext();) {
736 const UChar32 c = it.next32PostInc();
737 if (script.all_letters->contains(c))
738 label_characters_belonging_to_script.add(c);
739 }
740 return !label_characters_belonging_to_script.isEmpty() &&
741 script.latin_lookalike_letters->containsAll(
742 label_characters_belonging_to_script);
743 }
744
745 // static
SetTrieParamsForTesting(const HuffmanTrieParams & trie_params)746 void IDNSpoofChecker::SetTrieParamsForTesting(
747 const HuffmanTrieParams& trie_params) {
748 g_trie_params = trie_params;
749 }
750
751 // static
RestoreTrieParamsForTesting()752 void IDNSpoofChecker::RestoreTrieParamsForTesting() {
753 g_trie_params = HuffmanTrieParams{
754 kTopDomainsHuffmanTree, sizeof(kTopDomainsHuffmanTree), kTopDomainsTrie,
755 kTopDomainsTrieBits, kTopDomainsRootPosition};
756 }
757
758 } // namespace url_formatter
759