1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_INTL_SUPPORT
6 #error Internationalization is expected to be enabled.
7 #endif // V8_INTL_SUPPORT
8
9 #include "src/intl.h"
10
11 #include <memory>
12
13 #include "src/heap/factory.h"
14 #include "src/isolate.h"
15 #include "src/objects-inl.h"
16 #include "src/string-case.h"
17 #include "unicode/basictz.h"
18 #include "unicode/calendar.h"
19 #include "unicode/gregocal.h"
20 #include "unicode/timezone.h"
21 #include "unicode/ustring.h"
22 #include "unicode/uvernum.h"
23 #include "unicode/uversion.h"
24
25 namespace v8 {
26 namespace internal {
27
28 namespace {
IsASCIIUpper(uint16_t ch)29 inline bool IsASCIIUpper(uint16_t ch) { return ch >= 'A' && ch <= 'Z'; }
30
31 const uint8_t kToLower[256] = {
32 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
33 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
34 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
35 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
36 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
37 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
38 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73,
39 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
40 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B,
41 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
42 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
43 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
44 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
45 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
46 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
47 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
48 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB,
49 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xD7,
50 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
51 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
52 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
53 0xFC, 0xFD, 0xFE, 0xFF,
54 };
55
ToLatin1Lower(uint16_t ch)56 inline uint16_t ToLatin1Lower(uint16_t ch) {
57 return static_cast<uint16_t>(kToLower[ch]);
58 }
59
ToASCIIUpper(uint16_t ch)60 inline uint16_t ToASCIIUpper(uint16_t ch) {
61 return ch & ~((ch >= 'a' && ch <= 'z') << 5);
62 }
63
64 // Does not work for U+00DF (sharp-s), U+00B5 (micron), U+00FF.
ToLatin1Upper(uint16_t ch)65 inline uint16_t ToLatin1Upper(uint16_t ch) {
66 DCHECK(ch != 0xDF && ch != 0xB5 && ch != 0xFF);
67 return ch &
68 ~(((ch >= 'a' && ch <= 'z') || (((ch & 0xE0) == 0xE0) && ch != 0xF7))
69 << 5);
70 }
71
72 template <typename Char>
ToUpperFastASCII(const Vector<const Char> & src,Handle<SeqOneByteString> result)73 bool ToUpperFastASCII(const Vector<const Char>& src,
74 Handle<SeqOneByteString> result) {
75 // Do a faster loop for the case where all the characters are ASCII.
76 uint16_t ored = 0;
77 int32_t index = 0;
78 for (auto it = src.begin(); it != src.end(); ++it) {
79 uint16_t ch = static_cast<uint16_t>(*it);
80 ored |= ch;
81 result->SeqOneByteStringSet(index++, ToASCIIUpper(ch));
82 }
83 return !(ored & ~0x7F);
84 }
85
86 const uint16_t sharp_s = 0xDF;
87
88 template <typename Char>
ToUpperOneByte(const Vector<const Char> & src,uint8_t * dest,int * sharp_s_count)89 bool ToUpperOneByte(const Vector<const Char>& src, uint8_t* dest,
90 int* sharp_s_count) {
91 // Still pretty-fast path for the input with non-ASCII Latin-1 characters.
92
93 // There are two special cases.
94 // 1. U+00B5 and U+00FF are mapped to a character beyond U+00FF.
95 // 2. Lower case sharp-S converts to "SS" (two characters)
96 *sharp_s_count = 0;
97 for (auto it = src.begin(); it != src.end(); ++it) {
98 uint16_t ch = static_cast<uint16_t>(*it);
99 if (V8_UNLIKELY(ch == sharp_s)) {
100 ++(*sharp_s_count);
101 continue;
102 }
103 if (V8_UNLIKELY(ch == 0xB5 || ch == 0xFF)) {
104 // Since this upper-cased character does not fit in an 8-bit string, we
105 // need to take the 16-bit path.
106 return false;
107 }
108 *dest++ = ToLatin1Upper(ch);
109 }
110
111 return true;
112 }
113
114 template <typename Char>
ToUpperWithSharpS(const Vector<const Char> & src,Handle<SeqOneByteString> result)115 void ToUpperWithSharpS(const Vector<const Char>& src,
116 Handle<SeqOneByteString> result) {
117 int32_t dest_index = 0;
118 for (auto it = src.begin(); it != src.end(); ++it) {
119 uint16_t ch = static_cast<uint16_t>(*it);
120 if (ch == sharp_s) {
121 result->SeqOneByteStringSet(dest_index++, 'S');
122 result->SeqOneByteStringSet(dest_index++, 'S');
123 } else {
124 result->SeqOneByteStringSet(dest_index++, ToLatin1Upper(ch));
125 }
126 }
127 }
128
FindFirstUpperOrNonAscii(String * s,int length)129 inline int FindFirstUpperOrNonAscii(String* s, int length) {
130 for (int index = 0; index < length; ++index) {
131 uint16_t ch = s->Get(index);
132 if (V8_UNLIKELY(IsASCIIUpper(ch) || ch & ~0x7F)) {
133 return index;
134 }
135 }
136 return length;
137 }
138
139 } // namespace
140
ToLatin1LowerTable()141 const uint8_t* ToLatin1LowerTable() { return &kToLower[0]; }
142
GetUCharBufferFromFlat(const String::FlatContent & flat,std::unique_ptr<uc16[]> * dest,int32_t length)143 const UChar* GetUCharBufferFromFlat(const String::FlatContent& flat,
144 std::unique_ptr<uc16[]>* dest,
145 int32_t length) {
146 DCHECK(flat.IsFlat());
147 if (flat.IsOneByte()) {
148 if (!*dest) {
149 dest->reset(NewArray<uc16>(length));
150 CopyChars(dest->get(), flat.ToOneByteVector().start(), length);
151 }
152 return reinterpret_cast<const UChar*>(dest->get());
153 } else {
154 return reinterpret_cast<const UChar*>(flat.ToUC16Vector().start());
155 }
156 }
157
LocaleConvertCase(Handle<String> s,Isolate * isolate,bool is_to_upper,const char * lang)158 V8_WARN_UNUSED_RESULT Object* LocaleConvertCase(Handle<String> s,
159 Isolate* isolate,
160 bool is_to_upper,
161 const char* lang) {
162 auto case_converter = is_to_upper ? u_strToUpper : u_strToLower;
163 int32_t src_length = s->length();
164 int32_t dest_length = src_length;
165 UErrorCode status;
166 Handle<SeqTwoByteString> result;
167 std::unique_ptr<uc16[]> sap;
168
169 if (dest_length == 0) return isolate->heap()->empty_string();
170
171 // This is not a real loop. It'll be executed only once (no overflow) or
172 // twice (overflow).
173 for (int i = 0; i < 2; ++i) {
174 // Case conversion can increase the string length (e.g. sharp-S => SS) so
175 // that we have to handle RangeError exceptions here.
176 ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
177 isolate, result, isolate->factory()->NewRawTwoByteString(dest_length));
178 DisallowHeapAllocation no_gc;
179 DCHECK(s->IsFlat());
180 String::FlatContent flat = s->GetFlatContent();
181 const UChar* src = GetUCharBufferFromFlat(flat, &sap, src_length);
182 status = U_ZERO_ERROR;
183 dest_length = case_converter(reinterpret_cast<UChar*>(result->GetChars()),
184 dest_length, src, src_length, lang, &status);
185 if (status != U_BUFFER_OVERFLOW_ERROR) break;
186 }
187
188 // In most cases, the output will fill the destination buffer completely
189 // leading to an unterminated string (U_STRING_NOT_TERMINATED_WARNING).
190 // Only in rare cases, it'll be shorter than the destination buffer and
191 // |result| has to be truncated.
192 DCHECK(U_SUCCESS(status));
193 if (V8_LIKELY(status == U_STRING_NOT_TERMINATED_WARNING)) {
194 DCHECK(dest_length == result->length());
195 return *result;
196 }
197 if (U_SUCCESS(status)) {
198 DCHECK(dest_length < result->length());
199 return *Handle<SeqTwoByteString>::cast(
200 SeqString::Truncate(result, dest_length));
201 }
202 return *s;
203 }
204
205 // A stripped-down version of ConvertToLower that can only handle flat one-byte
206 // strings and does not allocate. Note that {src} could still be, e.g., a
207 // one-byte sliced string with a two-byte parent string.
208 // Called from TF builtins.
ConvertOneByteToLower(String * src,String * dst,Isolate * isolate)209 V8_WARN_UNUSED_RESULT Object* ConvertOneByteToLower(String* src, String* dst,
210 Isolate* isolate) {
211 DCHECK_EQ(src->length(), dst->length());
212 DCHECK(src->HasOnlyOneByteChars());
213 DCHECK(src->IsFlat());
214 DCHECK(dst->IsSeqOneByteString());
215
216 DisallowHeapAllocation no_gc;
217
218 const int length = src->length();
219 String::FlatContent src_flat = src->GetFlatContent();
220 uint8_t* dst_data = SeqOneByteString::cast(dst)->GetChars();
221
222 if (src_flat.IsOneByte()) {
223 const uint8_t* src_data = src_flat.ToOneByteVector().start();
224
225 bool has_changed_character = false;
226 int index_to_first_unprocessed =
227 FastAsciiConvert<true>(reinterpret_cast<char*>(dst_data),
228 reinterpret_cast<const char*>(src_data), length,
229 &has_changed_character);
230
231 if (index_to_first_unprocessed == length) {
232 return has_changed_character ? dst : src;
233 }
234
235 // If not ASCII, we keep the result up to index_to_first_unprocessed and
236 // process the rest.
237 for (int index = index_to_first_unprocessed; index < length; ++index) {
238 dst_data[index] = ToLatin1Lower(static_cast<uint16_t>(src_data[index]));
239 }
240 } else {
241 DCHECK(src_flat.IsTwoByte());
242 int index_to_first_unprocessed = FindFirstUpperOrNonAscii(src, length);
243 if (index_to_first_unprocessed == length) return src;
244
245 const uint16_t* src_data = src_flat.ToUC16Vector().start();
246 CopyChars(dst_data, src_data, index_to_first_unprocessed);
247 for (int index = index_to_first_unprocessed; index < length; ++index) {
248 dst_data[index] = ToLatin1Lower(static_cast<uint16_t>(src_data[index]));
249 }
250 }
251
252 return dst;
253 }
254
ConvertToLower(Handle<String> s,Isolate * isolate)255 V8_WARN_UNUSED_RESULT Object* ConvertToLower(Handle<String> s,
256 Isolate* isolate) {
257 if (!s->HasOnlyOneByteChars()) {
258 // Use a slower implementation for strings with characters beyond U+00FF.
259 return LocaleConvertCase(s, isolate, false, "");
260 }
261
262 int length = s->length();
263
264 // We depend here on the invariant that the length of a Latin1
265 // string is invariant under ToLowerCase, and the result always
266 // fits in the Latin1 range in the *root locale*. It does not hold
267 // for ToUpperCase even in the root locale.
268
269 // Scan the string for uppercase and non-ASCII characters for strings
270 // shorter than a machine-word without any memory allocation overhead.
271 // TODO(jshin): Apply this to a longer input by breaking FastAsciiConvert()
272 // to two parts, one for scanning the prefix with no change and the other for
273 // handling ASCII-only characters.
274
275 bool is_short = length < static_cast<int>(sizeof(uintptr_t));
276 if (is_short) {
277 bool is_lower_ascii = FindFirstUpperOrNonAscii(*s, length) == length;
278 if (is_lower_ascii) return *s;
279 }
280
281 Handle<SeqOneByteString> result =
282 isolate->factory()->NewRawOneByteString(length).ToHandleChecked();
283
284 return ConvertOneByteToLower(*s, *result, isolate);
285 }
286
ConvertToUpper(Handle<String> s,Isolate * isolate)287 V8_WARN_UNUSED_RESULT Object* ConvertToUpper(Handle<String> s,
288 Isolate* isolate) {
289 int32_t length = s->length();
290 if (s->HasOnlyOneByteChars() && length > 0) {
291 Handle<SeqOneByteString> result =
292 isolate->factory()->NewRawOneByteString(length).ToHandleChecked();
293
294 DCHECK(s->IsFlat());
295 int sharp_s_count;
296 bool is_result_single_byte;
297 {
298 DisallowHeapAllocation no_gc;
299 String::FlatContent flat = s->GetFlatContent();
300 uint8_t* dest = result->GetChars();
301 if (flat.IsOneByte()) {
302 Vector<const uint8_t> src = flat.ToOneByteVector();
303 bool has_changed_character = false;
304 int index_to_first_unprocessed =
305 FastAsciiConvert<false>(reinterpret_cast<char*>(result->GetChars()),
306 reinterpret_cast<const char*>(src.start()),
307 length, &has_changed_character);
308 if (index_to_first_unprocessed == length)
309 return has_changed_character ? *result : *s;
310 // If not ASCII, we keep the result up to index_to_first_unprocessed and
311 // process the rest.
312 is_result_single_byte =
313 ToUpperOneByte(src.SubVector(index_to_first_unprocessed, length),
314 dest + index_to_first_unprocessed, &sharp_s_count);
315 } else {
316 DCHECK(flat.IsTwoByte());
317 Vector<const uint16_t> src = flat.ToUC16Vector();
318 if (ToUpperFastASCII(src, result)) return *result;
319 is_result_single_byte = ToUpperOneByte(src, dest, &sharp_s_count);
320 }
321 }
322
323 // Go to the full Unicode path if there are characters whose uppercase
324 // is beyond the Latin-1 range (cannot be represented in OneByteString).
325 if (V8_UNLIKELY(!is_result_single_byte)) {
326 return LocaleConvertCase(s, isolate, true, "");
327 }
328
329 if (sharp_s_count == 0) return *result;
330
331 // We have sharp_s_count sharp-s characters, but the result is still
332 // in the Latin-1 range.
333 ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
334 isolate, result,
335 isolate->factory()->NewRawOneByteString(length + sharp_s_count));
336 DisallowHeapAllocation no_gc;
337 String::FlatContent flat = s->GetFlatContent();
338 if (flat.IsOneByte()) {
339 ToUpperWithSharpS(flat.ToOneByteVector(), result);
340 } else {
341 ToUpperWithSharpS(flat.ToUC16Vector(), result);
342 }
343
344 return *result;
345 }
346
347 return LocaleConvertCase(s, isolate, true, "");
348 }
349
ConvertCase(Handle<String> s,bool is_upper,Isolate * isolate)350 V8_WARN_UNUSED_RESULT Object* ConvertCase(Handle<String> s, bool is_upper,
351 Isolate* isolate) {
352 return is_upper ? ConvertToUpper(s, isolate) : ConvertToLower(s, isolate);
353 }
354
ICUTimezoneCache()355 ICUTimezoneCache::ICUTimezoneCache() : timezone_(nullptr) { Clear(); }
356
~ICUTimezoneCache()357 ICUTimezoneCache::~ICUTimezoneCache() { Clear(); }
358
LocalTimezone(double time_ms)359 const char* ICUTimezoneCache::LocalTimezone(double time_ms) {
360 bool is_dst = DaylightSavingsOffset(time_ms) != 0;
361 std::string* name = is_dst ? &dst_timezone_name_ : &timezone_name_;
362 if (name->empty()) {
363 icu::UnicodeString result;
364 GetTimeZone()->getDisplayName(is_dst, icu::TimeZone::LONG, result);
365 result += '\0';
366
367 icu::StringByteSink<std::string> byte_sink(name);
368 result.toUTF8(byte_sink);
369 }
370 DCHECK(!name->empty());
371 return name->c_str();
372 }
373
GetTimeZone()374 icu::TimeZone* ICUTimezoneCache::GetTimeZone() {
375 if (timezone_ == nullptr) {
376 timezone_ = icu::TimeZone::createDefault();
377 }
378 return timezone_;
379 }
380
GetOffsets(double time_ms,bool is_utc,int32_t * raw_offset,int32_t * dst_offset)381 bool ICUTimezoneCache::GetOffsets(double time_ms, bool is_utc,
382 int32_t* raw_offset, int32_t* dst_offset) {
383 UErrorCode status = U_ZERO_ERROR;
384 // TODO(jshin): ICU TimeZone class handles skipped time differently from
385 // Ecma 262 (https://github.com/tc39/ecma262/pull/778) and icu::TimeZone
386 // class does not expose the necessary API. Fixing
387 // http://bugs.icu-project.org/trac/ticket/13268 would make it easy to
388 // implement the proposed spec change. A proposed fix for ICU is
389 // https://chromium-review.googlesource.com/851265 .
390 // In the meantime, use an internal (still public) API of icu::BasicTimeZone.
391 // Once it's accepted by the upstream, get rid of cast. Note that casting
392 // TimeZone to BasicTimeZone is safe because we know that icu::TimeZone used
393 // here is a BasicTimeZone.
394 if (is_utc) {
395 GetTimeZone()->getOffset(time_ms, false, *raw_offset, *dst_offset, status);
396 } else {
397 static_cast<const icu::BasicTimeZone*>(GetTimeZone())
398 ->getOffsetFromLocal(time_ms, icu::BasicTimeZone::kFormer,
399 icu::BasicTimeZone::kFormer, *raw_offset,
400 *dst_offset, status);
401 }
402
403 return U_SUCCESS(status);
404 }
405
DaylightSavingsOffset(double time_ms)406 double ICUTimezoneCache::DaylightSavingsOffset(double time_ms) {
407 int32_t raw_offset, dst_offset;
408 if (!GetOffsets(time_ms, true, &raw_offset, &dst_offset)) return 0;
409 return dst_offset;
410 }
411
LocalTimeOffset(double time_ms,bool is_utc)412 double ICUTimezoneCache::LocalTimeOffset(double time_ms, bool is_utc) {
413 int32_t raw_offset, dst_offset;
414 if (!GetOffsets(time_ms, is_utc, &raw_offset, &dst_offset)) return 0;
415 return raw_offset + dst_offset;
416 }
417
Clear()418 void ICUTimezoneCache::Clear() {
419 delete timezone_;
420 timezone_ = nullptr;
421 timezone_name_.clear();
422 dst_timezone_name_.clear();
423 }
424
425 } // namespace internal
426 } // namespace v8
427