1 // Copyright 2007-2010 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_STRINGS_UNICODE_INL_H_
6 #define V8_STRINGS_UNICODE_INL_H_
7
8 #include "src/base/logging.h"
9 #include "src/strings/unicode.h"
10 #include "src/utils/utils.h"
11
12 namespace unibrow {
13
14 #ifndef V8_INTL_SUPPORT
15 template <class T, int s>
get(uchar code_point)16 bool Predicate<T, s>::get(uchar code_point) {
17 CacheEntry entry = entries_[code_point & kMask];
18 if (entry.code_point() == code_point) return entry.value();
19 return CalculateValue(code_point);
20 }
21
22 template <class T, int s>
CalculateValue(uchar code_point)23 bool Predicate<T, s>::CalculateValue(uchar code_point) {
24 bool result = T::Is(code_point);
25 entries_[code_point & kMask] = CacheEntry(code_point, result);
26 return result;
27 }
28
29 template <class T, int s>
get(uchar c,uchar n,uchar * result)30 int Mapping<T, s>::get(uchar c, uchar n, uchar* result) {
31 CacheEntry entry = entries_[c & kMask];
32 if (entry.code_point_ == c) {
33 if (entry.offset_ == 0) {
34 return 0;
35 } else {
36 result[0] = c + entry.offset_;
37 return 1;
38 }
39 } else {
40 return CalculateValue(c, n, result);
41 }
42 }
43
44 template <class T, int s>
CalculateValue(uchar c,uchar n,uchar * result)45 int Mapping<T, s>::CalculateValue(uchar c, uchar n, uchar* result) {
46 bool allow_caching = true;
47 int length = T::Convert(c, n, result, &allow_caching);
48 if (allow_caching) {
49 if (length == 1) {
50 entries_[c & kMask] = CacheEntry(c, result[0] - c);
51 return 1;
52 } else {
53 entries_[c & kMask] = CacheEntry(c, 0);
54 return 0;
55 }
56 } else {
57 return length;
58 }
59 }
60 #endif // !V8_INTL_SUPPORT
61
62 // Decodes UTF-8 bytes incrementally, allowing the decoding of bytes as they
63 // stream in. This **must** be followed by a call to ValueOfIncrementalFinish
64 // when the stream is complete, to ensure incomplete sequences are handled.
ValueOfIncremental(const byte ** cursor,State * state,Utf8IncrementalBuffer * buffer)65 uchar Utf8::ValueOfIncremental(const byte** cursor, State* state,
66 Utf8IncrementalBuffer* buffer) {
67 DCHECK_NOT_NULL(buffer);
68 State old_state = *state;
69 byte next = **cursor;
70 *cursor += 1;
71
72 if (V8_LIKELY(next <= kMaxOneByteChar && old_state == State::kAccept)) {
73 DCHECK_EQ(0u, *buffer);
74 return static_cast<uchar>(next);
75 }
76
77 // So we're at the lead byte of a 2/3/4 sequence, or we're at a continuation
78 // char in that sequence.
79 Utf8DfaDecoder::Decode(next, state, buffer);
80
81 switch (*state) {
82 case State::kAccept: {
83 uchar t = *buffer;
84 *buffer = 0;
85 return t;
86 }
87
88 case State::kReject:
89 *state = State::kAccept;
90 *buffer = 0;
91
92 // If we hit a bad byte, we need to determine if we were trying to start
93 // a sequence or continue one. If we were trying to start a sequence,
94 // that means it's just an invalid lead byte and we need to continue to
95 // the next (which we already did above). If we were already in a
96 // sequence, we need to reprocess this same byte after resetting to the
97 // initial state.
98 if (old_state != State::kAccept) {
99 // We were trying to continue a sequence, so let's reprocess this byte
100 // next time.
101 *cursor -= 1;
102 }
103 return kBadChar;
104
105 default:
106 return kIncomplete;
107 }
108 }
109
EncodeOneByte(char * str,uint8_t c)110 unsigned Utf8::EncodeOneByte(char* str, uint8_t c) {
111 static const int kMask = ~(1 << 6);
112 if (c <= kMaxOneByteChar) {
113 str[0] = c;
114 return 1;
115 }
116 str[0] = 0xC0 | (c >> 6);
117 str[1] = 0x80 | (c & kMask);
118 return 2;
119 }
120
121 // Encode encodes the UTF-16 code units c and previous into the given str
122 // buffer, and combines surrogate code units into single code points. If
123 // replace_invalid is set to true, orphan surrogate code units will be replaced
124 // with kBadChar.
Encode(char * str,uchar c,int previous,bool replace_invalid)125 unsigned Utf8::Encode(char* str, uchar c, int previous, bool replace_invalid) {
126 static const int kMask = ~(1 << 6);
127 if (c <= kMaxOneByteChar) {
128 str[0] = c;
129 return 1;
130 } else if (c <= kMaxTwoByteChar) {
131 str[0] = 0xC0 | (c >> 6);
132 str[1] = 0x80 | (c & kMask);
133 return 2;
134 } else if (c <= kMaxThreeByteChar) {
135 DCHECK(!Utf16::IsLeadSurrogate(Utf16::kNoPreviousCharacter));
136 if (Utf16::IsSurrogatePair(previous, c)) {
137 const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
138 return Encode(str - kUnmatchedSize,
139 Utf16::CombineSurrogatePair(previous, c),
140 Utf16::kNoPreviousCharacter, replace_invalid) -
141 kUnmatchedSize;
142 } else if (replace_invalid &&
143 (Utf16::IsLeadSurrogate(c) || Utf16::IsTrailSurrogate(c))) {
144 c = kBadChar;
145 }
146 str[0] = 0xE0 | (c >> 12);
147 str[1] = 0x80 | ((c >> 6) & kMask);
148 str[2] = 0x80 | (c & kMask);
149 return 3;
150 } else {
151 str[0] = 0xF0 | (c >> 18);
152 str[1] = 0x80 | ((c >> 12) & kMask);
153 str[2] = 0x80 | ((c >> 6) & kMask);
154 str[3] = 0x80 | (c & kMask);
155 return 4;
156 }
157 }
158
ValueOf(const byte * bytes,size_t length,size_t * cursor)159 uchar Utf8::ValueOf(const byte* bytes, size_t length, size_t* cursor) {
160 if (length <= 0) return kBadChar;
161 byte first = bytes[0];
162 // Characters between 0000 and 007F are encoded as a single character
163 if (V8_LIKELY(first <= kMaxOneByteChar)) {
164 *cursor += 1;
165 return first;
166 }
167 return CalculateValue(bytes, length, cursor);
168 }
169
Length(uchar c,int previous)170 unsigned Utf8::Length(uchar c, int previous) {
171 if (c <= kMaxOneByteChar) {
172 return 1;
173 } else if (c <= kMaxTwoByteChar) {
174 return 2;
175 } else if (c <= kMaxThreeByteChar) {
176 DCHECK(!Utf16::IsLeadSurrogate(Utf16::kNoPreviousCharacter));
177 if (Utf16::IsSurrogatePair(previous, c)) {
178 return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
179 }
180 return 3;
181 } else {
182 return 4;
183 }
184 }
185
IsValidCharacter(uchar c)186 bool Utf8::IsValidCharacter(uchar c) {
187 return c < 0xD800u || (c >= 0xE000u && c < 0xFDD0u) ||
188 (c > 0xFDEFu && c <= 0x10FFFFu && (c & 0xFFFEu) != 0xFFFEu &&
189 c != kBadChar);
190 }
191
192 } // namespace unibrow
193
194 #endif // V8_STRINGS_UNICODE_INL_H_
195