1 /*
2 Copyright 2015 Google Inc. All rights reserved.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 #ifndef JSONNET_UNICODE_H
18 #define JSONNET_UNICODE_H
19
20 /** Substituted when a unicode translation format encoding error is encountered. */
21 #define JSONNET_CODEPOINT_ERROR 0xfffd
22 #define JSONNET_CODEPOINT_MAX 0x110000
23
24 /** Convert a unicode codepoint to UTF8.
25 *
26 * \param x The unicode codepoint.
27 * \param s The UTF-8 string to append to.
28 * \returns The number of characters appended.
29 */
encode_utf8(char32_t x,std::string & s)30 static inline int encode_utf8(char32_t x, std::string &s)
31 {
32 if (x >= JSONNET_CODEPOINT_MAX)
33 x = JSONNET_CODEPOINT_ERROR;
34
35 // 00ZZZzzz 00zzYYYY 00Yyyyxx 00xxxxxx
36 long bytes = ((x & 0x1C0000) << 6) | ((x & 0x03F000) << 4) | ((x & 0x0FC0) << 2) | (x & 0x3F);
37
38 if (x < 0x80) {
39 s.push_back((char)x);
40 return 1;
41 } else if (x < 0x800) { // note that capital 'Y' bits must be 0
42 bytes |= 0xC080;
43 s.push_back((bytes >> 8) & 0xFF);
44 s.push_back((bytes >> 0) & 0xFF);
45 return 2;
46 } else if (x < 0x10000) { // note that 'z' bits must be 0
47 bytes |= 0xE08080;
48 s.push_back((bytes >> 16) & 0xFF);
49 s.push_back((bytes >> 8) & 0xFF);
50 s.push_back((bytes >> 0) & 0xFF);
51 return 3;
52 } else if (x < 0x110000) { // note that capital 'Z' bits must be 0
53 bytes |= 0xF0808080;
54 s.push_back((bytes >> 24) & 0xFF);
55 s.push_back((bytes >> 16) & 0xFF);
56 s.push_back((bytes >> 8) & 0xFF);
57 s.push_back((bytes >> 0) & 0xFF);
58 return 4;
59 } else {
60 std::cerr << "Should never get here." << std::endl;
61 abort();
62 }
63 }
64
65 /** Convert the UTF8 byte sequence in the given string to a unicode code point.
66 *
67 * \param str The string.
68 * \param i The index of the string from which to start decoding and returns the index of the last
69 * byte of the encoded codepoint.
70 * \returns The decoded unicode codepoint.
71 */
decode_utf8(const std::string & str,size_t & i)72 static inline char32_t decode_utf8(const std::string &str, size_t &i)
73 {
74 char c0 = str[i];
75 if ((c0 & 0x80) == 0) { // 0xxxxxxx
76 return c0;
77 } else if ((c0 & 0xE0) == 0xC0) { // 110yyyxx 10xxxxxx
78 if (i + 1 >= str.length()) {
79 return JSONNET_CODEPOINT_ERROR;
80 }
81 char c1 = str[++i];
82 if ((c1 & 0xC0) != 0x80) {
83 return JSONNET_CODEPOINT_ERROR;
84 }
85 return ((c0 & 0x1F) << 6ul) | (c1 & 0x3F);
86 } else if ((c0 & 0xF0) == 0xE0) { // 1110yyyy 10yyyyxx 10xxxxxx
87 if (i + 2 >= str.length()) {
88 return JSONNET_CODEPOINT_ERROR;
89 }
90 char c1 = str[++i];
91 if ((c1 & 0xC0) != 0x80) {
92 return JSONNET_CODEPOINT_ERROR;
93 }
94 char c2 = str[++i];
95 if ((c2 & 0xC0) != 0x80) {
96 return JSONNET_CODEPOINT_ERROR;
97 }
98 return ((c0 & 0xF) << 12ul) | ((c1 & 0x3F) << 6) | (c2 & 0x3F);
99 } else if ((c0 & 0xF8) == 0xF0) { // 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx
100 if (i + 3 >= str.length()) {
101 return JSONNET_CODEPOINT_ERROR;
102 }
103 char c1 = str[++i];
104 if ((c1 & 0xC0) != 0x80) {
105 return JSONNET_CODEPOINT_ERROR;
106 }
107 char c2 = str[++i];
108 if ((c2 & 0xC0) != 0x80) {
109 return JSONNET_CODEPOINT_ERROR;
110 }
111 char c3 = str[++i];
112 if ((c3 & 0xC0) != 0x80) {
113 return JSONNET_CODEPOINT_ERROR;
114 }
115 return ((c0 & 0x7) << 24ul) | ((c1 & 0x3F) << 12ul) | ((c2 & 0x3F) << 6) | (c3 & 0x3F);
116 } else {
117 return JSONNET_CODEPOINT_ERROR;
118 }
119 }
120
121 /** A string class capable of holding unicode codepoints. */
122 typedef std::basic_string<char32_t> UString;
123
encode_utf8(const UString & s,std::string & r)124 static inline void encode_utf8(const UString &s, std::string &r)
125 {
126 for (char32_t cp : s)
127 encode_utf8(cp, r);
128 }
129
encode_utf8(const UString & s)130 static inline std::string encode_utf8(const UString &s)
131 {
132 std::string r;
133 encode_utf8(s, r);
134 return r;
135 }
136
decode_utf8(const std::string & s)137 static inline UString decode_utf8(const std::string &s)
138 {
139 UString r;
140 for (size_t i = 0; i < s.length(); ++i)
141 r.push_back(decode_utf8(s, i));
142 return r;
143 }
144
145 /** A stringstream-like class capable of holding unicode codepoints.
146 * The C++ standard does not support std::basic_stringstream<char32_t.
147 */
148 class UStringStream {
149 UString buf;
150
151 public:
152 UStringStream &operator<<(const UString &s)
153 {
154 buf.append(s);
155 return *this;
156 }
157 UStringStream &operator<<(const char32_t *s)
158 {
159 buf.append(s);
160 return *this;
161 }
162 UStringStream &operator<<(char32_t c)
163 {
164 buf.push_back(c);
165 return *this;
166 }
167 template <class T>
168 UStringStream &operator<<(T c)
169 {
170 std::stringstream ss;
171 ss << c;
172 for (char c : ss.str())
173 buf.push_back(char32_t(c));
174 return *this;
175 }
str()176 UString str()
177 {
178 return buf;
179 }
180 };
181
182 #endif // JSONNET_UNICODE_H
183