1 // Copyright 2006-2016 Nemanja Trifunovic
2 
3 /*
4 Permission is hereby granted, free of charge, to any person or organization
5 obtaining a copy of the software and accompanying documentation covered by
6 this license (the "Software") to use, reproduce, display, distribute,
7 execute, and transmit the Software, and to prepare derivative works of the
8 Software, and to permit third-parties to whom the Software is furnished to
9 do so, all subject to the following:
10 
11 The copyright notices in the Software and this entire statement, including
12 the above license grant, this restriction and the following disclaimer,
13 must be included in all copies of the Software, in whole or in part, and
14 all derivative works of the Software, unless such copies or derivative
15 works are solely in the form of machine-executable object code generated by
16 a source language processor.
17 
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
21 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
22 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
23 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 DEALINGS IN THE SOFTWARE.
25 */
26 
27 
28 #ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
29 #define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
30 
31 #include "core.h"
32 #include <stdexcept>
33 
34 namespace utf8
35 {
36     // Base for the exceptions that may be thrown from the library
37     class exception : public ::std::exception {
38     };
39 
40     // Exceptions that may be thrown from the library functions.
41     class invalid_code_point : public exception {
42         uint32_t cp;
43     public:
invalid_code_point(uint32_t codepoint)44         invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
what()45         virtual const char* what() const throw() { return "Invalid code point"; }
code_point()46         uint32_t code_point() const {return cp;}
47     };
48 
49     class invalid_utf8 : public exception {
50         uint8_t u8;
51     public:
invalid_utf8(uint8_t u)52         invalid_utf8 (uint8_t u) : u8(u) {}
what()53         virtual const char* what() const throw() { return "Invalid UTF-8"; }
utf8_octet()54         uint8_t utf8_octet() const {return u8;}
55     };
56 
57     class invalid_utf16 : public exception {
58         uint16_t u16;
59     public:
invalid_utf16(uint16_t u)60         invalid_utf16 (uint16_t u) : u16(u) {}
what()61         virtual const char* what() const throw() { return "Invalid UTF-16"; }
utf16_word()62         uint16_t utf16_word() const {return u16;}
63     };
64 
65     class not_enough_room : public exception {
66     public:
what()67         virtual const char* what() const throw() { return "Not enough space"; }
68     };
69 
70     /// The library API - functions intended to be called by the users
71 
72     template <typename octet_iterator>
append(uint32_t cp,octet_iterator result)73     octet_iterator append(uint32_t cp, octet_iterator result)
74     {
75         if (!utf8::internal::is_code_point_valid(cp))
76             throw invalid_code_point(cp);
77 
78         if (cp < 0x80)                        // one octet
79             *(result++) = static_cast<uint8_t>(cp);
80         else if (cp < 0x800) {                // two octets
81             *(result++) = static_cast<uint8_t>((cp >> 6)            | 0xc0);
82             *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
83         }
84         else if (cp < 0x10000) {              // three octets
85             *(result++) = static_cast<uint8_t>((cp >> 12)           | 0xe0);
86             *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
87             *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
88         }
89         else {                                // four octets
90             *(result++) = static_cast<uint8_t>((cp >> 18)           | 0xf0);
91             *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)  | 0x80);
92             *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
93             *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
94         }
95         return result;
96     }
97 
98     template <typename octet_iterator, typename output_iterator>
replace_invalid(octet_iterator start,octet_iterator end,output_iterator out,uint32_t replacement)99     output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
100     {
101         while (start != end) {
102             octet_iterator sequence_start = start;
103             internal::utf_error err_code = utf8::internal::validate_next(start, end);
104             switch (err_code) {
105                 case internal::UTF8_OK :
106                     for (octet_iterator it = sequence_start; it != start; ++it)
107                         *out++ = *it;
108                     break;
109                 case internal::NOT_ENOUGH_ROOM:
110                     out = utf8::append (replacement, out);
111                     start = end;
112                     break;
113                 case internal::INVALID_LEAD:
114                     out = utf8::append (replacement, out);
115                     ++start;
116                     break;
117                 case internal::INCOMPLETE_SEQUENCE:
118                 case internal::OVERLONG_SEQUENCE:
119                 case internal::INVALID_CODE_POINT:
120                     out = utf8::append (replacement, out);
121                     ++start;
122                     // just one replacement mark for the sequence
123                     while (start != end && utf8::internal::is_trail(*start))
124                         ++start;
125                     break;
126             }
127         }
128         return out;
129     }
130 
131     template <typename octet_iterator, typename output_iterator>
replace_invalid(octet_iterator start,octet_iterator end,output_iterator out)132     inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
133     {
134         static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
135         return utf8::replace_invalid(start, end, out, replacement_marker);
136     }
137 
138     template <typename octet_iterator>
next(octet_iterator & it,octet_iterator end)139     uint32_t next(octet_iterator& it, octet_iterator end)
140     {
141         uint32_t cp = 0;
142         internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
143         switch (err_code) {
144             case internal::UTF8_OK :
145                 break;
146             case internal::NOT_ENOUGH_ROOM :
147                 throw not_enough_room();
148             case internal::INVALID_LEAD :
149             case internal::INCOMPLETE_SEQUENCE :
150             case internal::OVERLONG_SEQUENCE :
151                 throw invalid_utf8(*it);
152             case internal::INVALID_CODE_POINT :
153                 throw invalid_code_point(cp);
154         }
155         return cp;
156     }
157 
158     template <typename octet_iterator>
peek_next(octet_iterator it,octet_iterator end)159     uint32_t peek_next(octet_iterator it, octet_iterator end)
160     {
161         return utf8::next(it, end);
162     }
163 
164     template <typename octet_iterator>
prior(octet_iterator & it,octet_iterator start)165     uint32_t prior(octet_iterator& it, octet_iterator start)
166     {
167         // can't do much if it == start
168         if (it == start)
169             throw not_enough_room();
170 
171         octet_iterator end = it;
172         // Go back until we hit either a lead octet or start
173         while (utf8::internal::is_trail(*(--it)))
174             if (it == start)
175                 throw invalid_utf8(*it); // error - no lead byte in the sequence
176         return utf8::peek_next(it, end);
177     }
178 
179     template <typename octet_iterator, typename distance_type>
advance(octet_iterator & it,distance_type n,octet_iterator end)180     void advance (octet_iterator& it, distance_type n, octet_iterator end)
181     {
182         const distance_type zero(0);
183         if (n < zero) {
184             // backward
185             for (distance_type i = n; i < zero; ++i)
186                 utf8::prior(it, end);
187         } else {
188             // forward
189             for (distance_type i = zero; i < n; ++i)
190                 utf8::next(it, end);
191         }
192     }
193 
194     template <typename octet_iterator>
195     typename std::iterator_traits<octet_iterator>::difference_type
distance(octet_iterator first,octet_iterator last)196     distance (octet_iterator first, octet_iterator last)
197     {
198         typename std::iterator_traits<octet_iterator>::difference_type dist;
199         for (dist = 0; first < last; ++dist)
200             utf8::next(first, last);
201         return dist;
202     }
203 
204     template <typename u16bit_iterator, typename octet_iterator>
utf16to8(u16bit_iterator start,u16bit_iterator end,octet_iterator result)205     octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
206     {
207         while (start != end) {
208             uint32_t cp = utf8::internal::mask16(*start++);
209             // Take care of surrogate pairs first
210             if (utf8::internal::is_lead_surrogate(cp)) {
211                 if (start != end) {
212                     uint32_t trail_surrogate = utf8::internal::mask16(*start++);
213                     if (utf8::internal::is_trail_surrogate(trail_surrogate))
214                         cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
215                     else
216                         throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
217                 }
218                 else
219                     throw invalid_utf16(static_cast<uint16_t>(cp));
220 
221             }
222             // Lone trail surrogate
223             else if (utf8::internal::is_trail_surrogate(cp))
224                 throw invalid_utf16(static_cast<uint16_t>(cp));
225 
226             result = utf8::append(cp, result);
227         }
228         return result;
229     }
230 
231     template <typename u16bit_iterator, typename octet_iterator>
utf8to16(octet_iterator start,octet_iterator end,u16bit_iterator result)232     u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
233     {
234         while (start < end) {
235             uint32_t cp = utf8::next(start, end);
236             if (cp > 0xffff) { //make a surrogate pair
237                 *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
238                 *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
239             }
240             else
241                 *result++ = static_cast<uint16_t>(cp);
242         }
243         return result;
244     }
245 
246     template <typename octet_iterator, typename u32bit_iterator>
utf32to8(u32bit_iterator start,u32bit_iterator end,octet_iterator result)247     octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
248     {
249         while (start != end)
250             result = utf8::append(*(start++), result);
251 
252         return result;
253     }
254 
255     template <typename octet_iterator, typename u32bit_iterator>
utf8to32(octet_iterator start,octet_iterator end,u32bit_iterator result)256     u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
257     {
258         while (start < end)
259             (*result++) = utf8::next(start, end);
260 
261         return result;
262     }
263 
264     // The iterator class
265     template <typename octet_iterator>
266     class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
267       octet_iterator it;
268       octet_iterator range_start;
269       octet_iterator range_end;
270       public:
iterator()271       iterator () {}
iterator(const octet_iterator & octet_it,const octet_iterator & rangestart,const octet_iterator & rangeend)272       explicit iterator (const octet_iterator& octet_it,
273                          const octet_iterator& rangestart,
274                          const octet_iterator& rangeend) :
275                it(octet_it), range_start(rangestart), range_end(rangeend)
276       {
277           if (it < range_start || it > range_end)
278               throw std::out_of_range("Invalid utf-8 iterator position");
279       }
280       // the default "big three" are OK
base()281       octet_iterator base () const { return it; }
282       uint32_t operator * () const
283       {
284           octet_iterator temp = it;
285           return utf8::next(temp, range_end);
286       }
287       bool operator == (const iterator& rhs) const
288       {
289           if (range_start != rhs.range_start || range_end != rhs.range_end)
290               throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
291           return (it == rhs.it);
292       }
293       bool operator != (const iterator& rhs) const
294       {
295           return !(operator == (rhs));
296       }
297       iterator& operator ++ ()
298       {
299           utf8::next(it, range_end);
300           return *this;
301       }
302       iterator operator ++ (int)
303       {
304           iterator temp = *this;
305           utf8::next(it, range_end);
306           return temp;
307       }
308       iterator& operator -- ()
309       {
310           utf8::prior(it, range_start);
311           return *this;
312       }
313       iterator operator -- (int)
314       {
315           iterator temp = *this;
316           utf8::prior(it, range_start);
317           return temp;
318       }
319     }; // class iterator
320 
321 } // namespace utf8
322 
323 #endif //header guard
324 
325