1package yaml 2 3import ( 4 "io" 5) 6 7// Set the reader error and return 0. 8func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { 9 parser.error = yaml_READER_ERROR 10 parser.problem = problem 11 parser.problem_offset = offset 12 parser.problem_value = value 13 return false 14} 15 16// Byte order marks. 17const ( 18 bom_UTF8 = "\xef\xbb\xbf" 19 bom_UTF16LE = "\xff\xfe" 20 bom_UTF16BE = "\xfe\xff" 21) 22 23// Determine the input stream encoding by checking the BOM symbol. If no BOM is 24// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. 25func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { 26 // Ensure that we had enough bytes in the raw buffer. 27 for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { 28 if !yaml_parser_update_raw_buffer(parser) { 29 return false 30 } 31 } 32 33 // Determine the encoding. 34 buf := parser.raw_buffer 35 pos := parser.raw_buffer_pos 36 avail := len(buf) - pos 37 if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { 38 parser.encoding = yaml_UTF16LE_ENCODING 39 parser.raw_buffer_pos += 2 40 parser.offset += 2 41 } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { 42 parser.encoding = yaml_UTF16BE_ENCODING 43 parser.raw_buffer_pos += 2 44 parser.offset += 2 45 } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { 46 parser.encoding = yaml_UTF8_ENCODING 47 parser.raw_buffer_pos += 3 48 parser.offset += 3 49 } else { 50 parser.encoding = yaml_UTF8_ENCODING 51 } 52 return true 53} 54 55// Update the raw buffer. 56func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { 57 size_read := 0 58 59 // Return if the raw buffer is full. 60 if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { 61 return true 62 } 63 64 // Return on EOF. 65 if parser.eof { 66 return true 67 } 68 69 // Move the remaining bytes in the raw buffer to the beginning. 70 if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { 71 copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) 72 } 73 parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] 74 parser.raw_buffer_pos = 0 75 76 // Call the read handler to fill the buffer. 77 size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) 78 parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] 79 if err == io.EOF { 80 parser.eof = true 81 } else if err != nil { 82 return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) 83 } 84 return true 85} 86 87// Ensure that the buffer contains at least `length` characters. 88// Return true on success, false on failure. 89// 90// The length is supposed to be significantly less that the buffer size. 91func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { 92 if parser.read_handler == nil { 93 panic("read handler must be set") 94 } 95 96 // [Go] This function was changed to guarantee the requested length size at EOF. 97 // The fact we need to do this is pretty awful, but the description above implies 98 // for that to be the case, and there are tests 99 100 // If the EOF flag is set and the raw buffer is empty, do nothing. 101 if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { 102 // [Go] ACTUALLY! Read the documentation of this function above. 103 // This is just broken. To return true, we need to have the 104 // given length in the buffer. Not doing that means every single 105 // check that calls this function to make sure the buffer has a 106 // given length is Go) panicking; or C) accessing invalid memory. 107 //return true 108 } 109 110 // Return if the buffer contains enough characters. 111 if parser.unread >= length { 112 return true 113 } 114 115 // Determine the input encoding if it is not known yet. 116 if parser.encoding == yaml_ANY_ENCODING { 117 if !yaml_parser_determine_encoding(parser) { 118 return false 119 } 120 } 121 122 // Move the unread characters to the beginning of the buffer. 123 buffer_len := len(parser.buffer) 124 if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { 125 copy(parser.buffer, parser.buffer[parser.buffer_pos:]) 126 buffer_len -= parser.buffer_pos 127 parser.buffer_pos = 0 128 } else if parser.buffer_pos == buffer_len { 129 buffer_len = 0 130 parser.buffer_pos = 0 131 } 132 133 // Open the whole buffer for writing, and cut it before returning. 134 parser.buffer = parser.buffer[:cap(parser.buffer)] 135 136 // Fill the buffer until it has enough characters. 137 first := true 138 for parser.unread < length { 139 140 // Fill the raw buffer if necessary. 141 if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { 142 if !yaml_parser_update_raw_buffer(parser) { 143 parser.buffer = parser.buffer[:buffer_len] 144 return false 145 } 146 } 147 first = false 148 149 // Decode the raw buffer. 150 inner: 151 for parser.raw_buffer_pos != len(parser.raw_buffer) { 152 var value rune 153 var width int 154 155 raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos 156 157 // Decode the next character. 158 switch parser.encoding { 159 case yaml_UTF8_ENCODING: 160 // Decode a UTF-8 character. Check RFC 3629 161 // (http://www.ietf.org/rfc/rfc3629.txt) for more details. 162 // 163 // The following table (taken from the RFC) is used for 164 // decoding. 165 // 166 // Char. number range | UTF-8 octet sequence 167 // (hexadecimal) | (binary) 168 // --------------------+------------------------------------ 169 // 0000 0000-0000 007F | 0xxxxxxx 170 // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx 171 // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx 172 // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 173 // 174 // Additionally, the characters in the range 0xD800-0xDFFF 175 // are prohibited as they are reserved for use with UTF-16 176 // surrogate pairs. 177 178 // Determine the length of the UTF-8 sequence. 179 octet := parser.raw_buffer[parser.raw_buffer_pos] 180 switch { 181 case octet&0x80 == 0x00: 182 width = 1 183 case octet&0xE0 == 0xC0: 184 width = 2 185 case octet&0xF0 == 0xE0: 186 width = 3 187 case octet&0xF8 == 0xF0: 188 width = 4 189 default: 190 // The leading octet is invalid. 191 return yaml_parser_set_reader_error(parser, 192 "invalid leading UTF-8 octet", 193 parser.offset, int(octet)) 194 } 195 196 // Check if the raw buffer contains an incomplete character. 197 if width > raw_unread { 198 if parser.eof { 199 return yaml_parser_set_reader_error(parser, 200 "incomplete UTF-8 octet sequence", 201 parser.offset, -1) 202 } 203 break inner 204 } 205 206 // Decode the leading octet. 207 switch { 208 case octet&0x80 == 0x00: 209 value = rune(octet & 0x7F) 210 case octet&0xE0 == 0xC0: 211 value = rune(octet & 0x1F) 212 case octet&0xF0 == 0xE0: 213 value = rune(octet & 0x0F) 214 case octet&0xF8 == 0xF0: 215 value = rune(octet & 0x07) 216 default: 217 value = 0 218 } 219 220 // Check and decode the trailing octets. 221 for k := 1; k < width; k++ { 222 octet = parser.raw_buffer[parser.raw_buffer_pos+k] 223 224 // Check if the octet is valid. 225 if (octet & 0xC0) != 0x80 { 226 return yaml_parser_set_reader_error(parser, 227 "invalid trailing UTF-8 octet", 228 parser.offset+k, int(octet)) 229 } 230 231 // Decode the octet. 232 value = (value << 6) + rune(octet&0x3F) 233 } 234 235 // Check the length of the sequence against the value. 236 switch { 237 case width == 1: 238 case width == 2 && value >= 0x80: 239 case width == 3 && value >= 0x800: 240 case width == 4 && value >= 0x10000: 241 default: 242 return yaml_parser_set_reader_error(parser, 243 "invalid length of a UTF-8 sequence", 244 parser.offset, -1) 245 } 246 247 // Check the range of the value. 248 if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { 249 return yaml_parser_set_reader_error(parser, 250 "invalid Unicode character", 251 parser.offset, int(value)) 252 } 253 254 case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: 255 var low, high int 256 if parser.encoding == yaml_UTF16LE_ENCODING { 257 low, high = 0, 1 258 } else { 259 low, high = 1, 0 260 } 261 262 // The UTF-16 encoding is not as simple as one might 263 // naively think. Check RFC 2781 264 // (http://www.ietf.org/rfc/rfc2781.txt). 265 // 266 // Normally, two subsequent bytes describe a Unicode 267 // character. However a special technique (called a 268 // surrogate pair) is used for specifying character 269 // values larger than 0xFFFF. 270 // 271 // A surrogate pair consists of two pseudo-characters: 272 // high surrogate area (0xD800-0xDBFF) 273 // low surrogate area (0xDC00-0xDFFF) 274 // 275 // The following formulas are used for decoding 276 // and encoding characters using surrogate pairs: 277 // 278 // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) 279 // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) 280 // W1 = 110110yyyyyyyyyy 281 // W2 = 110111xxxxxxxxxx 282 // 283 // where U is the character value, W1 is the high surrogate 284 // area, W2 is the low surrogate area. 285 286 // Check for incomplete UTF-16 character. 287 if raw_unread < 2 { 288 if parser.eof { 289 return yaml_parser_set_reader_error(parser, 290 "incomplete UTF-16 character", 291 parser.offset, -1) 292 } 293 break inner 294 } 295 296 // Get the character. 297 value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + 298 (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) 299 300 // Check for unexpected low surrogate area. 301 if value&0xFC00 == 0xDC00 { 302 return yaml_parser_set_reader_error(parser, 303 "unexpected low surrogate area", 304 parser.offset, int(value)) 305 } 306 307 // Check for a high surrogate area. 308 if value&0xFC00 == 0xD800 { 309 width = 4 310 311 // Check for incomplete surrogate pair. 312 if raw_unread < 4 { 313 if parser.eof { 314 return yaml_parser_set_reader_error(parser, 315 "incomplete UTF-16 surrogate pair", 316 parser.offset, -1) 317 } 318 break inner 319 } 320 321 // Get the next character. 322 value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + 323 (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) 324 325 // Check for a low surrogate area. 326 if value2&0xFC00 != 0xDC00 { 327 return yaml_parser_set_reader_error(parser, 328 "expected low surrogate area", 329 parser.offset+2, int(value2)) 330 } 331 332 // Generate the value of the surrogate pair. 333 value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) 334 } else { 335 width = 2 336 } 337 338 default: 339 panic("impossible") 340 } 341 342 // Check if the character is in the allowed range: 343 // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) 344 // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) 345 // | [#x10000-#x10FFFF] (32 bit) 346 switch { 347 case value == 0x09: 348 case value == 0x0A: 349 case value == 0x0D: 350 case value >= 0x20 && value <= 0x7E: 351 case value == 0x85: 352 case value >= 0xA0 && value <= 0xD7FF: 353 case value >= 0xE000 && value <= 0xFFFD: 354 case value >= 0x10000 && value <= 0x10FFFF: 355 default: 356 return yaml_parser_set_reader_error(parser, 357 "control characters are not allowed", 358 parser.offset, int(value)) 359 } 360 361 // Move the raw pointers. 362 parser.raw_buffer_pos += width 363 parser.offset += width 364 365 // Finally put the character into the buffer. 366 if value <= 0x7F { 367 // 0000 0000-0000 007F . 0xxxxxxx 368 parser.buffer[buffer_len+0] = byte(value) 369 buffer_len += 1 370 } else if value <= 0x7FF { 371 // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx 372 parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) 373 parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) 374 buffer_len += 2 375 } else if value <= 0xFFFF { 376 // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx 377 parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) 378 parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) 379 parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) 380 buffer_len += 3 381 } else { 382 // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 383 parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) 384 parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) 385 parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) 386 parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) 387 buffer_len += 4 388 } 389 390 parser.unread++ 391 } 392 393 // On EOF, put NUL into the buffer and return. 394 if parser.eof { 395 parser.buffer[buffer_len] = 0 396 buffer_len++ 397 parser.unread++ 398 break 399 } 400 } 401 // [Go] Read the documentation of this function above. To return true, 402 // we need to have the given length in the buffer. Not doing that means 403 // every single check that calls this function to make sure the buffer 404 // has a given length is Go) panicking; or C) accessing invalid memory. 405 // This happens here due to the EOF above breaking early. 406 for buffer_len < length { 407 parser.buffer[buffer_len] = 0 408 buffer_len++ 409 } 410 parser.buffer = parser.buffer[:buffer_len] 411 return true 412} 413