1// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. 2// Use of this source code is governed by a MIT license found in the LICENSE file. 3 4package codec 5 6import ( 7 "encoding" 8 "errors" 9 "fmt" 10 "io" 11 "math" 12 "reflect" 13 "strconv" 14 "time" 15) 16 17// Some tagging information for error messages. 18const ( 19 msgBadDesc = "unrecognized descriptor byte" 20 // msgDecCannotExpandArr = "cannot expand go array from %v to stream length: %v" 21) 22 23const ( 24 decDefMaxDepth = 1024 // maximum depth 25 decDefSliceCap = 8 26 decDefChanCap = 64 // should be large, as cap cannot be expanded 27 decScratchByteArrayLen = (6 * 8) // ??? cacheLineSize + 28 29 // decContainerLenUnknown is length returned from Read(Map|Array)Len 30 // when a format doesn't know apiori. 31 // For example, json doesn't pre-determine the length of a container (sequence/map). 32 decContainerLenUnknown = -1 33 34 // decContainerLenNil is length returned from Read(Map|Array)Len 35 // when a 'nil' was encountered in the stream. 36 decContainerLenNil = math.MinInt32 37 38 // decFailNonEmptyIntf configures whether we error 39 // when decoding naked into a non-empty interface. 40 // 41 // Typically, we cannot decode non-nil stream value into 42 // nil interface with methods (e.g. io.Reader). 43 // However, in some scenarios, this should be allowed: 44 // - MapType 45 // - SliceType 46 // - Extensions 47 // 48 // Consequently, we should relax this. Put it behind a const flag for now. 49 decFailNonEmptyIntf = false 50) 51 52var ( 53 errstrOnlyMapOrArrayCanDecodeIntoStruct = "only encoded map or array can be decoded into a struct" 54 errstrCannotDecodeIntoNil = "cannot decode into nil" 55 56 // errmsgExpandSliceOverflow = "expand slice: slice overflow" 57 errmsgExpandSliceCannotChange = "expand slice: cannot change" 58 59 errDecoderNotInitialized = errors.New("Decoder not initialized") 60 61 errDecUnreadByteNothingToRead = errors.New("cannot unread - nothing has been read") 62 errDecUnreadByteLastByteNotRead = errors.New("cannot unread - last byte has not been read") 63 errDecUnreadByteUnknown = errors.New("cannot unread - reason unknown") 64 errMaxDepthExceeded = errors.New("maximum decoding depth exceeded") 65 66 errBytesDecReaderCannotUnread = errors.New("cannot unread last byte read") 67) 68 69type decDriver interface { 70 // this will check if the next token is a break. 71 CheckBreak() bool 72 73 // TryNil tries to decode as nil. 74 TryNil() bool 75 76 // ContainerType returns one of: Bytes, String, Nil, Slice or Map. 77 // 78 // Return unSet if not known. 79 // 80 // Note: Implementations MUST fully consume sentinel container types, specifically Nil. 81 ContainerType() (vt valueType) 82 83 // DecodeNaked will decode primitives (number, bool, string, []byte) and RawExt. 84 // For maps and arrays, it will not do the decoding in-band, but will signal 85 // the decoder, so that is done later, by setting the decNaked.valueType field. 86 // 87 // Note: Numbers are decoded as int64, uint64, float64 only (no smaller sized number types). 88 // for extensions, DecodeNaked must read the tag and the []byte if it exists. 89 // if the []byte is not read, then kInterfaceNaked will treat it as a Handle 90 // that stores the subsequent value in-band, and complete reading the RawExt. 91 // 92 // extensions should also use readx to decode them, for efficiency. 93 // kInterface will extract the detached byte slice if it has to pass it outside its realm. 94 DecodeNaked() 95 96 DecodeInt64() (i int64) 97 DecodeUint64() (ui uint64) 98 99 DecodeFloat64() (f float64) 100 DecodeBool() (b bool) 101 102 // DecodeStringAsBytes returns the bytes representing a string. 103 // By definition, it will return a view into a scratch buffer. 104 // 105 // Note: This can also decode symbols, if supported. 106 // 107 // Users should consume it right away and not store it for later use. 108 DecodeStringAsBytes() (v []byte) 109 110 // DecodeBytes may be called directly, without going through reflection. 111 // Consequently, it must be designed to handle possible nil. 112 DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) 113 // DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOut []byte) 114 115 // DecodeExt will decode into a *RawExt or into an extension. 116 DecodeExt(v interface{}, xtag uint64, ext Ext) 117 // decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) 118 119 DecodeTime() (t time.Time) 120 121 // ReadArrayStart will return the length of the array. 122 // If the format doesn't prefix the length, it returns decContainerLenUnknown. 123 // If the expected array was a nil in the stream, it returns decContainerLenNil. 124 ReadArrayStart() int 125 ReadArrayEnd() 126 127 // ReadMapStart will return the length of the array. 128 // If the format doesn't prefix the length, it returns decContainerLenUnknown. 129 // If the expected array was a nil in the stream, it returns decContainerLenNil. 130 ReadMapStart() int 131 ReadMapEnd() 132 133 reset() 134 atEndOfDecode() 135 uncacheRead() 136 137 decoder() *Decoder 138} 139 140type decDriverContainerTracker interface { 141 ReadArrayElem() 142 ReadMapElemKey() 143 ReadMapElemValue() 144} 145 146type decodeError struct { 147 codecError 148 pos int 149} 150 151func (d decodeError) Error() string { 152 return fmt.Sprintf("%s decode error [pos %d]: %v", d.name, d.pos, d.err) 153} 154 155type decDriverNoopContainerReader struct{} 156 157func (x decDriverNoopContainerReader) ReadArrayStart() (v int) { return } 158func (x decDriverNoopContainerReader) ReadArrayEnd() {} 159func (x decDriverNoopContainerReader) ReadMapStart() (v int) { return } 160func (x decDriverNoopContainerReader) ReadMapEnd() {} 161func (x decDriverNoopContainerReader) CheckBreak() (v bool) { return } 162func (x decDriverNoopContainerReader) atEndOfDecode() {} 163 164// DecodeOptions captures configuration options during decode. 165type DecodeOptions struct { 166 // MapType specifies type to use during schema-less decoding of a map in the stream. 167 // If nil (unset), we default to map[string]interface{} iff json handle and MapStringAsKey=true, 168 // else map[interface{}]interface{}. 169 MapType reflect.Type 170 171 // SliceType specifies type to use during schema-less decoding of an array in the stream. 172 // If nil (unset), we default to []interface{} for all formats. 173 SliceType reflect.Type 174 175 // MaxInitLen defines the maxinum initial length that we "make" a collection 176 // (string, slice, map, chan). If 0 or negative, we default to a sensible value 177 // based on the size of an element in the collection. 178 // 179 // For example, when decoding, a stream may say that it has 2^64 elements. 180 // We should not auto-matically provision a slice of that size, to prevent Out-Of-Memory crash. 181 // Instead, we provision up to MaxInitLen, fill that up, and start appending after that. 182 MaxInitLen int 183 184 // ReaderBufferSize is the size of the buffer used when reading. 185 // 186 // if > 0, we use a smart buffer internally for performance purposes. 187 ReaderBufferSize int 188 189 // MaxDepth defines the maximum depth when decoding nested 190 // maps and slices. If 0 or negative, we default to a suitably large number (currently 1024). 191 MaxDepth int16 192 193 // If ErrorIfNoField, return an error when decoding a map 194 // from a codec stream into a struct, and no matching struct field is found. 195 ErrorIfNoField bool 196 197 // If ErrorIfNoArrayExpand, return an error when decoding a slice/array that cannot be expanded. 198 // For example, the stream contains an array of 8 items, but you are decoding into a [4]T array, 199 // or you are decoding into a slice of length 4 which is non-addressable (and so cannot be set). 200 ErrorIfNoArrayExpand bool 201 202 // If SignedInteger, use the int64 during schema-less decoding of unsigned values (not uint64). 203 SignedInteger bool 204 205 // MapValueReset controls how we decode into a map value. 206 // 207 // By default, we MAY retrieve the mapping for a key, and then decode into that. 208 // However, especially with big maps, that retrieval may be expensive and unnecessary 209 // if the stream already contains all that is necessary to recreate the value. 210 // 211 // If true, we will never retrieve the previous mapping, 212 // but rather decode into a new value and set that in the map. 213 // 214 // If false, we will retrieve the previous mapping if necessary e.g. 215 // the previous mapping is a pointer, or is a struct or array with pre-set state, 216 // or is an interface. 217 MapValueReset bool 218 219 // SliceElementReset: on decoding a slice, reset the element to a zero value first. 220 // 221 // concern: if the slice already contained some garbage, we will decode into that garbage. 222 SliceElementReset bool 223 224 // InterfaceReset controls how we decode into an interface. 225 // 226 // By default, when we see a field that is an interface{...}, 227 // or a map with interface{...} value, we will attempt decoding into the 228 // "contained" value. 229 // 230 // However, this prevents us from reading a string into an interface{} 231 // that formerly contained a number. 232 // 233 // If true, we will decode into a new "blank" value, and set that in the interface. 234 // If false, we will decode into whatever is contained in the interface. 235 InterfaceReset bool 236 237 // InternString controls interning of strings during decoding. 238 // 239 // Some handles, e.g. json, typically will read map keys as strings. 240 // If the set of keys are finite, it may help reduce allocation to 241 // look them up from a map (than to allocate them afresh). 242 // 243 // Note: Handles will be smart when using the intern functionality. 244 // Every string should not be interned. 245 // An excellent use-case for interning is struct field names, 246 // or map keys where key type is string. 247 InternString bool 248 249 // PreferArrayOverSlice controls whether to decode to an array or a slice. 250 // 251 // This only impacts decoding into a nil interface{}. 252 // 253 // Consequently, it has no effect on codecgen. 254 // 255 // *Note*: This only applies if using go1.5 and above, 256 // as it requires reflect.ArrayOf support which was absent before go1.5. 257 PreferArrayOverSlice bool 258 259 // DeleteOnNilMapValue controls how to decode a nil value in the stream. 260 // 261 // If true, we will delete the mapping of the key. 262 // Else, just set the mapping to the zero value of the type. 263 // 264 // Deprecated: This does NOTHING and is left behind for compiling compatibility. 265 // This change is necessitated because 'nil' in a stream now consistently 266 // means the zero value (ie reset the value to its zero state). 267 DeleteOnNilMapValue bool 268 269 // RawToString controls how raw bytes in a stream are decoded into a nil interface{}. 270 // By default, they are decoded as []byte, but can be decoded as string (if configured). 271 RawToString bool 272} 273 274// ---------------------------------------- 275 276func (d *Decoder) rawExt(f *codecFnInfo, rv reflect.Value) { 277 d.d.DecodeExt(rv2i(rv), 0, nil) 278} 279 280func (d *Decoder) ext(f *codecFnInfo, rv reflect.Value) { 281 d.d.DecodeExt(rv2i(rv), f.xfTag, f.xfFn) 282} 283 284func (d *Decoder) selferUnmarshal(f *codecFnInfo, rv reflect.Value) { 285 rv2i(rv).(Selfer).CodecDecodeSelf(d) 286} 287 288func (d *Decoder) binaryUnmarshal(f *codecFnInfo, rv reflect.Value) { 289 bm := rv2i(rv).(encoding.BinaryUnmarshaler) 290 xbs := d.d.DecodeBytes(nil, true) 291 if fnerr := bm.UnmarshalBinary(xbs); fnerr != nil { 292 panic(fnerr) 293 } 294} 295 296func (d *Decoder) textUnmarshal(f *codecFnInfo, rv reflect.Value) { 297 tm := rv2i(rv).(encoding.TextUnmarshaler) 298 fnerr := tm.UnmarshalText(d.d.DecodeStringAsBytes()) 299 if fnerr != nil { 300 panic(fnerr) 301 } 302} 303 304func (d *Decoder) jsonUnmarshal(f *codecFnInfo, rv reflect.Value) { 305 tm := rv2i(rv).(jsonUnmarshaler) 306 // bs := d.d.DecodeBytes(d.b[:], true, true) 307 // grab the bytes to be read, as UnmarshalJSON needs the full JSON so as to unmarshal it itself. 308 fnerr := tm.UnmarshalJSON(d.nextValueBytes()) 309 if fnerr != nil { 310 panic(fnerr) 311 } 312} 313 314func (d *Decoder) kErr(f *codecFnInfo, rv reflect.Value) { 315 d.errorf("no decoding function defined for kind %v", rv.Kind()) 316} 317 318func (d *Decoder) raw(f *codecFnInfo, rv reflect.Value) { 319 rvSetBytes(rv, d.rawBytes()) 320} 321 322func (d *Decoder) kString(f *codecFnInfo, rv reflect.Value) { 323 rvSetString(rv, string(d.d.DecodeStringAsBytes())) 324} 325 326func (d *Decoder) kBool(f *codecFnInfo, rv reflect.Value) { 327 rvSetBool(rv, d.d.DecodeBool()) 328} 329 330func (d *Decoder) kTime(f *codecFnInfo, rv reflect.Value) { 331 rvSetTime(rv, d.d.DecodeTime()) 332} 333 334func (d *Decoder) kFloat32(f *codecFnInfo, rv reflect.Value) { 335 rvSetFloat32(rv, d.decodeFloat32()) 336} 337 338func (d *Decoder) kFloat64(f *codecFnInfo, rv reflect.Value) { 339 rvSetFloat64(rv, d.d.DecodeFloat64()) 340} 341 342func (d *Decoder) kInt(f *codecFnInfo, rv reflect.Value) { 343 rvSetInt(rv, int(chkOvf.IntV(d.d.DecodeInt64(), intBitsize))) 344} 345 346func (d *Decoder) kInt8(f *codecFnInfo, rv reflect.Value) { 347 rvSetInt8(rv, int8(chkOvf.IntV(d.d.DecodeInt64(), 8))) 348} 349 350func (d *Decoder) kInt16(f *codecFnInfo, rv reflect.Value) { 351 rvSetInt16(rv, int16(chkOvf.IntV(d.d.DecodeInt64(), 16))) 352} 353 354func (d *Decoder) kInt32(f *codecFnInfo, rv reflect.Value) { 355 rvSetInt32(rv, int32(chkOvf.IntV(d.d.DecodeInt64(), 32))) 356} 357 358func (d *Decoder) kInt64(f *codecFnInfo, rv reflect.Value) { 359 rvSetInt64(rv, d.d.DecodeInt64()) 360} 361 362func (d *Decoder) kUint(f *codecFnInfo, rv reflect.Value) { 363 rvSetUint(rv, uint(chkOvf.UintV(d.d.DecodeUint64(), uintBitsize))) 364} 365 366func (d *Decoder) kUintptr(f *codecFnInfo, rv reflect.Value) { 367 rvSetUintptr(rv, uintptr(chkOvf.UintV(d.d.DecodeUint64(), uintBitsize))) 368} 369 370func (d *Decoder) kUint8(f *codecFnInfo, rv reflect.Value) { 371 rvSetUint8(rv, uint8(chkOvf.UintV(d.d.DecodeUint64(), 8))) 372} 373 374func (d *Decoder) kUint16(f *codecFnInfo, rv reflect.Value) { 375 rvSetUint16(rv, uint16(chkOvf.UintV(d.d.DecodeUint64(), 16))) 376} 377 378func (d *Decoder) kUint32(f *codecFnInfo, rv reflect.Value) { 379 rvSetUint32(rv, uint32(chkOvf.UintV(d.d.DecodeUint64(), 32))) 380} 381 382func (d *Decoder) kUint64(f *codecFnInfo, rv reflect.Value) { 383 rvSetUint64(rv, d.d.DecodeUint64()) 384} 385 386func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) { 387 // nil interface: 388 // use some hieristics to decode it appropriately 389 // based on the detected next value in the stream. 390 n := d.naked() 391 d.d.DecodeNaked() 392 393 // We cannot decode non-nil stream value into nil interface with methods (e.g. io.Reader). 394 // Howver, it is possible that the user has ways to pass in a type for a given interface 395 // - MapType 396 // - SliceType 397 // - Extensions 398 // 399 // Consequently, we should relax this. Put it behind a const flag for now. 400 if decFailNonEmptyIntf && f.ti.numMeth > 0 { 401 d.errorf("cannot decode non-nil codec value into nil %v (%v methods)", f.ti.rt, f.ti.numMeth) 402 return 403 } 404 switch n.v { 405 case valueTypeMap: 406 // if json, default to a map type with string keys 407 mtid := d.mtid 408 if mtid == 0 { 409 if d.jsms { 410 mtid = mapStrIntfTypId 411 } else { 412 mtid = mapIntfIntfTypId 413 } 414 } 415 if mtid == mapIntfIntfTypId { 416 var v2 map[interface{}]interface{} 417 d.decode(&v2) 418 rvn = rv4i(&v2).Elem() 419 } else if mtid == mapStrIntfTypId { // for json performance 420 var v2 map[string]interface{} 421 d.decode(&v2) 422 rvn = rv4i(&v2).Elem() 423 } else { 424 if d.mtr { 425 rvn = reflect.New(d.h.MapType) 426 d.decode(rv2i(rvn)) 427 rvn = rvn.Elem() 428 } else { 429 rvn = rvZeroAddrK(d.h.MapType, reflect.Map) 430 d.decodeValue(rvn, nil) 431 } 432 } 433 case valueTypeArray: 434 if d.stid == 0 || d.stid == intfSliceTypId { 435 var v2 []interface{} 436 d.decode(&v2) 437 rvn = rv4i(&v2).Elem() 438 } else { 439 if d.str { 440 rvn = reflect.New(d.h.SliceType) 441 d.decode(rv2i(rvn)) 442 rvn = rvn.Elem() 443 } else { 444 rvn = rvZeroAddrK(d.h.SliceType, reflect.Slice) 445 d.decodeValue(rvn, nil) 446 } 447 } 448 if reflectArrayOfSupported && d.h.PreferArrayOverSlice { 449 rvn = rvGetArray4Slice(rvn) 450 } 451 case valueTypeExt: 452 tag, bytes := n.u, n.l // calling decode below might taint the values 453 bfn := d.h.getExtForTag(tag) 454 var re = RawExt{Tag: tag} 455 if bytes == nil { 456 // it is one of the InterfaceExt ones: json and cbor. 457 // most likely cbor, as json decoding never reveals valueTypeExt (no tagging support) 458 if bfn == nil { 459 d.decode(&re.Value) 460 rvn = rv4i(&re).Elem() 461 } else { 462 if bfn.ext == SelfExt { 463 rvn = rvZeroAddrK(bfn.rt, bfn.rt.Kind()) 464 d.decodeValue(rvn, d.h.fnNoExt(bfn.rt)) 465 } else { 466 rvn = reflect.New(bfn.rt) 467 d.interfaceExtConvertAndDecode(rv2i(rvn), bfn.ext) 468 rvn = rvn.Elem() 469 } 470 } 471 } else { 472 // one of the BytesExt ones: binc, msgpack, simple 473 if bfn == nil { 474 re.Data = detachZeroCopyBytes(d.bytes, nil, bytes) 475 rvn = rv4i(&re).Elem() 476 } else { 477 rvn = reflect.New(bfn.rt) 478 if bfn.ext == SelfExt { 479 d.sideDecode(rv2i(rvn), bytes) 480 } else { 481 bfn.ext.ReadExt(rv2i(rvn), bytes) 482 } 483 rvn = rvn.Elem() 484 } 485 } 486 case valueTypeNil: 487 // rvn = reflect.Zero(f.ti.rt) 488 // no-op 489 case valueTypeInt: 490 rvn = n.ri() 491 case valueTypeUint: 492 rvn = n.ru() 493 case valueTypeFloat: 494 rvn = n.rf() 495 case valueTypeBool: 496 rvn = n.rb() 497 case valueTypeString, valueTypeSymbol: 498 rvn = n.rs() 499 case valueTypeBytes: 500 rvn = n.rl() 501 case valueTypeTime: 502 rvn = n.rt() 503 default: 504 panicv.errorf("kInterfaceNaked: unexpected valueType: %d", n.v) 505 } 506 return 507} 508 509func (d *Decoder) kInterface(f *codecFnInfo, rv reflect.Value) { 510 // Note: 511 // A consequence of how kInterface works, is that 512 // if an interface already contains something, we try 513 // to decode into what was there before. 514 // We do not replace with a generic value (as got from decodeNaked). 515 516 // every interface passed here MUST be settable. 517 var rvn reflect.Value 518 if rvIsNil(rv) || d.h.InterfaceReset { 519 // check if mapping to a type: if so, initialize it and move on 520 rvn = d.h.intf2impl(f.ti.rtid) 521 if rvn.IsValid() { 522 rv.Set(rvn) 523 } else { 524 rvn = d.kInterfaceNaked(f) 525 // xdebugf("kInterface: %v", rvn) 526 if rvn.IsValid() { 527 rv.Set(rvn) 528 } else if d.h.InterfaceReset { 529 // reset to zero value based on current type in there. 530 if rvelem := rv.Elem(); rvelem.IsValid() { 531 rv.Set(reflect.Zero(rvelem.Type())) 532 } 533 } 534 return 535 } 536 } else { 537 // now we have a non-nil interface value, meaning it contains a type 538 rvn = rv.Elem() 539 } 540 541 // Note: interface{} is settable, but underlying type may not be. 542 // Consequently, we MAY have to create a decodable value out of the underlying value, 543 // decode into it, and reset the interface itself. 544 // fmt.Printf(">>>> kInterface: rvn type: %v, rv type: %v\n", rvn.Type(), rv.Type()) 545 546 if isDecodeable(rvn) { 547 d.decodeValue(rvn, nil) 548 return 549 } 550 551 rvn2 := rvZeroAddrK(rvn.Type(), rvn.Kind()) 552 rvSetDirect(rvn2, rvn) 553 d.decodeValue(rvn2, nil) 554 rv.Set(rvn2) 555} 556 557func decStructFieldKey(dd decDriver, keyType valueType, b *[decScratchByteArrayLen]byte) (rvkencname []byte) { 558 // use if-else-if, not switch (which compiles to binary-search) 559 // since keyType is typically valueTypeString, branch prediction is pretty good. 560 561 if keyType == valueTypeString { 562 rvkencname = dd.DecodeStringAsBytes() 563 } else if keyType == valueTypeInt { 564 rvkencname = strconv.AppendInt(b[:0], dd.DecodeInt64(), 10) 565 } else if keyType == valueTypeUint { 566 rvkencname = strconv.AppendUint(b[:0], dd.DecodeUint64(), 10) 567 } else if keyType == valueTypeFloat { 568 rvkencname = strconv.AppendFloat(b[:0], dd.DecodeFloat64(), 'f', -1, 64) 569 } else { 570 rvkencname = dd.DecodeStringAsBytes() 571 } 572 return 573} 574 575func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) { 576 sfn := structFieldNode{v: rv, update: true} 577 ctyp := d.d.ContainerType() 578 if ctyp == valueTypeNil { 579 rvSetDirect(rv, f.ti.rv0) 580 return 581 } 582 var mf MissingFielder 583 if f.ti.isFlag(tiflagMissingFielder) { 584 mf = rv2i(rv).(MissingFielder) 585 } else if f.ti.isFlag(tiflagMissingFielderPtr) { 586 mf = rv2i(rv.Addr()).(MissingFielder) 587 } 588 if ctyp == valueTypeMap { 589 containerLen := d.mapStart() 590 if containerLen == 0 { 591 d.mapEnd() 592 return 593 } 594 tisfi := f.ti.sfiSort 595 hasLen := containerLen >= 0 596 597 var rvkencname []byte 598 for j := 0; (hasLen && j < containerLen) || !(hasLen || d.checkBreak()); j++ { 599 d.mapElemKey() 600 rvkencname = decStructFieldKey(d.d, f.ti.keyType, &d.b) 601 d.mapElemValue() 602 if k := f.ti.indexForEncName(rvkencname); k > -1 { 603 si := tisfi[k] 604 d.decodeValue(sfn.field(si), nil) 605 } else if mf != nil { 606 // store rvkencname in new []byte, as it previously shares Decoder.b, which is used in decode 607 name2 := rvkencname 608 rvkencname = make([]byte, len(rvkencname)) 609 copy(rvkencname, name2) 610 611 var f interface{} 612 d.decode(&f) 613 if !mf.CodecMissingField(rvkencname, f) && d.h.ErrorIfNoField { 614 d.errorf("no matching struct field found when decoding stream map with key: %s ", 615 stringView(rvkencname)) 616 } 617 } else { 618 d.structFieldNotFound(-1, stringView(rvkencname)) 619 } 620 // keepAlive4StringView(rvkencnameB) // not needed, as reference is outside loop 621 } 622 d.mapEnd() 623 } else if ctyp == valueTypeArray { 624 containerLen := d.arrayStart() 625 if containerLen == 0 { 626 d.arrayEnd() 627 return 628 } 629 // Not much gain from doing it two ways for array. 630 // Arrays are not used as much for structs. 631 hasLen := containerLen >= 0 632 var checkbreak bool 633 for j, si := range f.ti.sfiSrc { 634 if hasLen && j == containerLen { 635 break 636 } 637 if !hasLen && d.checkBreak() { 638 checkbreak = true 639 break 640 } 641 d.arrayElem() 642 d.decodeValue(sfn.field(si), nil) 643 } 644 if (hasLen && containerLen > len(f.ti.sfiSrc)) || (!hasLen && !checkbreak) { 645 // read remaining values and throw away 646 for j := len(f.ti.sfiSrc); ; j++ { 647 if (hasLen && j == containerLen) || (!hasLen && d.checkBreak()) { 648 break 649 } 650 d.arrayElem() 651 d.structFieldNotFound(j, "") 652 } 653 } 654 d.arrayEnd() 655 } else { 656 d.errorstr(errstrOnlyMapOrArrayCanDecodeIntoStruct) 657 return 658 } 659} 660 661func (d *Decoder) kSlice(f *codecFnInfo, rv reflect.Value) { 662 // A slice can be set from a map or array in stream. 663 // This way, the order can be kept (as order is lost with map). 664 665 // Note: rv is a slice type here - guaranteed 666 667 rtelem0 := f.ti.elem 668 ctyp := d.d.ContainerType() 669 if ctyp == valueTypeNil { 670 if rv.CanSet() { 671 rvSetDirect(rv, f.ti.rv0) 672 } 673 return 674 } 675 if ctyp == valueTypeBytes || ctyp == valueTypeString { 676 // you can only decode bytes or string in the stream into a slice or array of bytes 677 if !(f.ti.rtid == uint8SliceTypId || rtelem0.Kind() == reflect.Uint8) { 678 d.errorf("bytes/string in stream must decode into slice/array of bytes, not %v", f.ti.rt) 679 } 680 rvbs := rvGetBytes(rv) 681 bs2 := d.d.DecodeBytes(rvbs, false) 682 // if rvbs == nil && bs2 != nil || rvbs != nil && bs2 == nil || len(bs2) != len(rvbs) { 683 if !(len(bs2) > 0 && len(bs2) == len(rvbs) && &bs2[0] == &rvbs[0]) { 684 if rv.CanSet() { 685 rvSetBytes(rv, bs2) 686 } else if len(rvbs) > 0 && len(bs2) > 0 { 687 copy(rvbs, bs2) 688 } 689 } 690 return 691 } 692 693 slh, containerLenS := d.decSliceHelperStart() // only expects valueType(Array|Map) - never Nil 694 695 // an array can never return a nil slice. so no need to check f.array here. 696 if containerLenS == 0 { 697 if rv.CanSet() { 698 if rvIsNil(rv) { 699 rvSetDirect(rv, reflect.MakeSlice(f.ti.rt, 0, 0)) 700 } else { 701 rvSetSliceLen(rv, 0) 702 } 703 } 704 slh.End() 705 return 706 } 707 708 rtelem0Size := int(rtelem0.Size()) 709 rtElem0Kind := rtelem0.Kind() 710 rtelem0Mut := !isImmutableKind(rtElem0Kind) 711 rtelem := rtelem0 712 rtelemkind := rtelem.Kind() 713 for rtelemkind == reflect.Ptr { 714 rtelem = rtelem.Elem() 715 rtelemkind = rtelem.Kind() 716 } 717 718 var fn *codecFn 719 720 var rv0 = rv 721 var rvChanged bool 722 var rvCanset = rv.CanSet() 723 var rv9 reflect.Value 724 725 rvlen := rvGetSliceLen(rv) 726 rvcap := rvGetSliceCap(rv) 727 hasLen := containerLenS > 0 728 if hasLen { 729 if containerLenS > rvcap { 730 oldRvlenGtZero := rvlen > 0 731 rvlen = decInferLen(containerLenS, d.h.MaxInitLen, int(rtelem0.Size())) 732 if rvlen <= rvcap { 733 if rvCanset { 734 rvSetSliceLen(rv, rvlen) 735 } 736 } else if rvCanset { 737 rv = reflect.MakeSlice(f.ti.rt, rvlen, rvlen) 738 rvcap = rvlen 739 rvChanged = true 740 } else { 741 d.errorf("cannot decode into non-settable slice") 742 } 743 if rvChanged && oldRvlenGtZero && rtelem0Mut { // !isImmutableKind(rtelem0.Kind()) { 744 rvCopySlice(rv, rv0) // only copy up to length NOT cap i.e. rv0.Slice(0, rvcap) 745 } 746 } else if containerLenS != rvlen { 747 rvlen = containerLenS 748 if rvCanset { 749 rvSetSliceLen(rv, rvlen) 750 } 751 } 752 } 753 754 // consider creating new element once, and just decoding into it. 755 var rtelem0Zero reflect.Value 756 var rtelem0ZeroValid bool 757 var j int 758 759 for ; (hasLen && j < containerLenS) || !(hasLen || d.checkBreak()); j++ { 760 if j == 0 && f.seq == seqTypeSlice && rvIsNil(rv) { 761 if hasLen { 762 rvlen = decInferLen(containerLenS, d.h.MaxInitLen, rtelem0Size) 763 } else { 764 rvlen = decDefSliceCap 765 } 766 if rvCanset { 767 rv = reflect.MakeSlice(f.ti.rt, rvlen, rvlen) 768 rvcap = rvlen 769 rvChanged = true 770 } else { 771 d.errorf("cannot decode into non-settable slice") 772 } 773 } 774 slh.ElemContainerState(j) 775 // if indefinite, etc, then expand the slice if necessary 776 if j >= rvlen { 777 if f.seq == seqTypeArray { 778 d.arrayCannotExpand(rvlen, j+1) 779 // drain completely and return 780 d.swallow() 781 j++ 782 for ; (hasLen && j < containerLenS) || !(hasLen || d.checkBreak()); j++ { 783 slh.ElemContainerState(j) 784 d.swallow() 785 } 786 slh.End() 787 return 788 } 789 // rv = reflect.Append(rv, reflect.Zero(rtelem0)) // append logic + varargs 790 791 // expand the slice up to the cap. 792 // Note that we did, so we have to reset it later. 793 794 if rvlen < rvcap { 795 if rv.CanSet() { 796 rvSetSliceLen(rv, rvcap) 797 } else if rvCanset { 798 rv = rvSlice(rv, rvcap) 799 rvChanged = true 800 } else { 801 d.errorf(errmsgExpandSliceCannotChange) 802 return 803 } 804 rvlen = rvcap 805 } else { 806 if !rvCanset { 807 d.errorf(errmsgExpandSliceCannotChange) 808 return 809 } 810 rvcap = growCap(rvcap, rtelem0Size, rvcap) 811 rv9 = reflect.MakeSlice(f.ti.rt, rvcap, rvcap) 812 rvCopySlice(rv9, rv) 813 rv = rv9 814 rvChanged = true 815 rvlen = rvcap 816 } 817 } 818 rv9 = rvSliceIndex(rv, j, f.ti) 819 if d.h.SliceElementReset { 820 if !rtelem0ZeroValid { 821 rtelem0ZeroValid = true 822 rtelem0Zero = reflect.Zero(rtelem0) 823 } 824 rv9.Set(rtelem0Zero) 825 } 826 827 if fn == nil { 828 fn = d.h.fn(rtelem) 829 } 830 d.decodeValue(rv9, fn) 831 } 832 if j < rvlen { 833 if rv.CanSet() { 834 rvSetSliceLen(rv, j) 835 } else if rvCanset { 836 rv = rvSlice(rv, j) 837 rvChanged = true 838 } 839 rvlen = j 840 } else if j == 0 && rvIsNil(rv) { 841 if rvCanset { 842 rv = reflect.MakeSlice(f.ti.rt, 0, 0) 843 rvChanged = true 844 } 845 } 846 slh.End() 847 848 if rvChanged { // infers rvCanset=true, so it can be reset 849 rv0.Set(rv) 850 } 851 852} 853 854func (d *Decoder) kSliceForChan(f *codecFnInfo, rv reflect.Value) { 855 // A slice can be set from a map or array in stream. 856 // This way, the order can be kept (as order is lost with map). 857 858 if f.ti.chandir&uint8(reflect.SendDir) == 0 { 859 d.errorf("receive-only channel cannot be decoded") 860 } 861 rtelem0 := f.ti.elem 862 ctyp := d.d.ContainerType() 863 if ctyp == valueTypeNil { 864 rvSetDirect(rv, f.ti.rv0) 865 return 866 } 867 if ctyp == valueTypeBytes || ctyp == valueTypeString { 868 // you can only decode bytes or string in the stream into a slice or array of bytes 869 if !(f.ti.rtid == uint8SliceTypId || rtelem0.Kind() == reflect.Uint8) { 870 d.errorf("bytes/string in stream must decode into slice/array of bytes, not %v", f.ti.rt) 871 } 872 bs2 := d.d.DecodeBytes(nil, true) 873 irv := rv2i(rv) 874 ch, ok := irv.(chan<- byte) 875 if !ok { 876 ch = irv.(chan byte) 877 } 878 for _, b := range bs2 { 879 ch <- b 880 } 881 return 882 } 883 884 // only expects valueType(Array|Map - nil handled above) 885 slh, containerLenS := d.decSliceHelperStart() 886 887 // an array can never return a nil slice. so no need to check f.array here. 888 if containerLenS == 0 { 889 if rv.CanSet() && rvIsNil(rv) { 890 rvSetDirect(rv, reflect.MakeChan(f.ti.rt, 0)) 891 } 892 slh.End() 893 return 894 } 895 896 rtelem0Size := int(rtelem0.Size()) 897 rtElem0Kind := rtelem0.Kind() 898 rtelem0Mut := !isImmutableKind(rtElem0Kind) 899 rtelem := rtelem0 900 rtelemkind := rtelem.Kind() 901 for rtelemkind == reflect.Ptr { 902 rtelem = rtelem.Elem() 903 rtelemkind = rtelem.Kind() 904 } 905 906 var fn *codecFn 907 908 var rvCanset = rv.CanSet() 909 var rvChanged bool 910 var rv0 = rv 911 var rv9 reflect.Value 912 913 var rvlen int // := rv.Len() 914 hasLen := containerLenS > 0 915 916 var j int 917 918 for ; (hasLen && j < containerLenS) || !(hasLen || d.checkBreak()); j++ { 919 if j == 0 && rvIsNil(rv) { 920 if hasLen { 921 rvlen = decInferLen(containerLenS, d.h.MaxInitLen, rtelem0Size) 922 } else { 923 rvlen = decDefChanCap 924 } 925 if rvCanset { 926 rv = reflect.MakeChan(f.ti.rt, rvlen) 927 rvChanged = true 928 } else { 929 d.errorf("cannot decode into non-settable chan") 930 } 931 } 932 slh.ElemContainerState(j) 933 if rtelem0Mut || !rv9.IsValid() { // || (rtElem0Kind == reflect.Ptr && rvIsNil(rv9)) { 934 rv9 = rvZeroAddrK(rtelem0, rtElem0Kind) 935 } 936 if fn == nil { 937 fn = d.h.fn(rtelem) 938 } 939 d.decodeValue(rv9, fn) 940 rv.Send(rv9) 941 } 942 slh.End() 943 944 if rvChanged { // infers rvCanset=true, so it can be reset 945 rv0.Set(rv) 946 } 947 948} 949 950func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) { 951 containerLen := d.mapStart() 952 if containerLen == decContainerLenNil { 953 rvSetDirect(rv, f.ti.rv0) 954 return 955 } 956 ti := f.ti 957 if rvIsNil(rv) { 958 rvlen := decInferLen(containerLen, d.h.MaxInitLen, int(ti.key.Size()+ti.elem.Size())) 959 rvSetDirect(rv, makeMapReflect(ti.rt, rvlen)) 960 } 961 962 if containerLen == 0 { 963 d.mapEnd() 964 return 965 } 966 967 ktype, vtype := ti.key, ti.elem 968 ktypeId := rt2id(ktype) 969 vtypeKind := vtype.Kind() 970 ktypeKind := ktype.Kind() 971 972 var vtypeElem reflect.Type 973 974 var keyFn, valFn *codecFn 975 var ktypeLo, vtypeLo reflect.Type 976 977 for ktypeLo = ktype; ktypeLo.Kind() == reflect.Ptr; ktypeLo = ktypeLo.Elem() { 978 } 979 980 for vtypeLo = vtype; vtypeLo.Kind() == reflect.Ptr; vtypeLo = vtypeLo.Elem() { 981 } 982 983 rvvMut := !isImmutableKind(vtypeKind) 984 985 // we do a doMapGet if kind is mutable, and InterfaceReset=true if interface 986 var doMapGet, doMapSet bool 987 if !d.h.MapValueReset { 988 if rvvMut { 989 if vtypeKind == reflect.Interface { 990 if !d.h.InterfaceReset { 991 doMapGet = true 992 } 993 } else { 994 doMapGet = true 995 } 996 } 997 } 998 999 var rvk, rvkn, rvv, rvvn, rvva reflect.Value 1000 var rvvaSet bool 1001 rvkMut := !isImmutableKind(ktype.Kind()) // if ktype is immutable, then re-use the same rvk. 1002 ktypeIsString := ktypeId == stringTypId 1003 ktypeIsIntf := ktypeId == intfTypId 1004 hasLen := containerLen > 0 1005 var kstrbs []byte 1006 1007 for j := 0; (hasLen && j < containerLen) || !(hasLen || d.checkBreak()); j++ { 1008 if j == 0 { 1009 if !rvkMut { 1010 rvkn = rvZeroAddrK(ktype, ktypeKind) 1011 } 1012 if !rvvMut { 1013 rvvn = rvZeroAddrK(vtype, vtypeKind) 1014 } 1015 } 1016 1017 if rvkMut { 1018 rvk = rvZeroAddrK(ktype, ktypeKind) 1019 } else { 1020 rvk = rvkn 1021 } 1022 1023 d.mapElemKey() 1024 1025 if ktypeIsString { 1026 kstrbs = d.d.DecodeStringAsBytes() 1027 rvk.SetString(stringView(kstrbs)) // NOTE: if doing an insert, use real string (not stringview) 1028 } else { 1029 if keyFn == nil { 1030 keyFn = d.h.fn(ktypeLo) 1031 } 1032 d.decodeValue(rvk, keyFn) 1033 } 1034 1035 // special case if interface wrapping a byte array. 1036 if ktypeIsIntf { 1037 if rvk2 := rvk.Elem(); rvk2.IsValid() && rvk2.Type() == uint8SliceTyp { 1038 rvk.Set(rv4i(d.string(rvGetBytes(rvk2)))) 1039 } 1040 // NOTE: consider failing early if map/slice/func 1041 } 1042 1043 d.mapElemValue() 1044 1045 doMapSet = true // set to false if u do a get, and its a non-nil pointer 1046 if doMapGet { 1047 if !rvvaSet { 1048 rvva = mapAddressableRV(vtype, vtypeKind) 1049 rvvaSet = true 1050 } 1051 rvv = mapGet(rv, rvk, rvva) // reflect.Value{}) 1052 if vtypeKind == reflect.Ptr { 1053 if rvv.IsValid() && !rvIsNil(rvv) { 1054 doMapSet = false 1055 } else { 1056 if vtypeElem == nil { 1057 vtypeElem = vtype.Elem() 1058 } 1059 rvv = reflect.New(vtypeElem) 1060 } 1061 } else if rvv.IsValid() && vtypeKind == reflect.Interface && !rvIsNil(rvv) { 1062 rvvn = rvZeroAddrK(vtype, vtypeKind) 1063 rvvn.Set(rvv) 1064 rvv = rvvn 1065 } else if rvvMut { 1066 rvv = rvZeroAddrK(vtype, vtypeKind) 1067 } else { 1068 rvv = rvvn 1069 } 1070 } else if rvvMut { 1071 rvv = rvZeroAddrK(vtype, vtypeKind) 1072 } else { 1073 rvv = rvvn 1074 } 1075 1076 if valFn == nil { 1077 valFn = d.h.fn(vtypeLo) 1078 } 1079 1080 // We MUST be done with the stringview of the key, BEFORE decoding the value (rvv) 1081 // so that we don't unknowingly reuse the rvk backing buffer during rvv decode. 1082 if doMapSet && ktypeIsString { // set to a real string (not string view) 1083 rvk.SetString(d.string(kstrbs)) 1084 } 1085 d.decodeValue(rvv, valFn) 1086 if doMapSet { 1087 mapSet(rv, rvk, rvv) 1088 } 1089 } 1090 1091 d.mapEnd() 1092 1093} 1094 1095// decNaked is used to keep track of the primitives decoded. 1096// Without it, we would have to decode each primitive and wrap it 1097// in an interface{}, causing an allocation. 1098// In this model, the primitives are decoded in a "pseudo-atomic" fashion, 1099// so we can rest assured that no other decoding happens while these 1100// primitives are being decoded. 1101// 1102// maps and arrays are not handled by this mechanism. 1103// However, RawExt is, and we accommodate for extensions that decode 1104// RawExt from DecodeNaked, but need to decode the value subsequently. 1105// kInterfaceNaked and swallow, which call DecodeNaked, handle this caveat. 1106// 1107// However, decNaked also keeps some arrays of default maps and slices 1108// used in DecodeNaked. This way, we can get a pointer to it 1109// without causing a new heap allocation. 1110// 1111// kInterfaceNaked will ensure that there is no allocation for the common 1112// uses. 1113 1114type decNaked struct { 1115 // r RawExt // used for RawExt, uint, []byte. 1116 1117 // primitives below 1118 u uint64 1119 i int64 1120 f float64 1121 l []byte 1122 s string 1123 1124 // ---- cpu cache line boundary? 1125 t time.Time 1126 b bool 1127 1128 // state 1129 v valueType 1130} 1131 1132// Decoder reads and decodes an object from an input stream in a supported format. 1133// 1134// Decoder is NOT safe for concurrent use i.e. a Decoder cannot be used 1135// concurrently in multiple goroutines. 1136// 1137// However, as Decoder could be allocation heavy to initialize, a Reset method is provided 1138// so its state can be reused to decode new input streams repeatedly. 1139// This is the idiomatic way to use. 1140type Decoder struct { 1141 panicHdl 1142 // hopefully, reduce derefencing cost by laying the decReader inside the Decoder. 1143 // Try to put things that go together to fit within a cache line (8 words). 1144 1145 d decDriver 1146 1147 // cache the mapTypeId and sliceTypeId for faster comparisons 1148 mtid uintptr 1149 stid uintptr 1150 1151 h *BasicHandle 1152 1153 blist bytesFreelist 1154 1155 // ---- cpu cache line boundary? 1156 decRd 1157 1158 // ---- cpu cache line boundary? 1159 n decNaked 1160 1161 hh Handle 1162 err error 1163 1164 // ---- cpu cache line boundary? 1165 is map[string]string // used for interning strings 1166 1167 // ---- writable fields during execution --- *try* to keep in sep cache line 1168 maxdepth int16 1169 depth int16 1170 1171 // Extensions can call Decode() within a current Decode() call. 1172 // We need to know when the top level Decode() call returns, 1173 // so we can decide whether to Release() or not. 1174 calls uint16 // what depth in mustDecode are we in now. 1175 1176 c containerState 1177 _ [1]byte // padding 1178 1179 // ---- cpu cache line boundary? 1180 1181 // b is an always-available scratch buffer used by Decoder and decDrivers. 1182 // By being always-available, it can be used for one-off things without 1183 // having to get from freelist, use, and return back to freelist. 1184 b [decScratchByteArrayLen]byte 1185} 1186 1187// NewDecoder returns a Decoder for decoding a stream of bytes from an io.Reader. 1188// 1189// For efficiency, Users are encouraged to configure ReaderBufferSize on the handle 1190// OR pass in a memory buffered reader (eg bufio.Reader, bytes.Buffer). 1191func NewDecoder(r io.Reader, h Handle) *Decoder { 1192 d := h.newDecDriver().decoder() 1193 d.Reset(r) 1194 return d 1195} 1196 1197// NewDecoderBytes returns a Decoder which efficiently decodes directly 1198// from a byte slice with zero copying. 1199func NewDecoderBytes(in []byte, h Handle) *Decoder { 1200 d := h.newDecDriver().decoder() 1201 d.ResetBytes(in) 1202 return d 1203} 1204 1205func (d *Decoder) r() *decRd { 1206 return &d.decRd 1207} 1208 1209func (d *Decoder) init(h Handle) { 1210 d.bytes = true 1211 d.err = errDecoderNotInitialized 1212 d.h = basicHandle(h) 1213 d.hh = h 1214 d.be = h.isBinary() 1215 // NOTE: do not initialize d.n here. It is lazily initialized in d.naked() 1216 if d.h.InternString { 1217 d.is = make(map[string]string, 32) 1218 } 1219} 1220 1221func (d *Decoder) resetCommon() { 1222 d.d.reset() 1223 d.err = nil 1224 d.depth = 0 1225 d.calls = 0 1226 d.maxdepth = d.h.MaxDepth 1227 if d.maxdepth <= 0 { 1228 d.maxdepth = decDefMaxDepth 1229 } 1230 // reset all things which were cached from the Handle, but could change 1231 d.mtid, d.stid = 0, 0 1232 d.mtr, d.str = false, false 1233 if d.h.MapType != nil { 1234 d.mtid = rt2id(d.h.MapType) 1235 d.mtr = fastpathAV.index(d.mtid) != -1 1236 } 1237 if d.h.SliceType != nil { 1238 d.stid = rt2id(d.h.SliceType) 1239 d.str = fastpathAV.index(d.stid) != -1 1240 } 1241} 1242 1243// Reset the Decoder with a new Reader to decode from, 1244// clearing all state from last run(s). 1245func (d *Decoder) Reset(r io.Reader) { 1246 if r == nil { 1247 return 1248 } 1249 d.bytes = false 1250 if d.h.ReaderBufferSize > 0 { 1251 if d.bi == nil { 1252 d.bi = new(bufioDecReader) 1253 } 1254 d.bi.reset(r, d.h.ReaderBufferSize, &d.blist) 1255 d.bufio = true 1256 } else { 1257 if d.ri == nil { 1258 d.ri = new(ioDecReader) 1259 } 1260 d.ri.reset(r, &d.blist) 1261 d.bufio = false 1262 } 1263 d.resetCommon() 1264} 1265 1266// ResetBytes resets the Decoder with a new []byte to decode from, 1267// clearing all state from last run(s). 1268func (d *Decoder) ResetBytes(in []byte) { 1269 if in == nil { 1270 return 1271 } 1272 d.bytes = true 1273 d.bufio = false 1274 d.rb.reset(in) 1275 d.resetCommon() 1276} 1277 1278func (d *Decoder) naked() *decNaked { 1279 return &d.n 1280} 1281 1282// Decode decodes the stream from reader and stores the result in the 1283// value pointed to by v. v cannot be a nil pointer. v can also be 1284// a reflect.Value of a pointer. 1285// 1286// Note that a pointer to a nil interface is not a nil pointer. 1287// If you do not know what type of stream it is, pass in a pointer to a nil interface. 1288// We will decode and store a value in that nil interface. 1289// 1290// Sample usages: 1291// // Decoding into a non-nil typed value 1292// var f float32 1293// err = codec.NewDecoder(r, handle).Decode(&f) 1294// 1295// // Decoding into nil interface 1296// var v interface{} 1297// dec := codec.NewDecoder(r, handle) 1298// err = dec.Decode(&v) 1299// 1300// When decoding into a nil interface{}, we will decode into an appropriate value based 1301// on the contents of the stream: 1302// - Numbers are decoded as float64, int64 or uint64. 1303// - Other values are decoded appropriately depending on the type: 1304// bool, string, []byte, time.Time, etc 1305// - Extensions are decoded as RawExt (if no ext function registered for the tag) 1306// Configurations exist on the Handle to override defaults 1307// (e.g. for MapType, SliceType and how to decode raw bytes). 1308// 1309// When decoding into a non-nil interface{} value, the mode of encoding is based on the 1310// type of the value. When a value is seen: 1311// - If an extension is registered for it, call that extension function 1312// - If it implements BinaryUnmarshaler, call its UnmarshalBinary(data []byte) error 1313// - Else decode it based on its reflect.Kind 1314// 1315// There are some special rules when decoding into containers (slice/array/map/struct). 1316// Decode will typically use the stream contents to UPDATE the container i.e. the values 1317// in these containers will not be zero'ed before decoding. 1318// - A map can be decoded from a stream map, by updating matching keys. 1319// - A slice can be decoded from a stream array, 1320// by updating the first n elements, where n is length of the stream. 1321// - A slice can be decoded from a stream map, by decoding as if 1322// it contains a sequence of key-value pairs. 1323// - A struct can be decoded from a stream map, by updating matching fields. 1324// - A struct can be decoded from a stream array, 1325// by updating fields as they occur in the struct (by index). 1326// 1327// This in-place update maintains consistency in the decoding philosophy (i.e. we ALWAYS update 1328// in place by default). However, the consequence of this is that values in slices or maps 1329// which are not zero'ed before hand, will have part of the prior values in place after decode 1330// if the stream doesn't contain an update for those parts. 1331// 1332// This in-place update can be disabled by configuring the MapValueReset and SliceElementReset 1333// decode options available on every handle. 1334// 1335// Furthermore, when decoding a stream map or array with length of 0 into a nil map or slice, 1336// we reset the destination map or slice to a zero-length value. 1337// 1338// However, when decoding a stream nil, we reset the destination container 1339// to its "zero" value (e.g. nil for slice/map, etc). 1340// 1341// Note: we allow nil values in the stream anywhere except for map keys. 1342// A nil value in the encoded stream where a map key is expected is treated as an error. 1343func (d *Decoder) Decode(v interface{}) (err error) { 1344 // tried to use closure, as runtime optimizes defer with no params. 1345 // This seemed to be causing weird issues (like circular reference found, unexpected panic, etc). 1346 // Also, see https://github.com/golang/go/issues/14939#issuecomment-417836139 1347 // defer func() { d.deferred(&err) }() 1348 // { x, y := d, &err; defer func() { x.deferred(y) }() } 1349 if d.err != nil { 1350 return d.err 1351 } 1352 if recoverPanicToErr { 1353 defer func() { 1354 if x := recover(); x != nil { 1355 panicValToErr(d, x, &d.err) 1356 if d.err != err { 1357 err = d.err 1358 } 1359 } 1360 }() 1361 } 1362 1363 // defer d.deferred(&err) 1364 d.mustDecode(v) 1365 return 1366} 1367 1368// MustDecode is like Decode, but panics if unable to Decode. 1369// This provides insight to the code location that triggered the error. 1370func (d *Decoder) MustDecode(v interface{}) { 1371 if d.err != nil { 1372 panic(d.err) 1373 } 1374 d.mustDecode(v) 1375} 1376 1377// MustDecode is like Decode, but panics if unable to Decode. 1378// This provides insight to the code location that triggered the error. 1379func (d *Decoder) mustDecode(v interface{}) { 1380 // Top-level: v is a pointer and not nil. 1381 1382 d.calls++ 1383 d.decode(v) 1384 d.calls-- 1385 if d.calls == 0 { 1386 d.d.atEndOfDecode() 1387 } 1388} 1389 1390// Release releases shared (pooled) resources. 1391// 1392// It is important to call Release() when done with a Decoder, so those resources 1393// are released instantly for use by subsequently created Decoders. 1394// 1395// By default, Release() is automatically called unless the option ExplicitRelease is set. 1396// 1397// Deprecated: Release is a no-op as pooled resources are not used with an Decoder. 1398// This method is kept for compatibility reasons only. 1399func (d *Decoder) Release() { 1400} 1401 1402func (d *Decoder) swallow() { 1403 switch d.d.ContainerType() { 1404 case valueTypeNil: 1405 case valueTypeMap: 1406 containerLen := d.mapStart() 1407 hasLen := containerLen >= 0 1408 for j := 0; (hasLen && j < containerLen) || !(hasLen || d.checkBreak()); j++ { 1409 d.mapElemKey() 1410 d.swallow() 1411 d.mapElemValue() 1412 d.swallow() 1413 } 1414 d.mapEnd() 1415 case valueTypeArray: 1416 containerLen := d.arrayStart() 1417 hasLen := containerLen >= 0 1418 for j := 0; (hasLen && j < containerLen) || !(hasLen || d.checkBreak()); j++ { 1419 d.arrayElem() 1420 d.swallow() 1421 } 1422 d.arrayEnd() 1423 case valueTypeBytes: 1424 d.d.DecodeBytes(d.b[:], true) 1425 case valueTypeString: 1426 d.d.DecodeStringAsBytes() 1427 default: 1428 // these are all primitives, which we can get from decodeNaked 1429 // if RawExt using Value, complete the processing. 1430 n := d.naked() 1431 d.d.DecodeNaked() 1432 if n.v == valueTypeExt && n.l == nil { 1433 var v2 interface{} 1434 d.decode(&v2) 1435 } 1436 } 1437} 1438 1439func setZero(iv interface{}) { 1440 if iv == nil { 1441 return 1442 } 1443 if _, ok := isNil(iv); ok { 1444 return 1445 } 1446 // var canDecode bool 1447 switch v := iv.(type) { 1448 case *string: 1449 *v = "" 1450 case *bool: 1451 *v = false 1452 case *int: 1453 *v = 0 1454 case *int8: 1455 *v = 0 1456 case *int16: 1457 *v = 0 1458 case *int32: 1459 *v = 0 1460 case *int64: 1461 *v = 0 1462 case *uint: 1463 *v = 0 1464 case *uint8: 1465 *v = 0 1466 case *uint16: 1467 *v = 0 1468 case *uint32: 1469 *v = 0 1470 case *uint64: 1471 *v = 0 1472 case *float32: 1473 *v = 0 1474 case *float64: 1475 *v = 0 1476 case *[]uint8: 1477 *v = nil 1478 case *Raw: 1479 *v = nil 1480 case *time.Time: 1481 *v = time.Time{} 1482 case reflect.Value: 1483 setZeroRV(v) 1484 default: 1485 if !fastpathDecodeSetZeroTypeSwitch(iv) { 1486 setZeroRV(rv4i(iv)) 1487 } 1488 } 1489} 1490 1491func setZeroRV(v reflect.Value) { 1492 // It not decodeable, we do not touch it. 1493 // We considered empty'ing it if not decodeable e.g. 1494 // - if chan, drain it 1495 // - if map, clear it 1496 // - if slice or array, zero all elements up to len 1497 // 1498 // However, we decided instead that we either will set the 1499 // whole value to the zero value, or leave AS IS. 1500 if isDecodeable(v) { 1501 if v.Kind() == reflect.Ptr { 1502 v = v.Elem() 1503 } 1504 if v.CanSet() { 1505 v.Set(reflect.Zero(v.Type())) 1506 } 1507 } 1508} 1509 1510func (d *Decoder) decode(iv interface{}) { 1511 // a switch with only concrete types can be optimized. 1512 // consequently, we deal with nil and interfaces outside the switch. 1513 1514 if iv == nil { 1515 d.errorstr(errstrCannotDecodeIntoNil) 1516 return 1517 } 1518 1519 switch v := iv.(type) { 1520 // case nil: 1521 // case Selfer: 1522 case reflect.Value: 1523 d.ensureDecodeable(v) 1524 d.decodeValue(v, nil) 1525 1526 case *string: 1527 *v = string(d.d.DecodeStringAsBytes()) 1528 case *bool: 1529 *v = d.d.DecodeBool() 1530 case *int: 1531 *v = int(chkOvf.IntV(d.d.DecodeInt64(), intBitsize)) 1532 case *int8: 1533 *v = int8(chkOvf.IntV(d.d.DecodeInt64(), 8)) 1534 case *int16: 1535 *v = int16(chkOvf.IntV(d.d.DecodeInt64(), 16)) 1536 case *int32: 1537 *v = int32(chkOvf.IntV(d.d.DecodeInt64(), 32)) 1538 case *int64: 1539 *v = d.d.DecodeInt64() 1540 case *uint: 1541 *v = uint(chkOvf.UintV(d.d.DecodeUint64(), uintBitsize)) 1542 case *uint8: 1543 *v = uint8(chkOvf.UintV(d.d.DecodeUint64(), 8)) 1544 case *uint16: 1545 *v = uint16(chkOvf.UintV(d.d.DecodeUint64(), 16)) 1546 case *uint32: 1547 *v = uint32(chkOvf.UintV(d.d.DecodeUint64(), 32)) 1548 case *uint64: 1549 *v = d.d.DecodeUint64() 1550 case *float32: 1551 *v = float32(d.decodeFloat32()) 1552 case *float64: 1553 *v = d.d.DecodeFloat64() 1554 case *[]uint8: 1555 *v = d.d.DecodeBytes(*v, false) 1556 case []uint8: 1557 b := d.d.DecodeBytes(v, false) 1558 if !(len(b) > 0 && len(b) == len(v) && &b[0] == &v[0]) { 1559 copy(v, b) 1560 } 1561 case *time.Time: 1562 *v = d.d.DecodeTime() 1563 case *Raw: 1564 *v = d.rawBytes() 1565 1566 case *interface{}: 1567 d.decodeValue(rv4i(iv), nil) 1568 1569 default: 1570 if v, ok := iv.(Selfer); ok { 1571 v.CodecDecodeSelf(d) 1572 } else if !fastpathDecodeTypeSwitch(iv, d) { 1573 v := rv4i(iv) 1574 d.ensureDecodeable(v) 1575 d.decodeValue(v, nil) 1576 } 1577 } 1578} 1579 1580// decodeValue MUST be called by the actual value we want to decode into, 1581// not its addr or a reference to it. 1582// 1583// This way, we know if it is itself a pointer, and can handle nil in 1584// the stream effectively. 1585func (d *Decoder) decodeValue(rv reflect.Value, fn *codecFn) { 1586 // If stream is not containing a nil value, then we can deref to the base 1587 // non-pointer value, and decode into that. 1588 var rvp reflect.Value 1589 var rvpValid bool 1590 if rv.Kind() == reflect.Ptr { 1591 if d.d.TryNil() { 1592 if rvelem := rv.Elem(); rvelem.CanSet() { 1593 rvelem.Set(reflect.Zero(rvelem.Type())) 1594 } 1595 return 1596 } 1597 rvpValid = true 1598 for rv.Kind() == reflect.Ptr { 1599 if rvIsNil(rv) { 1600 rvSetDirect(rv, reflect.New(rv.Type().Elem())) 1601 } 1602 rvp = rv 1603 rv = rv.Elem() 1604 } 1605 } 1606 1607 if fn == nil { 1608 fn = d.h.fn(rv.Type()) 1609 } 1610 if fn.i.addrD { 1611 if rvpValid { 1612 fn.fd(d, &fn.i, rvp) 1613 } else if rv.CanAddr() { 1614 fn.fd(d, &fn.i, rv.Addr()) 1615 } else if !fn.i.addrF { 1616 fn.fd(d, &fn.i, rv) 1617 } else { 1618 d.errorf("cannot decode into a non-pointer value") 1619 } 1620 } else { 1621 fn.fd(d, &fn.i, rv) 1622 } 1623} 1624 1625func (d *Decoder) structFieldNotFound(index int, rvkencname string) { 1626 // NOTE: rvkencname may be a stringView, so don't pass it to another function. 1627 if d.h.ErrorIfNoField { 1628 if index >= 0 { 1629 d.errorf("no matching struct field found when decoding stream array at index %v", index) 1630 return 1631 } else if rvkencname != "" { 1632 d.errorf("no matching struct field found when decoding stream map with key " + rvkencname) 1633 return 1634 } 1635 } 1636 d.swallow() 1637} 1638 1639func (d *Decoder) arrayCannotExpand(sliceLen, streamLen int) { 1640 if d.h.ErrorIfNoArrayExpand { 1641 d.errorf("cannot expand array len during decode from %v to %v", sliceLen, streamLen) 1642 } 1643} 1644 1645func isDecodeable(rv reflect.Value) (canDecode bool) { 1646 switch rv.Kind() { 1647 case reflect.Array: 1648 return rv.CanAddr() 1649 case reflect.Ptr: 1650 if !rvIsNil(rv) { 1651 return true 1652 } 1653 case reflect.Slice, reflect.Chan, reflect.Map: 1654 if !rvIsNil(rv) { 1655 return true 1656 } 1657 } 1658 return 1659} 1660 1661func (d *Decoder) ensureDecodeable(rv reflect.Value) { 1662 // decode can take any reflect.Value that is a inherently addressable i.e. 1663 // - array 1664 // - non-nil chan (we will SEND to it) 1665 // - non-nil slice (we will set its elements) 1666 // - non-nil map (we will put into it) 1667 // - non-nil pointer (we can "update" it) 1668 if isDecodeable(rv) { 1669 return 1670 } 1671 if !rv.IsValid() { 1672 d.errorstr(errstrCannotDecodeIntoNil) 1673 return 1674 } 1675 if !rv.CanInterface() { 1676 d.errorf("cannot decode into a value without an interface: %v", rv) 1677 return 1678 } 1679 rvi := rv2i(rv) 1680 rvk := rv.Kind() 1681 d.errorf("cannot decode into value of kind: %v, type: %T, %#v", rvk, rvi, rvi) 1682} 1683 1684func (d *Decoder) depthIncr() { 1685 d.depth++ 1686 if d.depth >= d.maxdepth { 1687 panic(errMaxDepthExceeded) 1688 } 1689} 1690 1691func (d *Decoder) depthDecr() { 1692 d.depth-- 1693} 1694 1695// Possibly get an interned version of a string 1696// 1697// This should mostly be used for map keys, where the key type is string. 1698// This is because keys of a map/struct are typically reused across many objects. 1699func (d *Decoder) string(v []byte) (s string) { 1700 if v == nil { 1701 return 1702 } 1703 if d.is == nil { 1704 return string(v) // don't return stringView, as we need a real string here. 1705 } 1706 s, ok := d.is[string(v)] // no allocation here, per go implementation 1707 if !ok { 1708 s = string(v) // new allocation here 1709 d.is[s] = s 1710 } 1711 return 1712} 1713 1714// nextValueBytes returns the next value in the stream as a set of bytes. 1715func (d *Decoder) nextValueBytes() (bs []byte) { 1716 d.d.uncacheRead() 1717 d.r().track() 1718 d.swallow() 1719 bs = d.r().stopTrack() 1720 return 1721} 1722 1723func (d *Decoder) rawBytes() []byte { 1724 // ensure that this is not a view into the bytes 1725 // i.e. make new copy always. 1726 bs := d.nextValueBytes() 1727 bs2 := make([]byte, len(bs)) 1728 copy(bs2, bs) 1729 return bs2 1730} 1731 1732func (d *Decoder) wrapErr(v interface{}, err *error) { 1733 *err = decodeError{codecError: codecError{name: d.hh.Name(), err: v}, pos: d.NumBytesRead()} 1734} 1735 1736// NumBytesRead returns the number of bytes read 1737func (d *Decoder) NumBytesRead() int { 1738 return int(d.r().numread()) 1739} 1740 1741// decodeFloat32 will delegate to an appropriate DecodeFloat32 implementation (if exists), 1742// else if will call DecodeFloat64 and ensure the value doesn't overflow. 1743// 1744// Note that we return float64 to reduce unnecessary conversions 1745func (d *Decoder) decodeFloat32() float32 { 1746 if d.js { 1747 return d.jsondriver().DecodeFloat32() // custom implementation for 32-bit 1748 } 1749 return float32(chkOvf.Float32V(d.d.DecodeFloat64())) 1750} 1751 1752// ---- container tracking 1753// Note: We update the .c after calling the callback. 1754// This way, the callback can know what the last status was. 1755 1756// Note: if you call mapStart and it returns decContainerLenNil, 1757// then do NOT call mapEnd. 1758 1759func (d *Decoder) mapStart() (v int) { 1760 v = d.d.ReadMapStart() 1761 if v != decContainerLenNil { 1762 d.depthIncr() 1763 d.c = containerMapStart 1764 } 1765 return 1766} 1767 1768func (d *Decoder) mapElemKey() { 1769 if d.js { 1770 d.jsondriver().ReadMapElemKey() 1771 } 1772 d.c = containerMapKey 1773} 1774 1775func (d *Decoder) mapElemValue() { 1776 if d.js { 1777 d.jsondriver().ReadMapElemValue() 1778 } 1779 d.c = containerMapValue 1780} 1781 1782func (d *Decoder) mapEnd() { 1783 d.d.ReadMapEnd() 1784 d.depthDecr() 1785 // d.c = containerMapEnd 1786 d.c = 0 1787} 1788 1789func (d *Decoder) arrayStart() (v int) { 1790 v = d.d.ReadArrayStart() 1791 if v != decContainerLenNil { 1792 d.depthIncr() 1793 d.c = containerArrayStart 1794 } 1795 return 1796} 1797 1798func (d *Decoder) arrayElem() { 1799 if d.js { 1800 d.jsondriver().ReadArrayElem() 1801 } 1802 d.c = containerArrayElem 1803} 1804 1805func (d *Decoder) arrayEnd() { 1806 d.d.ReadArrayEnd() 1807 d.depthDecr() 1808 // d.c = containerArrayEnd 1809 d.c = 0 1810} 1811 1812func (d *Decoder) interfaceExtConvertAndDecode(v interface{}, ext Ext) { 1813 // var v interface{} = ext.ConvertExt(rv) 1814 // d.d.decode(&v) 1815 // ext.UpdateExt(rv, v) 1816 1817 // assume v is a pointer: 1818 // - if struct|array, pass as is to ConvertExt 1819 // - else make it non-addressable and pass to ConvertExt 1820 // - make return value from ConvertExt addressable 1821 // - decode into it 1822 // - return the interface for passing into UpdateExt. 1823 // - interface should be a pointer if struct|array, else a value 1824 1825 var s interface{} 1826 rv := rv4i(v) 1827 rv2 := rv.Elem() 1828 rvk := rv2.Kind() 1829 if rvk == reflect.Struct || rvk == reflect.Array { 1830 s = ext.ConvertExt(v) 1831 } else { 1832 s = ext.ConvertExt(rv2i(rv2)) 1833 } 1834 rv = rv4i(s) 1835 if !rv.CanAddr() { 1836 if rv.Kind() == reflect.Ptr { 1837 rv2 = reflect.New(rv.Type().Elem()) 1838 } else { 1839 rv2 = rvZeroAddrK(rv.Type(), rv.Kind()) 1840 } 1841 rvSetDirect(rv2, rv) 1842 rv = rv2 1843 } 1844 d.decodeValue(rv, nil) 1845 ext.UpdateExt(v, rv2i(rv)) 1846} 1847 1848func (d *Decoder) sideDecode(v interface{}, bs []byte) { 1849 rv := baseRV(v) 1850 NewDecoderBytes(bs, d.hh).decodeValue(rv, d.h.fnNoExt(rv.Type())) 1851} 1852 1853// -------------------------------------------------- 1854 1855// decSliceHelper assists when decoding into a slice, from a map or an array in the stream. 1856// A slice can be set from a map or array in stream. This supports the MapBySlice interface. 1857// 1858// Note: if IsNil, do not call ElemContainerState. 1859type decSliceHelper struct { 1860 d *Decoder 1861 ct valueType 1862 Array bool 1863 IsNil bool 1864} 1865 1866func (d *Decoder) decSliceHelperStart() (x decSliceHelper, clen int) { 1867 x.ct = d.d.ContainerType() 1868 x.d = d 1869 switch x.ct { 1870 case valueTypeNil: 1871 x.IsNil = true 1872 case valueTypeArray: 1873 x.Array = true 1874 clen = d.arrayStart() 1875 case valueTypeMap: 1876 clen = d.mapStart() * 2 1877 default: 1878 d.errorf("only encoded map or array can be decoded into a slice (%d)", x.ct) 1879 } 1880 return 1881} 1882 1883func (x decSliceHelper) End() { 1884 if x.IsNil { 1885 } else if x.Array { 1886 x.d.arrayEnd() 1887 } else { 1888 x.d.mapEnd() 1889 } 1890} 1891 1892func (x decSliceHelper) ElemContainerState(index int) { 1893 // Note: if isnil, clen=0, so we never call into ElemContainerState 1894 1895 if x.Array { 1896 x.d.arrayElem() 1897 } else { 1898 if index%2 == 0 { 1899 x.d.mapElemKey() 1900 } else { 1901 x.d.mapElemValue() 1902 } 1903 } 1904} 1905 1906func decByteSlice(r *decRd, clen, maxInitLen int, bs []byte) (bsOut []byte) { 1907 if clen == 0 { 1908 return zeroByteSlice 1909 } 1910 if len(bs) == clen { 1911 bsOut = bs 1912 r.readb(bsOut) 1913 } else if cap(bs) >= clen { 1914 bsOut = bs[:clen] 1915 r.readb(bsOut) 1916 } else { 1917 len2 := decInferLen(clen, maxInitLen, 1) 1918 bsOut = make([]byte, len2) 1919 r.readb(bsOut) 1920 for len2 < clen { 1921 len3 := decInferLen(clen-len2, maxInitLen, 1) 1922 bs3 := bsOut 1923 bsOut = make([]byte, len2+len3) 1924 copy(bsOut, bs3) 1925 r.readb(bsOut[len2:]) 1926 len2 += len3 1927 } 1928 } 1929 return 1930} 1931 1932// detachZeroCopyBytes will copy the in bytes into dest, 1933// or create a new one if not large enough. 1934// 1935// It is used to ensure that the []byte returned is not 1936// part of the input stream or input stream buffers. 1937func detachZeroCopyBytes(isBytesReader bool, dest []byte, in []byte) (out []byte) { 1938 if len(in) > 0 { 1939 // if isBytesReader || len(in) <= scratchByteArrayLen { 1940 // if cap(dest) >= len(in) { 1941 // out = dest[:len(in)] 1942 // } else { 1943 // out = make([]byte, len(in)) 1944 // } 1945 // copy(out, in) 1946 // return 1947 // } 1948 if cap(dest) >= len(in) { 1949 out = dest[:len(in)] 1950 } else { 1951 out = make([]byte, len(in)) 1952 } 1953 copy(out, in) 1954 return 1955 } 1956 return in 1957} 1958 1959// decInferLen will infer a sensible length, given the following: 1960// - clen: length wanted. 1961// - maxlen: max length to be returned. 1962// if <= 0, it is unset, and we infer it based on the unit size 1963// - unit: number of bytes for each element of the collection 1964func decInferLen(clen, maxlen, unit int) (rvlen int) { 1965 const maxLenIfUnset = 8 // 64 1966 // handle when maxlen is not set i.e. <= 0 1967 1968 // clen==0: use 0 1969 // maxlen<=0, clen<0: use default 1970 // maxlen> 0, clen<0: use default 1971 // maxlen<=0, clen>0: infer maxlen, and cap on it 1972 // maxlen> 0, clen>0: cap at maxlen 1973 1974 if clen == 0 { 1975 return 1976 } 1977 if clen < 0 { 1978 if clen == decContainerLenNil { 1979 return 0 1980 } 1981 return maxLenIfUnset 1982 } 1983 if unit == 0 { 1984 return clen 1985 } 1986 if maxlen <= 0 { 1987 // no maxlen defined. Use maximum of 256K memory, with a floor of 4K items. 1988 // maxlen = 256 * 1024 / unit 1989 // if maxlen < (4 * 1024) { 1990 // maxlen = 4 * 1024 1991 // } 1992 if unit < (256 / 4) { 1993 maxlen = 256 * 1024 / unit 1994 } else { 1995 maxlen = 4 * 1024 1996 } 1997 // if maxlen > maxLenIfUnset { 1998 // maxlen = maxLenIfUnset 1999 // } 2000 } 2001 if clen > maxlen { 2002 rvlen = maxlen 2003 } else { 2004 rvlen = clen 2005 } 2006 return 2007} 2008 2009func decReadFull(r io.Reader, bs []byte) (n uint, err error) { 2010 var nn int 2011 for n < uint(len(bs)) && err == nil { 2012 nn, err = r.Read(bs[n:]) 2013 if nn > 0 { 2014 if err == io.EOF { 2015 // leave EOF for next time 2016 err = nil 2017 } 2018 n += uint(nn) 2019 } 2020 } 2021 // do not do this - it serves no purpose 2022 // if n != len(bs) && err == io.EOF { err = io.ErrUnexpectedEOF } 2023 return 2024} 2025 2026func decNakedReadRawBytes(dr decDriver, d *Decoder, n *decNaked, rawToString bool) { 2027 if rawToString { 2028 n.v = valueTypeString 2029 n.s = string(dr.DecodeBytes(d.b[:], true)) 2030 } else { 2031 n.v = valueTypeBytes 2032 n.l = dr.DecodeBytes(nil, false) 2033 } 2034} 2035