1// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. 2// Use of this source code is governed by a MIT license found in the LICENSE file. 3 4package codec 5 6import ( 7 "math" 8 "reflect" 9 "time" 10) 11 12const bincDoPrune = true // No longer needed. Needed before as C lib did not support pruning. 13 14// vd as low 4 bits (there are 16 slots) 15const ( 16 bincVdSpecial byte = iota 17 bincVdPosInt 18 bincVdNegInt 19 bincVdFloat 20 21 bincVdString 22 bincVdByteArray 23 bincVdArray 24 bincVdMap 25 26 bincVdTimestamp 27 bincVdSmallInt 28 bincVdUnicodeOther 29 bincVdSymbol 30 31 bincVdDecimal 32 _ // open slot 33 _ // open slot 34 bincVdCustomExt = 0x0f 35) 36 37const ( 38 bincSpNil byte = iota 39 bincSpFalse 40 bincSpTrue 41 bincSpNan 42 bincSpPosInf 43 bincSpNegInf 44 bincSpZeroFloat 45 bincSpZero 46 bincSpNegOne 47) 48 49const ( 50 bincFlBin16 byte = iota 51 bincFlBin32 52 _ // bincFlBin32e 53 bincFlBin64 54 _ // bincFlBin64e 55 // others not currently supported 56) 57 58func bincdesc(vd, vs byte) string { 59 switch vd { 60 case bincVdSpecial: 61 switch vs { 62 case bincSpNil: 63 return "nil" 64 case bincSpFalse: 65 return "false" 66 case bincSpTrue: 67 return "true" 68 case bincSpNan, bincSpPosInf, bincSpNegInf, bincSpZeroFloat: 69 return "float" 70 case bincSpZero: 71 return "uint" 72 case bincSpNegOne: 73 return "int" 74 default: 75 return "unknown" 76 } 77 case bincVdSmallInt, bincVdPosInt: 78 return "uint" 79 case bincVdNegInt: 80 return "int" 81 case bincVdFloat: 82 return "float" 83 case bincVdSymbol: 84 return "string" 85 case bincVdString: 86 return "string" 87 case bincVdByteArray: 88 return "bytes" 89 case bincVdTimestamp: 90 return "time" 91 case bincVdCustomExt: 92 return "ext" 93 case bincVdArray: 94 return "array" 95 case bincVdMap: 96 return "map" 97 default: 98 return "unknown" 99 } 100} 101 102type bincEncDriver struct { 103 e *Encoder 104 h *BincHandle 105 w *encWriterSwitch 106 m map[string]uint16 // symbols 107 b [16]byte // scratch, used for encoding numbers - bigendian style 108 s uint16 // symbols sequencer 109 // c containerState 110 encDriverTrackContainerWriter 111 noBuiltInTypes 112 // encNoSeparator 113 _ [1]uint64 // padding 114} 115 116func (e *bincEncDriver) EncodeNil() { 117 e.w.writen1(bincVdSpecial<<4 | bincSpNil) 118} 119 120func (e *bincEncDriver) EncodeTime(t time.Time) { 121 if t.IsZero() { 122 e.EncodeNil() 123 } else { 124 bs := bincEncodeTime(t) 125 e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs))) 126 e.w.writeb(bs) 127 } 128} 129 130func (e *bincEncDriver) EncodeBool(b bool) { 131 if b { 132 e.w.writen1(bincVdSpecial<<4 | bincSpTrue) 133 } else { 134 e.w.writen1(bincVdSpecial<<4 | bincSpFalse) 135 } 136} 137 138func (e *bincEncDriver) EncodeFloat32(f float32) { 139 if f == 0 { 140 e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat) 141 return 142 } 143 e.w.writen1(bincVdFloat<<4 | bincFlBin32) 144 bigenHelper{e.b[:4], e.w}.writeUint32(math.Float32bits(f)) 145} 146 147func (e *bincEncDriver) EncodeFloat64(f float64) { 148 if f == 0 { 149 e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat) 150 return 151 } 152 bigen.PutUint64(e.b[:8], math.Float64bits(f)) 153 if bincDoPrune { 154 i := 7 155 for ; i >= 0 && (e.b[i] == 0); i-- { 156 } 157 i++ 158 if i <= 6 { 159 e.w.writen1(bincVdFloat<<4 | 0x8 | bincFlBin64) 160 e.w.writen1(byte(i)) 161 e.w.writeb(e.b[:i]) 162 return 163 } 164 } 165 e.w.writen1(bincVdFloat<<4 | bincFlBin64) 166 e.w.writeb(e.b[:8]) 167} 168 169func (e *bincEncDriver) encIntegerPrune(bd byte, pos bool, v uint64, lim uint8) { 170 if lim == 4 { 171 bigen.PutUint32(e.b[:lim], uint32(v)) 172 } else { 173 bigen.PutUint64(e.b[:lim], v) 174 } 175 if bincDoPrune { 176 i := pruneSignExt(e.b[:lim], pos) 177 e.w.writen1(bd | lim - 1 - byte(i)) 178 e.w.writeb(e.b[i:lim]) 179 } else { 180 e.w.writen1(bd | lim - 1) 181 e.w.writeb(e.b[:lim]) 182 } 183} 184 185func (e *bincEncDriver) EncodeInt(v int64) { 186 // const nbd byte = bincVdNegInt << 4 187 if v >= 0 { 188 e.encUint(bincVdPosInt<<4, true, uint64(v)) 189 } else if v == -1 { 190 e.w.writen1(bincVdSpecial<<4 | bincSpNegOne) 191 } else { 192 e.encUint(bincVdNegInt<<4, false, uint64(-v)) 193 } 194} 195 196func (e *bincEncDriver) EncodeUint(v uint64) { 197 e.encUint(bincVdPosInt<<4, true, v) 198} 199 200func (e *bincEncDriver) encUint(bd byte, pos bool, v uint64) { 201 if v == 0 { 202 e.w.writen1(bincVdSpecial<<4 | bincSpZero) 203 } else if pos && v >= 1 && v <= 16 { 204 e.w.writen1(bincVdSmallInt<<4 | byte(v-1)) 205 } else if v <= math.MaxUint8 { 206 e.w.writen2(bd|0x0, byte(v)) 207 } else if v <= math.MaxUint16 { 208 e.w.writen1(bd | 0x01) 209 bigenHelper{e.b[:2], e.w}.writeUint16(uint16(v)) 210 } else if v <= math.MaxUint32 { 211 e.encIntegerPrune(bd, pos, v, 4) 212 } else { 213 e.encIntegerPrune(bd, pos, v, 8) 214 } 215} 216 217func (e *bincEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, _ *Encoder) { 218 bs := ext.WriteExt(rv) 219 if bs == nil { 220 e.EncodeNil() 221 return 222 } 223 e.encodeExtPreamble(uint8(xtag), len(bs)) 224 e.w.writeb(bs) 225} 226 227func (e *bincEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) { 228 e.encodeExtPreamble(uint8(re.Tag), len(re.Data)) 229 e.w.writeb(re.Data) 230} 231 232func (e *bincEncDriver) encodeExtPreamble(xtag byte, length int) { 233 e.encLen(bincVdCustomExt<<4, uint64(length)) 234 e.w.writen1(xtag) 235} 236 237func (e *bincEncDriver) WriteArrayStart(length int) { 238 e.encLen(bincVdArray<<4, uint64(length)) 239 e.c = containerArrayStart 240} 241 242func (e *bincEncDriver) WriteMapStart(length int) { 243 e.encLen(bincVdMap<<4, uint64(length)) 244 e.c = containerMapStart 245} 246 247func (e *bincEncDriver) EncodeSymbol(v string) { 248 // if WriteSymbolsNoRefs { 249 // e.encodeString(cUTF8, v) 250 // return 251 // } 252 253 //symbols only offer benefit when string length > 1. 254 //This is because strings with length 1 take only 2 bytes to store 255 //(bd with embedded length, and single byte for string val). 256 257 l := len(v) 258 if l == 0 { 259 e.encBytesLen(cUTF8, 0) 260 return 261 } else if l == 1 { 262 e.encBytesLen(cUTF8, 1) 263 e.w.writen1(v[0]) 264 return 265 } 266 if e.m == nil { 267 e.m = make(map[string]uint16, 16) 268 } 269 ui, ok := e.m[v] 270 if ok { 271 if ui <= math.MaxUint8 { 272 e.w.writen2(bincVdSymbol<<4, byte(ui)) 273 } else { 274 e.w.writen1(bincVdSymbol<<4 | 0x8) 275 bigenHelper{e.b[:2], e.w}.writeUint16(ui) 276 } 277 } else { 278 e.s++ 279 ui = e.s 280 //ui = uint16(atomic.AddUint32(&e.s, 1)) 281 e.m[v] = ui 282 var lenprec uint8 283 if l <= math.MaxUint8 { 284 // lenprec = 0 285 } else if l <= math.MaxUint16 { 286 lenprec = 1 287 } else if int64(l) <= math.MaxUint32 { 288 lenprec = 2 289 } else { 290 lenprec = 3 291 } 292 if ui <= math.MaxUint8 { 293 e.w.writen2(bincVdSymbol<<4|0x0|0x4|lenprec, byte(ui)) 294 } else { 295 e.w.writen1(bincVdSymbol<<4 | 0x8 | 0x4 | lenprec) 296 bigenHelper{e.b[:2], e.w}.writeUint16(ui) 297 } 298 if lenprec == 0 { 299 e.w.writen1(byte(l)) 300 } else if lenprec == 1 { 301 bigenHelper{e.b[:2], e.w}.writeUint16(uint16(l)) 302 } else if lenprec == 2 { 303 bigenHelper{e.b[:4], e.w}.writeUint32(uint32(l)) 304 } else { 305 bigenHelper{e.b[:8], e.w}.writeUint64(uint64(l)) 306 } 307 e.w.writestr(v) 308 } 309} 310 311func (e *bincEncDriver) EncodeString(c charEncoding, v string) { 312 if e.c == containerMapKey && c == cUTF8 && (e.h.AsSymbols == 0 || e.h.AsSymbols == 1) { 313 e.EncodeSymbol(v) 314 return 315 } 316 l := uint64(len(v)) 317 e.encBytesLen(c, l) 318 if l > 0 { 319 e.w.writestr(v) 320 } 321} 322 323func (e *bincEncDriver) EncodeStringEnc(c charEncoding, v string) { 324 if e.c == containerMapKey && c == cUTF8 && (e.h.AsSymbols == 0 || e.h.AsSymbols == 1) { 325 e.EncodeSymbol(v) 326 return 327 } 328 l := uint64(len(v)) 329 e.encLen(bincVdString<<4, l) // e.encBytesLen(c, l) 330 if l > 0 { 331 e.w.writestr(v) 332 } 333 334} 335 336func (e *bincEncDriver) EncodeStringBytes(c charEncoding, v []byte) { 337 if v == nil { 338 e.EncodeNil() 339 return 340 } 341 l := uint64(len(v)) 342 e.encBytesLen(c, l) 343 if l > 0 { 344 e.w.writeb(v) 345 } 346} 347 348func (e *bincEncDriver) EncodeStringBytesRaw(v []byte) { 349 if v == nil { 350 e.EncodeNil() 351 return 352 } 353 l := uint64(len(v)) 354 e.encLen(bincVdByteArray<<4, l) // e.encBytesLen(c, l) 355 if l > 0 { 356 e.w.writeb(v) 357 } 358} 359 360func (e *bincEncDriver) encBytesLen(c charEncoding, length uint64) { 361 //TODO: support bincUnicodeOther (for now, just use string or bytearray) 362 if c == cRAW { 363 e.encLen(bincVdByteArray<<4, length) 364 } else { 365 e.encLen(bincVdString<<4, length) 366 } 367} 368 369func (e *bincEncDriver) encLen(bd byte, l uint64) { 370 if l < 12 { 371 e.w.writen1(bd | uint8(l+4)) 372 } else { 373 e.encLenNumber(bd, l) 374 } 375} 376 377func (e *bincEncDriver) encLenNumber(bd byte, v uint64) { 378 if v <= math.MaxUint8 { 379 e.w.writen2(bd, byte(v)) 380 } else if v <= math.MaxUint16 { 381 e.w.writen1(bd | 0x01) 382 bigenHelper{e.b[:2], e.w}.writeUint16(uint16(v)) 383 } else if v <= math.MaxUint32 { 384 e.w.writen1(bd | 0x02) 385 bigenHelper{e.b[:4], e.w}.writeUint32(uint32(v)) 386 } else { 387 e.w.writen1(bd | 0x03) 388 bigenHelper{e.b[:8], e.w}.writeUint64(uint64(v)) 389 } 390} 391 392//------------------------------------ 393 394type bincDecSymbol struct { 395 s string 396 b []byte 397 i uint16 398} 399 400type bincDecDriver struct { 401 decDriverNoopContainerReader 402 noBuiltInTypes 403 404 d *Decoder 405 h *BincHandle 406 r *decReaderSwitch 407 br bool // bytes reader 408 bdRead bool 409 bd byte 410 vd byte 411 vs byte 412 _ [3]byte // padding 413 // linear searching on this slice is ok, 414 // because we typically expect < 32 symbols in each stream. 415 s []bincDecSymbol 416 417 // noStreamingCodec 418 // decNoSeparator 419 420 b [(8 + 1) * 8]byte // scratch 421} 422 423func (d *bincDecDriver) readNextBd() { 424 d.bd = d.r.readn1() 425 d.vd = d.bd >> 4 426 d.vs = d.bd & 0x0f 427 d.bdRead = true 428} 429 430func (d *bincDecDriver) uncacheRead() { 431 if d.bdRead { 432 d.r.unreadn1() 433 d.bdRead = false 434 } 435} 436 437func (d *bincDecDriver) ContainerType() (vt valueType) { 438 if !d.bdRead { 439 d.readNextBd() 440 } 441 if d.vd == bincVdSpecial && d.vs == bincSpNil { 442 return valueTypeNil 443 } else if d.vd == bincVdByteArray { 444 return valueTypeBytes 445 } else if d.vd == bincVdString { 446 return valueTypeString 447 } else if d.vd == bincVdArray { 448 return valueTypeArray 449 } else if d.vd == bincVdMap { 450 return valueTypeMap 451 } 452 // else { 453 // d.d.errorf("isContainerType: unsupported parameter: %v", vt) 454 // } 455 return valueTypeUnset 456} 457 458func (d *bincDecDriver) TryDecodeAsNil() bool { 459 if !d.bdRead { 460 d.readNextBd() 461 } 462 if d.bd == bincVdSpecial<<4|bincSpNil { 463 d.bdRead = false 464 return true 465 } 466 return false 467} 468 469func (d *bincDecDriver) DecodeTime() (t time.Time) { 470 if !d.bdRead { 471 d.readNextBd() 472 } 473 if d.bd == bincVdSpecial<<4|bincSpNil { 474 d.bdRead = false 475 return 476 } 477 if d.vd != bincVdTimestamp { 478 d.d.errorf("cannot decode time - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 479 return 480 } 481 t, err := bincDecodeTime(d.r.readx(uint(d.vs))) 482 if err != nil { 483 panic(err) 484 } 485 d.bdRead = false 486 return 487} 488 489func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) { 490 if vs&0x8 == 0 { 491 d.r.readb(d.b[0:defaultLen]) 492 } else { 493 l := d.r.readn1() 494 if l > 8 { 495 d.d.errorf("cannot read float - at most 8 bytes used to represent float - received %v bytes", l) 496 return 497 } 498 for i := l; i < 8; i++ { 499 d.b[i] = 0 500 } 501 d.r.readb(d.b[0:l]) 502 } 503} 504 505func (d *bincDecDriver) decFloat() (f float64) { 506 //if true { f = math.Float64frombits(bigen.Uint64(d.r.readx(8))); break; } 507 if x := d.vs & 0x7; x == bincFlBin32 { 508 d.decFloatPre(d.vs, 4) 509 f = float64(math.Float32frombits(bigen.Uint32(d.b[0:4]))) 510 } else if x == bincFlBin64 { 511 d.decFloatPre(d.vs, 8) 512 f = math.Float64frombits(bigen.Uint64(d.b[0:8])) 513 } else { 514 d.d.errorf("read float - only float32 and float64 are supported - %s %x-%x/%s", 515 msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 516 return 517 } 518 return 519} 520 521func (d *bincDecDriver) decUint() (v uint64) { 522 // need to inline the code (interface conversion and type assertion expensive) 523 switch d.vs { 524 case 0: 525 v = uint64(d.r.readn1()) 526 case 1: 527 d.r.readb(d.b[6:8]) 528 v = uint64(bigen.Uint16(d.b[6:8])) 529 case 2: 530 d.b[4] = 0 531 d.r.readb(d.b[5:8]) 532 v = uint64(bigen.Uint32(d.b[4:8])) 533 case 3: 534 d.r.readb(d.b[4:8]) 535 v = uint64(bigen.Uint32(d.b[4:8])) 536 case 4, 5, 6: 537 lim := 7 - d.vs 538 d.r.readb(d.b[lim:8]) 539 for i := uint8(0); i < lim; i++ { 540 d.b[i] = 0 541 } 542 v = uint64(bigen.Uint64(d.b[:8])) 543 case 7: 544 d.r.readb(d.b[:8]) 545 v = uint64(bigen.Uint64(d.b[:8])) 546 default: 547 d.d.errorf("unsigned integers with greater than 64 bits of precision not supported") 548 return 549 } 550 return 551} 552 553func (d *bincDecDriver) decCheckInteger() (ui uint64, neg bool) { 554 if !d.bdRead { 555 d.readNextBd() 556 } 557 vd, vs := d.vd, d.vs 558 if vd == bincVdPosInt { 559 ui = d.decUint() 560 } else if vd == bincVdNegInt { 561 ui = d.decUint() 562 neg = true 563 } else if vd == bincVdSmallInt { 564 ui = uint64(d.vs) + 1 565 } else if vd == bincVdSpecial { 566 if vs == bincSpZero { 567 //i = 0 568 } else if vs == bincSpNegOne { 569 neg = true 570 ui = 1 571 } else { 572 d.d.errorf("integer decode fails - invalid special value from descriptor %x-%x/%s", 573 d.vd, d.vs, bincdesc(d.vd, d.vs)) 574 return 575 } 576 } else { 577 d.d.errorf("integer can only be decoded from int/uint. d.bd: 0x%x, d.vd: 0x%x", d.bd, d.vd) 578 return 579 } 580 return 581} 582 583func (d *bincDecDriver) DecodeInt64() (i int64) { 584 ui, neg := d.decCheckInteger() 585 i = chkOvf.SignedIntV(ui) 586 if neg { 587 i = -i 588 } 589 d.bdRead = false 590 return 591} 592 593func (d *bincDecDriver) DecodeUint64() (ui uint64) { 594 ui, neg := d.decCheckInteger() 595 if neg { 596 d.d.errorf("assigning negative signed value to unsigned integer type") 597 return 598 } 599 d.bdRead = false 600 return 601} 602 603func (d *bincDecDriver) DecodeFloat64() (f float64) { 604 if !d.bdRead { 605 d.readNextBd() 606 } 607 vd, vs := d.vd, d.vs 608 if vd == bincVdSpecial { 609 d.bdRead = false 610 if vs == bincSpNan { 611 return math.NaN() 612 } else if vs == bincSpPosInf { 613 return math.Inf(1) 614 } else if vs == bincSpZeroFloat || vs == bincSpZero { 615 return 616 } else if vs == bincSpNegInf { 617 return math.Inf(-1) 618 } else { 619 d.d.errorf("float - invalid special value from descriptor %x-%x/%s", 620 d.vd, d.vs, bincdesc(d.vd, d.vs)) 621 return 622 } 623 } else if vd == bincVdFloat { 624 f = d.decFloat() 625 } else { 626 f = float64(d.DecodeInt64()) 627 } 628 d.bdRead = false 629 return 630} 631 632// bool can be decoded from bool only (single byte). 633func (d *bincDecDriver) DecodeBool() (b bool) { 634 if !d.bdRead { 635 d.readNextBd() 636 } 637 if bd := d.bd; bd == (bincVdSpecial | bincSpFalse) { 638 // b = false 639 } else if bd == (bincVdSpecial | bincSpTrue) { 640 b = true 641 } else { 642 d.d.errorf("bool - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 643 return 644 } 645 d.bdRead = false 646 return 647} 648 649func (d *bincDecDriver) ReadMapStart() (length int) { 650 if !d.bdRead { 651 d.readNextBd() 652 } 653 if d.vd != bincVdMap { 654 d.d.errorf("map - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 655 return 656 } 657 length = d.decLen() 658 d.bdRead = false 659 return 660} 661 662func (d *bincDecDriver) ReadArrayStart() (length int) { 663 if !d.bdRead { 664 d.readNextBd() 665 } 666 if d.vd != bincVdArray { 667 d.d.errorf("array - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 668 return 669 } 670 length = d.decLen() 671 d.bdRead = false 672 return 673} 674 675func (d *bincDecDriver) decLen() int { 676 if d.vs > 3 { 677 return int(d.vs - 4) 678 } 679 return int(d.decLenNumber()) 680} 681 682func (d *bincDecDriver) decLenNumber() (v uint64) { 683 if x := d.vs; x == 0 { 684 v = uint64(d.r.readn1()) 685 } else if x == 1 { 686 d.r.readb(d.b[6:8]) 687 v = uint64(bigen.Uint16(d.b[6:8])) 688 } else if x == 2 { 689 d.r.readb(d.b[4:8]) 690 v = uint64(bigen.Uint32(d.b[4:8])) 691 } else { 692 d.r.readb(d.b[:8]) 693 v = bigen.Uint64(d.b[:8]) 694 } 695 return 696} 697 698func (d *bincDecDriver) decStringAndBytes(bs []byte, withString, zerocopy bool) ( 699 bs2 []byte, s string) { 700 if !d.bdRead { 701 d.readNextBd() 702 } 703 if d.bd == bincVdSpecial<<4|bincSpNil { 704 d.bdRead = false 705 return 706 } 707 var slen = -1 708 // var ok bool 709 switch d.vd { 710 case bincVdString, bincVdByteArray: 711 slen = d.decLen() 712 if zerocopy { 713 if d.br { 714 bs2 = d.r.readx(uint(slen)) 715 } else if len(bs) == 0 { 716 bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, d.b[:]) 717 } else { 718 bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, bs) 719 } 720 } else { 721 bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, bs) 722 } 723 if withString { 724 s = string(bs2) 725 } 726 case bincVdSymbol: 727 // zerocopy doesn't apply for symbols, 728 // as the values must be stored in a table for later use. 729 // 730 //from vs: extract numSymbolBytes, containsStringVal, strLenPrecision, 731 //extract symbol 732 //if containsStringVal, read it and put in map 733 //else look in map for string value 734 var symbol uint16 735 vs := d.vs 736 if vs&0x8 == 0 { 737 symbol = uint16(d.r.readn1()) 738 } else { 739 symbol = uint16(bigen.Uint16(d.r.readx(2))) 740 } 741 if d.s == nil { 742 d.s = make([]bincDecSymbol, 0, 16) 743 } 744 745 if vs&0x4 == 0 { 746 for i := range d.s { 747 j := &d.s[i] 748 if j.i == symbol { 749 bs2 = j.b 750 if withString { 751 if j.s == "" && bs2 != nil { 752 j.s = string(bs2) 753 } 754 s = j.s 755 } 756 break 757 } 758 } 759 } else { 760 switch vs & 0x3 { 761 case 0: 762 slen = int(d.r.readn1()) 763 case 1: 764 slen = int(bigen.Uint16(d.r.readx(2))) 765 case 2: 766 slen = int(bigen.Uint32(d.r.readx(4))) 767 case 3: 768 slen = int(bigen.Uint64(d.r.readx(8))) 769 } 770 // since using symbols, do not store any part of 771 // the parameter bs in the map, as it might be a shared buffer. 772 // bs2 = decByteSlice(d.r, slen, bs) 773 bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, nil) 774 if withString { 775 s = string(bs2) 776 } 777 d.s = append(d.s, bincDecSymbol{i: symbol, s: s, b: bs2}) 778 } 779 default: 780 d.d.errorf("string/bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 781 return 782 } 783 d.bdRead = false 784 return 785} 786 787func (d *bincDecDriver) DecodeString() (s string) { 788 // DecodeBytes does not accommodate symbols, whose impl stores string version in map. 789 // Use decStringAndBytes directly. 790 // return string(d.DecodeBytes(d.b[:], true, true)) 791 _, s = d.decStringAndBytes(d.b[:], true, true) 792 return 793} 794 795func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) { 796 s, _ = d.decStringAndBytes(d.b[:], false, true) 797 return 798} 799 800func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) { 801 if !d.bdRead { 802 d.readNextBd() 803 } 804 if d.bd == bincVdSpecial<<4|bincSpNil { 805 d.bdRead = false 806 return nil 807 } 808 // check if an "array" of uint8's (see ContainerType for how to infer if an array) 809 if d.vd == bincVdArray { 810 bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d) 811 return 812 } 813 var clen int 814 if d.vd == bincVdString || d.vd == bincVdByteArray { 815 clen = d.decLen() 816 } else { 817 d.d.errorf("bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 818 return 819 } 820 d.bdRead = false 821 if zerocopy { 822 if d.br { 823 return d.r.readx(uint(clen)) 824 } else if len(bs) == 0 { 825 bs = d.b[:] 826 } 827 } 828 return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs) 829} 830 831func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { 832 if xtag > 0xff { 833 d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag) 834 return 835 } 836 realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag)) 837 realxtag = uint64(realxtag1) 838 if ext == nil { 839 re := rv.(*RawExt) 840 re.Tag = realxtag 841 re.Data = detachZeroCopyBytes(d.br, re.Data, xbs) 842 } else { 843 ext.ReadExt(rv, xbs) 844 } 845 return 846} 847 848func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) { 849 if !d.bdRead { 850 d.readNextBd() 851 } 852 if d.vd == bincVdCustomExt { 853 l := d.decLen() 854 xtag = d.r.readn1() 855 if verifyTag && xtag != tag { 856 d.d.errorf("wrong extension tag - got %b, expecting: %v", xtag, tag) 857 return 858 } 859 if d.br { 860 xbs = d.r.readx(uint(l)) 861 } else { 862 xbs = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) 863 } 864 } else if d.vd == bincVdByteArray { 865 xbs = d.DecodeBytes(nil, true) 866 } else { 867 d.d.errorf("ext - expecting extensions or byte array - %s %x-%x/%s", 868 msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 869 return 870 } 871 d.bdRead = false 872 return 873} 874 875func (d *bincDecDriver) DecodeNaked() { 876 if !d.bdRead { 877 d.readNextBd() 878 } 879 880 n := d.d.naked() 881 var decodeFurther bool 882 883 switch d.vd { 884 case bincVdSpecial: 885 switch d.vs { 886 case bincSpNil: 887 n.v = valueTypeNil 888 case bincSpFalse: 889 n.v = valueTypeBool 890 n.b = false 891 case bincSpTrue: 892 n.v = valueTypeBool 893 n.b = true 894 case bincSpNan: 895 n.v = valueTypeFloat 896 n.f = math.NaN() 897 case bincSpPosInf: 898 n.v = valueTypeFloat 899 n.f = math.Inf(1) 900 case bincSpNegInf: 901 n.v = valueTypeFloat 902 n.f = math.Inf(-1) 903 case bincSpZeroFloat: 904 n.v = valueTypeFloat 905 n.f = float64(0) 906 case bincSpZero: 907 n.v = valueTypeUint 908 n.u = uint64(0) // int8(0) 909 case bincSpNegOne: 910 n.v = valueTypeInt 911 n.i = int64(-1) // int8(-1) 912 default: 913 d.d.errorf("cannot infer value - unrecognized special value from descriptor %x-%x/%s", 914 d.vd, d.vs, bincdesc(d.vd, d.vs)) 915 } 916 case bincVdSmallInt: 917 n.v = valueTypeUint 918 n.u = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1 919 case bincVdPosInt: 920 n.v = valueTypeUint 921 n.u = d.decUint() 922 case bincVdNegInt: 923 n.v = valueTypeInt 924 n.i = -(int64(d.decUint())) 925 case bincVdFloat: 926 n.v = valueTypeFloat 927 n.f = d.decFloat() 928 case bincVdSymbol: 929 n.v = valueTypeSymbol 930 n.s = d.DecodeString() 931 case bincVdString: 932 n.v = valueTypeString 933 n.s = d.DecodeString() 934 case bincVdByteArray: 935 decNakedReadRawBytes(d, d.d, n, d.h.RawToString) 936 case bincVdTimestamp: 937 n.v = valueTypeTime 938 tt, err := bincDecodeTime(d.r.readx(uint(d.vs))) 939 if err != nil { 940 panic(err) 941 } 942 n.t = tt 943 case bincVdCustomExt: 944 n.v = valueTypeExt 945 l := d.decLen() 946 n.u = uint64(d.r.readn1()) 947 if d.br { 948 n.l = d.r.readx(uint(l)) 949 } else { 950 n.l = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) 951 } 952 case bincVdArray: 953 n.v = valueTypeArray 954 decodeFurther = true 955 case bincVdMap: 956 n.v = valueTypeMap 957 decodeFurther = true 958 default: 959 d.d.errorf("cannot infer value - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) 960 } 961 962 if !decodeFurther { 963 d.bdRead = false 964 } 965 if n.v == valueTypeUint && d.h.SignedInteger { 966 n.v = valueTypeInt 967 n.i = int64(n.u) 968 } 969} 970 971//------------------------------------ 972 973//BincHandle is a Handle for the Binc Schema-Free Encoding Format 974//defined at https://github.com/ugorji/binc . 975// 976//BincHandle currently supports all Binc features with the following EXCEPTIONS: 977// - only integers up to 64 bits of precision are supported. 978// big integers are unsupported. 979// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types). 980// extended precision and decimal IEEE 754 floats are unsupported. 981// - Only UTF-8 strings supported. 982// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported. 983// 984//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon. 985type BincHandle struct { 986 BasicHandle 987 binaryEncodingType 988 noElemSeparators 989 990 // AsSymbols defines what should be encoded as symbols. 991 // 992 // Encoding as symbols can reduce the encoded size significantly. 993 // 994 // However, during decoding, each string to be encoded as a symbol must 995 // be checked to see if it has been seen before. Consequently, encoding time 996 // will increase if using symbols, because string comparisons has a clear cost. 997 // 998 // Values: 999 // - 0: default: library uses best judgement 1000 // - 1: use symbols 1001 // - 2: do not use symbols 1002 AsSymbols uint8 1003 1004 // AsSymbols: may later on introduce more options ... 1005 // - m: map keys 1006 // - s: struct fields 1007 // - n: none 1008 // - a: all: same as m, s, ... 1009 1010 // _ [1]uint64 // padding 1011} 1012 1013// Name returns the name of the handle: binc 1014func (h *BincHandle) Name() string { return "binc" } 1015 1016// SetBytesExt sets an extension 1017func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) { 1018 return h.SetExt(rt, tag, &extWrapper{ext, interfaceExtFailer{}}) 1019} 1020 1021func (h *BincHandle) newEncDriver(e *Encoder) encDriver { 1022 return &bincEncDriver{e: e, h: h, w: e.w} 1023} 1024 1025func (h *BincHandle) newDecDriver(d *Decoder) decDriver { 1026 return &bincDecDriver{d: d, h: h, r: d.r, br: d.bytes} 1027} 1028 1029func (e *bincEncDriver) reset() { 1030 e.w = e.e.w 1031 e.s = 0 1032 e.c = 0 1033 e.m = nil 1034} 1035 1036func (d *bincDecDriver) reset() { 1037 d.r, d.br = d.d.r, d.d.bytes 1038 d.s = nil 1039 d.bd, d.bdRead, d.vd, d.vs = 0, false, 0, 0 1040} 1041 1042// var timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'} 1043 1044// EncodeTime encodes a time.Time as a []byte, including 1045// information on the instant in time and UTC offset. 1046// 1047// Format Description 1048// 1049// A timestamp is composed of 3 components: 1050// 1051// - secs: signed integer representing seconds since unix epoch 1052// - nsces: unsigned integer representing fractional seconds as a 1053// nanosecond offset within secs, in the range 0 <= nsecs < 1e9 1054// - tz: signed integer representing timezone offset in minutes east of UTC, 1055// and a dst (daylight savings time) flag 1056// 1057// When encoding a timestamp, the first byte is the descriptor, which 1058// defines which components are encoded and how many bytes are used to 1059// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it 1060// is not encoded in the byte array explicitly*. 1061// 1062// Descriptor 8 bits are of the form `A B C DDD EE`: 1063// A: Is secs component encoded? 1 = true 1064// B: Is nsecs component encoded? 1 = true 1065// C: Is tz component encoded? 1 = true 1066// DDD: Number of extra bytes for secs (range 0-7). 1067// If A = 1, secs encoded in DDD+1 bytes. 1068// If A = 0, secs is not encoded, and is assumed to be 0. 1069// If A = 1, then we need at least 1 byte to encode secs. 1070// DDD says the number of extra bytes beyond that 1. 1071// E.g. if DDD=0, then secs is represented in 1 byte. 1072// if DDD=2, then secs is represented in 3 bytes. 1073// EE: Number of extra bytes for nsecs (range 0-3). 1074// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above) 1075// 1076// Following the descriptor bytes, subsequent bytes are: 1077// 1078// secs component encoded in `DDD + 1` bytes (if A == 1) 1079// nsecs component encoded in `EE + 1` bytes (if B == 1) 1080// tz component encoded in 2 bytes (if C == 1) 1081// 1082// secs and nsecs components are integers encoded in a BigEndian 1083// 2-complement encoding format. 1084// 1085// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to 1086// Least significant bit 0 are described below: 1087// 1088// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes). 1089// Bit 15 = have\_dst: set to 1 if we set the dst flag. 1090// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not. 1091// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format. 1092// 1093func bincEncodeTime(t time.Time) []byte { 1094 //t := rv.Interface().(time.Time) 1095 tsecs, tnsecs := t.Unix(), t.Nanosecond() 1096 var ( 1097 bd byte 1098 btmp [8]byte 1099 bs [16]byte 1100 i int = 1 1101 ) 1102 l := t.Location() 1103 if l == time.UTC { 1104 l = nil 1105 } 1106 if tsecs != 0 { 1107 bd = bd | 0x80 1108 bigen.PutUint64(btmp[:], uint64(tsecs)) 1109 f := pruneSignExt(btmp[:], tsecs >= 0) 1110 bd = bd | (byte(7-f) << 2) 1111 copy(bs[i:], btmp[f:]) 1112 i = i + (8 - f) 1113 } 1114 if tnsecs != 0 { 1115 bd = bd | 0x40 1116 bigen.PutUint32(btmp[:4], uint32(tnsecs)) 1117 f := pruneSignExt(btmp[:4], true) 1118 bd = bd | byte(3-f) 1119 copy(bs[i:], btmp[f:4]) 1120 i = i + (4 - f) 1121 } 1122 if l != nil { 1123 bd = bd | 0x20 1124 // Note that Go Libs do not give access to dst flag. 1125 _, zoneOffset := t.Zone() 1126 //zoneName, zoneOffset := t.Zone() 1127 zoneOffset /= 60 1128 z := uint16(zoneOffset) 1129 bigen.PutUint16(btmp[:2], z) 1130 // clear dst flags 1131 bs[i] = btmp[0] & 0x3f 1132 bs[i+1] = btmp[1] 1133 i = i + 2 1134 } 1135 bs[0] = bd 1136 return bs[0:i] 1137} 1138 1139// bincDecodeTime decodes a []byte into a time.Time. 1140func bincDecodeTime(bs []byte) (tt time.Time, err error) { 1141 bd := bs[0] 1142 var ( 1143 tsec int64 1144 tnsec uint32 1145 tz uint16 1146 i byte = 1 1147 i2 byte 1148 n byte 1149 ) 1150 if bd&(1<<7) != 0 { 1151 var btmp [8]byte 1152 n = ((bd >> 2) & 0x7) + 1 1153 i2 = i + n 1154 copy(btmp[8-n:], bs[i:i2]) 1155 //if first bit of bs[i] is set, then fill btmp[0..8-n] with 0xff (ie sign extend it) 1156 if bs[i]&(1<<7) != 0 { 1157 copy(btmp[0:8-n], bsAll0xff) 1158 //for j,k := byte(0), 8-n; j < k; j++ { btmp[j] = 0xff } 1159 } 1160 i = i2 1161 tsec = int64(bigen.Uint64(btmp[:])) 1162 } 1163 if bd&(1<<6) != 0 { 1164 var btmp [4]byte 1165 n = (bd & 0x3) + 1 1166 i2 = i + n 1167 copy(btmp[4-n:], bs[i:i2]) 1168 i = i2 1169 tnsec = bigen.Uint32(btmp[:]) 1170 } 1171 if bd&(1<<5) == 0 { 1172 tt = time.Unix(tsec, int64(tnsec)).UTC() 1173 return 1174 } 1175 // In stdlib time.Parse, when a date is parsed without a zone name, it uses "" as zone name. 1176 // However, we need name here, so it can be shown when time is printf.d. 1177 // Zone name is in form: UTC-08:00. 1178 // Note that Go Libs do not give access to dst flag, so we ignore dst bits 1179 1180 i2 = i + 2 1181 tz = bigen.Uint16(bs[i:i2]) 1182 // i = i2 1183 // sign extend sign bit into top 2 MSB (which were dst bits): 1184 if tz&(1<<13) == 0 { // positive 1185 tz = tz & 0x3fff //clear 2 MSBs: dst bits 1186 } else { // negative 1187 tz = tz | 0xc000 //set 2 MSBs: dst bits 1188 } 1189 tzint := int16(tz) 1190 if tzint == 0 { 1191 tt = time.Unix(tsec, int64(tnsec)).UTC() 1192 } else { 1193 // For Go Time, do not use a descriptive timezone. 1194 // It's unnecessary, and makes it harder to do a reflect.DeepEqual. 1195 // The Offset already tells what the offset should be, if not on UTC and unknown zone name. 1196 // var zoneName = timeLocUTCName(tzint) 1197 tt = time.Unix(tsec, int64(tnsec)).In(time.FixedZone("", int(tzint)*60)) 1198 } 1199 return 1200} 1201 1202var _ decDriver = (*bincDecDriver)(nil) 1203var _ encDriver = (*bincEncDriver)(nil) 1204