1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2010 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32/* 33 Package proto converts data structures to and from the wire format of 34 protocol buffers. It works in concert with the Go source code generated 35 for .proto files by the protocol compiler. 36 37 A summary of the properties of the protocol buffer interface 38 for a protocol buffer variable v: 39 40 - Names are turned from camel_case to CamelCase for export. 41 - There are no methods on v to set fields; just treat 42 them as structure fields. 43 - There are getters that return a field's value if set, 44 and return the field's default value if unset. 45 The getters work even if the receiver is a nil message. 46 - The zero value for a struct is its correct initialization state. 47 All desired fields must be set before marshaling. 48 - A Reset() method will restore a protobuf struct to its zero state. 49 - Non-repeated fields are pointers to the values; nil means unset. 50 That is, optional or required field int32 f becomes F *int32. 51 - Repeated fields are slices. 52 - Helper functions are available to aid the setting of fields. 53 msg.Foo = proto.String("hello") // set field 54 - Constants are defined to hold the default values of all fields that 55 have them. They have the form Default_StructName_FieldName. 56 Because the getter methods handle defaulted values, 57 direct use of these constants should be rare. 58 - Enums are given type names and maps from names to values. 59 Enum values are prefixed by the enclosing message's name, or by the 60 enum's type name if it is a top-level enum. Enum types have a String 61 method, and a Enum method to assist in message construction. 62 - Nested messages, groups and enums have type names prefixed with the name of 63 the surrounding message type. 64 - Extensions are given descriptor names that start with E_, 65 followed by an underscore-delimited list of the nested messages 66 that contain it (if any) followed by the CamelCased name of the 67 extension field itself. HasExtension, ClearExtension, GetExtension 68 and SetExtension are functions for manipulating extensions. 69 - Marshal and Unmarshal are functions to encode and decode the wire format. 70 71 The simplest way to describe this is to see an example. 72 Given file test.proto, containing 73 74 package example; 75 76 enum FOO { X = 17; } 77 78 message Test { 79 required string label = 1; 80 optional int32 type = 2 [default=77]; 81 repeated int64 reps = 3; 82 optional group OptionalGroup = 4 { 83 required string RequiredField = 5; 84 } 85 } 86 87 The resulting file, test.pb.go, is: 88 89 package example 90 91 import proto "github.com/golang/protobuf/proto" 92 import math "math" 93 94 type FOO int32 95 const ( 96 FOO_X FOO = 17 97 ) 98 var FOO_name = map[int32]string{ 99 17: "X", 100 } 101 var FOO_value = map[string]int32{ 102 "X": 17, 103 } 104 105 func (x FOO) Enum() *FOO { 106 p := new(FOO) 107 *p = x 108 return p 109 } 110 func (x FOO) String() string { 111 return proto.EnumName(FOO_name, int32(x)) 112 } 113 func (x *FOO) UnmarshalJSON(data []byte) error { 114 value, err := proto.UnmarshalJSONEnum(FOO_value, data) 115 if err != nil { 116 return err 117 } 118 *x = FOO(value) 119 return nil 120 } 121 122 type Test struct { 123 Label *string `protobuf:"bytes,1,req,name=label" json:"label,omitempty"` 124 Type *int32 `protobuf:"varint,2,opt,name=type,def=77" json:"type,omitempty"` 125 Reps []int64 `protobuf:"varint,3,rep,name=reps" json:"reps,omitempty"` 126 Optionalgroup *Test_OptionalGroup `protobuf:"group,4,opt,name=OptionalGroup" json:"optionalgroup,omitempty"` 127 XXX_unrecognized []byte `json:"-"` 128 } 129 func (m *Test) Reset() { *m = Test{} } 130 func (m *Test) String() string { return proto.CompactTextString(m) } 131 func (*Test) ProtoMessage() {} 132 const Default_Test_Type int32 = 77 133 134 func (m *Test) GetLabel() string { 135 if m != nil && m.Label != nil { 136 return *m.Label 137 } 138 return "" 139 } 140 141 func (m *Test) GetType() int32 { 142 if m != nil && m.Type != nil { 143 return *m.Type 144 } 145 return Default_Test_Type 146 } 147 148 func (m *Test) GetOptionalgroup() *Test_OptionalGroup { 149 if m != nil { 150 return m.Optionalgroup 151 } 152 return nil 153 } 154 155 type Test_OptionalGroup struct { 156 RequiredField *string `protobuf:"bytes,5,req" json:"RequiredField,omitempty"` 157 } 158 func (m *Test_OptionalGroup) Reset() { *m = Test_OptionalGroup{} } 159 func (m *Test_OptionalGroup) String() string { return proto.CompactTextString(m) } 160 161 func (m *Test_OptionalGroup) GetRequiredField() string { 162 if m != nil && m.RequiredField != nil { 163 return *m.RequiredField 164 } 165 return "" 166 } 167 168 func init() { 169 proto.RegisterEnum("example.FOO", FOO_name, FOO_value) 170 } 171 172 To create and play with a Test object: 173 174 package main 175 176 import ( 177 "log" 178 179 "github.com/golang/protobuf/proto" 180 pb "./example.pb" 181 ) 182 183 func main() { 184 test := &pb.Test{ 185 Label: proto.String("hello"), 186 Type: proto.Int32(17), 187 Optionalgroup: &pb.Test_OptionalGroup{ 188 RequiredField: proto.String("good bye"), 189 }, 190 } 191 data, err := proto.Marshal(test) 192 if err != nil { 193 log.Fatal("marshaling error: ", err) 194 } 195 newTest := &pb.Test{} 196 err = proto.Unmarshal(data, newTest) 197 if err != nil { 198 log.Fatal("unmarshaling error: ", err) 199 } 200 // Now test and newTest contain the same data. 201 if test.GetLabel() != newTest.GetLabel() { 202 log.Fatalf("data mismatch %q != %q", test.GetLabel(), newTest.GetLabel()) 203 } 204 // etc. 205 } 206*/ 207package proto 208 209import ( 210 "encoding/json" 211 "fmt" 212 "log" 213 "reflect" 214 "strconv" 215 "sync" 216) 217 218// Message is implemented by generated protocol buffer messages. 219type Message interface { 220 Reset() 221 String() string 222 ProtoMessage() 223} 224 225// Stats records allocation details about the protocol buffer encoders 226// and decoders. Useful for tuning the library itself. 227type Stats struct { 228 Emalloc uint64 // mallocs in encode 229 Dmalloc uint64 // mallocs in decode 230 Encode uint64 // number of encodes 231 Decode uint64 // number of decodes 232 Chit uint64 // number of cache hits 233 Cmiss uint64 // number of cache misses 234 Size uint64 // number of sizes 235} 236 237// Set to true to enable stats collection. 238const collectStats = false 239 240var stats Stats 241 242// GetStats returns a copy of the global Stats structure. 243func GetStats() Stats { return stats } 244 245// A Buffer is a buffer manager for marshaling and unmarshaling 246// protocol buffers. It may be reused between invocations to 247// reduce memory usage. It is not necessary to use a Buffer; 248// the global functions Marshal and Unmarshal create a 249// temporary Buffer and are fine for most applications. 250type Buffer struct { 251 buf []byte // encode/decode byte stream 252 index int // write point 253 254 // pools of basic types to amortize allocation. 255 bools []bool 256 uint32s []uint32 257 uint64s []uint64 258 259 // extra pools, only used with pointer_reflect.go 260 int32s []int32 261 int64s []int64 262 float32s []float32 263 float64s []float64 264} 265 266// NewBuffer allocates a new Buffer and initializes its internal data to 267// the contents of the argument slice. 268func NewBuffer(e []byte) *Buffer { 269 return &Buffer{buf: e} 270} 271 272// Reset resets the Buffer, ready for marshaling a new protocol buffer. 273func (p *Buffer) Reset() { 274 p.buf = p.buf[0:0] // for reading/writing 275 p.index = 0 // for reading 276} 277 278// SetBuf replaces the internal buffer with the slice, 279// ready for unmarshaling the contents of the slice. 280func (p *Buffer) SetBuf(s []byte) { 281 p.buf = s 282 p.index = 0 283} 284 285// Bytes returns the contents of the Buffer. 286func (p *Buffer) Bytes() []byte { return p.buf } 287 288/* 289 * Helper routines for simplifying the creation of optional fields of basic type. 290 */ 291 292// Bool is a helper routine that allocates a new bool value 293// to store v and returns a pointer to it. 294func Bool(v bool) *bool { 295 return &v 296} 297 298// Int32 is a helper routine that allocates a new int32 value 299// to store v and returns a pointer to it. 300func Int32(v int32) *int32 { 301 return &v 302} 303 304// Int is a helper routine that allocates a new int32 value 305// to store v and returns a pointer to it, but unlike Int32 306// its argument value is an int. 307func Int(v int) *int32 { 308 p := new(int32) 309 *p = int32(v) 310 return p 311} 312 313// Int64 is a helper routine that allocates a new int64 value 314// to store v and returns a pointer to it. 315func Int64(v int64) *int64 { 316 return &v 317} 318 319// Float32 is a helper routine that allocates a new float32 value 320// to store v and returns a pointer to it. 321func Float32(v float32) *float32 { 322 return &v 323} 324 325// Float64 is a helper routine that allocates a new float64 value 326// to store v and returns a pointer to it. 327func Float64(v float64) *float64 { 328 return &v 329} 330 331// Uint32 is a helper routine that allocates a new uint32 value 332// to store v and returns a pointer to it. 333func Uint32(v uint32) *uint32 { 334 return &v 335} 336 337// Uint64 is a helper routine that allocates a new uint64 value 338// to store v and returns a pointer to it. 339func Uint64(v uint64) *uint64 { 340 return &v 341} 342 343// String is a helper routine that allocates a new string value 344// to store v and returns a pointer to it. 345func String(v string) *string { 346 return &v 347} 348 349// EnumName is a helper function to simplify printing protocol buffer enums 350// by name. Given an enum map and a value, it returns a useful string. 351func EnumName(m map[int32]string, v int32) string { 352 s, ok := m[v] 353 if ok { 354 return s 355 } 356 return strconv.Itoa(int(v)) 357} 358 359// UnmarshalJSONEnum is a helper function to simplify recovering enum int values 360// from their JSON-encoded representation. Given a map from the enum's symbolic 361// names to its int values, and a byte buffer containing the JSON-encoded 362// value, it returns an int32 that can be cast to the enum type by the caller. 363// 364// The function can deal with both JSON representations, numeric and symbolic. 365func UnmarshalJSONEnum(m map[string]int32, data []byte, enumName string) (int32, error) { 366 if data[0] == '"' { 367 // New style: enums are strings. 368 var repr string 369 if err := json.Unmarshal(data, &repr); err != nil { 370 return -1, err 371 } 372 val, ok := m[repr] 373 if !ok { 374 return 0, fmt.Errorf("unrecognized enum %s value %q", enumName, repr) 375 } 376 return val, nil 377 } 378 // Old style: enums are ints. 379 var val int32 380 if err := json.Unmarshal(data, &val); err != nil { 381 return 0, fmt.Errorf("cannot unmarshal %#q into enum %s", data, enumName) 382 } 383 return val, nil 384} 385 386// DebugPrint dumps the encoded data in b in a debugging format with a header 387// including the string s. Used in testing but made available for general debugging. 388func (o *Buffer) DebugPrint(s string, b []byte) { 389 var u uint64 390 391 obuf := o.buf 392 index := o.index 393 o.buf = b 394 o.index = 0 395 depth := 0 396 397 fmt.Printf("\n--- %s ---\n", s) 398 399out: 400 for { 401 for i := 0; i < depth; i++ { 402 fmt.Print(" ") 403 } 404 405 index := o.index 406 if index == len(o.buf) { 407 break 408 } 409 410 op, err := o.DecodeVarint() 411 if err != nil { 412 fmt.Printf("%3d: fetching op err %v\n", index, err) 413 break out 414 } 415 tag := op >> 3 416 wire := op & 7 417 418 switch wire { 419 default: 420 fmt.Printf("%3d: t=%3d unknown wire=%d\n", 421 index, tag, wire) 422 break out 423 424 case WireBytes: 425 var r []byte 426 427 r, err = o.DecodeRawBytes(false) 428 if err != nil { 429 break out 430 } 431 fmt.Printf("%3d: t=%3d bytes [%d]", index, tag, len(r)) 432 if len(r) <= 6 { 433 for i := 0; i < len(r); i++ { 434 fmt.Printf(" %.2x", r[i]) 435 } 436 } else { 437 for i := 0; i < 3; i++ { 438 fmt.Printf(" %.2x", r[i]) 439 } 440 fmt.Printf(" ..") 441 for i := len(r) - 3; i < len(r); i++ { 442 fmt.Printf(" %.2x", r[i]) 443 } 444 } 445 fmt.Printf("\n") 446 447 case WireFixed32: 448 u, err = o.DecodeFixed32() 449 if err != nil { 450 fmt.Printf("%3d: t=%3d fix32 err %v\n", index, tag, err) 451 break out 452 } 453 fmt.Printf("%3d: t=%3d fix32 %d\n", index, tag, u) 454 455 case WireFixed64: 456 u, err = o.DecodeFixed64() 457 if err != nil { 458 fmt.Printf("%3d: t=%3d fix64 err %v\n", index, tag, err) 459 break out 460 } 461 fmt.Printf("%3d: t=%3d fix64 %d\n", index, tag, u) 462 break 463 464 case WireVarint: 465 u, err = o.DecodeVarint() 466 if err != nil { 467 fmt.Printf("%3d: t=%3d varint err %v\n", index, tag, err) 468 break out 469 } 470 fmt.Printf("%3d: t=%3d varint %d\n", index, tag, u) 471 472 case WireStartGroup: 473 if err != nil { 474 fmt.Printf("%3d: t=%3d start err %v\n", index, tag, err) 475 break out 476 } 477 fmt.Printf("%3d: t=%3d start\n", index, tag) 478 depth++ 479 480 case WireEndGroup: 481 depth-- 482 if err != nil { 483 fmt.Printf("%3d: t=%3d end err %v\n", index, tag, err) 484 break out 485 } 486 fmt.Printf("%3d: t=%3d end\n", index, tag) 487 } 488 } 489 490 if depth != 0 { 491 fmt.Printf("%3d: start-end not balanced %d\n", o.index, depth) 492 } 493 fmt.Printf("\n") 494 495 o.buf = obuf 496 o.index = index 497} 498 499// SetDefaults sets unset protocol buffer fields to their default values. 500// It only modifies fields that are both unset and have defined defaults. 501// It recursively sets default values in any non-nil sub-messages. 502func SetDefaults(pb Message) { 503 setDefaults(reflect.ValueOf(pb), true, false) 504} 505 506// v is a pointer to a struct. 507func setDefaults(v reflect.Value, recur, zeros bool) { 508 v = v.Elem() 509 510 defaultMu.RLock() 511 dm, ok := defaults[v.Type()] 512 defaultMu.RUnlock() 513 if !ok { 514 dm = buildDefaultMessage(v.Type()) 515 defaultMu.Lock() 516 defaults[v.Type()] = dm 517 defaultMu.Unlock() 518 } 519 520 for _, sf := range dm.scalars { 521 f := v.Field(sf.index) 522 if !f.IsNil() { 523 // field already set 524 continue 525 } 526 dv := sf.value 527 if dv == nil && !zeros { 528 // no explicit default, and don't want to set zeros 529 continue 530 } 531 fptr := f.Addr().Interface() // **T 532 // TODO: Consider batching the allocations we do here. 533 switch sf.kind { 534 case reflect.Bool: 535 b := new(bool) 536 if dv != nil { 537 *b = dv.(bool) 538 } 539 *(fptr.(**bool)) = b 540 case reflect.Float32: 541 f := new(float32) 542 if dv != nil { 543 *f = dv.(float32) 544 } 545 *(fptr.(**float32)) = f 546 case reflect.Float64: 547 f := new(float64) 548 if dv != nil { 549 *f = dv.(float64) 550 } 551 *(fptr.(**float64)) = f 552 case reflect.Int32: 553 // might be an enum 554 if ft := f.Type(); ft != int32PtrType { 555 // enum 556 f.Set(reflect.New(ft.Elem())) 557 if dv != nil { 558 f.Elem().SetInt(int64(dv.(int32))) 559 } 560 } else { 561 // int32 field 562 i := new(int32) 563 if dv != nil { 564 *i = dv.(int32) 565 } 566 *(fptr.(**int32)) = i 567 } 568 case reflect.Int64: 569 i := new(int64) 570 if dv != nil { 571 *i = dv.(int64) 572 } 573 *(fptr.(**int64)) = i 574 case reflect.String: 575 s := new(string) 576 if dv != nil { 577 *s = dv.(string) 578 } 579 *(fptr.(**string)) = s 580 case reflect.Uint8: 581 // exceptional case: []byte 582 var b []byte 583 if dv != nil { 584 db := dv.([]byte) 585 b = make([]byte, len(db)) 586 copy(b, db) 587 } else { 588 b = []byte{} 589 } 590 *(fptr.(*[]byte)) = b 591 case reflect.Uint32: 592 u := new(uint32) 593 if dv != nil { 594 *u = dv.(uint32) 595 } 596 *(fptr.(**uint32)) = u 597 case reflect.Uint64: 598 u := new(uint64) 599 if dv != nil { 600 *u = dv.(uint64) 601 } 602 *(fptr.(**uint64)) = u 603 default: 604 log.Printf("proto: can't set default for field %v (sf.kind=%v)", f, sf.kind) 605 } 606 } 607 608 for _, ni := range dm.nested { 609 f := v.Field(ni) 610 if f.IsNil() { 611 continue 612 } 613 // f is *T or []*T 614 if f.Kind() == reflect.Ptr { 615 setDefaults(f, recur, zeros) 616 } else { 617 for i := 0; i < f.Len(); i++ { 618 e := f.Index(i) 619 if e.IsNil() { 620 continue 621 } 622 setDefaults(e, recur, zeros) 623 } 624 } 625 } 626} 627 628var ( 629 // defaults maps a protocol buffer struct type to a slice of the fields, 630 // with its scalar fields set to their proto-declared non-zero default values. 631 defaultMu sync.RWMutex 632 defaults = make(map[reflect.Type]defaultMessage) 633 634 int32PtrType = reflect.TypeOf((*int32)(nil)) 635) 636 637// defaultMessage represents information about the default values of a message. 638type defaultMessage struct { 639 scalars []scalarField 640 nested []int // struct field index of nested messages 641} 642 643type scalarField struct { 644 index int // struct field index 645 kind reflect.Kind // element type (the T in *T or []T) 646 value interface{} // the proto-declared default value, or nil 647} 648 649func ptrToStruct(t reflect.Type) bool { 650 return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct 651} 652 653// t is a struct type. 654func buildDefaultMessage(t reflect.Type) (dm defaultMessage) { 655 sprop := GetProperties(t) 656 for _, prop := range sprop.Prop { 657 fi, ok := sprop.decoderTags.get(prop.Tag) 658 if !ok { 659 // XXX_unrecognized 660 continue 661 } 662 ft := t.Field(fi).Type 663 664 // nested messages 665 if ptrToStruct(ft) || (ft.Kind() == reflect.Slice && ptrToStruct(ft.Elem())) { 666 dm.nested = append(dm.nested, fi) 667 continue 668 } 669 670 sf := scalarField{ 671 index: fi, 672 kind: ft.Elem().Kind(), 673 } 674 675 // scalar fields without defaults 676 if !prop.HasDefault { 677 dm.scalars = append(dm.scalars, sf) 678 continue 679 } 680 681 // a scalar field: either *T or []byte 682 switch ft.Elem().Kind() { 683 case reflect.Bool: 684 x, err := strconv.ParseBool(prop.Default) 685 if err != nil { 686 log.Printf("proto: bad default bool %q: %v", prop.Default, err) 687 continue 688 } 689 sf.value = x 690 case reflect.Float32: 691 x, err := strconv.ParseFloat(prop.Default, 32) 692 if err != nil { 693 log.Printf("proto: bad default float32 %q: %v", prop.Default, err) 694 continue 695 } 696 sf.value = float32(x) 697 case reflect.Float64: 698 x, err := strconv.ParseFloat(prop.Default, 64) 699 if err != nil { 700 log.Printf("proto: bad default float64 %q: %v", prop.Default, err) 701 continue 702 } 703 sf.value = x 704 case reflect.Int32: 705 x, err := strconv.ParseInt(prop.Default, 10, 32) 706 if err != nil { 707 log.Printf("proto: bad default int32 %q: %v", prop.Default, err) 708 continue 709 } 710 sf.value = int32(x) 711 case reflect.Int64: 712 x, err := strconv.ParseInt(prop.Default, 10, 64) 713 if err != nil { 714 log.Printf("proto: bad default int64 %q: %v", prop.Default, err) 715 continue 716 } 717 sf.value = x 718 case reflect.String: 719 sf.value = prop.Default 720 case reflect.Uint8: 721 // []byte (not *uint8) 722 sf.value = []byte(prop.Default) 723 case reflect.Uint32: 724 x, err := strconv.ParseUint(prop.Default, 10, 32) 725 if err != nil { 726 log.Printf("proto: bad default uint32 %q: %v", prop.Default, err) 727 continue 728 } 729 sf.value = uint32(x) 730 case reflect.Uint64: 731 x, err := strconv.ParseUint(prop.Default, 10, 64) 732 if err != nil { 733 log.Printf("proto: bad default uint64 %q: %v", prop.Default, err) 734 continue 735 } 736 sf.value = x 737 default: 738 log.Printf("proto: unhandled def kind %v", ft.Elem().Kind()) 739 continue 740 } 741 742 dm.scalars = append(dm.scalars, sf) 743 } 744 745 return dm 746} 747 748// Map fields may have key types of non-float scalars, strings and enums. 749// The easiest way to sort them in some deterministic order is to use fmt. 750// If this turns out to be inefficient we can always consider other options, 751// such as doing a Schwartzian transform. 752 753type mapKeys []reflect.Value 754 755func (s mapKeys) Len() int { return len(s) } 756func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 757func (s mapKeys) Less(i, j int) bool { 758 return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) 759} 760