1// Protocol Buffers for Go with Gadgets 2// 3// Copyright (c) 2013, The GoGo Authors. All rights reserved. 4// http://github.com/gogo/protobuf 5// 6// Go support for Protocol Buffers - Google's data interchange format 7// 8// Copyright 2010 The Go Authors. All rights reserved. 9// https://github.com/golang/protobuf 10// 11// Redistribution and use in source and binary forms, with or without 12// modification, are permitted provided that the following conditions are 13// met: 14// 15// * Redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer. 17// * Redistributions in binary form must reproduce the above 18// copyright notice, this list of conditions and the following disclaimer 19// in the documentation and/or other materials provided with the 20// distribution. 21// * Neither the name of Google Inc. nor the names of its 22// contributors may be used to endorse or promote products derived from 23// this software without specific prior written permission. 24// 25// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 37package proto 38 39/* 40 * Routines for encoding data into the wire format for protocol buffers. 41 */ 42 43import ( 44 "fmt" 45 "log" 46 "os" 47 "reflect" 48 "sort" 49 "strconv" 50 "strings" 51 "sync" 52) 53 54const debug bool = false 55 56// Constants that identify the encoding of a value on the wire. 57const ( 58 WireVarint = 0 59 WireFixed64 = 1 60 WireBytes = 2 61 WireStartGroup = 3 62 WireEndGroup = 4 63 WireFixed32 = 5 64) 65 66// tagMap is an optimization over map[int]int for typical protocol buffer 67// use-cases. Encoded protocol buffers are often in tag order with small tag 68// numbers. 69type tagMap struct { 70 fastTags []int 71 slowTags map[int]int 72} 73 74// tagMapFastLimit is the upper bound on the tag number that will be stored in 75// the tagMap slice rather than its map. 76const tagMapFastLimit = 1024 77 78func (p *tagMap) get(t int) (int, bool) { 79 if t > 0 && t < tagMapFastLimit { 80 if t >= len(p.fastTags) { 81 return 0, false 82 } 83 fi := p.fastTags[t] 84 return fi, fi >= 0 85 } 86 fi, ok := p.slowTags[t] 87 return fi, ok 88} 89 90func (p *tagMap) put(t int, fi int) { 91 if t > 0 && t < tagMapFastLimit { 92 for len(p.fastTags) < t+1 { 93 p.fastTags = append(p.fastTags, -1) 94 } 95 p.fastTags[t] = fi 96 return 97 } 98 if p.slowTags == nil { 99 p.slowTags = make(map[int]int) 100 } 101 p.slowTags[t] = fi 102} 103 104// StructProperties represents properties for all the fields of a struct. 105// decoderTags and decoderOrigNames should only be used by the decoder. 106type StructProperties struct { 107 Prop []*Properties // properties for each field 108 reqCount int // required count 109 decoderTags tagMap // map from proto tag to struct field number 110 decoderOrigNames map[string]int // map from original name to struct field number 111 order []int // list of struct field numbers in tag order 112 113 // OneofTypes contains information about the oneof fields in this message. 114 // It is keyed by the original name of a field. 115 OneofTypes map[string]*OneofProperties 116} 117 118// OneofProperties represents information about a specific field in a oneof. 119type OneofProperties struct { 120 Type reflect.Type // pointer to generated struct type for this oneof field 121 Field int // struct field number of the containing oneof in the message 122 Prop *Properties 123} 124 125// Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec. 126// See encode.go, (*Buffer).enc_struct. 127 128func (sp *StructProperties) Len() int { return len(sp.order) } 129func (sp *StructProperties) Less(i, j int) bool { 130 return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag 131} 132func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] } 133 134// Properties represents the protocol-specific behavior of a single struct field. 135type Properties struct { 136 Name string // name of the field, for error messages 137 OrigName string // original name before protocol compiler (always set) 138 JSONName string // name to use for JSON; determined by protoc 139 Wire string 140 WireType int 141 Tag int 142 Required bool 143 Optional bool 144 Repeated bool 145 Packed bool // relevant for repeated primitives only 146 Enum string // set for enum types only 147 proto3 bool // whether this is known to be a proto3 field 148 oneof bool // whether this is a oneof field 149 150 Default string // default value 151 HasDefault bool // whether an explicit default was provided 152 CustomType string 153 CastType string 154 StdTime bool 155 StdDuration bool 156 WktPointer bool 157 158 stype reflect.Type // set for struct types only 159 ctype reflect.Type // set for custom types only 160 sprop *StructProperties // set for struct types only 161 162 mtype reflect.Type // set for map types only 163 MapKeyProp *Properties // set for map types only 164 MapValProp *Properties // set for map types only 165} 166 167// String formats the properties in the protobuf struct field tag style. 168func (p *Properties) String() string { 169 s := p.Wire 170 s += "," 171 s += strconv.Itoa(p.Tag) 172 if p.Required { 173 s += ",req" 174 } 175 if p.Optional { 176 s += ",opt" 177 } 178 if p.Repeated { 179 s += ",rep" 180 } 181 if p.Packed { 182 s += ",packed" 183 } 184 s += ",name=" + p.OrigName 185 if p.JSONName != p.OrigName { 186 s += ",json=" + p.JSONName 187 } 188 if p.proto3 { 189 s += ",proto3" 190 } 191 if p.oneof { 192 s += ",oneof" 193 } 194 if len(p.Enum) > 0 { 195 s += ",enum=" + p.Enum 196 } 197 if p.HasDefault { 198 s += ",def=" + p.Default 199 } 200 return s 201} 202 203// Parse populates p by parsing a string in the protobuf struct field tag style. 204func (p *Properties) Parse(s string) { 205 // "bytes,49,opt,name=foo,def=hello!" 206 fields := strings.Split(s, ",") // breaks def=, but handled below. 207 if len(fields) < 2 { 208 fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s) 209 return 210 } 211 212 p.Wire = fields[0] 213 switch p.Wire { 214 case "varint": 215 p.WireType = WireVarint 216 case "fixed32": 217 p.WireType = WireFixed32 218 case "fixed64": 219 p.WireType = WireFixed64 220 case "zigzag32": 221 p.WireType = WireVarint 222 case "zigzag64": 223 p.WireType = WireVarint 224 case "bytes", "group": 225 p.WireType = WireBytes 226 // no numeric converter for non-numeric types 227 default: 228 fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s) 229 return 230 } 231 232 var err error 233 p.Tag, err = strconv.Atoi(fields[1]) 234 if err != nil { 235 return 236 } 237 238outer: 239 for i := 2; i < len(fields); i++ { 240 f := fields[i] 241 switch { 242 case f == "req": 243 p.Required = true 244 case f == "opt": 245 p.Optional = true 246 case f == "rep": 247 p.Repeated = true 248 case f == "packed": 249 p.Packed = true 250 case strings.HasPrefix(f, "name="): 251 p.OrigName = f[5:] 252 case strings.HasPrefix(f, "json="): 253 p.JSONName = f[5:] 254 case strings.HasPrefix(f, "enum="): 255 p.Enum = f[5:] 256 case f == "proto3": 257 p.proto3 = true 258 case f == "oneof": 259 p.oneof = true 260 case strings.HasPrefix(f, "def="): 261 p.HasDefault = true 262 p.Default = f[4:] // rest of string 263 if i+1 < len(fields) { 264 // Commas aren't escaped, and def is always last. 265 p.Default += "," + strings.Join(fields[i+1:], ",") 266 break outer 267 } 268 case strings.HasPrefix(f, "embedded="): 269 p.OrigName = strings.Split(f, "=")[1] 270 case strings.HasPrefix(f, "customtype="): 271 p.CustomType = strings.Split(f, "=")[1] 272 case strings.HasPrefix(f, "casttype="): 273 p.CastType = strings.Split(f, "=")[1] 274 case f == "stdtime": 275 p.StdTime = true 276 case f == "stdduration": 277 p.StdDuration = true 278 case f == "wktptr": 279 p.WktPointer = true 280 } 281 } 282} 283 284var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() 285 286// setFieldProps initializes the field properties for submessages and maps. 287func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { 288 isMap := typ.Kind() == reflect.Map 289 if len(p.CustomType) > 0 && !isMap { 290 p.ctype = typ 291 p.setTag(lockGetProp) 292 return 293 } 294 if p.StdTime && !isMap { 295 p.setTag(lockGetProp) 296 return 297 } 298 if p.StdDuration && !isMap { 299 p.setTag(lockGetProp) 300 return 301 } 302 if p.WktPointer && !isMap { 303 p.setTag(lockGetProp) 304 return 305 } 306 switch t1 := typ; t1.Kind() { 307 case reflect.Struct: 308 p.stype = typ 309 case reflect.Ptr: 310 if t1.Elem().Kind() == reflect.Struct { 311 p.stype = t1.Elem() 312 } 313 case reflect.Slice: 314 switch t2 := t1.Elem(); t2.Kind() { 315 case reflect.Ptr: 316 switch t3 := t2.Elem(); t3.Kind() { 317 case reflect.Struct: 318 p.stype = t3 319 } 320 case reflect.Struct: 321 p.stype = t2 322 } 323 324 case reflect.Map: 325 326 p.mtype = t1 327 p.MapKeyProp = &Properties{} 328 p.MapKeyProp.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) 329 p.MapValProp = &Properties{} 330 vtype := p.mtype.Elem() 331 if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice { 332 // The value type is not a message (*T) or bytes ([]byte), 333 // so we need encoders for the pointer to this type. 334 vtype = reflect.PtrTo(vtype) 335 } 336 337 p.MapValProp.CustomType = p.CustomType 338 p.MapValProp.StdDuration = p.StdDuration 339 p.MapValProp.StdTime = p.StdTime 340 p.MapValProp.WktPointer = p.WktPointer 341 p.MapValProp.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) 342 } 343 p.setTag(lockGetProp) 344} 345 346func (p *Properties) setTag(lockGetProp bool) { 347 if p.stype != nil { 348 if lockGetProp { 349 p.sprop = GetProperties(p.stype) 350 } else { 351 p.sprop = getPropertiesLocked(p.stype) 352 } 353 } 354} 355 356var ( 357 marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() 358) 359 360// Init populates the properties from a protocol buffer struct tag. 361func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { 362 p.init(typ, name, tag, f, true) 363} 364 365func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) { 366 // "bytes,49,opt,def=hello!" 367 p.Name = name 368 p.OrigName = name 369 if tag == "" { 370 return 371 } 372 p.Parse(tag) 373 p.setFieldProps(typ, f, lockGetProp) 374} 375 376var ( 377 propertiesMu sync.RWMutex 378 propertiesMap = make(map[reflect.Type]*StructProperties) 379) 380 381// GetProperties returns the list of properties for the type represented by t. 382// t must represent a generated struct type of a protocol message. 383func GetProperties(t reflect.Type) *StructProperties { 384 if t.Kind() != reflect.Struct { 385 panic("proto: type must have kind struct") 386 } 387 388 // Most calls to GetProperties in a long-running program will be 389 // retrieving details for types we have seen before. 390 propertiesMu.RLock() 391 sprop, ok := propertiesMap[t] 392 propertiesMu.RUnlock() 393 if ok { 394 return sprop 395 } 396 397 propertiesMu.Lock() 398 sprop = getPropertiesLocked(t) 399 propertiesMu.Unlock() 400 return sprop 401} 402 403// getPropertiesLocked requires that propertiesMu is held. 404func getPropertiesLocked(t reflect.Type) *StructProperties { 405 if prop, ok := propertiesMap[t]; ok { 406 return prop 407 } 408 409 prop := new(StructProperties) 410 // in case of recursive protos, fill this in now. 411 propertiesMap[t] = prop 412 413 // build properties 414 prop.Prop = make([]*Properties, t.NumField()) 415 prop.order = make([]int, t.NumField()) 416 417 isOneofMessage := false 418 for i := 0; i < t.NumField(); i++ { 419 f := t.Field(i) 420 p := new(Properties) 421 name := f.Name 422 p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) 423 424 oneof := f.Tag.Get("protobuf_oneof") // special case 425 if oneof != "" { 426 isOneofMessage = true 427 // Oneof fields don't use the traditional protobuf tag. 428 p.OrigName = oneof 429 } 430 prop.Prop[i] = p 431 prop.order[i] = i 432 if debug { 433 print(i, " ", f.Name, " ", t.String(), " ") 434 if p.Tag > 0 { 435 print(p.String()) 436 } 437 print("\n") 438 } 439 } 440 441 // Re-order prop.order. 442 sort.Sort(prop) 443 444 type oneofMessage interface { 445 XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) 446 } 447 if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); isOneofMessage && ok { 448 var oots []interface{} 449 _, _, _, oots = om.XXX_OneofFuncs() 450 451 // Interpret oneof metadata. 452 prop.OneofTypes = make(map[string]*OneofProperties) 453 for _, oot := range oots { 454 oop := &OneofProperties{ 455 Type: reflect.ValueOf(oot).Type(), // *T 456 Prop: new(Properties), 457 } 458 sft := oop.Type.Elem().Field(0) 459 oop.Prop.Name = sft.Name 460 oop.Prop.Parse(sft.Tag.Get("protobuf")) 461 // There will be exactly one interface field that 462 // this new value is assignable to. 463 for i := 0; i < t.NumField(); i++ { 464 f := t.Field(i) 465 if f.Type.Kind() != reflect.Interface { 466 continue 467 } 468 if !oop.Type.AssignableTo(f.Type) { 469 continue 470 } 471 oop.Field = i 472 break 473 } 474 prop.OneofTypes[oop.Prop.OrigName] = oop 475 } 476 } 477 478 // build required counts 479 // build tags 480 reqCount := 0 481 prop.decoderOrigNames = make(map[string]int) 482 for i, p := range prop.Prop { 483 if strings.HasPrefix(p.Name, "XXX_") { 484 // Internal fields should not appear in tags/origNames maps. 485 // They are handled specially when encoding and decoding. 486 continue 487 } 488 if p.Required { 489 reqCount++ 490 } 491 prop.decoderTags.put(p.Tag, i) 492 prop.decoderOrigNames[p.OrigName] = i 493 } 494 prop.reqCount = reqCount 495 496 return prop 497} 498 499// A global registry of enum types. 500// The generated code will register the generated maps by calling RegisterEnum. 501 502var enumValueMaps = make(map[string]map[string]int32) 503var enumStringMaps = make(map[string]map[int32]string) 504 505// RegisterEnum is called from the generated code to install the enum descriptor 506// maps into the global table to aid parsing text format protocol buffers. 507func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) { 508 if _, ok := enumValueMaps[typeName]; ok { 509 panic("proto: duplicate enum registered: " + typeName) 510 } 511 enumValueMaps[typeName] = valueMap 512 if _, ok := enumStringMaps[typeName]; ok { 513 panic("proto: duplicate enum registered: " + typeName) 514 } 515 enumStringMaps[typeName] = unusedNameMap 516} 517 518// EnumValueMap returns the mapping from names to integers of the 519// enum type enumType, or a nil if not found. 520func EnumValueMap(enumType string) map[string]int32 { 521 return enumValueMaps[enumType] 522} 523 524// A registry of all linked message types. 525// The string is a fully-qualified proto name ("pkg.Message"). 526var ( 527 protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers 528 protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types 529 revProtoTypes = make(map[reflect.Type]string) 530) 531 532// RegisterType is called from generated code and maps from the fully qualified 533// proto name to the type (pointer to struct) of the protocol buffer. 534func RegisterType(x Message, name string) { 535 if _, ok := protoTypedNils[name]; ok { 536 // TODO: Some day, make this a panic. 537 log.Printf("proto: duplicate proto type registered: %s", name) 538 return 539 } 540 t := reflect.TypeOf(x) 541 if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 { 542 // Generated code always calls RegisterType with nil x. 543 // This check is just for extra safety. 544 protoTypedNils[name] = x 545 } else { 546 protoTypedNils[name] = reflect.Zero(t).Interface().(Message) 547 } 548 revProtoTypes[t] = name 549} 550 551// RegisterMapType is called from generated code and maps from the fully qualified 552// proto name to the native map type of the proto map definition. 553func RegisterMapType(x interface{}, name string) { 554 if reflect.TypeOf(x).Kind() != reflect.Map { 555 panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name)) 556 } 557 if _, ok := protoMapTypes[name]; ok { 558 log.Printf("proto: duplicate proto type registered: %s", name) 559 return 560 } 561 t := reflect.TypeOf(x) 562 protoMapTypes[name] = t 563 revProtoTypes[t] = name 564} 565 566// MessageName returns the fully-qualified proto name for the given message type. 567func MessageName(x Message) string { 568 type xname interface { 569 XXX_MessageName() string 570 } 571 if m, ok := x.(xname); ok { 572 return m.XXX_MessageName() 573 } 574 return revProtoTypes[reflect.TypeOf(x)] 575} 576 577// MessageType returns the message type (pointer to struct) for a named message. 578// The type is not guaranteed to implement proto.Message if the name refers to a 579// map entry. 580func MessageType(name string) reflect.Type { 581 if t, ok := protoTypedNils[name]; ok { 582 return reflect.TypeOf(t) 583 } 584 return protoMapTypes[name] 585} 586 587// A registry of all linked proto files. 588var ( 589 protoFiles = make(map[string][]byte) // file name => fileDescriptor 590) 591 592// RegisterFile is called from generated code and maps from the 593// full file name of a .proto file to its compressed FileDescriptorProto. 594func RegisterFile(filename string, fileDescriptor []byte) { 595 protoFiles[filename] = fileDescriptor 596} 597 598// FileDescriptor returns the compressed FileDescriptorProto for a .proto file. 599func FileDescriptor(filename string) []byte { return protoFiles[filename] } 600