1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2015 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32/* 33Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON. 34It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json. 35 36This package produces a different output than the standard "encoding/json" package, 37which does not operate correctly on protocol buffers. 38*/ 39package jsonpb 40 41import ( 42 "bytes" 43 "encoding/json" 44 "errors" 45 "fmt" 46 "io" 47 "math" 48 "reflect" 49 "sort" 50 "strconv" 51 "strings" 52 "time" 53 54 "github.com/gogo/protobuf/proto" 55 "github.com/gogo/protobuf/types" 56) 57 58const secondInNanos = int64(time.Second / time.Nanosecond) 59 60// Marshaler is a configurable object for converting between 61// protocol buffer objects and a JSON representation for them. 62type Marshaler struct { 63 // Whether to render enum values as integers, as opposed to string values. 64 EnumsAsInts bool 65 66 // Whether to render fields with zero values. 67 EmitDefaults bool 68 69 // A string to indent each level by. The presence of this field will 70 // also cause a space to appear between the field separator and 71 // value, and for newlines to be appear between fields and array 72 // elements. 73 Indent string 74 75 // Whether to use the original (.proto) name for fields. 76 OrigName bool 77 78 // A custom URL resolver to use when marshaling Any messages to JSON. 79 // If unset, the default resolution strategy is to extract the 80 // fully-qualified type name from the type URL and pass that to 81 // proto.MessageType(string). 82 AnyResolver AnyResolver 83} 84 85// AnyResolver takes a type URL, present in an Any message, and resolves it into 86// an instance of the associated message. 87type AnyResolver interface { 88 Resolve(typeUrl string) (proto.Message, error) 89} 90 91func defaultResolveAny(typeUrl string) (proto.Message, error) { 92 // Only the part of typeUrl after the last slash is relevant. 93 mname := typeUrl 94 if slash := strings.LastIndex(mname, "/"); slash >= 0 { 95 mname = mname[slash+1:] 96 } 97 mt := proto.MessageType(mname) 98 if mt == nil { 99 return nil, fmt.Errorf("unknown message type %q", mname) 100 } 101 return reflect.New(mt.Elem()).Interface().(proto.Message), nil 102} 103 104// JSONPBMarshaler is implemented by protobuf messages that customize the 105// way they are marshaled to JSON. Messages that implement this should 106// also implement JSONPBUnmarshaler so that the custom format can be 107// parsed. 108// 109// The JSON marshaling must follow the proto to JSON specification: 110// https://developers.google.com/protocol-buffers/docs/proto3#json 111type JSONPBMarshaler interface { 112 MarshalJSONPB(*Marshaler) ([]byte, error) 113} 114 115// JSONPBUnmarshaler is implemented by protobuf messages that customize 116// the way they are unmarshaled from JSON. Messages that implement this 117// should also implement JSONPBMarshaler so that the custom format can be 118// produced. 119// 120// The JSON unmarshaling must follow the JSON to proto specification: 121// https://developers.google.com/protocol-buffers/docs/proto3#json 122type JSONPBUnmarshaler interface { 123 UnmarshalJSONPB(*Unmarshaler, []byte) error 124} 125 126// Marshal marshals a protocol buffer into JSON. 127func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error { 128 v := reflect.ValueOf(pb) 129 if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) { 130 return errors.New("Marshal called with nil") 131 } 132 // Check for unset required fields first. 133 if err := checkRequiredFields(pb); err != nil { 134 return err 135 } 136 writer := &errWriter{writer: out} 137 return m.marshalObject(writer, pb, "", "") 138} 139 140// MarshalToString converts a protocol buffer object to JSON string. 141func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) { 142 var buf bytes.Buffer 143 if err := m.Marshal(&buf, pb); err != nil { 144 return "", err 145 } 146 return buf.String(), nil 147} 148 149type int32Slice []int32 150 151var nonFinite = map[string]float64{ 152 `"NaN"`: math.NaN(), 153 `"Infinity"`: math.Inf(1), 154 `"-Infinity"`: math.Inf(-1), 155} 156 157// For sorting extensions ids to ensure stable output. 158func (s int32Slice) Len() int { return len(s) } 159func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } 160func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 161 162type isWkt interface { 163 XXX_WellKnownType() string 164} 165 166// marshalObject writes a struct to the Writer. 167func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { 168 if jsm, ok := v.(JSONPBMarshaler); ok { 169 b, err := jsm.MarshalJSONPB(m) 170 if err != nil { 171 return err 172 } 173 if typeURL != "" { 174 // we are marshaling this object to an Any type 175 var js map[string]*json.RawMessage 176 if err = json.Unmarshal(b, &js); err != nil { 177 return fmt.Errorf("type %T produced invalid JSON: %v", v, err) 178 } 179 turl, err := json.Marshal(typeURL) 180 if err != nil { 181 return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err) 182 } 183 js["@type"] = (*json.RawMessage)(&turl) 184 if b, err = json.Marshal(js); err != nil { 185 return err 186 } 187 } 188 189 out.write(string(b)) 190 return out.err 191 } 192 193 s := reflect.ValueOf(v).Elem() 194 195 // Handle well-known types. 196 if wkt, ok := v.(isWkt); ok { 197 switch wkt.XXX_WellKnownType() { 198 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 199 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 200 // "Wrappers use the same representation in JSON 201 // as the wrapped primitive type, ..." 202 sprop := proto.GetProperties(s.Type()) 203 return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent) 204 case "Any": 205 // Any is a bit more involved. 206 return m.marshalAny(out, v, indent) 207 case "Duration": 208 // "Generated output always contains 0, 3, 6, or 9 fractional digits, 209 // depending on required precision." 210 s, ns := s.Field(0).Int(), s.Field(1).Int() 211 if ns <= -secondInNanos || ns >= secondInNanos { 212 return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos) 213 } 214 if (s > 0 && ns < 0) || (s < 0 && ns > 0) { 215 return errors.New("signs of seconds and nanos do not match") 216 } 217 if s < 0 { 218 ns = -ns 219 } 220 x := fmt.Sprintf("%d.%09d", s, ns) 221 x = strings.TrimSuffix(x, "000") 222 x = strings.TrimSuffix(x, "000") 223 x = strings.TrimSuffix(x, ".000") 224 out.write(`"`) 225 out.write(x) 226 out.write(`s"`) 227 return out.err 228 case "Struct", "ListValue": 229 // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice. 230 // TODO: pass the correct Properties if needed. 231 return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent) 232 case "Timestamp": 233 // "RFC 3339, where generated output will always be Z-normalized 234 // and uses 0, 3, 6 or 9 fractional digits." 235 s, ns := s.Field(0).Int(), s.Field(1).Int() 236 if ns < 0 || ns >= secondInNanos { 237 return fmt.Errorf("ns out of range [0, %v)", secondInNanos) 238 } 239 t := time.Unix(s, ns).UTC() 240 // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits). 241 x := t.Format("2006-01-02T15:04:05.000000000") 242 x = strings.TrimSuffix(x, "000") 243 x = strings.TrimSuffix(x, "000") 244 x = strings.TrimSuffix(x, ".000") 245 out.write(`"`) 246 out.write(x) 247 out.write(`Z"`) 248 return out.err 249 case "Value": 250 // Value has a single oneof. 251 kind := s.Field(0) 252 if kind.IsNil() { 253 // "absence of any variant indicates an error" 254 return errors.New("nil Value") 255 } 256 // oneof -> *T -> T -> T.F 257 x := kind.Elem().Elem().Field(0) 258 // TODO: pass the correct Properties if needed. 259 return m.marshalValue(out, &proto.Properties{}, x, indent) 260 } 261 } 262 263 out.write("{") 264 if m.Indent != "" { 265 out.write("\n") 266 } 267 268 firstField := true 269 270 if typeURL != "" { 271 if err := m.marshalTypeURL(out, indent, typeURL); err != nil { 272 return err 273 } 274 firstField = false 275 } 276 277 for i := 0; i < s.NumField(); i++ { 278 value := s.Field(i) 279 valueField := s.Type().Field(i) 280 if strings.HasPrefix(valueField.Name, "XXX_") { 281 continue 282 } 283 284 //this is not a protobuf field 285 if valueField.Tag.Get("protobuf") == "" && valueField.Tag.Get("protobuf_oneof") == "" { 286 continue 287 } 288 289 // IsNil will panic on most value kinds. 290 switch value.Kind() { 291 case reflect.Chan, reflect.Func, reflect.Interface: 292 if value.IsNil() { 293 continue 294 } 295 } 296 297 if !m.EmitDefaults { 298 switch value.Kind() { 299 case reflect.Bool: 300 if !value.Bool() { 301 continue 302 } 303 case reflect.Int32, reflect.Int64: 304 if value.Int() == 0 { 305 continue 306 } 307 case reflect.Uint32, reflect.Uint64: 308 if value.Uint() == 0 { 309 continue 310 } 311 case reflect.Float32, reflect.Float64: 312 if value.Float() == 0 { 313 continue 314 } 315 case reflect.String: 316 if value.Len() == 0 { 317 continue 318 } 319 case reflect.Map, reflect.Ptr, reflect.Slice: 320 if value.IsNil() { 321 continue 322 } 323 } 324 } 325 326 // Oneof fields need special handling. 327 if valueField.Tag.Get("protobuf_oneof") != "" { 328 // value is an interface containing &T{real_value}. 329 sv := value.Elem().Elem() // interface -> *T -> T 330 value = sv.Field(0) 331 valueField = sv.Type().Field(0) 332 } 333 prop := jsonProperties(valueField, m.OrigName) 334 if !firstField { 335 m.writeSep(out) 336 } 337 // If the map value is a cast type, it may not implement proto.Message, therefore 338 // allow the struct tag to declare the underlying message type. Change the property 339 // of the child types, use CustomType as a passer. CastType currently property is 340 // not used in json encoding. 341 if value.Kind() == reflect.Map { 342 if tag := valueField.Tag.Get("protobuf"); tag != "" { 343 for _, v := range strings.Split(tag, ",") { 344 if !strings.HasPrefix(v, "castvaluetype=") { 345 continue 346 } 347 v = strings.TrimPrefix(v, "castvaluetype=") 348 prop.MapValProp.CustomType = v 349 break 350 } 351 } 352 } 353 if err := m.marshalField(out, prop, value, indent); err != nil { 354 return err 355 } 356 firstField = false 357 } 358 359 // Handle proto2 extensions. 360 if ep, ok := v.(proto.Message); ok { 361 extensions := proto.RegisteredExtensions(v) 362 // Sort extensions for stable output. 363 ids := make([]int32, 0, len(extensions)) 364 for id, desc := range extensions { 365 if !proto.HasExtension(ep, desc) { 366 continue 367 } 368 ids = append(ids, id) 369 } 370 sort.Sort(int32Slice(ids)) 371 for _, id := range ids { 372 desc := extensions[id] 373 if desc == nil { 374 // unknown extension 375 continue 376 } 377 ext, extErr := proto.GetExtension(ep, desc) 378 if extErr != nil { 379 return extErr 380 } 381 value := reflect.ValueOf(ext) 382 var prop proto.Properties 383 prop.Parse(desc.Tag) 384 prop.JSONName = fmt.Sprintf("[%s]", desc.Name) 385 if !firstField { 386 m.writeSep(out) 387 } 388 if err := m.marshalField(out, &prop, value, indent); err != nil { 389 return err 390 } 391 firstField = false 392 } 393 394 } 395 396 if m.Indent != "" { 397 out.write("\n") 398 out.write(indent) 399 } 400 out.write("}") 401 return out.err 402} 403 404func (m *Marshaler) writeSep(out *errWriter) { 405 if m.Indent != "" { 406 out.write(",\n") 407 } else { 408 out.write(",") 409 } 410} 411 412func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error { 413 // "If the Any contains a value that has a special JSON mapping, 414 // it will be converted as follows: {"@type": xxx, "value": yyy}. 415 // Otherwise, the value will be converted into a JSON object, 416 // and the "@type" field will be inserted to indicate the actual data type." 417 v := reflect.ValueOf(any).Elem() 418 turl := v.Field(0).String() 419 val := v.Field(1).Bytes() 420 421 var msg proto.Message 422 var err error 423 if m.AnyResolver != nil { 424 msg, err = m.AnyResolver.Resolve(turl) 425 } else { 426 msg, err = defaultResolveAny(turl) 427 } 428 if err != nil { 429 return err 430 } 431 432 if err := proto.Unmarshal(val, msg); err != nil { 433 return err 434 } 435 436 if _, ok := msg.(isWkt); ok { 437 out.write("{") 438 if m.Indent != "" { 439 out.write("\n") 440 } 441 if err := m.marshalTypeURL(out, indent, turl); err != nil { 442 return err 443 } 444 m.writeSep(out) 445 if m.Indent != "" { 446 out.write(indent) 447 out.write(m.Indent) 448 out.write(`"value": `) 449 } else { 450 out.write(`"value":`) 451 } 452 if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil { 453 return err 454 } 455 if m.Indent != "" { 456 out.write("\n") 457 out.write(indent) 458 } 459 out.write("}") 460 return out.err 461 } 462 463 return m.marshalObject(out, msg, indent, turl) 464} 465 466func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error { 467 if m.Indent != "" { 468 out.write(indent) 469 out.write(m.Indent) 470 } 471 out.write(`"@type":`) 472 if m.Indent != "" { 473 out.write(" ") 474 } 475 b, err := json.Marshal(typeURL) 476 if err != nil { 477 return err 478 } 479 out.write(string(b)) 480 return out.err 481} 482 483// marshalField writes field description and value to the Writer. 484func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 485 if m.Indent != "" { 486 out.write(indent) 487 out.write(m.Indent) 488 } 489 out.write(`"`) 490 out.write(prop.JSONName) 491 out.write(`":`) 492 if m.Indent != "" { 493 out.write(" ") 494 } 495 if err := m.marshalValue(out, prop, v, indent); err != nil { 496 return err 497 } 498 return nil 499} 500 501// marshalValue writes the value to the Writer. 502func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 503 504 v = reflect.Indirect(v) 505 506 // Handle nil pointer 507 if v.Kind() == reflect.Invalid { 508 out.write("null") 509 return out.err 510 } 511 512 // Handle repeated elements. 513 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { 514 out.write("[") 515 comma := "" 516 for i := 0; i < v.Len(); i++ { 517 sliceVal := v.Index(i) 518 out.write(comma) 519 if m.Indent != "" { 520 out.write("\n") 521 out.write(indent) 522 out.write(m.Indent) 523 out.write(m.Indent) 524 } 525 if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil { 526 return err 527 } 528 comma = "," 529 } 530 if m.Indent != "" { 531 out.write("\n") 532 out.write(indent) 533 out.write(m.Indent) 534 } 535 out.write("]") 536 return out.err 537 } 538 539 // Handle well-known types. 540 // Most are handled up in marshalObject (because 99% are messages). 541 if wkt, ok := v.Interface().(isWkt); ok { 542 switch wkt.XXX_WellKnownType() { 543 case "NullValue": 544 out.write("null") 545 return out.err 546 } 547 } 548 549 if t, ok := v.Interface().(time.Time); ok { 550 ts, err := types.TimestampProto(t) 551 if err != nil { 552 return err 553 } 554 return m.marshalValue(out, prop, reflect.ValueOf(ts), indent) 555 } 556 557 if d, ok := v.Interface().(time.Duration); ok { 558 dur := types.DurationProto(d) 559 return m.marshalValue(out, prop, reflect.ValueOf(dur), indent) 560 } 561 562 // Handle enumerations. 563 if !m.EnumsAsInts && prop.Enum != "" { 564 // Unknown enum values will are stringified by the proto library as their 565 // value. Such values should _not_ be quoted or they will be interpreted 566 // as an enum string instead of their value. 567 enumStr := v.Interface().(fmt.Stringer).String() 568 var valStr string 569 if v.Kind() == reflect.Ptr { 570 valStr = strconv.Itoa(int(v.Elem().Int())) 571 } else { 572 valStr = strconv.Itoa(int(v.Int())) 573 } 574 575 if m, ok := v.Interface().(interface { 576 MarshalJSON() ([]byte, error) 577 }); ok { 578 data, err := m.MarshalJSON() 579 if err != nil { 580 return err 581 } 582 enumStr = string(data) 583 enumStr, err = strconv.Unquote(enumStr) 584 if err != nil { 585 return err 586 } 587 } 588 589 isKnownEnum := enumStr != valStr 590 591 if isKnownEnum { 592 out.write(`"`) 593 } 594 out.write(enumStr) 595 if isKnownEnum { 596 out.write(`"`) 597 } 598 return out.err 599 } 600 601 // Handle nested messages. 602 if v.Kind() == reflect.Struct { 603 i := v 604 if v.CanAddr() { 605 i = v.Addr() 606 } else { 607 i = reflect.New(v.Type()) 608 i.Elem().Set(v) 609 } 610 iface := i.Interface() 611 if iface == nil { 612 out.write(`null`) 613 return out.err 614 } 615 616 if m, ok := v.Interface().(interface { 617 MarshalJSON() ([]byte, error) 618 }); ok { 619 data, err := m.MarshalJSON() 620 if err != nil { 621 return err 622 } 623 out.write(string(data)) 624 return nil 625 } 626 627 pm, ok := iface.(proto.Message) 628 if !ok { 629 if prop.CustomType == "" { 630 return fmt.Errorf("%v does not implement proto.Message", v.Type()) 631 } 632 t := proto.MessageType(prop.CustomType) 633 if t == nil || !i.Type().ConvertibleTo(t) { 634 return fmt.Errorf("%v declared custom type %s but it is not convertible to %v", v.Type(), prop.CustomType, t) 635 } 636 pm = i.Convert(t).Interface().(proto.Message) 637 } 638 return m.marshalObject(out, pm, indent+m.Indent, "") 639 } 640 641 // Handle maps. 642 // Since Go randomizes map iteration, we sort keys for stable output. 643 if v.Kind() == reflect.Map { 644 out.write(`{`) 645 keys := v.MapKeys() 646 sort.Sort(mapKeys(keys)) 647 for i, k := range keys { 648 if i > 0 { 649 out.write(`,`) 650 } 651 if m.Indent != "" { 652 out.write("\n") 653 out.write(indent) 654 out.write(m.Indent) 655 out.write(m.Indent) 656 } 657 658 // TODO handle map key prop properly 659 b, err := json.Marshal(k.Interface()) 660 if err != nil { 661 return err 662 } 663 s := string(b) 664 665 // If the JSON is not a string value, encode it again to make it one. 666 if !strings.HasPrefix(s, `"`) { 667 b, err := json.Marshal(s) 668 if err != nil { 669 return err 670 } 671 s = string(b) 672 } 673 674 out.write(s) 675 out.write(`:`) 676 if m.Indent != "" { 677 out.write(` `) 678 } 679 680 vprop := prop 681 if prop != nil && prop.MapValProp != nil { 682 vprop = prop.MapValProp 683 } 684 if err := m.marshalValue(out, vprop, v.MapIndex(k), indent+m.Indent); err != nil { 685 return err 686 } 687 } 688 if m.Indent != "" { 689 out.write("\n") 690 out.write(indent) 691 out.write(m.Indent) 692 } 693 out.write(`}`) 694 return out.err 695 } 696 697 // Handle non-finite floats, e.g. NaN, Infinity and -Infinity. 698 if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { 699 f := v.Float() 700 var sval string 701 switch { 702 case math.IsInf(f, 1): 703 sval = `"Infinity"` 704 case math.IsInf(f, -1): 705 sval = `"-Infinity"` 706 case math.IsNaN(f): 707 sval = `"NaN"` 708 } 709 if sval != "" { 710 out.write(sval) 711 return out.err 712 } 713 } 714 715 // Default handling defers to the encoding/json library. 716 b, err := json.Marshal(v.Interface()) 717 if err != nil { 718 return err 719 } 720 needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64) 721 if needToQuote { 722 out.write(`"`) 723 } 724 out.write(string(b)) 725 if needToQuote { 726 out.write(`"`) 727 } 728 return out.err 729} 730 731// Unmarshaler is a configurable object for converting from a JSON 732// representation to a protocol buffer object. 733type Unmarshaler struct { 734 // Whether to allow messages to contain unknown fields, as opposed to 735 // failing to unmarshal. 736 AllowUnknownFields bool 737 738 // A custom URL resolver to use when unmarshaling Any messages from JSON. 739 // If unset, the default resolution strategy is to extract the 740 // fully-qualified type name from the type URL and pass that to 741 // proto.MessageType(string). 742 AnyResolver AnyResolver 743} 744 745// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 746// This function is lenient and will decode any options permutations of the 747// related Marshaler. 748func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 749 inputValue := json.RawMessage{} 750 if err := dec.Decode(&inputValue); err != nil { 751 return err 752 } 753 if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil { 754 return err 755 } 756 return checkRequiredFields(pb) 757} 758 759// Unmarshal unmarshals a JSON object stream into a protocol 760// buffer. This function is lenient and will decode any options 761// permutations of the related Marshaler. 762func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error { 763 dec := json.NewDecoder(r) 764 return u.UnmarshalNext(dec, pb) 765} 766 767// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 768// This function is lenient and will decode any options permutations of the 769// related Marshaler. 770func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 771 return new(Unmarshaler).UnmarshalNext(dec, pb) 772} 773 774// Unmarshal unmarshals a JSON object stream into a protocol 775// buffer. This function is lenient and will decode any options 776// permutations of the related Marshaler. 777func Unmarshal(r io.Reader, pb proto.Message) error { 778 return new(Unmarshaler).Unmarshal(r, pb) 779} 780 781// UnmarshalString will populate the fields of a protocol buffer based 782// on a JSON string. This function is lenient and will decode any options 783// permutations of the related Marshaler. 784func UnmarshalString(str string, pb proto.Message) error { 785 return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb) 786} 787 788// unmarshalValue converts/copies a value into the target. 789// prop may be nil. 790func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { 791 targetType := target.Type() 792 793 // Allocate memory for pointer fields. 794 if targetType.Kind() == reflect.Ptr { 795 // If input value is "null" and target is a pointer type, then the field should be treated as not set 796 // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue. 797 _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler) 798 if string(inputValue) == "null" && targetType != reflect.TypeOf(&types.Value{}) && !isJSONPBUnmarshaler { 799 return nil 800 } 801 target.Set(reflect.New(targetType.Elem())) 802 803 return u.unmarshalValue(target.Elem(), inputValue, prop) 804 } 805 806 if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok { 807 return jsu.UnmarshalJSONPB(u, []byte(inputValue)) 808 } 809 810 // Handle well-known types that are not pointers. 811 if w, ok := target.Addr().Interface().(isWkt); ok { 812 switch w.XXX_WellKnownType() { 813 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 814 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 815 return u.unmarshalValue(target.Field(0), inputValue, prop) 816 case "Any": 817 // Use json.RawMessage pointer type instead of value to support pre-1.8 version. 818 // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see 819 // https://github.com/golang/go/issues/14493 820 var jsonFields map[string]*json.RawMessage 821 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 822 return err 823 } 824 825 val, ok := jsonFields["@type"] 826 if !ok || val == nil { 827 return errors.New("Any JSON doesn't have '@type'") 828 } 829 830 var turl string 831 if err := json.Unmarshal([]byte(*val), &turl); err != nil { 832 return fmt.Errorf("can't unmarshal Any's '@type': %q", *val) 833 } 834 target.Field(0).SetString(turl) 835 836 var m proto.Message 837 var err error 838 if u.AnyResolver != nil { 839 m, err = u.AnyResolver.Resolve(turl) 840 } else { 841 m, err = defaultResolveAny(turl) 842 } 843 if err != nil { 844 return err 845 } 846 847 if _, ok := m.(isWkt); ok { 848 val, ok := jsonFields["value"] 849 if !ok { 850 return errors.New("Any JSON doesn't have 'value'") 851 } 852 853 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil { 854 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 855 } 856 } else { 857 delete(jsonFields, "@type") 858 nestedProto, uerr := json.Marshal(jsonFields) 859 if uerr != nil { 860 return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", uerr) 861 } 862 863 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil { 864 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 865 } 866 } 867 868 b, err := proto.Marshal(m) 869 if err != nil { 870 return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err) 871 } 872 target.Field(1).SetBytes(b) 873 874 return nil 875 case "Duration": 876 unq, err := unquote(string(inputValue)) 877 if err != nil { 878 return err 879 } 880 881 d, err := time.ParseDuration(unq) 882 if err != nil { 883 return fmt.Errorf("bad Duration: %v", err) 884 } 885 886 ns := d.Nanoseconds() 887 s := ns / 1e9 888 ns %= 1e9 889 target.Field(0).SetInt(s) 890 target.Field(1).SetInt(ns) 891 return nil 892 case "Timestamp": 893 unq, err := unquote(string(inputValue)) 894 if err != nil { 895 return err 896 } 897 898 t, err := time.Parse(time.RFC3339Nano, unq) 899 if err != nil { 900 return fmt.Errorf("bad Timestamp: %v", err) 901 } 902 903 target.Field(0).SetInt(t.Unix()) 904 target.Field(1).SetInt(int64(t.Nanosecond())) 905 return nil 906 case "Struct": 907 var m map[string]json.RawMessage 908 if err := json.Unmarshal(inputValue, &m); err != nil { 909 return fmt.Errorf("bad StructValue: %v", err) 910 } 911 target.Field(0).Set(reflect.ValueOf(map[string]*types.Value{})) 912 for k, jv := range m { 913 pv := &types.Value{} 914 if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil { 915 return fmt.Errorf("bad value in StructValue for key %q: %v", k, err) 916 } 917 target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv)) 918 } 919 return nil 920 case "ListValue": 921 var s []json.RawMessage 922 if err := json.Unmarshal(inputValue, &s); err != nil { 923 return fmt.Errorf("bad ListValue: %v", err) 924 } 925 926 target.Field(0).Set(reflect.ValueOf(make([]*types.Value, len(s)))) 927 for i, sv := range s { 928 if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil { 929 return err 930 } 931 } 932 return nil 933 case "Value": 934 ivStr := string(inputValue) 935 if ivStr == "null" { 936 target.Field(0).Set(reflect.ValueOf(&types.Value_NullValue{})) 937 } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil { 938 target.Field(0).Set(reflect.ValueOf(&types.Value_NumberValue{NumberValue: v})) 939 } else if v, err := unquote(ivStr); err == nil { 940 target.Field(0).Set(reflect.ValueOf(&types.Value_StringValue{StringValue: v})) 941 } else if v, err := strconv.ParseBool(ivStr); err == nil { 942 target.Field(0).Set(reflect.ValueOf(&types.Value_BoolValue{BoolValue: v})) 943 } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil { 944 lv := &types.ListValue{} 945 target.Field(0).Set(reflect.ValueOf(&types.Value_ListValue{ListValue: lv})) 946 return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop) 947 } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil { 948 sv := &types.Struct{} 949 target.Field(0).Set(reflect.ValueOf(&types.Value_StructValue{StructValue: sv})) 950 return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop) 951 } else { 952 return fmt.Errorf("unrecognized type for Value %q", ivStr) 953 } 954 return nil 955 } 956 } 957 958 if t, ok := target.Addr().Interface().(*time.Time); ok { 959 ts := &types.Timestamp{} 960 if err := u.unmarshalValue(reflect.ValueOf(ts).Elem(), inputValue, prop); err != nil { 961 return err 962 } 963 tt, err := types.TimestampFromProto(ts) 964 if err != nil { 965 return err 966 } 967 *t = tt 968 return nil 969 } 970 971 if d, ok := target.Addr().Interface().(*time.Duration); ok { 972 dur := &types.Duration{} 973 if err := u.unmarshalValue(reflect.ValueOf(dur).Elem(), inputValue, prop); err != nil { 974 return err 975 } 976 dd, err := types.DurationFromProto(dur) 977 if err != nil { 978 return err 979 } 980 *d = dd 981 return nil 982 } 983 984 // Handle enums, which have an underlying type of int32, 985 // and may appear as strings. 986 // The case of an enum appearing as a number is handled 987 // at the bottom of this function. 988 if inputValue[0] == '"' && prop != nil && prop.Enum != "" { 989 vmap := proto.EnumValueMap(prop.Enum) 990 // Don't need to do unquoting; valid enum names 991 // are from a limited character set. 992 s := inputValue[1 : len(inputValue)-1] 993 n, ok := vmap[string(s)] 994 if !ok { 995 return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum) 996 } 997 if target.Kind() == reflect.Ptr { // proto2 998 target.Set(reflect.New(targetType.Elem())) 999 target = target.Elem() 1000 } 1001 if targetType.Kind() != reflect.Int32 { 1002 return fmt.Errorf("invalid target %q for enum %s", targetType.Kind(), prop.Enum) 1003 } 1004 target.SetInt(int64(n)) 1005 return nil 1006 } 1007 1008 if prop != nil && len(prop.CustomType) > 0 && target.CanAddr() { 1009 if m, ok := target.Addr().Interface().(interface { 1010 UnmarshalJSON([]byte) error 1011 }); ok { 1012 return json.Unmarshal(inputValue, m) 1013 } 1014 } 1015 1016 // Handle nested messages. 1017 if targetType.Kind() == reflect.Struct { 1018 var jsonFields map[string]json.RawMessage 1019 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 1020 return err 1021 } 1022 1023 consumeField := func(prop *proto.Properties) (json.RawMessage, bool) { 1024 // Be liberal in what names we accept; both orig_name and camelName are okay. 1025 fieldNames := acceptedJSONFieldNames(prop) 1026 1027 vOrig, okOrig := jsonFields[fieldNames.orig] 1028 vCamel, okCamel := jsonFields[fieldNames.camel] 1029 if !okOrig && !okCamel { 1030 return nil, false 1031 } 1032 // If, for some reason, both are present in the data, favour the camelName. 1033 var raw json.RawMessage 1034 if okOrig { 1035 raw = vOrig 1036 delete(jsonFields, fieldNames.orig) 1037 } 1038 if okCamel { 1039 raw = vCamel 1040 delete(jsonFields, fieldNames.camel) 1041 } 1042 return raw, true 1043 } 1044 1045 sprops := proto.GetProperties(targetType) 1046 for i := 0; i < target.NumField(); i++ { 1047 ft := target.Type().Field(i) 1048 if strings.HasPrefix(ft.Name, "XXX_") { 1049 continue 1050 } 1051 valueForField, ok := consumeField(sprops.Prop[i]) 1052 if !ok { 1053 continue 1054 } 1055 1056 if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { 1057 return err 1058 } 1059 } 1060 // Check for any oneof fields. 1061 if len(jsonFields) > 0 { 1062 for _, oop := range sprops.OneofTypes { 1063 raw, ok := consumeField(oop.Prop) 1064 if !ok { 1065 continue 1066 } 1067 nv := reflect.New(oop.Type.Elem()) 1068 target.Field(oop.Field).Set(nv) 1069 if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { 1070 return err 1071 } 1072 } 1073 } 1074 // Handle proto2 extensions. 1075 if len(jsonFields) > 0 { 1076 if ep, ok := target.Addr().Interface().(proto.Message); ok { 1077 for _, ext := range proto.RegisteredExtensions(ep) { 1078 name := fmt.Sprintf("[%s]", ext.Name) 1079 raw, ok := jsonFields[name] 1080 if !ok { 1081 continue 1082 } 1083 delete(jsonFields, name) 1084 nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem()) 1085 if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil { 1086 return err 1087 } 1088 if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil { 1089 return err 1090 } 1091 } 1092 } 1093 } 1094 if !u.AllowUnknownFields && len(jsonFields) > 0 { 1095 // Pick any field to be the scapegoat. 1096 var f string 1097 for fname := range jsonFields { 1098 f = fname 1099 break 1100 } 1101 return fmt.Errorf("unknown field %q in %v", f, targetType) 1102 } 1103 return nil 1104 } 1105 1106 // Handle arrays 1107 if targetType.Kind() == reflect.Slice { 1108 if targetType.Elem().Kind() == reflect.Uint8 { 1109 outRef := reflect.New(targetType) 1110 outVal := outRef.Interface() 1111 //CustomType with underlying type []byte 1112 if _, ok := outVal.(interface { 1113 UnmarshalJSON([]byte) error 1114 }); ok { 1115 if err := json.Unmarshal(inputValue, outVal); err != nil { 1116 return err 1117 } 1118 target.Set(outRef.Elem()) 1119 return nil 1120 } 1121 // Special case for encoded bytes. Pre-go1.5 doesn't support unmarshalling 1122 // strings into aliased []byte types. 1123 // https://github.com/golang/go/commit/4302fd0409da5e4f1d71471a6770dacdc3301197 1124 // https://github.com/golang/go/commit/c60707b14d6be26bf4213114d13070bff00d0b0a 1125 var out []byte 1126 if err := json.Unmarshal(inputValue, &out); err != nil { 1127 return err 1128 } 1129 target.SetBytes(out) 1130 return nil 1131 } 1132 1133 var slc []json.RawMessage 1134 if err := json.Unmarshal(inputValue, &slc); err != nil { 1135 return err 1136 } 1137 if slc != nil { 1138 l := len(slc) 1139 target.Set(reflect.MakeSlice(targetType, l, l)) 1140 for i := 0; i < l; i++ { 1141 if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil { 1142 return err 1143 } 1144 } 1145 } 1146 return nil 1147 } 1148 1149 // Handle maps (whose keys are always strings) 1150 if targetType.Kind() == reflect.Map { 1151 var mp map[string]json.RawMessage 1152 if err := json.Unmarshal(inputValue, &mp); err != nil { 1153 return err 1154 } 1155 if mp != nil { 1156 target.Set(reflect.MakeMap(targetType)) 1157 for ks, raw := range mp { 1158 // Unmarshal map key. The core json library already decoded the key into a 1159 // string, so we handle that specially. Other types were quoted post-serialization. 1160 var k reflect.Value 1161 if targetType.Key().Kind() == reflect.String { 1162 k = reflect.ValueOf(ks) 1163 } else { 1164 k = reflect.New(targetType.Key()).Elem() 1165 var kprop *proto.Properties 1166 if prop != nil && prop.MapKeyProp != nil { 1167 kprop = prop.MapKeyProp 1168 } 1169 if err := u.unmarshalValue(k, json.RawMessage(ks), kprop); err != nil { 1170 return err 1171 } 1172 } 1173 1174 if !k.Type().AssignableTo(targetType.Key()) { 1175 k = k.Convert(targetType.Key()) 1176 } 1177 1178 // Unmarshal map value. 1179 v := reflect.New(targetType.Elem()).Elem() 1180 var vprop *proto.Properties 1181 if prop != nil && prop.MapValProp != nil { 1182 vprop = prop.MapValProp 1183 } 1184 if err := u.unmarshalValue(v, raw, vprop); err != nil { 1185 return err 1186 } 1187 target.SetMapIndex(k, v) 1188 } 1189 } 1190 return nil 1191 } 1192 1193 // Non-finite numbers can be encoded as strings. 1194 isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1195 if isFloat { 1196 if num, ok := nonFinite[string(inputValue)]; ok { 1197 target.SetFloat(num) 1198 return nil 1199 } 1200 } 1201 1202 // integers & floats can be encoded as strings. In this case we drop 1203 // the quotes and proceed as normal. 1204 isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64 || 1205 targetType.Kind() == reflect.Int32 || targetType.Kind() == reflect.Uint32 || 1206 targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1207 if isNum && strings.HasPrefix(string(inputValue), `"`) { 1208 inputValue = inputValue[1 : len(inputValue)-1] 1209 } 1210 1211 // Use the encoding/json for parsing other value types. 1212 return json.Unmarshal(inputValue, target.Addr().Interface()) 1213} 1214 1215func unquote(s string) (string, error) { 1216 var ret string 1217 err := json.Unmarshal([]byte(s), &ret) 1218 return ret, err 1219} 1220 1221// jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute. 1222func jsonProperties(f reflect.StructField, origName bool) *proto.Properties { 1223 var prop proto.Properties 1224 prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f) 1225 if origName || prop.JSONName == "" { 1226 prop.JSONName = prop.OrigName 1227 } 1228 return &prop 1229} 1230 1231type fieldNames struct { 1232 orig, camel string 1233} 1234 1235func acceptedJSONFieldNames(prop *proto.Properties) fieldNames { 1236 opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName} 1237 if prop.JSONName != "" { 1238 opts.camel = prop.JSONName 1239 } 1240 return opts 1241} 1242 1243// Writer wrapper inspired by https://blog.golang.org/errors-are-values 1244type errWriter struct { 1245 writer io.Writer 1246 err error 1247} 1248 1249func (w *errWriter) write(str string) { 1250 if w.err != nil { 1251 return 1252 } 1253 _, w.err = w.writer.Write([]byte(str)) 1254} 1255 1256// Map fields may have key types of non-float scalars, strings and enums. 1257// The easiest way to sort them in some deterministic order is to use fmt. 1258// If this turns out to be inefficient we can always consider other options, 1259// such as doing a Schwartzian transform. 1260// 1261// Numeric keys are sorted in numeric order per 1262// https://developers.google.com/protocol-buffers/docs/proto#maps. 1263type mapKeys []reflect.Value 1264 1265func (s mapKeys) Len() int { return len(s) } 1266func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 1267func (s mapKeys) Less(i, j int) bool { 1268 if k := s[i].Kind(); k == s[j].Kind() { 1269 switch k { 1270 case reflect.String: 1271 return s[i].String() < s[j].String() 1272 case reflect.Int32, reflect.Int64: 1273 return s[i].Int() < s[j].Int() 1274 case reflect.Uint32, reflect.Uint64: 1275 return s[i].Uint() < s[j].Uint() 1276 } 1277 } 1278 return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) 1279} 1280 1281// checkRequiredFields returns an error if any required field in the given proto message is not set. 1282// This function is used by both Marshal and Unmarshal. While required fields only exist in a 1283// proto2 message, a proto3 message can contain proto2 message(s). 1284func checkRequiredFields(pb proto.Message) error { 1285 // Most well-known type messages do not contain required fields. The "Any" type may contain 1286 // a message that has required fields. 1287 // 1288 // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value 1289 // field in order to transform that into JSON, and that should have returned an error if a 1290 // required field is not set in the embedded message. 1291 // 1292 // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the 1293 // embedded message to store the serialized message in Any.Value field, and that should have 1294 // returned an error if a required field is not set. 1295 if _, ok := pb.(isWkt); ok { 1296 return nil 1297 } 1298 1299 v := reflect.ValueOf(pb) 1300 // Skip message if it is not a struct pointer. 1301 if v.Kind() != reflect.Ptr { 1302 return nil 1303 } 1304 v = v.Elem() 1305 if v.Kind() != reflect.Struct { 1306 return nil 1307 } 1308 1309 for i := 0; i < v.NumField(); i++ { 1310 field := v.Field(i) 1311 sfield := v.Type().Field(i) 1312 1313 if sfield.PkgPath != "" { 1314 // blank PkgPath means the field is exported; skip if not exported 1315 continue 1316 } 1317 1318 if strings.HasPrefix(sfield.Name, "XXX_") { 1319 continue 1320 } 1321 1322 // Oneof field is an interface implemented by wrapper structs containing the actual oneof 1323 // field, i.e. an interface containing &T{real_value}. 1324 if sfield.Tag.Get("protobuf_oneof") != "" { 1325 if field.Kind() != reflect.Interface { 1326 continue 1327 } 1328 v := field.Elem() 1329 if v.Kind() != reflect.Ptr || v.IsNil() { 1330 continue 1331 } 1332 v = v.Elem() 1333 if v.Kind() != reflect.Struct || v.NumField() < 1 { 1334 continue 1335 } 1336 field = v.Field(0) 1337 sfield = v.Type().Field(0) 1338 } 1339 1340 protoTag := sfield.Tag.Get("protobuf") 1341 if protoTag == "" { 1342 continue 1343 } 1344 var prop proto.Properties 1345 prop.Init(sfield.Type, sfield.Name, protoTag, &sfield) 1346 1347 switch field.Kind() { 1348 case reflect.Map: 1349 if field.IsNil() { 1350 continue 1351 } 1352 // Check each map value. 1353 keys := field.MapKeys() 1354 for _, k := range keys { 1355 v := field.MapIndex(k) 1356 if err := checkRequiredFieldsInValue(v); err != nil { 1357 return err 1358 } 1359 } 1360 case reflect.Slice: 1361 // Handle non-repeated type, e.g. bytes. 1362 if !prop.Repeated { 1363 if prop.Required && field.IsNil() { 1364 return fmt.Errorf("required field %q is not set", prop.Name) 1365 } 1366 continue 1367 } 1368 1369 // Handle repeated type. 1370 if field.IsNil() { 1371 continue 1372 } 1373 // Check each slice item. 1374 for i := 0; i < field.Len(); i++ { 1375 v := field.Index(i) 1376 if err := checkRequiredFieldsInValue(v); err != nil { 1377 return err 1378 } 1379 } 1380 case reflect.Ptr: 1381 if field.IsNil() { 1382 if prop.Required { 1383 return fmt.Errorf("required field %q is not set", prop.Name) 1384 } 1385 continue 1386 } 1387 if err := checkRequiredFieldsInValue(field); err != nil { 1388 return err 1389 } 1390 } 1391 } 1392 1393 // Handle proto2 extensions. 1394 for _, ext := range proto.RegisteredExtensions(pb) { 1395 if !proto.HasExtension(pb, ext) { 1396 continue 1397 } 1398 ep, err := proto.GetExtension(pb, ext) 1399 if err != nil { 1400 return err 1401 } 1402 err = checkRequiredFieldsInValue(reflect.ValueOf(ep)) 1403 if err != nil { 1404 return err 1405 } 1406 } 1407 1408 return nil 1409} 1410 1411func checkRequiredFieldsInValue(v reflect.Value) error { 1412 if pm, ok := v.Interface().(proto.Message); ok { 1413 return checkRequiredFields(pm) 1414 } 1415 return nil 1416} 1417