1// Go support for Protocol Buffers - Google's data interchange format 2// 3// Copyright 2015 The Go Authors. All rights reserved. 4// https://github.com/golang/protobuf 5// 6// Redistribution and use in source and binary forms, with or without 7// modification, are permitted provided that the following conditions are 8// met: 9// 10// * Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// * Redistributions in binary form must reproduce the above 13// copyright notice, this list of conditions and the following disclaimer 14// in the documentation and/or other materials provided with the 15// distribution. 16// * Neither the name of Google Inc. nor the names of its 17// contributors may be used to endorse or promote products derived from 18// this software without specific prior written permission. 19// 20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32/* 33Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON. 34It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json. 35 36This package produces a different output than the standard "encoding/json" package, 37which does not operate correctly on protocol buffers. 38*/ 39package jsonpb 40 41import ( 42 "bytes" 43 "encoding/json" 44 "errors" 45 "fmt" 46 "io" 47 "math" 48 "reflect" 49 "sort" 50 "strconv" 51 "strings" 52 "time" 53 54 "github.com/gogo/protobuf/proto" 55 "github.com/gogo/protobuf/types" 56) 57 58const secondInNanos = int64(time.Second / time.Nanosecond) 59 60// Marshaler is a configurable object for converting between 61// protocol buffer objects and a JSON representation for them. 62type Marshaler struct { 63 // Whether to render enum values as integers, as opposed to string values. 64 EnumsAsInts bool 65 66 // Whether to render fields with zero values. 67 EmitDefaults bool 68 69 // A string to indent each level by. The presence of this field will 70 // also cause a space to appear between the field separator and 71 // value, and for newlines to be appear between fields and array 72 // elements. 73 Indent string 74 75 // Whether to use the original (.proto) name for fields. 76 OrigName bool 77 78 // A custom URL resolver to use when marshaling Any messages to JSON. 79 // If unset, the default resolution strategy is to extract the 80 // fully-qualified type name from the type URL and pass that to 81 // proto.MessageType(string). 82 AnyResolver AnyResolver 83} 84 85// AnyResolver takes a type URL, present in an Any message, and resolves it into 86// an instance of the associated message. 87type AnyResolver interface { 88 Resolve(typeUrl string) (proto.Message, error) 89} 90 91func defaultResolveAny(typeUrl string) (proto.Message, error) { 92 // Only the part of typeUrl after the last slash is relevant. 93 mname := typeUrl 94 if slash := strings.LastIndex(mname, "/"); slash >= 0 { 95 mname = mname[slash+1:] 96 } 97 mt := proto.MessageType(mname) 98 if mt == nil { 99 return nil, fmt.Errorf("unknown message type %q", mname) 100 } 101 return reflect.New(mt.Elem()).Interface().(proto.Message), nil 102} 103 104// JSONPBMarshaler is implemented by protobuf messages that customize the 105// way they are marshaled to JSON. Messages that implement this should 106// also implement JSONPBUnmarshaler so that the custom format can be 107// parsed. 108type JSONPBMarshaler interface { 109 MarshalJSONPB(*Marshaler) ([]byte, error) 110} 111 112// JSONPBUnmarshaler is implemented by protobuf messages that customize 113// the way they are unmarshaled from JSON. Messages that implement this 114// should also implement JSONPBMarshaler so that the custom format can be 115// produced. 116type JSONPBUnmarshaler interface { 117 UnmarshalJSONPB(*Unmarshaler, []byte) error 118} 119 120// Marshal marshals a protocol buffer into JSON. 121func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error { 122 v := reflect.ValueOf(pb) 123 if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) { 124 return errors.New("Marshal called with nil") 125 } 126 // Check for unset required fields first. 127 if err := checkRequiredFields(pb); err != nil { 128 return err 129 } 130 writer := &errWriter{writer: out} 131 return m.marshalObject(writer, pb, "", "") 132} 133 134// MarshalToString converts a protocol buffer object to JSON string. 135func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) { 136 var buf bytes.Buffer 137 if err := m.Marshal(&buf, pb); err != nil { 138 return "", err 139 } 140 return buf.String(), nil 141} 142 143type int32Slice []int32 144 145var nonFinite = map[string]float64{ 146 `"NaN"`: math.NaN(), 147 `"Infinity"`: math.Inf(1), 148 `"-Infinity"`: math.Inf(-1), 149} 150 151// For sorting extensions ids to ensure stable output. 152func (s int32Slice) Len() int { return len(s) } 153func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } 154func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 155 156type isWkt interface { 157 XXX_WellKnownType() string 158} 159 160// marshalObject writes a struct to the Writer. 161func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { 162 if jsm, ok := v.(JSONPBMarshaler); ok { 163 b, err := jsm.MarshalJSONPB(m) 164 if err != nil { 165 return err 166 } 167 if typeURL != "" { 168 // we are marshaling this object to an Any type 169 var js map[string]*json.RawMessage 170 if err = json.Unmarshal(b, &js); err != nil { 171 return fmt.Errorf("type %T produced invalid JSON: %v", v, err) 172 } 173 turl, err := json.Marshal(typeURL) 174 if err != nil { 175 return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err) 176 } 177 js["@type"] = (*json.RawMessage)(&turl) 178 if b, err = json.Marshal(js); err != nil { 179 return err 180 } 181 } 182 183 out.write(string(b)) 184 return out.err 185 } 186 187 s := reflect.ValueOf(v).Elem() 188 189 // Handle well-known types. 190 if wkt, ok := v.(isWkt); ok { 191 switch wkt.XXX_WellKnownType() { 192 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 193 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 194 // "Wrappers use the same representation in JSON 195 // as the wrapped primitive type, ..." 196 sprop := proto.GetProperties(s.Type()) 197 return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent) 198 case "Any": 199 // Any is a bit more involved. 200 return m.marshalAny(out, v, indent) 201 case "Duration": 202 // "Generated output always contains 0, 3, 6, or 9 fractional digits, 203 // depending on required precision." 204 s, ns := s.Field(0).Int(), s.Field(1).Int() 205 if ns <= -secondInNanos || ns >= secondInNanos { 206 return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos) 207 } 208 if (s > 0 && ns < 0) || (s < 0 && ns > 0) { 209 return errors.New("signs of seconds and nanos do not match") 210 } 211 if s < 0 { 212 ns = -ns 213 } 214 x := fmt.Sprintf("%d.%09d", s, ns) 215 x = strings.TrimSuffix(x, "000") 216 x = strings.TrimSuffix(x, "000") 217 x = strings.TrimSuffix(x, ".000") 218 out.write(`"`) 219 out.write(x) 220 out.write(`s"`) 221 return out.err 222 case "Struct", "ListValue": 223 // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice. 224 // TODO: pass the correct Properties if needed. 225 return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent) 226 case "Timestamp": 227 // "RFC 3339, where generated output will always be Z-normalized 228 // and uses 0, 3, 6 or 9 fractional digits." 229 s, ns := s.Field(0).Int(), s.Field(1).Int() 230 if ns < 0 || ns >= secondInNanos { 231 return fmt.Errorf("ns out of range [0, %v)", secondInNanos) 232 } 233 t := time.Unix(s, ns).UTC() 234 // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits). 235 x := t.Format("2006-01-02T15:04:05.000000000") 236 x = strings.TrimSuffix(x, "000") 237 x = strings.TrimSuffix(x, "000") 238 x = strings.TrimSuffix(x, ".000") 239 out.write(`"`) 240 out.write(x) 241 out.write(`Z"`) 242 return out.err 243 case "Value": 244 // Value has a single oneof. 245 kind := s.Field(0) 246 if kind.IsNil() { 247 // "absence of any variant indicates an error" 248 return errors.New("nil Value") 249 } 250 // oneof -> *T -> T -> T.F 251 x := kind.Elem().Elem().Field(0) 252 // TODO: pass the correct Properties if needed. 253 return m.marshalValue(out, &proto.Properties{}, x, indent) 254 } 255 } 256 257 out.write("{") 258 if m.Indent != "" { 259 out.write("\n") 260 } 261 262 firstField := true 263 264 if typeURL != "" { 265 if err := m.marshalTypeURL(out, indent, typeURL); err != nil { 266 return err 267 } 268 firstField = false 269 } 270 271 for i := 0; i < s.NumField(); i++ { 272 value := s.Field(i) 273 valueField := s.Type().Field(i) 274 if strings.HasPrefix(valueField.Name, "XXX_") { 275 continue 276 } 277 278 //this is not a protobuf field 279 if valueField.Tag.Get("protobuf") == "" && valueField.Tag.Get("protobuf_oneof") == "" { 280 continue 281 } 282 283 // IsNil will panic on most value kinds. 284 switch value.Kind() { 285 case reflect.Chan, reflect.Func, reflect.Interface: 286 if value.IsNil() { 287 continue 288 } 289 } 290 291 if !m.EmitDefaults { 292 switch value.Kind() { 293 case reflect.Bool: 294 if !value.Bool() { 295 continue 296 } 297 case reflect.Int32, reflect.Int64: 298 if value.Int() == 0 { 299 continue 300 } 301 case reflect.Uint32, reflect.Uint64: 302 if value.Uint() == 0 { 303 continue 304 } 305 case reflect.Float32, reflect.Float64: 306 if value.Float() == 0 { 307 continue 308 } 309 case reflect.String: 310 if value.Len() == 0 { 311 continue 312 } 313 case reflect.Map, reflect.Ptr, reflect.Slice: 314 if value.IsNil() { 315 continue 316 } 317 } 318 } 319 320 // Oneof fields need special handling. 321 if valueField.Tag.Get("protobuf_oneof") != "" { 322 // value is an interface containing &T{real_value}. 323 sv := value.Elem().Elem() // interface -> *T -> T 324 value = sv.Field(0) 325 valueField = sv.Type().Field(0) 326 } 327 prop := jsonProperties(valueField, m.OrigName) 328 if !firstField { 329 m.writeSep(out) 330 } 331 // If the map value is a cast type, it may not implement proto.Message, therefore 332 // allow the struct tag to declare the underlying message type. Instead of changing 333 // the signatures of the child types (and because prop.mvalue is not public), use 334 // CustomType as a passer. 335 if value.Kind() == reflect.Map { 336 if tag := valueField.Tag.Get("protobuf"); tag != "" { 337 for _, v := range strings.Split(tag, ",") { 338 if !strings.HasPrefix(v, "castvaluetype=") { 339 continue 340 } 341 v = strings.TrimPrefix(v, "castvaluetype=") 342 prop.CustomType = v 343 break 344 } 345 } 346 } 347 if err := m.marshalField(out, prop, value, indent); err != nil { 348 return err 349 } 350 firstField = false 351 } 352 353 // Handle proto2 extensions. 354 if ep, ok := v.(proto.Message); ok { 355 extensions := proto.RegisteredExtensions(v) 356 // Sort extensions for stable output. 357 ids := make([]int32, 0, len(extensions)) 358 for id, desc := range extensions { 359 if !proto.HasExtension(ep, desc) { 360 continue 361 } 362 ids = append(ids, id) 363 } 364 sort.Sort(int32Slice(ids)) 365 for _, id := range ids { 366 desc := extensions[id] 367 if desc == nil { 368 // unknown extension 369 continue 370 } 371 ext, extErr := proto.GetExtension(ep, desc) 372 if extErr != nil { 373 return extErr 374 } 375 value := reflect.ValueOf(ext) 376 var prop proto.Properties 377 prop.Parse(desc.Tag) 378 prop.JSONName = fmt.Sprintf("[%s]", desc.Name) 379 if !firstField { 380 m.writeSep(out) 381 } 382 if err := m.marshalField(out, &prop, value, indent); err != nil { 383 return err 384 } 385 firstField = false 386 } 387 388 } 389 390 if m.Indent != "" { 391 out.write("\n") 392 out.write(indent) 393 } 394 out.write("}") 395 return out.err 396} 397 398func (m *Marshaler) writeSep(out *errWriter) { 399 if m.Indent != "" { 400 out.write(",\n") 401 } else { 402 out.write(",") 403 } 404} 405 406func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error { 407 // "If the Any contains a value that has a special JSON mapping, 408 // it will be converted as follows: {"@type": xxx, "value": yyy}. 409 // Otherwise, the value will be converted into a JSON object, 410 // and the "@type" field will be inserted to indicate the actual data type." 411 v := reflect.ValueOf(any).Elem() 412 turl := v.Field(0).String() 413 val := v.Field(1).Bytes() 414 415 var msg proto.Message 416 var err error 417 if m.AnyResolver != nil { 418 msg, err = m.AnyResolver.Resolve(turl) 419 } else { 420 msg, err = defaultResolveAny(turl) 421 } 422 if err != nil { 423 return err 424 } 425 426 if err := proto.Unmarshal(val, msg); err != nil { 427 return err 428 } 429 430 if _, ok := msg.(isWkt); ok { 431 out.write("{") 432 if m.Indent != "" { 433 out.write("\n") 434 } 435 if err := m.marshalTypeURL(out, indent, turl); err != nil { 436 return err 437 } 438 m.writeSep(out) 439 if m.Indent != "" { 440 out.write(indent) 441 out.write(m.Indent) 442 out.write(`"value": `) 443 } else { 444 out.write(`"value":`) 445 } 446 if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil { 447 return err 448 } 449 if m.Indent != "" { 450 out.write("\n") 451 out.write(indent) 452 } 453 out.write("}") 454 return out.err 455 } 456 457 return m.marshalObject(out, msg, indent, turl) 458} 459 460func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error { 461 if m.Indent != "" { 462 out.write(indent) 463 out.write(m.Indent) 464 } 465 out.write(`"@type":`) 466 if m.Indent != "" { 467 out.write(" ") 468 } 469 b, err := json.Marshal(typeURL) 470 if err != nil { 471 return err 472 } 473 out.write(string(b)) 474 return out.err 475} 476 477// marshalField writes field description and value to the Writer. 478func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 479 if m.Indent != "" { 480 out.write(indent) 481 out.write(m.Indent) 482 } 483 out.write(`"`) 484 out.write(prop.JSONName) 485 out.write(`":`) 486 if m.Indent != "" { 487 out.write(" ") 488 } 489 if err := m.marshalValue(out, prop, v, indent); err != nil { 490 return err 491 } 492 return nil 493} 494 495// marshalValue writes the value to the Writer. 496func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error { 497 498 v = reflect.Indirect(v) 499 500 // Handle nil pointer 501 if v.Kind() == reflect.Invalid { 502 out.write("null") 503 return out.err 504 } 505 506 // Handle repeated elements. 507 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { 508 out.write("[") 509 comma := "" 510 for i := 0; i < v.Len(); i++ { 511 sliceVal := v.Index(i) 512 out.write(comma) 513 if m.Indent != "" { 514 out.write("\n") 515 out.write(indent) 516 out.write(m.Indent) 517 out.write(m.Indent) 518 } 519 if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil { 520 return err 521 } 522 comma = "," 523 } 524 if m.Indent != "" { 525 out.write("\n") 526 out.write(indent) 527 out.write(m.Indent) 528 } 529 out.write("]") 530 return out.err 531 } 532 533 // Handle well-known types. 534 // Most are handled up in marshalObject (because 99% are messages). 535 if wkt, ok := v.Interface().(isWkt); ok { 536 switch wkt.XXX_WellKnownType() { 537 case "NullValue": 538 out.write("null") 539 return out.err 540 } 541 } 542 543 if t, ok := v.Interface().(time.Time); ok { 544 ts, err := types.TimestampProto(t) 545 if err != nil { 546 return err 547 } 548 return m.marshalValue(out, prop, reflect.ValueOf(ts), indent) 549 } 550 551 if d, ok := v.Interface().(time.Duration); ok { 552 dur := types.DurationProto(d) 553 return m.marshalValue(out, prop, reflect.ValueOf(dur), indent) 554 } 555 556 // Handle enumerations. 557 if !m.EnumsAsInts && prop.Enum != "" { 558 // Unknown enum values will are stringified by the proto library as their 559 // value. Such values should _not_ be quoted or they will be interpreted 560 // as an enum string instead of their value. 561 enumStr := v.Interface().(fmt.Stringer).String() 562 var valStr string 563 if v.Kind() == reflect.Ptr { 564 valStr = strconv.Itoa(int(v.Elem().Int())) 565 } else { 566 valStr = strconv.Itoa(int(v.Int())) 567 } 568 569 if m, ok := v.Interface().(interface { 570 MarshalJSON() ([]byte, error) 571 }); ok { 572 data, err := m.MarshalJSON() 573 if err != nil { 574 return err 575 } 576 enumStr = string(data) 577 enumStr, err = strconv.Unquote(enumStr) 578 if err != nil { 579 return err 580 } 581 } 582 583 isKnownEnum := enumStr != valStr 584 585 if isKnownEnum { 586 out.write(`"`) 587 } 588 out.write(enumStr) 589 if isKnownEnum { 590 out.write(`"`) 591 } 592 return out.err 593 } 594 595 // Handle nested messages. 596 if v.Kind() == reflect.Struct { 597 i := v 598 if v.CanAddr() { 599 i = v.Addr() 600 } else { 601 i = reflect.New(v.Type()) 602 i.Elem().Set(v) 603 } 604 iface := i.Interface() 605 if iface == nil { 606 out.write(`null`) 607 return out.err 608 } 609 610 if m, ok := v.Interface().(interface { 611 MarshalJSON() ([]byte, error) 612 }); ok { 613 data, err := m.MarshalJSON() 614 if err != nil { 615 return err 616 } 617 out.write(string(data)) 618 return nil 619 } 620 621 pm, ok := iface.(proto.Message) 622 if !ok { 623 if prop.CustomType == "" { 624 return fmt.Errorf("%v does not implement proto.Message", v.Type()) 625 } 626 t := proto.MessageType(prop.CustomType) 627 if t == nil || !i.Type().ConvertibleTo(t) { 628 return fmt.Errorf("%v declared custom type %s but it is not convertible to %v", v.Type(), prop.CustomType, t) 629 } 630 pm = i.Convert(t).Interface().(proto.Message) 631 } 632 return m.marshalObject(out, pm, indent+m.Indent, "") 633 } 634 635 // Handle maps. 636 // Since Go randomizes map iteration, we sort keys for stable output. 637 if v.Kind() == reflect.Map { 638 out.write(`{`) 639 keys := v.MapKeys() 640 sort.Sort(mapKeys(keys)) 641 for i, k := range keys { 642 if i > 0 { 643 out.write(`,`) 644 } 645 if m.Indent != "" { 646 out.write("\n") 647 out.write(indent) 648 out.write(m.Indent) 649 out.write(m.Indent) 650 } 651 652 b, err := json.Marshal(k.Interface()) 653 if err != nil { 654 return err 655 } 656 s := string(b) 657 658 // If the JSON is not a string value, encode it again to make it one. 659 if !strings.HasPrefix(s, `"`) { 660 b, err := json.Marshal(s) 661 if err != nil { 662 return err 663 } 664 s = string(b) 665 } 666 667 out.write(s) 668 out.write(`:`) 669 if m.Indent != "" { 670 out.write(` `) 671 } 672 673 if err := m.marshalValue(out, prop, v.MapIndex(k), indent+m.Indent); err != nil { 674 return err 675 } 676 } 677 if m.Indent != "" { 678 out.write("\n") 679 out.write(indent) 680 out.write(m.Indent) 681 } 682 out.write(`}`) 683 return out.err 684 } 685 686 // Handle non-finite floats, e.g. NaN, Infinity and -Infinity. 687 if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { 688 f := v.Float() 689 var sval string 690 switch { 691 case math.IsInf(f, 1): 692 sval = `"Infinity"` 693 case math.IsInf(f, -1): 694 sval = `"-Infinity"` 695 case math.IsNaN(f): 696 sval = `"NaN"` 697 } 698 if sval != "" { 699 out.write(sval) 700 return out.err 701 } 702 } 703 704 // Default handling defers to the encoding/json library. 705 b, err := json.Marshal(v.Interface()) 706 if err != nil { 707 return err 708 } 709 needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64) 710 if needToQuote { 711 out.write(`"`) 712 } 713 out.write(string(b)) 714 if needToQuote { 715 out.write(`"`) 716 } 717 return out.err 718} 719 720// Unmarshaler is a configurable object for converting from a JSON 721// representation to a protocol buffer object. 722type Unmarshaler struct { 723 // Whether to allow messages to contain unknown fields, as opposed to 724 // failing to unmarshal. 725 AllowUnknownFields bool 726 727 // A custom URL resolver to use when unmarshaling Any messages from JSON. 728 // If unset, the default resolution strategy is to extract the 729 // fully-qualified type name from the type URL and pass that to 730 // proto.MessageType(string). 731 AnyResolver AnyResolver 732} 733 734// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 735// This function is lenient and will decode any options permutations of the 736// related Marshaler. 737func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 738 inputValue := json.RawMessage{} 739 if err := dec.Decode(&inputValue); err != nil { 740 return err 741 } 742 if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil { 743 return err 744 } 745 return checkRequiredFields(pb) 746} 747 748// Unmarshal unmarshals a JSON object stream into a protocol 749// buffer. This function is lenient and will decode any options 750// permutations of the related Marshaler. 751func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error { 752 dec := json.NewDecoder(r) 753 return u.UnmarshalNext(dec, pb) 754} 755 756// UnmarshalNext unmarshals the next protocol buffer from a JSON object stream. 757// This function is lenient and will decode any options permutations of the 758// related Marshaler. 759func UnmarshalNext(dec *json.Decoder, pb proto.Message) error { 760 return new(Unmarshaler).UnmarshalNext(dec, pb) 761} 762 763// Unmarshal unmarshals a JSON object stream into a protocol 764// buffer. This function is lenient and will decode any options 765// permutations of the related Marshaler. 766func Unmarshal(r io.Reader, pb proto.Message) error { 767 return new(Unmarshaler).Unmarshal(r, pb) 768} 769 770// UnmarshalString will populate the fields of a protocol buffer based 771// on a JSON string. This function is lenient and will decode any options 772// permutations of the related Marshaler. 773func UnmarshalString(str string, pb proto.Message) error { 774 return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb) 775} 776 777// unmarshalValue converts/copies a value into the target. 778// prop may be nil. 779func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error { 780 targetType := target.Type() 781 782 // Allocate memory for pointer fields. 783 if targetType.Kind() == reflect.Ptr { 784 // If input value is "null" and target is a pointer type, then the field should be treated as not set 785 // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue. 786 _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler) 787 if string(inputValue) == "null" && targetType != reflect.TypeOf(&types.Value{}) && !isJSONPBUnmarshaler { 788 return nil 789 } 790 target.Set(reflect.New(targetType.Elem())) 791 792 return u.unmarshalValue(target.Elem(), inputValue, prop) 793 } 794 795 if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok { 796 return jsu.UnmarshalJSONPB(u, []byte(inputValue)) 797 } 798 799 // Handle well-known types that are not pointers. 800 if w, ok := target.Addr().Interface().(isWkt); ok { 801 switch w.XXX_WellKnownType() { 802 case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value", 803 "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue": 804 return u.unmarshalValue(target.Field(0), inputValue, prop) 805 case "Any": 806 // Use json.RawMessage pointer type instead of value to support pre-1.8 version. 807 // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see 808 // https://github.com/golang/go/issues/14493 809 var jsonFields map[string]*json.RawMessage 810 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 811 return err 812 } 813 814 val, ok := jsonFields["@type"] 815 if !ok || val == nil { 816 return errors.New("Any JSON doesn't have '@type'") 817 } 818 819 var turl string 820 if err := json.Unmarshal([]byte(*val), &turl); err != nil { 821 return fmt.Errorf("can't unmarshal Any's '@type': %q", *val) 822 } 823 target.Field(0).SetString(turl) 824 825 var m proto.Message 826 var err error 827 if u.AnyResolver != nil { 828 m, err = u.AnyResolver.Resolve(turl) 829 } else { 830 m, err = defaultResolveAny(turl) 831 } 832 if err != nil { 833 return err 834 } 835 836 if _, ok := m.(isWkt); ok { 837 val, ok := jsonFields["value"] 838 if !ok { 839 return errors.New("Any JSON doesn't have 'value'") 840 } 841 842 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil { 843 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 844 } 845 } else { 846 delete(jsonFields, "@type") 847 nestedProto, uerr := json.Marshal(jsonFields) 848 if uerr != nil { 849 return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", uerr) 850 } 851 852 if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil { 853 return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err) 854 } 855 } 856 857 b, err := proto.Marshal(m) 858 if err != nil { 859 return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err) 860 } 861 target.Field(1).SetBytes(b) 862 863 return nil 864 case "Duration": 865 unq, err := strconv.Unquote(string(inputValue)) 866 if err != nil { 867 return err 868 } 869 870 d, err := time.ParseDuration(unq) 871 if err != nil { 872 return fmt.Errorf("bad Duration: %v", err) 873 } 874 875 ns := d.Nanoseconds() 876 s := ns / 1e9 877 ns %= 1e9 878 target.Field(0).SetInt(s) 879 target.Field(1).SetInt(ns) 880 return nil 881 case "Timestamp": 882 unq, err := strconv.Unquote(string(inputValue)) 883 if err != nil { 884 return err 885 } 886 887 t, err := time.Parse(time.RFC3339Nano, unq) 888 if err != nil { 889 return fmt.Errorf("bad Timestamp: %v", err) 890 } 891 892 target.Field(0).SetInt(t.Unix()) 893 target.Field(1).SetInt(int64(t.Nanosecond())) 894 return nil 895 case "Struct": 896 var m map[string]json.RawMessage 897 if err := json.Unmarshal(inputValue, &m); err != nil { 898 return fmt.Errorf("bad StructValue: %v", err) 899 } 900 target.Field(0).Set(reflect.ValueOf(map[string]*types.Value{})) 901 for k, jv := range m { 902 pv := &types.Value{} 903 if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil { 904 return fmt.Errorf("bad value in StructValue for key %q: %v", k, err) 905 } 906 target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv)) 907 } 908 return nil 909 case "ListValue": 910 var s []json.RawMessage 911 if err := json.Unmarshal(inputValue, &s); err != nil { 912 return fmt.Errorf("bad ListValue: %v", err) 913 } 914 915 target.Field(0).Set(reflect.ValueOf(make([]*types.Value, len(s)))) 916 for i, sv := range s { 917 if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil { 918 return err 919 } 920 } 921 return nil 922 case "Value": 923 ivStr := string(inputValue) 924 if ivStr == "null" { 925 target.Field(0).Set(reflect.ValueOf(&types.Value_NullValue{})) 926 } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil { 927 target.Field(0).Set(reflect.ValueOf(&types.Value_NumberValue{NumberValue: v})) 928 } else if v, err := strconv.Unquote(ivStr); err == nil { 929 target.Field(0).Set(reflect.ValueOf(&types.Value_StringValue{StringValue: v})) 930 } else if v, err := strconv.ParseBool(ivStr); err == nil { 931 target.Field(0).Set(reflect.ValueOf(&types.Value_BoolValue{BoolValue: v})) 932 } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil { 933 lv := &types.ListValue{} 934 target.Field(0).Set(reflect.ValueOf(&types.Value_ListValue{ListValue: lv})) 935 return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop) 936 } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil { 937 sv := &types.Struct{} 938 target.Field(0).Set(reflect.ValueOf(&types.Value_StructValue{StructValue: sv})) 939 return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop) 940 } else { 941 return fmt.Errorf("unrecognized type for Value %q", ivStr) 942 } 943 return nil 944 } 945 } 946 947 if t, ok := target.Addr().Interface().(*time.Time); ok { 948 ts := &types.Timestamp{} 949 if err := u.unmarshalValue(reflect.ValueOf(ts).Elem(), inputValue, prop); err != nil { 950 return err 951 } 952 tt, err := types.TimestampFromProto(ts) 953 if err != nil { 954 return err 955 } 956 *t = tt 957 return nil 958 } 959 960 if d, ok := target.Addr().Interface().(*time.Duration); ok { 961 dur := &types.Duration{} 962 if err := u.unmarshalValue(reflect.ValueOf(dur).Elem(), inputValue, prop); err != nil { 963 return err 964 } 965 dd, err := types.DurationFromProto(dur) 966 if err != nil { 967 return err 968 } 969 *d = dd 970 return nil 971 } 972 973 // Handle enums, which have an underlying type of int32, 974 // and may appear as strings. 975 // The case of an enum appearing as a number is handled 976 // at the bottom of this function. 977 if inputValue[0] == '"' && prop != nil && prop.Enum != "" { 978 vmap := proto.EnumValueMap(prop.Enum) 979 // Don't need to do unquoting; valid enum names 980 // are from a limited character set. 981 s := inputValue[1 : len(inputValue)-1] 982 n, ok := vmap[string(s)] 983 if !ok { 984 return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum) 985 } 986 if target.Kind() == reflect.Ptr { // proto2 987 target.Set(reflect.New(targetType.Elem())) 988 target = target.Elem() 989 } 990 target.SetInt(int64(n)) 991 return nil 992 } 993 994 // Handle nested messages. 995 if targetType.Kind() == reflect.Struct { 996 if prop != nil && len(prop.CustomType) > 0 && target.CanAddr() { 997 if m, ok := target.Addr().Interface().(interface { 998 UnmarshalJSON([]byte) error 999 }); ok { 1000 return json.Unmarshal(inputValue, m) 1001 } 1002 } 1003 1004 var jsonFields map[string]json.RawMessage 1005 if err := json.Unmarshal(inputValue, &jsonFields); err != nil { 1006 return err 1007 } 1008 1009 consumeField := func(prop *proto.Properties) (json.RawMessage, bool) { 1010 // Be liberal in what names we accept; both orig_name and camelName are okay. 1011 fieldNames := acceptedJSONFieldNames(prop) 1012 1013 vOrig, okOrig := jsonFields[fieldNames.orig] 1014 vCamel, okCamel := jsonFields[fieldNames.camel] 1015 if !okOrig && !okCamel { 1016 return nil, false 1017 } 1018 // If, for some reason, both are present in the data, favour the camelName. 1019 var raw json.RawMessage 1020 if okOrig { 1021 raw = vOrig 1022 delete(jsonFields, fieldNames.orig) 1023 } 1024 if okCamel { 1025 raw = vCamel 1026 delete(jsonFields, fieldNames.camel) 1027 } 1028 return raw, true 1029 } 1030 1031 sprops := proto.GetProperties(targetType) 1032 for i := 0; i < target.NumField(); i++ { 1033 ft := target.Type().Field(i) 1034 if strings.HasPrefix(ft.Name, "XXX_") { 1035 continue 1036 } 1037 valueForField, ok := consumeField(sprops.Prop[i]) 1038 if !ok { 1039 continue 1040 } 1041 1042 if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil { 1043 return err 1044 } 1045 } 1046 // Check for any oneof fields. 1047 if len(jsonFields) > 0 { 1048 for _, oop := range sprops.OneofTypes { 1049 raw, ok := consumeField(oop.Prop) 1050 if !ok { 1051 continue 1052 } 1053 nv := reflect.New(oop.Type.Elem()) 1054 target.Field(oop.Field).Set(nv) 1055 if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil { 1056 return err 1057 } 1058 } 1059 } 1060 // Handle proto2 extensions. 1061 if len(jsonFields) > 0 { 1062 if ep, ok := target.Addr().Interface().(proto.Message); ok { 1063 for _, ext := range proto.RegisteredExtensions(ep) { 1064 name := fmt.Sprintf("[%s]", ext.Name) 1065 raw, ok := jsonFields[name] 1066 if !ok { 1067 continue 1068 } 1069 delete(jsonFields, name) 1070 nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem()) 1071 if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil { 1072 return err 1073 } 1074 if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil { 1075 return err 1076 } 1077 } 1078 } 1079 } 1080 if !u.AllowUnknownFields && len(jsonFields) > 0 { 1081 // Pick any field to be the scapegoat. 1082 var f string 1083 for fname := range jsonFields { 1084 f = fname 1085 break 1086 } 1087 return fmt.Errorf("unknown field %q in %v", f, targetType) 1088 } 1089 return nil 1090 } 1091 1092 // Handle arrays 1093 if targetType.Kind() == reflect.Slice { 1094 if targetType.Elem().Kind() == reflect.Uint8 { 1095 outRef := reflect.New(targetType) 1096 outVal := outRef.Interface() 1097 //CustomType with underlying type []byte 1098 if _, ok := outVal.(interface { 1099 UnmarshalJSON([]byte) error 1100 }); ok { 1101 if err := json.Unmarshal(inputValue, outVal); err != nil { 1102 return err 1103 } 1104 target.Set(outRef.Elem()) 1105 return nil 1106 } 1107 // Special case for encoded bytes. Pre-go1.5 doesn't support unmarshalling 1108 // strings into aliased []byte types. 1109 // https://github.com/golang/go/commit/4302fd0409da5e4f1d71471a6770dacdc3301197 1110 // https://github.com/golang/go/commit/c60707b14d6be26bf4213114d13070bff00d0b0a 1111 var out []byte 1112 if err := json.Unmarshal(inputValue, &out); err != nil { 1113 return err 1114 } 1115 target.SetBytes(out) 1116 return nil 1117 } 1118 1119 var slc []json.RawMessage 1120 if err := json.Unmarshal(inputValue, &slc); err != nil { 1121 return err 1122 } 1123 if slc != nil { 1124 l := len(slc) 1125 target.Set(reflect.MakeSlice(targetType, l, l)) 1126 for i := 0; i < l; i++ { 1127 if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil { 1128 return err 1129 } 1130 } 1131 } 1132 return nil 1133 } 1134 1135 // Handle maps (whose keys are always strings) 1136 if targetType.Kind() == reflect.Map { 1137 var mp map[string]json.RawMessage 1138 if err := json.Unmarshal(inputValue, &mp); err != nil { 1139 return err 1140 } 1141 if mp != nil { 1142 target.Set(reflect.MakeMap(targetType)) 1143 for ks, raw := range mp { 1144 // Unmarshal map key. The core json library already decoded the key into a 1145 // string, so we handle that specially. Other types were quoted post-serialization. 1146 var k reflect.Value 1147 if targetType.Key().Kind() == reflect.String { 1148 k = reflect.ValueOf(ks) 1149 } else { 1150 k = reflect.New(targetType.Key()).Elem() 1151 // TODO: pass the correct Properties if needed. 1152 if err := u.unmarshalValue(k, json.RawMessage(ks), nil); err != nil { 1153 return err 1154 } 1155 } 1156 1157 if !k.Type().AssignableTo(targetType.Key()) { 1158 k = k.Convert(targetType.Key()) 1159 } 1160 1161 // Unmarshal map value. 1162 v := reflect.New(targetType.Elem()).Elem() 1163 // TODO: pass the correct Properties if needed. 1164 if err := u.unmarshalValue(v, raw, nil); err != nil { 1165 return err 1166 } 1167 target.SetMapIndex(k, v) 1168 } 1169 } 1170 return nil 1171 } 1172 1173 // 64-bit integers can be encoded as strings. In this case we drop 1174 // the quotes and proceed as normal. 1175 isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64 1176 if isNum && strings.HasPrefix(string(inputValue), `"`) { 1177 inputValue = inputValue[1 : len(inputValue)-1] 1178 } 1179 1180 // Non-finite numbers can be encoded as strings. 1181 isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64 1182 if isFloat { 1183 if num, ok := nonFinite[string(inputValue)]; ok { 1184 target.SetFloat(num) 1185 return nil 1186 } 1187 } 1188 1189 // Use the encoding/json for parsing other value types. 1190 return json.Unmarshal(inputValue, target.Addr().Interface()) 1191} 1192 1193// jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute. 1194func jsonProperties(f reflect.StructField, origName bool) *proto.Properties { 1195 var prop proto.Properties 1196 prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f) 1197 if origName || prop.JSONName == "" { 1198 prop.JSONName = prop.OrigName 1199 } 1200 return &prop 1201} 1202 1203type fieldNames struct { 1204 orig, camel string 1205} 1206 1207func acceptedJSONFieldNames(prop *proto.Properties) fieldNames { 1208 opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName} 1209 if prop.JSONName != "" { 1210 opts.camel = prop.JSONName 1211 } 1212 return opts 1213} 1214 1215// Writer wrapper inspired by https://blog.golang.org/errors-are-values 1216type errWriter struct { 1217 writer io.Writer 1218 err error 1219} 1220 1221func (w *errWriter) write(str string) { 1222 if w.err != nil { 1223 return 1224 } 1225 _, w.err = w.writer.Write([]byte(str)) 1226} 1227 1228// Map fields may have key types of non-float scalars, strings and enums. 1229// The easiest way to sort them in some deterministic order is to use fmt. 1230// If this turns out to be inefficient we can always consider other options, 1231// such as doing a Schwartzian transform. 1232// 1233// Numeric keys are sorted in numeric order per 1234// https://developers.google.com/protocol-buffers/docs/proto#maps. 1235type mapKeys []reflect.Value 1236 1237func (s mapKeys) Len() int { return len(s) } 1238func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 1239func (s mapKeys) Less(i, j int) bool { 1240 if k := s[i].Kind(); k == s[j].Kind() { 1241 switch k { 1242 case reflect.Int32, reflect.Int64: 1243 return s[i].Int() < s[j].Int() 1244 case reflect.Uint32, reflect.Uint64: 1245 return s[i].Uint() < s[j].Uint() 1246 } 1247 } 1248 return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) 1249} 1250 1251// checkRequiredFields returns an error if any required field in the given proto message is not set. 1252// This function is used by both Marshal and Unmarshal. While required fields only exist in a 1253// proto2 message, a proto3 message can contain proto2 message(s). 1254func checkRequiredFields(pb proto.Message) error { 1255 // Most well-known type messages do not contain required fields. The "Any" type may contain 1256 // a message that has required fields. 1257 // 1258 // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value 1259 // field in order to transform that into JSON, and that should have returned an error if a 1260 // required field is not set in the embedded message. 1261 // 1262 // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the 1263 // embedded message to store the serialized message in Any.Value field, and that should have 1264 // returned an error if a required field is not set. 1265 if _, ok := pb.(isWkt); ok { 1266 return nil 1267 } 1268 1269 v := reflect.ValueOf(pb) 1270 // Skip message if it is not a struct pointer. 1271 if v.Kind() != reflect.Ptr { 1272 return nil 1273 } 1274 v = v.Elem() 1275 if v.Kind() != reflect.Struct { 1276 return nil 1277 } 1278 1279 for i := 0; i < v.NumField(); i++ { 1280 field := v.Field(i) 1281 sfield := v.Type().Field(i) 1282 1283 if sfield.PkgPath != "" { 1284 // blank PkgPath means the field is exported; skip if not exported 1285 continue 1286 } 1287 1288 if strings.HasPrefix(sfield.Name, "XXX_") { 1289 continue 1290 } 1291 1292 // Oneof field is an interface implemented by wrapper structs containing the actual oneof 1293 // field, i.e. an interface containing &T{real_value}. 1294 if sfield.Tag.Get("protobuf_oneof") != "" { 1295 if field.Kind() != reflect.Interface { 1296 continue 1297 } 1298 v := field.Elem() 1299 if v.Kind() != reflect.Ptr || v.IsNil() { 1300 continue 1301 } 1302 v = v.Elem() 1303 if v.Kind() != reflect.Struct || v.NumField() < 1 { 1304 continue 1305 } 1306 field = v.Field(0) 1307 sfield = v.Type().Field(0) 1308 } 1309 1310 protoTag := sfield.Tag.Get("protobuf") 1311 if protoTag == "" { 1312 continue 1313 } 1314 var prop proto.Properties 1315 prop.Init(sfield.Type, sfield.Name, protoTag, &sfield) 1316 1317 switch field.Kind() { 1318 case reflect.Map: 1319 if field.IsNil() { 1320 continue 1321 } 1322 // Check each map value. 1323 keys := field.MapKeys() 1324 for _, k := range keys { 1325 v := field.MapIndex(k) 1326 if err := checkRequiredFieldsInValue(v); err != nil { 1327 return err 1328 } 1329 } 1330 case reflect.Slice: 1331 // Handle non-repeated type, e.g. bytes. 1332 if !prop.Repeated { 1333 if prop.Required && field.IsNil() { 1334 return fmt.Errorf("required field %q is not set", prop.Name) 1335 } 1336 continue 1337 } 1338 1339 // Handle repeated type. 1340 if field.IsNil() { 1341 continue 1342 } 1343 // Check each slice item. 1344 for i := 0; i < field.Len(); i++ { 1345 v := field.Index(i) 1346 if err := checkRequiredFieldsInValue(v); err != nil { 1347 return err 1348 } 1349 } 1350 case reflect.Ptr: 1351 if field.IsNil() { 1352 if prop.Required { 1353 return fmt.Errorf("required field %q is not set", prop.Name) 1354 } 1355 continue 1356 } 1357 if err := checkRequiredFieldsInValue(field); err != nil { 1358 return err 1359 } 1360 } 1361 } 1362 1363 // Handle proto2 extensions. 1364 for _, ext := range proto.RegisteredExtensions(pb) { 1365 if !proto.HasExtension(pb, ext) { 1366 continue 1367 } 1368 ep, err := proto.GetExtension(pb, ext) 1369 if err != nil { 1370 return err 1371 } 1372 err = checkRequiredFieldsInValue(reflect.ValueOf(ep)) 1373 if err != nil { 1374 return err 1375 } 1376 } 1377 1378 return nil 1379} 1380 1381func checkRequiredFieldsInValue(v reflect.Value) error { 1382 if pm, ok := v.Interface().(proto.Message); ok { 1383 return checkRequiredFields(pm) 1384 } 1385 return nil 1386} 1387