1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package proto 6 7import ( 8 "google.golang.org/protobuf/encoding/protowire" 9 "google.golang.org/protobuf/internal/encoding/messageset" 10 "google.golang.org/protobuf/internal/order" 11 "google.golang.org/protobuf/internal/pragma" 12 "google.golang.org/protobuf/reflect/protoreflect" 13 "google.golang.org/protobuf/runtime/protoiface" 14) 15 16// MarshalOptions configures the marshaler. 17// 18// Example usage: 19// b, err := MarshalOptions{Deterministic: true}.Marshal(m) 20type MarshalOptions struct { 21 pragma.NoUnkeyedLiterals 22 23 // AllowPartial allows messages that have missing required fields to marshal 24 // without returning an error. If AllowPartial is false (the default), 25 // Marshal will return an error if there are any missing required fields. 26 AllowPartial bool 27 28 // Deterministic controls whether the same message will always be 29 // serialized to the same bytes within the same binary. 30 // 31 // Setting this option guarantees that repeated serialization of 32 // the same message will return the same bytes, and that different 33 // processes of the same binary (which may be executing on different 34 // machines) will serialize equal messages to the same bytes. 35 // It has no effect on the resulting size of the encoded message compared 36 // to a non-deterministic marshal. 37 // 38 // Note that the deterministic serialization is NOT canonical across 39 // languages. It is not guaranteed to remain stable over time. It is 40 // unstable across different builds with schema changes due to unknown 41 // fields. Users who need canonical serialization (e.g., persistent 42 // storage in a canonical form, fingerprinting, etc.) must define 43 // their own canonicalization specification and implement their own 44 // serializer rather than relying on this API. 45 // 46 // If deterministic serialization is requested, map entries will be 47 // sorted by keys in lexographical order. This is an implementation 48 // detail and subject to change. 49 Deterministic bool 50 51 // UseCachedSize indicates that the result of a previous Size call 52 // may be reused. 53 // 54 // Setting this option asserts that: 55 // 56 // 1. Size has previously been called on this message with identical 57 // options (except for UseCachedSize itself). 58 // 59 // 2. The message and all its submessages have not changed in any 60 // way since the Size call. 61 // 62 // If either of these invariants is violated, 63 // the results are undefined and may include panics or corrupted output. 64 // 65 // Implementations MAY take this option into account to provide 66 // better performance, but there is no guarantee that they will do so. 67 // There is absolutely no guarantee that Size followed by Marshal with 68 // UseCachedSize set will perform equivalently to Marshal alone. 69 UseCachedSize bool 70} 71 72// Marshal returns the wire-format encoding of m. 73func Marshal(m Message) ([]byte, error) { 74 // Treat nil message interface as an empty message; nothing to output. 75 if m == nil { 76 return nil, nil 77 } 78 79 out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect()) 80 if len(out.Buf) == 0 && err == nil { 81 out.Buf = emptyBytesForMessage(m) 82 } 83 return out.Buf, err 84} 85 86// Marshal returns the wire-format encoding of m. 87func (o MarshalOptions) Marshal(m Message) ([]byte, error) { 88 // Treat nil message interface as an empty message; nothing to output. 89 if m == nil { 90 return nil, nil 91 } 92 93 out, err := o.marshal(nil, m.ProtoReflect()) 94 if len(out.Buf) == 0 && err == nil { 95 out.Buf = emptyBytesForMessage(m) 96 } 97 return out.Buf, err 98} 99 100// emptyBytesForMessage returns a nil buffer if and only if m is invalid, 101// otherwise it returns a non-nil empty buffer. 102// 103// This is to assist the edge-case where user-code does the following: 104// m1.OptionalBytes, _ = proto.Marshal(m2) 105// where they expect the proto2 "optional_bytes" field to be populated 106// if any only if m2 is a valid message. 107func emptyBytesForMessage(m Message) []byte { 108 if m == nil || !m.ProtoReflect().IsValid() { 109 return nil 110 } 111 return emptyBuf[:] 112} 113 114// MarshalAppend appends the wire-format encoding of m to b, 115// returning the result. 116func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) { 117 // Treat nil message interface as an empty message; nothing to append. 118 if m == nil { 119 return b, nil 120 } 121 122 out, err := o.marshal(b, m.ProtoReflect()) 123 return out.Buf, err 124} 125 126// MarshalState returns the wire-format encoding of a message. 127// 128// This method permits fine-grained control over the marshaler. 129// Most users should use Marshal instead. 130func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) { 131 return o.marshal(in.Buf, in.Message) 132} 133 134// marshal is a centralized function that all marshal operations go through. 135// For profiling purposes, avoid changing the name of this function or 136// introducing other code paths for marshal that do not go through this. 137func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) { 138 allowPartial := o.AllowPartial 139 o.AllowPartial = true 140 if methods := protoMethods(m); methods != nil && methods.Marshal != nil && 141 !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) { 142 in := protoiface.MarshalInput{ 143 Message: m, 144 Buf: b, 145 } 146 if o.Deterministic { 147 in.Flags |= protoiface.MarshalDeterministic 148 } 149 if o.UseCachedSize { 150 in.Flags |= protoiface.MarshalUseCachedSize 151 } 152 if methods.Size != nil { 153 sout := methods.Size(protoiface.SizeInput{ 154 Message: m, 155 Flags: in.Flags, 156 }) 157 if cap(b) < len(b)+sout.Size { 158 in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size)) 159 copy(in.Buf, b) 160 } 161 in.Flags |= protoiface.MarshalUseCachedSize 162 } 163 out, err = methods.Marshal(in) 164 } else { 165 out.Buf, err = o.marshalMessageSlow(b, m) 166 } 167 if err != nil { 168 return out, err 169 } 170 if allowPartial { 171 return out, nil 172 } 173 return out, checkInitialized(m) 174} 175 176func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) { 177 out, err := o.marshal(b, m) 178 return out.Buf, err 179} 180 181// growcap scales up the capacity of a slice. 182// 183// Given a slice with a current capacity of oldcap and a desired 184// capacity of wantcap, growcap returns a new capacity >= wantcap. 185// 186// The algorithm is mostly identical to the one used by append as of Go 1.14. 187func growcap(oldcap, wantcap int) (newcap int) { 188 if wantcap > oldcap*2 { 189 newcap = wantcap 190 } else if oldcap < 1024 { 191 // The Go 1.14 runtime takes this case when len(s) < 1024, 192 // not when cap(s) < 1024. The difference doesn't seem 193 // significant here. 194 newcap = oldcap * 2 195 } else { 196 newcap = oldcap 197 for 0 < newcap && newcap < wantcap { 198 newcap += newcap / 4 199 } 200 if newcap <= 0 { 201 newcap = wantcap 202 } 203 } 204 return newcap 205} 206 207func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) { 208 if messageset.IsMessageSet(m.Descriptor()) { 209 return o.marshalMessageSet(b, m) 210 } 211 fieldOrder := order.AnyFieldOrder 212 if o.Deterministic { 213 // TODO: This should use a more natural ordering like NumberFieldOrder, 214 // but doing so breaks golden tests that make invalid assumption about 215 // output stability of this implementation. 216 fieldOrder = order.LegacyFieldOrder 217 } 218 var err error 219 order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { 220 b, err = o.marshalField(b, fd, v) 221 return err == nil 222 }) 223 if err != nil { 224 return b, err 225 } 226 b = append(b, m.GetUnknown()...) 227 return b, nil 228} 229 230func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) { 231 switch { 232 case fd.IsList(): 233 return o.marshalList(b, fd, value.List()) 234 case fd.IsMap(): 235 return o.marshalMap(b, fd, value.Map()) 236 default: 237 b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()]) 238 return o.marshalSingular(b, fd, value) 239 } 240} 241 242func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) { 243 if fd.IsPacked() && list.Len() > 0 { 244 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) 245 b, pos := appendSpeculativeLength(b) 246 for i, llen := 0, list.Len(); i < llen; i++ { 247 var err error 248 b, err = o.marshalSingular(b, fd, list.Get(i)) 249 if err != nil { 250 return b, err 251 } 252 } 253 b = finishSpeculativeLength(b, pos) 254 return b, nil 255 } 256 257 kind := fd.Kind() 258 for i, llen := 0, list.Len(); i < llen; i++ { 259 var err error 260 b = protowire.AppendTag(b, fd.Number(), wireTypes[kind]) 261 b, err = o.marshalSingular(b, fd, list.Get(i)) 262 if err != nil { 263 return b, err 264 } 265 } 266 return b, nil 267} 268 269func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) { 270 keyf := fd.MapKey() 271 valf := fd.MapValue() 272 keyOrder := order.AnyKeyOrder 273 if o.Deterministic { 274 keyOrder = order.GenericKeyOrder 275 } 276 var err error 277 order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool { 278 b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) 279 var pos int 280 b, pos = appendSpeculativeLength(b) 281 282 b, err = o.marshalField(b, keyf, key.Value()) 283 if err != nil { 284 return false 285 } 286 b, err = o.marshalField(b, valf, value) 287 if err != nil { 288 return false 289 } 290 b = finishSpeculativeLength(b, pos) 291 return true 292 }) 293 return b, err 294} 295 296// When encoding length-prefixed fields, we speculatively set aside some number of bytes 297// for the length, encode the data, and then encode the length (shifting the data if necessary 298// to make room). 299const speculativeLength = 1 300 301func appendSpeculativeLength(b []byte) ([]byte, int) { 302 pos := len(b) 303 b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...) 304 return b, pos 305} 306 307func finishSpeculativeLength(b []byte, pos int) []byte { 308 mlen := len(b) - pos - speculativeLength 309 msiz := protowire.SizeVarint(uint64(mlen)) 310 if msiz != speculativeLength { 311 for i := 0; i < msiz-speculativeLength; i++ { 312 b = append(b, 0) 313 } 314 copy(b[pos+msiz:], b[pos+speculativeLength:]) 315 b = b[:pos+msiz+mlen] 316 } 317 protowire.AppendVarint(b[:pos], uint64(mlen)) 318 return b 319} 320