1/* 2Copyright 2020 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package value 18 19import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "reflect" 24 "sort" 25 "sync" 26 "sync/atomic" 27) 28 29// UnstructuredConverter defines how a type can be converted directly to unstructured. 30// Types that implement json.Marshaler may also optionally implement this interface to provide a more 31// direct and more efficient conversion. All types that choose to implement this interface must still 32// implement this same conversion via json.Marshaler. 33type UnstructuredConverter interface { 34 json.Marshaler // require that json.Marshaler is implemented 35 36 // ToUnstructured returns the unstructured representation. 37 ToUnstructured() interface{} 38} 39 40// TypeReflectCacheEntry keeps data gathered using reflection about how a type is converted to/from unstructured. 41type TypeReflectCacheEntry struct { 42 isJsonMarshaler bool 43 ptrIsJsonMarshaler bool 44 isJsonUnmarshaler bool 45 ptrIsJsonUnmarshaler bool 46 isStringConvertable bool 47 ptrIsStringConvertable bool 48 49 structFields map[string]*FieldCacheEntry 50 orderedStructFields []*FieldCacheEntry 51} 52 53// FieldCacheEntry keeps data gathered using reflection about how the field of a struct is converted to/from 54// unstructured. 55type FieldCacheEntry struct { 56 // JsonName returns the name of the field according to the json tags on the struct field. 57 JsonName string 58 // isOmitEmpty is true if the field has the json 'omitempty' tag. 59 isOmitEmpty bool 60 // fieldPath is a list of field indices (see FieldByIndex) to lookup the value of 61 // a field in a reflect.Value struct. The field indices in the list form a path used 62 // to traverse through intermediary 'inline' fields. 63 fieldPath [][]int 64 65 fieldType reflect.Type 66 TypeEntry *TypeReflectCacheEntry 67} 68 69func (f *FieldCacheEntry) CanOmit(fieldVal reflect.Value) bool { 70 return f.isOmitEmpty && (safeIsNil(fieldVal) || isZero(fieldVal)) 71} 72 73// GetFrom returns the field identified by this FieldCacheEntry from the provided struct. 74func (f *FieldCacheEntry) GetFrom(structVal reflect.Value) reflect.Value { 75 // field might be nested within 'inline' structs 76 for _, elem := range f.fieldPath { 77 structVal = dereference(structVal).FieldByIndex(elem) 78 } 79 return structVal 80} 81 82var marshalerType = reflect.TypeOf(new(json.Marshaler)).Elem() 83var unmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem() 84var unstructuredConvertableType = reflect.TypeOf(new(UnstructuredConverter)).Elem() 85var defaultReflectCache = newReflectCache() 86 87// TypeReflectEntryOf returns the TypeReflectCacheEntry of the provided reflect.Type. 88func TypeReflectEntryOf(t reflect.Type) *TypeReflectCacheEntry { 89 cm := defaultReflectCache.get() 90 if record, ok := cm[t]; ok { 91 return record 92 } 93 updates := reflectCacheMap{} 94 result := typeReflectEntryOf(cm, t, updates) 95 if len(updates) > 0 { 96 defaultReflectCache.update(updates) 97 } 98 return result 99} 100 101// TypeReflectEntryOf returns all updates needed to add provided reflect.Type, and the types its fields transitively 102// depend on, to the cache. 103func typeReflectEntryOf(cm reflectCacheMap, t reflect.Type, updates reflectCacheMap) *TypeReflectCacheEntry { 104 if record, ok := cm[t]; ok { 105 return record 106 } 107 if record, ok := updates[t]; ok { 108 return record 109 } 110 typeEntry := &TypeReflectCacheEntry{ 111 isJsonMarshaler: t.Implements(marshalerType), 112 ptrIsJsonMarshaler: reflect.PtrTo(t).Implements(marshalerType), 113 isJsonUnmarshaler: reflect.PtrTo(t).Implements(unmarshalerType), 114 isStringConvertable: t.Implements(unstructuredConvertableType), 115 ptrIsStringConvertable: reflect.PtrTo(t).Implements(unstructuredConvertableType), 116 } 117 if t.Kind() == reflect.Struct { 118 fieldEntries := map[string]*FieldCacheEntry{} 119 buildStructCacheEntry(t, fieldEntries, nil) 120 typeEntry.structFields = fieldEntries 121 sortedByJsonName := make([]*FieldCacheEntry, len(fieldEntries)) 122 i := 0 123 for _, entry := range fieldEntries { 124 sortedByJsonName[i] = entry 125 i++ 126 } 127 sort.Slice(sortedByJsonName, func(i, j int) bool { 128 return sortedByJsonName[i].JsonName < sortedByJsonName[j].JsonName 129 }) 130 typeEntry.orderedStructFields = sortedByJsonName 131 } 132 133 // cyclic type references are allowed, so we must add the typeEntry to the updates map before resolving 134 // the field.typeEntry references, or creating them if they are not already in the cache 135 updates[t] = typeEntry 136 137 for _, field := range typeEntry.structFields { 138 if field.TypeEntry == nil { 139 field.TypeEntry = typeReflectEntryOf(cm, field.fieldType, updates) 140 } 141 } 142 return typeEntry 143} 144 145func buildStructCacheEntry(t reflect.Type, infos map[string]*FieldCacheEntry, fieldPath [][]int) { 146 for i := 0; i < t.NumField(); i++ { 147 field := t.Field(i) 148 jsonName, omit, isInline, isOmitempty := lookupJsonTags(field) 149 if omit { 150 continue 151 } 152 if isInline { 153 e := field.Type 154 if field.Type.Kind() == reflect.Ptr { 155 e = field.Type.Elem() 156 } 157 buildStructCacheEntry(e, infos, append(fieldPath, field.Index)) 158 continue 159 } 160 info := &FieldCacheEntry{JsonName: jsonName, isOmitEmpty: isOmitempty, fieldPath: append(fieldPath, field.Index), fieldType: field.Type} 161 infos[jsonName] = info 162 } 163} 164 165// Fields returns a map of JSON field name to FieldCacheEntry for structs, or nil for non-structs. 166func (e TypeReflectCacheEntry) Fields() map[string]*FieldCacheEntry { 167 return e.structFields 168} 169 170// Fields returns a map of JSON field name to FieldCacheEntry for structs, or nil for non-structs. 171func (e TypeReflectCacheEntry) OrderedFields() []*FieldCacheEntry { 172 return e.orderedStructFields 173} 174 175// CanConvertToUnstructured returns true if this TypeReflectCacheEntry can convert values of its type to unstructured. 176func (e TypeReflectCacheEntry) CanConvertToUnstructured() bool { 177 return e.isJsonMarshaler || e.ptrIsJsonMarshaler || e.isStringConvertable || e.ptrIsStringConvertable 178} 179 180// ToUnstructured converts the provided value to unstructured and returns it. 181func (e TypeReflectCacheEntry) ToUnstructured(sv reflect.Value) (interface{}, error) { 182 // This is based on https://github.com/kubernetes/kubernetes/blob/82c9e5c814eb7acc6cc0a090c057294d0667ad66/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go#L505 183 // and is intended to replace it. 184 185 // Check if the object has a custom string converter and use it if available, since it is much more efficient 186 // than round tripping through json. 187 if converter, ok := e.getUnstructuredConverter(sv); ok { 188 return converter.ToUnstructured(), nil 189 } 190 // Check if the object has a custom JSON marshaller/unmarshaller. 191 if marshaler, ok := e.getJsonMarshaler(sv); ok { 192 if sv.Kind() == reflect.Ptr && sv.IsNil() { 193 // We're done - we don't need to store anything. 194 return nil, nil 195 } 196 197 data, err := marshaler.MarshalJSON() 198 if err != nil { 199 return nil, err 200 } 201 switch { 202 case len(data) == 0: 203 return nil, fmt.Errorf("error decoding from json: empty value") 204 205 case bytes.Equal(data, nullBytes): 206 // We're done - we don't need to store anything. 207 return nil, nil 208 209 case bytes.Equal(data, trueBytes): 210 return true, nil 211 212 case bytes.Equal(data, falseBytes): 213 return false, nil 214 215 case data[0] == '"': 216 var result string 217 err := unmarshal(data, &result) 218 if err != nil { 219 return nil, fmt.Errorf("error decoding string from json: %v", err) 220 } 221 return result, nil 222 223 case data[0] == '{': 224 result := make(map[string]interface{}) 225 err := unmarshal(data, &result) 226 if err != nil { 227 return nil, fmt.Errorf("error decoding object from json: %v", err) 228 } 229 return result, nil 230 231 case data[0] == '[': 232 result := make([]interface{}, 0) 233 err := unmarshal(data, &result) 234 if err != nil { 235 return nil, fmt.Errorf("error decoding array from json: %v", err) 236 } 237 return result, nil 238 239 default: 240 var ( 241 resultInt int64 242 resultFloat float64 243 err error 244 ) 245 if err = unmarshal(data, &resultInt); err == nil { 246 return resultInt, nil 247 } else if err = unmarshal(data, &resultFloat); err == nil { 248 return resultFloat, nil 249 } else { 250 return nil, fmt.Errorf("error decoding number from json: %v", err) 251 } 252 } 253 } 254 255 return nil, fmt.Errorf("provided type cannot be converted: %v", sv.Type()) 256} 257 258// CanConvertFromUnstructured returns true if this TypeReflectCacheEntry can convert objects of the type from unstructured. 259func (e TypeReflectCacheEntry) CanConvertFromUnstructured() bool { 260 return e.isJsonUnmarshaler 261} 262 263// FromUnstructured converts the provided source value from unstructured into the provided destination value. 264func (e TypeReflectCacheEntry) FromUnstructured(sv, dv reflect.Value) error { 265 // TODO: this could be made much more efficient using direct conversions like 266 // UnstructuredConverter.ToUnstructured provides. 267 st := dv.Type() 268 data, err := json.Marshal(sv.Interface()) 269 if err != nil { 270 return fmt.Errorf("error encoding %s to json: %v", st.String(), err) 271 } 272 if unmarshaler, ok := e.getJsonUnmarshaler(dv); ok { 273 return unmarshaler.UnmarshalJSON(data) 274 } 275 return fmt.Errorf("unable to unmarshal %v into %v", sv.Type(), dv.Type()) 276} 277 278var ( 279 nullBytes = []byte("null") 280 trueBytes = []byte("true") 281 falseBytes = []byte("false") 282) 283 284func (e TypeReflectCacheEntry) getJsonMarshaler(v reflect.Value) (json.Marshaler, bool) { 285 if e.isJsonMarshaler { 286 return v.Interface().(json.Marshaler), true 287 } 288 if e.ptrIsJsonMarshaler { 289 // Check pointer receivers if v is not a pointer 290 if v.Kind() != reflect.Ptr && v.CanAddr() { 291 v = v.Addr() 292 return v.Interface().(json.Marshaler), true 293 } 294 } 295 return nil, false 296} 297 298func (e TypeReflectCacheEntry) getJsonUnmarshaler(v reflect.Value) (json.Unmarshaler, bool) { 299 if !e.isJsonUnmarshaler { 300 return nil, false 301 } 302 return v.Addr().Interface().(json.Unmarshaler), true 303} 304 305func (e TypeReflectCacheEntry) getUnstructuredConverter(v reflect.Value) (UnstructuredConverter, bool) { 306 if e.isStringConvertable { 307 return v.Interface().(UnstructuredConverter), true 308 } 309 if e.ptrIsStringConvertable { 310 // Check pointer receivers if v is not a pointer 311 if v.CanAddr() { 312 v = v.Addr() 313 return v.Interface().(UnstructuredConverter), true 314 } 315 } 316 return nil, false 317} 318 319type typeReflectCache struct { 320 // use an atomic and copy-on-write since there are a fixed (typically very small) number of structs compiled into any 321 // go program using this cache 322 value atomic.Value 323 // mu is held by writers when performing load/modify/store operations on the cache, readers do not need to hold a 324 // read-lock since the atomic value is always read-only 325 mu sync.Mutex 326} 327 328func newReflectCache() *typeReflectCache { 329 cache := &typeReflectCache{} 330 cache.value.Store(make(reflectCacheMap)) 331 return cache 332} 333 334type reflectCacheMap map[reflect.Type]*TypeReflectCacheEntry 335 336// get returns the reflectCacheMap. 337func (c *typeReflectCache) get() reflectCacheMap { 338 return c.value.Load().(reflectCacheMap) 339} 340 341// update merges the provided updates into the cache. 342func (c *typeReflectCache) update(updates reflectCacheMap) { 343 c.mu.Lock() 344 defer c.mu.Unlock() 345 346 currentCacheMap := c.value.Load().(reflectCacheMap) 347 348 hasNewEntries := false 349 for t := range updates { 350 if _, ok := currentCacheMap[t]; !ok { 351 hasNewEntries = true 352 break 353 } 354 } 355 if !hasNewEntries { 356 // Bail if the updates have been set while waiting for lock acquisition. 357 // This is safe since setting entries is idempotent. 358 return 359 } 360 361 newCacheMap := make(reflectCacheMap, len(currentCacheMap)+len(updates)) 362 for k, v := range currentCacheMap { 363 newCacheMap[k] = v 364 } 365 for t, update := range updates { 366 newCacheMap[t] = update 367 } 368 c.value.Store(newCacheMap) 369} 370 371// Below json Unmarshal is fromk8s.io/apimachinery/pkg/util/json 372// to handle number conversions as expected by Kubernetes 373 374// limit recursive depth to prevent stack overflow errors 375const maxDepth = 10000 376 377// unmarshal unmarshals the given data 378// If v is a *map[string]interface{}, numbers are converted to int64 or float64 379func unmarshal(data []byte, v interface{}) error { 380 switch v := v.(type) { 381 case *map[string]interface{}: 382 // Build a decoder from the given data 383 decoder := json.NewDecoder(bytes.NewBuffer(data)) 384 // Preserve numbers, rather than casting to float64 automatically 385 decoder.UseNumber() 386 // Run the decode 387 if err := decoder.Decode(v); err != nil { 388 return err 389 } 390 // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 391 return convertMapNumbers(*v, 0) 392 393 case *[]interface{}: 394 // Build a decoder from the given data 395 decoder := json.NewDecoder(bytes.NewBuffer(data)) 396 // Preserve numbers, rather than casting to float64 automatically 397 decoder.UseNumber() 398 // Run the decode 399 if err := decoder.Decode(v); err != nil { 400 return err 401 } 402 // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 403 return convertSliceNumbers(*v, 0) 404 405 default: 406 return json.Unmarshal(data, v) 407 } 408} 409 410// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64. 411// values which are map[string]interface{} or []interface{} are recursively visited 412func convertMapNumbers(m map[string]interface{}, depth int) error { 413 if depth > maxDepth { 414 return fmt.Errorf("exceeded max depth of %d", maxDepth) 415 } 416 417 var err error 418 for k, v := range m { 419 switch v := v.(type) { 420 case json.Number: 421 m[k], err = convertNumber(v) 422 case map[string]interface{}: 423 err = convertMapNumbers(v, depth+1) 424 case []interface{}: 425 err = convertSliceNumbers(v, depth+1) 426 } 427 if err != nil { 428 return err 429 } 430 } 431 return nil 432} 433 434// convertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64. 435// values which are map[string]interface{} or []interface{} are recursively visited 436func convertSliceNumbers(s []interface{}, depth int) error { 437 if depth > maxDepth { 438 return fmt.Errorf("exceeded max depth of %d", maxDepth) 439 } 440 441 var err error 442 for i, v := range s { 443 switch v := v.(type) { 444 case json.Number: 445 s[i], err = convertNumber(v) 446 case map[string]interface{}: 447 err = convertMapNumbers(v, depth+1) 448 case []interface{}: 449 err = convertSliceNumbers(v, depth+1) 450 } 451 if err != nil { 452 return err 453 } 454 } 455 return nil 456} 457 458// convertNumber converts a json.Number to an int64 or float64, or returns an error 459func convertNumber(n json.Number) (interface{}, error) { 460 // Attempt to convert to an int64 first 461 if i, err := n.Int64(); err == nil { 462 return i, nil 463 } 464 // Return a float64 (default json.Decode() behavior) 465 // An overflow will return an error 466 return n.Float64() 467} 468