1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package datastore 16 17import ( 18 "errors" 19 "fmt" 20 "reflect" 21 "time" 22 "unicode/utf8" 23 24 "cloud.google.com/go/civil" 25 timepb "github.com/golang/protobuf/ptypes/timestamp" 26 pb "google.golang.org/genproto/googleapis/datastore/v1" 27 llpb "google.golang.org/genproto/googleapis/type/latlng" 28) 29 30type saveOpts struct { 31 noIndex bool 32 flatten bool 33 omitEmpty bool 34} 35 36// saveEntity saves an EntityProto into a PropertyLoadSaver or struct pointer. 37func saveEntity(key *Key, src interface{}) (*pb.Entity, error) { 38 var err error 39 var props []Property 40 if e, ok := src.(PropertyLoadSaver); ok { 41 props, err = e.Save() 42 } else { 43 props, err = SaveStruct(src) 44 } 45 if err != nil { 46 return nil, err 47 } 48 return propertiesToProto(key, props) 49} 50 51// reflectFieldSave extracts the underlying value of v by reflection, 52// and tries to extract a Property that'll be appended to props. 53func reflectFieldSave(props *[]Property, p Property, name string, opts saveOpts, v reflect.Value) error { 54 switch x := v.Interface().(type) { 55 case *Key, time.Time, GeoPoint: 56 p.Value = x 57 case civil.Date: 58 p.Value = x.In(time.UTC) 59 *props = append(*props, p) 60 return nil 61 case civil.Time: 62 var format string 63 if x.Nanosecond == 0 { 64 format = "15:04:05" 65 } else { 66 format = "15:04:05.000000000" 67 } 68 val, err := time.Parse(format, x.String()) 69 if err != nil { 70 return fmt.Errorf("datastore: error while parsing civil.Time: %v", err) 71 } 72 p.Value = val 73 *props = append(*props, p) 74 return nil 75 case civil.DateTime: 76 p.Value = x.In(time.UTC) 77 *props = append(*props, p) 78 return nil 79 default: 80 switch v.Kind() { 81 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 82 p.Value = v.Int() 83 case reflect.Bool: 84 p.Value = v.Bool() 85 case reflect.String: 86 p.Value = v.String() 87 case reflect.Float32, reflect.Float64: 88 p.Value = v.Float() 89 90 case reflect.Interface: 91 // Extract the interface's underlying value and then retry the save. 92 // See issue https://github.com/googleapis/google-cloud-go/issues/1474. 93 if v.IsNil() { 94 // Nil interface becomes a nil property value (unless 95 // omitEmpty is true, which is handled by the caller). 96 p.Value = nil 97 *props = append(*props, p) 98 return nil 99 } 100 return reflectFieldSave(props, p, name, opts, v.Elem()) 101 102 case reflect.Slice: 103 if v.Type().Elem().Kind() == reflect.Uint8 { 104 p.Value = v.Bytes() 105 } else { 106 return saveSliceProperty(props, name, opts, v) 107 } 108 case reflect.Ptr: 109 if isValidPointerType(v.Type().Elem()) { 110 if v.IsNil() { 111 // Nil pointer becomes a nil property value (unless 112 // omitEmpty is true, which is handled by the caller). 113 p.Value = nil 114 *props = append(*props, p) 115 return nil 116 } 117 // When we recurse on the derefenced pointer, omitempty no longer applies: 118 // we already know the pointer is not empty, it doesn't matter if its referent 119 // is empty or not. 120 opts.omitEmpty = false 121 return saveStructProperty(props, name, opts, v.Elem()) 122 } 123 if v.Type().Elem().Kind() != reflect.Struct { 124 return fmt.Errorf("datastore: unsupported struct field type: %s", v.Type()) 125 } 126 // Pointer to struct is a special case. 127 if v.IsNil() { 128 return nil 129 } 130 v = v.Elem() 131 fallthrough 132 case reflect.Struct: 133 if !v.CanAddr() { 134 return fmt.Errorf("datastore: unsupported struct field: value is unaddressable") 135 } 136 vi := v.Addr().Interface() 137 138 sub, err := newStructPLS(vi) 139 if err != nil { 140 return fmt.Errorf("datastore: unsupported struct field: %v", err) 141 } 142 143 if opts.flatten { 144 return sub.save(props, opts, name+".") 145 } 146 147 var subProps []Property 148 err = sub.save(&subProps, opts, "") 149 if err != nil { 150 return err 151 } 152 subKey, err := sub.key(v) 153 if err != nil { 154 return err 155 } 156 157 p.Value = &Entity{ 158 Key: subKey, 159 Properties: subProps, 160 } 161 } 162 } 163 164 if v.CanAddr() { 165 vi := v.Addr().Interface() 166 if pSaver, ok := vi.(PropertyLoadSaver); ok { 167 val, err := pSaver.Save() 168 if err != nil { 169 return fmt.Errorf("field save error: %v", err) 170 } 171 p.Value = val 172 } 173 } 174 175 if p.Value == nil { 176 return fmt.Errorf("datastore: unsupported struct field type: %v", v.Type()) 177 } 178 *props = append(*props, p) 179 return nil 180} 181 182// TODO(djd): Convert this and below to return ([]Property, error). 183func saveStructProperty(props *[]Property, name string, opts saveOpts, v reflect.Value) error { 184 p := Property{ 185 Name: name, 186 NoIndex: opts.noIndex, 187 } 188 189 if opts.omitEmpty && isEmptyValue(v) { 190 return nil 191 } 192 193 // First check if field type implements PLS. If so, use PLS to 194 // save. 195 ok, err := plsFieldSave(props, p, name, opts, v) 196 if err != nil { 197 return err 198 } 199 if ok { 200 return nil 201 } 202 203 return reflectFieldSave(props, p, name, opts, v) 204} 205 206// plsFieldSave first tries to converts v's value to a PLS, then v's addressed 207// value to a PLS. If neither succeeds, plsFieldSave returns false for first return 208// value. 209// If v is successfully converted to a PLS, plsFieldSave will then add the 210// Value to property p by way of the PLS's Save method, and append it to props. 211// 212// If the flatten option is present in opts, name must be prepended to each property's 213// name before it is appended to props. Eg. if name were "A" and a subproperty's name 214// were "B", the resultant name of the property to be appended to props would be "A.B". 215func plsFieldSave(props *[]Property, p Property, name string, opts saveOpts, v reflect.Value) (ok bool, err error) { 216 vpls, err := plsForSave(v) 217 if err != nil { 218 return false, err 219 } 220 221 if vpls == nil { 222 return false, nil 223 } 224 225 subProps, err := vpls.Save() 226 if err != nil { 227 return true, err 228 } 229 230 if opts.flatten { 231 for _, subp := range subProps { 232 subp.Name = name + "." + subp.Name 233 *props = append(*props, subp) 234 } 235 return true, nil 236 } 237 238 p.Value = &Entity{Properties: subProps} 239 *props = append(*props, p) 240 241 return true, nil 242} 243 244// key extracts the *Key struct field from struct v based on the structCodec of s. 245func (s structPLS) key(v reflect.Value) (*Key, error) { 246 if v.Kind() != reflect.Struct { 247 return nil, errors.New("datastore: cannot save key of non-struct type") 248 } 249 250 keyField := s.codec.Match(keyFieldName) 251 252 if keyField == nil { 253 return nil, nil 254 } 255 256 f := v.FieldByIndex(keyField.Index) 257 k, ok := f.Interface().(*Key) 258 if !ok { 259 return nil, fmt.Errorf("datastore: %s field on struct %T is not a *datastore.Key", keyFieldName, v.Interface()) 260 } 261 262 return k, nil 263} 264 265func saveSliceProperty(props *[]Property, name string, opts saveOpts, v reflect.Value) error { 266 // Easy case: if the slice is empty, we're done. 267 if v.Len() == 0 { 268 return nil 269 } 270 // Work out the properties generated by the first element in the slice. This will 271 // usually be a single property, but will be more if this is a slice of structs. 272 var headProps []Property 273 if err := saveStructProperty(&headProps, name, opts, v.Index(0)); err != nil { 274 return err 275 } 276 277 // Convert the first element's properties into slice properties, and 278 // keep track of the values in a map. 279 values := make(map[string][]interface{}, len(headProps)) 280 for _, p := range headProps { 281 values[p.Name] = append(make([]interface{}, 0, v.Len()), p.Value) 282 } 283 284 // Find the elements for the subsequent elements. 285 for i := 1; i < v.Len(); i++ { 286 elemProps := make([]Property, 0, len(headProps)) 287 if err := saveStructProperty(&elemProps, name, opts, v.Index(i)); err != nil { 288 return err 289 } 290 for _, p := range elemProps { 291 v, ok := values[p.Name] 292 if !ok { 293 return fmt.Errorf("datastore: unexpected property %q in elem %d of slice", p.Name, i) 294 } 295 values[p.Name] = append(v, p.Value) 296 } 297 } 298 299 // Convert to the final properties. 300 for _, p := range headProps { 301 p.Value = values[p.Name] 302 *props = append(*props, p) 303 } 304 return nil 305} 306 307func (s structPLS) Save() ([]Property, error) { 308 var props []Property 309 if err := s.save(&props, saveOpts{}, ""); err != nil { 310 return nil, err 311 } 312 return props, nil 313} 314 315func (s structPLS) save(props *[]Property, opts saveOpts, prefix string) error { 316 for _, f := range s.codec { 317 name := prefix + f.Name 318 v := getField(s.v, f.Index) 319 if !v.IsValid() || !v.CanSet() { 320 continue 321 } 322 323 var tagOpts saveOpts 324 if f.ParsedTag != nil { 325 tagOpts = f.ParsedTag.(saveOpts) 326 } 327 328 var opts1 saveOpts 329 opts1.noIndex = opts.noIndex || tagOpts.noIndex 330 opts1.flatten = opts.flatten || tagOpts.flatten 331 opts1.omitEmpty = tagOpts.omitEmpty // don't propagate 332 if err := saveStructProperty(props, name, opts1, v); err != nil { 333 return err 334 } 335 } 336 return nil 337} 338 339// getField returns the field from v at the given index path. 340// If it encounters a nil-valued field in the path, getField 341// stops and returns a zero-valued reflect.Value, preventing the 342// panic that would have been caused by reflect's FieldByIndex. 343func getField(v reflect.Value, index []int) reflect.Value { 344 var zero reflect.Value 345 if v.Type().Kind() != reflect.Struct { 346 return zero 347 } 348 349 for _, i := range index { 350 if v.Kind() == reflect.Ptr && v.Type().Elem().Kind() == reflect.Struct { 351 if v.IsNil() { 352 return zero 353 } 354 v = v.Elem() 355 } 356 v = v.Field(i) 357 } 358 return v 359} 360 361func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) { 362 e := &pb.Entity{ 363 Key: keyToProto(key), 364 Properties: map[string]*pb.Value{}, 365 } 366 indexedProps := 0 367 for _, p := range props { 368 // Do not send a Key value a field to datastore. 369 if p.Name == keyFieldName { 370 continue 371 } 372 373 val, err := interfaceToProto(p.Value, p.NoIndex) 374 if err != nil { 375 return nil, fmt.Errorf("datastore: %v for a Property with Name %q", err, p.Name) 376 } 377 if !p.NoIndex { 378 rVal := reflect.ValueOf(p.Value) 379 if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 { 380 indexedProps += rVal.Len() 381 } else { 382 indexedProps++ 383 } 384 } 385 if indexedProps > maxIndexedProperties { 386 return nil, errors.New("datastore: too many indexed properties") 387 } 388 389 if _, ok := e.Properties[p.Name]; ok { 390 return nil, fmt.Errorf("datastore: duplicate Property with Name %q", p.Name) 391 } 392 e.Properties[p.Name] = val 393 } 394 return e, nil 395} 396 397func interfaceToProto(iv interface{}, noIndex bool) (*pb.Value, error) { 398 val := &pb.Value{ExcludeFromIndexes: noIndex} 399 switch v := iv.(type) { 400 case int: 401 val.ValueType = &pb.Value_IntegerValue{IntegerValue: int64(v)} 402 case int32: 403 val.ValueType = &pb.Value_IntegerValue{IntegerValue: int64(v)} 404 case int64: 405 val.ValueType = &pb.Value_IntegerValue{IntegerValue: v} 406 case bool: 407 val.ValueType = &pb.Value_BooleanValue{BooleanValue: v} 408 case string: 409 if len(v) > 1500 && !noIndex { 410 return nil, errors.New("string property too long to index") 411 } 412 if !utf8.ValidString(v) { 413 return nil, fmt.Errorf("string is not valid utf8: %q", v) 414 } 415 val.ValueType = &pb.Value_StringValue{StringValue: v} 416 case float32: 417 val.ValueType = &pb.Value_DoubleValue{DoubleValue: float64(v)} 418 case float64: 419 val.ValueType = &pb.Value_DoubleValue{DoubleValue: v} 420 case *Key: 421 if v == nil { 422 val.ValueType = &pb.Value_NullValue{} 423 } else { 424 val.ValueType = &pb.Value_KeyValue{KeyValue: keyToProto(v)} 425 } 426 case GeoPoint: 427 if !v.Valid() { 428 return nil, errors.New("invalid GeoPoint value") 429 } 430 val.ValueType = &pb.Value_GeoPointValue{GeoPointValue: &llpb.LatLng{ 431 Latitude: v.Lat, 432 Longitude: v.Lng, 433 }} 434 case time.Time: 435 if v.Before(minTime) || v.After(maxTime) { 436 return nil, errors.New("time value out of range") 437 } 438 val.ValueType = &pb.Value_TimestampValue{TimestampValue: &timepb.Timestamp{ 439 Seconds: v.Unix(), 440 Nanos: int32(v.Nanosecond()), 441 }} 442 case []byte: 443 if len(v) > 1500 && !noIndex { 444 return nil, errors.New("[]byte property too long to index") 445 } 446 val.ValueType = &pb.Value_BlobValue{BlobValue: v} 447 case *Entity: 448 e, err := propertiesToProto(v.Key, v.Properties) 449 if err != nil { 450 return nil, err 451 } 452 val.ValueType = &pb.Value_EntityValue{EntityValue: e} 453 case []interface{}: 454 arr := make([]*pb.Value, 0, len(v)) 455 for i, v := range v { 456 elem, err := interfaceToProto(v, noIndex) 457 if err != nil { 458 return nil, fmt.Errorf("%v at index %d", err, i) 459 } 460 arr = append(arr, elem) 461 } 462 val.ValueType = &pb.Value_ArrayValue{ArrayValue: &pb.ArrayValue{Values: arr}} 463 // ArrayValues have ExcludeFromIndexes set on the individual items, rather 464 // than the top-level value. 465 val.ExcludeFromIndexes = false 466 default: 467 rv := reflect.ValueOf(iv) 468 if !rv.IsValid() { 469 val.ValueType = &pb.Value_NullValue{} 470 } else if rv.Kind() == reflect.Ptr { // non-nil pointer: dereference 471 if rv.IsNil() { 472 val.ValueType = &pb.Value_NullValue{} 473 return val, nil 474 } 475 return interfaceToProto(rv.Elem().Interface(), noIndex) 476 } else { 477 return nil, fmt.Errorf("invalid Value type %T", iv) 478 } 479 } 480 // TODO(jbd): Support EntityValue. 481 return val, nil 482} 483 484// isEmptyValue is taken from the encoding/json package in the 485// standard library. 486func isEmptyValue(v reflect.Value) bool { 487 switch v.Kind() { 488 case reflect.Array, reflect.Map, reflect.Slice, reflect.String: 489 return v.Len() == 0 490 case reflect.Bool: 491 return !v.Bool() 492 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 493 return v.Int() == 0 494 case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: 495 return v.Uint() == 0 496 case reflect.Float32, reflect.Float64: 497 return v.Float() == 0 498 case reflect.Interface, reflect.Ptr: 499 return v.IsNil() 500 case reflect.Struct: 501 if t, ok := v.Interface().(time.Time); ok { 502 return t.IsZero() 503 } 504 } 505 return false 506} 507 508// isValidPointerType reports whether a struct field can be a pointer to type t 509// for the purposes of saving and loading. 510func isValidPointerType(t reflect.Type) bool { 511 if t == typeOfTime || t == typeOfGeoPoint { 512 return true 513 } 514 switch t.Kind() { 515 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 516 return true 517 case reflect.Bool: 518 return true 519 case reflect.String: 520 return true 521 case reflect.Float32, reflect.Float64: 522 return true 523 } 524 return false 525} 526