1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package datastore 16 17import ( 18 "errors" 19 "fmt" 20 "reflect" 21 "time" 22 "unicode/utf8" 23 24 timepb "github.com/golang/protobuf/ptypes/timestamp" 25 pb "google.golang.org/genproto/googleapis/datastore/v1" 26 llpb "google.golang.org/genproto/googleapis/type/latlng" 27) 28 29type saveOpts struct { 30 noIndex bool 31 flatten bool 32 omitEmpty bool 33} 34 35// saveEntity saves an EntityProto into a PropertyLoadSaver or struct pointer. 36func saveEntity(key *Key, src interface{}) (*pb.Entity, error) { 37 var err error 38 var props []Property 39 if e, ok := src.(PropertyLoadSaver); ok { 40 props, err = e.Save() 41 } else { 42 props, err = SaveStruct(src) 43 } 44 if err != nil { 45 return nil, err 46 } 47 return propertiesToProto(key, props) 48} 49 50// TODO(djd): Convert this and below to return ([]Property, error). 51func saveStructProperty(props *[]Property, name string, opts saveOpts, v reflect.Value) error { 52 p := Property{ 53 Name: name, 54 NoIndex: opts.noIndex, 55 } 56 57 if opts.omitEmpty && isEmptyValue(v) { 58 return nil 59 } 60 61 // First check if field type implements PLS. If so, use PLS to 62 // save. 63 ok, err := plsFieldSave(props, p, name, opts, v) 64 if err != nil { 65 return err 66 } 67 if ok { 68 return nil 69 } 70 71 switch x := v.Interface().(type) { 72 case *Key, time.Time, GeoPoint: 73 p.Value = x 74 default: 75 switch v.Kind() { 76 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 77 p.Value = v.Int() 78 case reflect.Bool: 79 p.Value = v.Bool() 80 case reflect.String: 81 p.Value = v.String() 82 case reflect.Float32, reflect.Float64: 83 p.Value = v.Float() 84 case reflect.Slice: 85 if v.Type().Elem().Kind() == reflect.Uint8 { 86 p.Value = v.Bytes() 87 } else { 88 return saveSliceProperty(props, name, opts, v) 89 } 90 case reflect.Ptr: 91 if isValidPointerType(v.Type().Elem()) { 92 if v.IsNil() { 93 // Nil pointer becomes a nil property value (unless omitempty, handled above). 94 p.Value = nil 95 *props = append(*props, p) 96 return nil 97 } 98 // When we recurse on the derefenced pointer, omitempty no longer applies: 99 // we already know the pointer is not empty, it doesn't matter if its referent 100 // is empty or not. 101 opts.omitEmpty = false 102 return saveStructProperty(props, name, opts, v.Elem()) 103 } 104 if v.Type().Elem().Kind() != reflect.Struct { 105 return fmt.Errorf("datastore: unsupported struct field type: %s", v.Type()) 106 } 107 // Pointer to struct is a special case. 108 if v.IsNil() { 109 return nil 110 } 111 v = v.Elem() 112 fallthrough 113 case reflect.Struct: 114 if !v.CanAddr() { 115 return fmt.Errorf("datastore: unsupported struct field: value is unaddressable") 116 } 117 vi := v.Addr().Interface() 118 119 sub, err := newStructPLS(vi) 120 if err != nil { 121 return fmt.Errorf("datastore: unsupported struct field: %v", err) 122 } 123 124 if opts.flatten { 125 return sub.save(props, opts, name+".") 126 } 127 128 var subProps []Property 129 err = sub.save(&subProps, opts, "") 130 if err != nil { 131 return err 132 } 133 subKey, err := sub.key(v) 134 if err != nil { 135 return err 136 } 137 138 p.Value = &Entity{ 139 Key: subKey, 140 Properties: subProps, 141 } 142 } 143 } 144 if p.Value == nil { 145 return fmt.Errorf("datastore: unsupported struct field type: %v", v.Type()) 146 } 147 *props = append(*props, p) 148 return nil 149} 150 151// plsFieldSave first tries to converts v's value to a PLS, then v's addressed 152// value to a PLS. If neither succeeds, plsFieldSave returns false for first return 153// value. 154// If v is successfully converted to a PLS, plsFieldSave will then add the 155// Value to property p by way of the PLS's Save method, and append it to props. 156// 157// If the flatten option is present in opts, name must be prepended to each property's 158// name before it is appended to props. Eg. if name were "A" and a subproperty's name 159// were "B", the resultant name of the property to be appended to props would be "A.B". 160func plsFieldSave(props *[]Property, p Property, name string, opts saveOpts, v reflect.Value) (ok bool, err error) { 161 vpls, err := plsForSave(v) 162 if err != nil { 163 return false, err 164 } 165 166 if vpls == nil { 167 return false, nil 168 } 169 170 subProps, err := vpls.Save() 171 if err != nil { 172 return true, err 173 } 174 175 if opts.flatten { 176 for _, subp := range subProps { 177 subp.Name = name + "." + subp.Name 178 *props = append(*props, subp) 179 } 180 return true, nil 181 } 182 183 p.Value = &Entity{Properties: subProps} 184 *props = append(*props, p) 185 186 return true, nil 187} 188 189// key extracts the *Key struct field from struct v based on the structCodec of s. 190func (s structPLS) key(v reflect.Value) (*Key, error) { 191 if v.Kind() != reflect.Struct { 192 return nil, errors.New("datastore: cannot save key of non-struct type") 193 } 194 195 keyField := s.codec.Match(keyFieldName) 196 197 if keyField == nil { 198 return nil, nil 199 } 200 201 f := v.FieldByIndex(keyField.Index) 202 k, ok := f.Interface().(*Key) 203 if !ok { 204 return nil, fmt.Errorf("datastore: %s field on struct %T is not a *datastore.Key", keyFieldName, v.Interface()) 205 } 206 207 return k, nil 208} 209 210func saveSliceProperty(props *[]Property, name string, opts saveOpts, v reflect.Value) error { 211 // Easy case: if the slice is empty, we're done. 212 if v.Len() == 0 { 213 return nil 214 } 215 // Work out the properties generated by the first element in the slice. This will 216 // usually be a single property, but will be more if this is a slice of structs. 217 var headProps []Property 218 if err := saveStructProperty(&headProps, name, opts, v.Index(0)); err != nil { 219 return err 220 } 221 222 // Convert the first element's properties into slice properties, and 223 // keep track of the values in a map. 224 values := make(map[string][]interface{}, len(headProps)) 225 for _, p := range headProps { 226 values[p.Name] = append(make([]interface{}, 0, v.Len()), p.Value) 227 } 228 229 // Find the elements for the subsequent elements. 230 for i := 1; i < v.Len(); i++ { 231 elemProps := make([]Property, 0, len(headProps)) 232 if err := saveStructProperty(&elemProps, name, opts, v.Index(i)); err != nil { 233 return err 234 } 235 for _, p := range elemProps { 236 v, ok := values[p.Name] 237 if !ok { 238 return fmt.Errorf("datastore: unexpected property %q in elem %d of slice", p.Name, i) 239 } 240 values[p.Name] = append(v, p.Value) 241 } 242 } 243 244 // Convert to the final properties. 245 for _, p := range headProps { 246 p.Value = values[p.Name] 247 *props = append(*props, p) 248 } 249 return nil 250} 251 252func (s structPLS) Save() ([]Property, error) { 253 var props []Property 254 if err := s.save(&props, saveOpts{}, ""); err != nil { 255 return nil, err 256 } 257 return props, nil 258} 259 260func (s structPLS) save(props *[]Property, opts saveOpts, prefix string) error { 261 for _, f := range s.codec { 262 name := prefix + f.Name 263 v := getField(s.v, f.Index) 264 if !v.IsValid() || !v.CanSet() { 265 continue 266 } 267 268 var tagOpts saveOpts 269 if f.ParsedTag != nil { 270 tagOpts = f.ParsedTag.(saveOpts) 271 } 272 273 var opts1 saveOpts 274 opts1.noIndex = opts.noIndex || tagOpts.noIndex 275 opts1.flatten = opts.flatten || tagOpts.flatten 276 opts1.omitEmpty = tagOpts.omitEmpty // don't propagate 277 if err := saveStructProperty(props, name, opts1, v); err != nil { 278 return err 279 } 280 } 281 return nil 282} 283 284// getField returns the field from v at the given index path. 285// If it encounters a nil-valued field in the path, getField 286// stops and returns a zero-valued reflect.Value, preventing the 287// panic that would have been caused by reflect's FieldByIndex. 288func getField(v reflect.Value, index []int) reflect.Value { 289 var zero reflect.Value 290 if v.Type().Kind() != reflect.Struct { 291 return zero 292 } 293 294 for _, i := range index { 295 if v.Kind() == reflect.Ptr && v.Type().Elem().Kind() == reflect.Struct { 296 if v.IsNil() { 297 return zero 298 } 299 v = v.Elem() 300 } 301 v = v.Field(i) 302 } 303 return v 304} 305 306func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) { 307 e := &pb.Entity{ 308 Key: keyToProto(key), 309 Properties: map[string]*pb.Value{}, 310 } 311 indexedProps := 0 312 for _, p := range props { 313 // Do not send a Key value a field to datastore. 314 if p.Name == keyFieldName { 315 continue 316 } 317 318 val, err := interfaceToProto(p.Value, p.NoIndex) 319 if err != nil { 320 return nil, fmt.Errorf("datastore: %v for a Property with Name %q", err, p.Name) 321 } 322 if !p.NoIndex { 323 rVal := reflect.ValueOf(p.Value) 324 if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 { 325 indexedProps += rVal.Len() 326 } else { 327 indexedProps++ 328 } 329 } 330 if indexedProps > maxIndexedProperties { 331 return nil, errors.New("datastore: too many indexed properties") 332 } 333 334 if _, ok := e.Properties[p.Name]; ok { 335 return nil, fmt.Errorf("datastore: duplicate Property with Name %q", p.Name) 336 } 337 e.Properties[p.Name] = val 338 } 339 return e, nil 340} 341 342func interfaceToProto(iv interface{}, noIndex bool) (*pb.Value, error) { 343 val := &pb.Value{ExcludeFromIndexes: noIndex} 344 switch v := iv.(type) { 345 case int: 346 val.ValueType = &pb.Value_IntegerValue{IntegerValue: int64(v)} 347 case int32: 348 val.ValueType = &pb.Value_IntegerValue{IntegerValue: int64(v)} 349 case int64: 350 val.ValueType = &pb.Value_IntegerValue{IntegerValue: v} 351 case bool: 352 val.ValueType = &pb.Value_BooleanValue{BooleanValue: v} 353 case string: 354 if len(v) > 1500 && !noIndex { 355 return nil, errors.New("string property too long to index") 356 } 357 if !utf8.ValidString(v) { 358 return nil, fmt.Errorf("string is not valid utf8: %q", v) 359 } 360 val.ValueType = &pb.Value_StringValue{StringValue: v} 361 case float32: 362 val.ValueType = &pb.Value_DoubleValue{DoubleValue: float64(v)} 363 case float64: 364 val.ValueType = &pb.Value_DoubleValue{DoubleValue: v} 365 case *Key: 366 if v == nil { 367 val.ValueType = &pb.Value_NullValue{} 368 } else { 369 val.ValueType = &pb.Value_KeyValue{KeyValue: keyToProto(v)} 370 } 371 case GeoPoint: 372 if !v.Valid() { 373 return nil, errors.New("invalid GeoPoint value") 374 } 375 val.ValueType = &pb.Value_GeoPointValue{GeoPointValue: &llpb.LatLng{ 376 Latitude: v.Lat, 377 Longitude: v.Lng, 378 }} 379 case time.Time: 380 if v.Before(minTime) || v.After(maxTime) { 381 return nil, errors.New("time value out of range") 382 } 383 val.ValueType = &pb.Value_TimestampValue{TimestampValue: &timepb.Timestamp{ 384 Seconds: v.Unix(), 385 Nanos: int32(v.Nanosecond()), 386 }} 387 case []byte: 388 if len(v) > 1500 && !noIndex { 389 return nil, errors.New("[]byte property too long to index") 390 } 391 val.ValueType = &pb.Value_BlobValue{BlobValue: v} 392 case *Entity: 393 e, err := propertiesToProto(v.Key, v.Properties) 394 if err != nil { 395 return nil, err 396 } 397 val.ValueType = &pb.Value_EntityValue{EntityValue: e} 398 case []interface{}: 399 arr := make([]*pb.Value, 0, len(v)) 400 for i, v := range v { 401 elem, err := interfaceToProto(v, noIndex) 402 if err != nil { 403 return nil, fmt.Errorf("%v at index %d", err, i) 404 } 405 arr = append(arr, elem) 406 } 407 val.ValueType = &pb.Value_ArrayValue{ArrayValue: &pb.ArrayValue{Values: arr}} 408 // ArrayValues have ExcludeFromIndexes set on the individual items, rather 409 // than the top-level value. 410 val.ExcludeFromIndexes = false 411 default: 412 rv := reflect.ValueOf(iv) 413 if !rv.IsValid() { 414 val.ValueType = &pb.Value_NullValue{} 415 } else if rv.Kind() == reflect.Ptr { // non-nil pointer: dereference 416 if rv.IsNil() { 417 val.ValueType = &pb.Value_NullValue{} 418 return val, nil 419 } 420 return interfaceToProto(rv.Elem().Interface(), noIndex) 421 } else { 422 return nil, fmt.Errorf("invalid Value type %T", iv) 423 } 424 } 425 // TODO(jbd): Support EntityValue. 426 return val, nil 427} 428 429// isEmptyValue is taken from the encoding/json package in the 430// standard library. 431func isEmptyValue(v reflect.Value) bool { 432 switch v.Kind() { 433 case reflect.Array, reflect.Map, reflect.Slice, reflect.String: 434 return v.Len() == 0 435 case reflect.Bool: 436 return !v.Bool() 437 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 438 return v.Int() == 0 439 case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: 440 return v.Uint() == 0 441 case reflect.Float32, reflect.Float64: 442 return v.Float() == 0 443 case reflect.Interface, reflect.Ptr: 444 return v.IsNil() 445 case reflect.Struct: 446 if t, ok := v.Interface().(time.Time); ok { 447 return t.IsZero() 448 } 449 } 450 return false 451} 452 453// isValidPointerType reports whether a struct field can be a pointer to type t 454// for the purposes of saving and loading. 455func isValidPointerType(t reflect.Type) bool { 456 if t == typeOfTime || t == typeOfGeoPoint { 457 return true 458 } 459 switch t.Kind() { 460 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 461 return true 462 case reflect.Bool: 463 return true 464 case reflect.String: 465 return true 466 case reflect.Float32, reflect.Float64: 467 return true 468 } 469 return false 470} 471