1// Copyright 2014 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package datastore 16 17import ( 18 "fmt" 19 "reflect" 20 "strings" 21 "unicode" 22 23 "cloud.google.com/go/internal/fields" 24) 25 26// Entities with more than this many indexed properties will not be saved. 27const maxIndexedProperties = 20000 28 29// Property is a name/value pair plus some metadata. A datastore entity's 30// contents are loaded and saved as a sequence of Properties. Each property 31// name must be unique within an entity. 32type Property struct { 33 // Name is the property name. 34 Name string 35 // Value is the property value. The valid types are: 36 // - int64 37 // - bool 38 // - string 39 // - float64 40 // - *Key 41 // - time.Time (retrieved as local time) 42 // - GeoPoint 43 // - []byte (up to 1 megabyte in length) 44 // - *Entity (representing a nested struct) 45 // Value can also be: 46 // - []interface{} where each element is one of the above types 47 // This set is smaller than the set of valid struct field types that the 48 // datastore can load and save. A Value's type must be explicitly on 49 // the list above; it is not sufficient for the underlying type to be 50 // on that list. For example, a Value of "type myInt64 int64" is 51 // invalid. Smaller-width integers and floats are also invalid. Again, 52 // this is more restrictive than the set of valid struct field types. 53 // 54 // A Value will have an opaque type when loading entities from an index, 55 // such as via a projection query. Load entities into a struct instead 56 // of a PropertyLoadSaver when using a projection query. 57 // 58 // A Value may also be the nil interface value; this is equivalent to 59 // Python's None but not directly representable by a Go struct. Loading 60 // a nil-valued property into a struct will set that field to the zero 61 // value. 62 Value interface{} 63 // NoIndex is whether the datastore cannot index this property. 64 // If NoIndex is set to false, []byte and string values are limited to 65 // 1500 bytes. 66 NoIndex bool 67} 68 69// An Entity is the value type for a nested struct. 70// This type is only used for a Property's Value. 71type Entity struct { 72 Key *Key 73 Properties []Property 74} 75 76// PropertyLoadSaver can be converted from and to a slice of Properties. 77type PropertyLoadSaver interface { 78 Load([]Property) error 79 Save() ([]Property, error) 80} 81 82// KeyLoader can store a Key. 83type KeyLoader interface { 84 // PropertyLoadSaver is embedded because a KeyLoader 85 // must also always implement PropertyLoadSaver. 86 PropertyLoadSaver 87 LoadKey(k *Key) error 88} 89 90// PropertyList converts a []Property to implement PropertyLoadSaver. 91type PropertyList []Property 92 93var ( 94 typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem() 95 typeOfPropertyList = reflect.TypeOf(PropertyList(nil)) 96) 97 98// Load loads all of the provided properties into l. 99// It does not first reset *l to an empty slice. 100func (l *PropertyList) Load(p []Property) error { 101 *l = append(*l, p...) 102 return nil 103} 104 105// Save saves all of l's properties as a slice of Properties. 106func (l *PropertyList) Save() ([]Property, error) { 107 return *l, nil 108} 109 110// validPropertyName returns whether name consists of one or more valid Go 111// identifiers joined by ".". 112func validPropertyName(name string) bool { 113 if name == "" { 114 return false 115 } 116 for _, s := range strings.Split(name, ".") { 117 if s == "" { 118 return false 119 } 120 first := true 121 for _, c := range s { 122 if first { 123 first = false 124 if c != '_' && !unicode.IsLetter(c) { 125 return false 126 } 127 } else { 128 if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) { 129 return false 130 } 131 } 132 } 133 } 134 return true 135} 136 137// parseTag interprets datastore struct field tags 138func parseTag(t reflect.StructTag) (name string, keep bool, other interface{}, err error) { 139 s := t.Get("datastore") 140 parts := strings.Split(s, ",") 141 if parts[0] == "-" && len(parts) == 1 { 142 return "", false, nil, nil 143 } 144 if parts[0] != "" && !validPropertyName(parts[0]) { 145 err = fmt.Errorf("datastore: struct tag has invalid property name: %q", parts[0]) 146 return "", false, nil, err 147 } 148 149 var opts saveOpts 150 if len(parts) > 1 { 151 for _, p := range parts[1:] { 152 switch p { 153 case "flatten": 154 opts.flatten = true 155 case "omitempty": 156 opts.omitEmpty = true 157 case "noindex": 158 opts.noIndex = true 159 default: 160 err = fmt.Errorf("datastore: struct tag has invalid option: %q", p) 161 return "", false, nil, err 162 } 163 } 164 other = opts 165 } 166 return parts[0], true, other, nil 167} 168 169func validateType(t reflect.Type) error { 170 if t.Kind() != reflect.Struct { 171 return fmt.Errorf("datastore: validate called with non-struct type %s", t) 172 } 173 174 return validateChildType(t, "", false, false, map[reflect.Type]bool{}) 175} 176 177// validateChildType is a recursion helper func for validateType 178func validateChildType(t reflect.Type, fieldName string, flatten, prevSlice bool, prevTypes map[reflect.Type]bool) error { 179 if prevTypes[t] { 180 return nil 181 } 182 prevTypes[t] = true 183 184 switch t.Kind() { 185 case reflect.Slice: 186 if flatten && prevSlice { 187 return fmt.Errorf("datastore: flattening nested structs leads to a slice of slices: field %q", fieldName) 188 } 189 return validateChildType(t.Elem(), fieldName, flatten, true, prevTypes) 190 case reflect.Struct: 191 if t == typeOfTime || t == typeOfGeoPoint { 192 return nil 193 } 194 195 for i := 0; i < t.NumField(); i++ { 196 f := t.Field(i) 197 198 // If a named field is unexported, ignore it. An anonymous 199 // unexported field is processed, because it may contain 200 // exported fields, which are visible. 201 exported := (f.PkgPath == "") 202 if !exported && !f.Anonymous { 203 continue 204 } 205 206 _, keep, other, err := parseTag(f.Tag) 207 // Handle error from parseTag now instead of later (in cache.Fields call). 208 if err != nil { 209 return err 210 } 211 if !keep { 212 continue 213 } 214 if other != nil { 215 opts := other.(saveOpts) 216 flatten = flatten || opts.flatten 217 } 218 if err := validateChildType(f.Type, f.Name, flatten, prevSlice, prevTypes); err != nil { 219 return err 220 } 221 } 222 case reflect.Ptr: 223 if t == typeOfKeyPtr { 224 return nil 225 } 226 return validateChildType(t.Elem(), fieldName, flatten, prevSlice, prevTypes) 227 } 228 return nil 229} 230 231// isLeafType determines whether or not a type is a 'leaf type' 232// and should not be recursed into, but considered one field. 233func isLeafType(t reflect.Type) bool { 234 return t == typeOfTime || t == typeOfGeoPoint 235} 236 237// structCache collects the structs whose fields have already been calculated. 238var structCache = fields.NewCache(parseTag, validateType, isLeafType) 239 240// structPLS adapts a struct to be a PropertyLoadSaver. 241type structPLS struct { 242 v reflect.Value 243 codec fields.List 244} 245 246// newStructPLS returns a structPLS, which implements the 247// PropertyLoadSaver interface, for the struct pointer p. 248func newStructPLS(p interface{}) (*structPLS, error) { 249 v := reflect.ValueOf(p) 250 if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct { 251 return nil, ErrInvalidEntityType 252 } 253 v = v.Elem() 254 f, err := structCache.Fields(v.Type()) 255 if err != nil { 256 return nil, err 257 } 258 return &structPLS{v, f}, nil 259} 260 261// LoadStruct loads the properties from p to dst. 262// dst must be a struct pointer. 263// 264// The values of dst's unmatched struct fields are not modified, 265// and matching slice-typed fields are not reset before appending to 266// them. In particular, it is recommended to pass a pointer to a zero 267// valued struct on each LoadStruct call. 268func LoadStruct(dst interface{}, p []Property) error { 269 x, err := newStructPLS(dst) 270 if err != nil { 271 return err 272 } 273 return x.Load(p) 274} 275 276// SaveStruct returns the properties from src as a slice of Properties. 277// src must be a struct pointer. 278func SaveStruct(src interface{}) ([]Property, error) { 279 x, err := newStructPLS(src) 280 if err != nil { 281 return nil, err 282 } 283 return x.Save() 284} 285 286// plsForLoad tries to convert v to a PropertyLoadSaver. 287// If successful, plsForLoad returns a settable v as a PropertyLoadSaver. 288// 289// plsForLoad is intended to be used with nested struct fields which 290// may implement PropertyLoadSaver. 291// 292// v must be settable. 293func plsForLoad(v reflect.Value) (PropertyLoadSaver, error) { 294 var nilPtr bool 295 if v.Kind() == reflect.Ptr && v.IsNil() { 296 nilPtr = true 297 v.Set(reflect.New(v.Type().Elem())) 298 } 299 300 vpls, err := pls(v) 301 if nilPtr && (vpls == nil || err != nil) { 302 // unset v 303 v.Set(reflect.Zero(v.Type())) 304 } 305 306 return vpls, err 307} 308 309// plsForSave tries to convert v to a PropertyLoadSaver. 310// If successful, plsForSave returns v as a PropertyLoadSaver. 311// 312// plsForSave is intended to be used with nested struct fields which 313// may implement PropertyLoadSaver. 314// 315// v must be settable. 316func plsForSave(v reflect.Value) (PropertyLoadSaver, error) { 317 switch v.Kind() { 318 case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface, reflect.Chan, reflect.Func: 319 // If v is nil, return early. v contains no data to save. 320 if v.IsNil() { 321 return nil, nil 322 } 323 } 324 325 return pls(v) 326} 327 328func pls(v reflect.Value) (PropertyLoadSaver, error) { 329 if v.Kind() != reflect.Ptr { 330 if _, ok := v.Interface().(PropertyLoadSaver); ok { 331 return nil, fmt.Errorf("datastore: PropertyLoadSaver methods must be implemented on a pointer to %T", v.Interface()) 332 } 333 334 v = v.Addr() 335 } 336 337 vpls, _ := v.Interface().(PropertyLoadSaver) 338 return vpls, nil 339} 340