1// Copyright 2016 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15/* 16Package datastore provides a client for Google Cloud Datastore. 17 18See https://godoc.org/cloud.google.com/go for authentication, timeouts, 19connection pooling and similar aspects of this package. 20 21 22Basic Operations 23 24Entities are the unit of storage and are associated with a key. A key 25consists of an optional parent key, a string application ID, a string kind 26(also known as an entity type), and either a StringID or an IntID. A 27StringID is also known as an entity name or key name. 28 29It is valid to create a key with a zero StringID and a zero IntID; this is 30called an incomplete key, and does not refer to any saved entity. Putting an 31entity into the datastore under an incomplete key will cause a unique key 32to be generated for that entity, with a non-zero IntID. 33 34An entity's contents are a mapping from case-sensitive field names to values. 35Valid value types are: 36 - Signed integers (int, int8, int16, int32 and int64) 37 - bool 38 - string 39 - float32 and float64 40 - []byte (up to 1 megabyte in length) 41 - Any type whose underlying type is one of the above predeclared types 42 - *Key 43 - GeoPoint 44 - time.Time (stored with microsecond precision, retrieved as local time) 45 - Structs whose fields are all valid value types 46 - Pointers to structs whose fields are all valid value types 47 - Slices of any of the above 48 - Pointers to a signed integer, bool, string, float32, or float64 49 50Slices of structs are valid, as are structs that contain slices. 51 52The Get and Put functions load and save an entity's contents. An entity's 53contents are typically represented by a struct pointer. 54 55Example code: 56 57 type Entity struct { 58 Value string 59 } 60 61 func main() { 62 ctx := context.Background() 63 64 // Create a datastore client. In a typical application, you would create 65 // a single client which is reused for every datastore operation. 66 dsClient, err := datastore.NewClient(ctx, "my-project") 67 if err != nil { 68 // Handle error. 69 } 70 defer dsClient.Close() 71 72 k := datastore.NameKey("Entity", "stringID", nil) 73 e := new(Entity) 74 if err := dsClient.Get(ctx, k, e); err != nil { 75 // Handle error. 76 } 77 78 old := e.Value 79 e.Value = "Hello World!" 80 81 if _, err := dsClient.Put(ctx, k, e); err != nil { 82 // Handle error. 83 } 84 85 fmt.Printf("Updated value from %q to %q\n", old, e.Value) 86 } 87 88GetMulti, PutMulti and DeleteMulti are batch versions of the Get, Put and 89Delete functions. They take a []*Key instead of a *Key, and may return a 90datastore.MultiError when encountering partial failure. 91 92Mutate generalizes PutMulti and DeleteMulti to a sequence of any Datastore 93mutations. It takes a series of mutations created with NewInsert, NewUpdate, 94NewUpsert and NewDelete and applies them. Datastore.Mutate uses 95non-transactional mode; if atomicity is required, use Transaction.Mutate 96instead. 97 98 99Properties 100 101An entity's contents can be represented by a variety of types. These are 102typically struct pointers, but can also be any type that implements the 103PropertyLoadSaver interface. If using a struct pointer, you do not have to 104explicitly implement the PropertyLoadSaver interface; the datastore will 105automatically convert via reflection. If a struct pointer does implement 106PropertyLoadSaver then those methods will be used in preference to the default 107behavior for struct pointers. Struct pointers are more strongly typed and are 108easier to use; PropertyLoadSavers are more flexible. 109 110The actual types passed do not have to match between Get and Put calls or even 111across different calls to datastore. It is valid to put a *PropertyList and 112get that same entity as a *myStruct, or put a *myStruct0 and get a *myStruct1. 113Conceptually, any entity is saved as a sequence of properties, and is loaded 114into the destination value on a property-by-property basis. When loading into 115a struct pointer, an entity that cannot be completely represented (such as a 116missing field) will result in an ErrFieldMismatch error but it is up to the 117caller whether this error is fatal, recoverable or ignorable. 118 119By default, for struct pointers, all properties are potentially indexed, and 120the property name is the same as the field name (and hence must start with an 121upper case letter). 122 123Fields may have a `datastore:"name,options"` tag. The tag name is the 124property name, which must be one or more valid Go identifiers joined by ".", 125but may start with a lower case letter. An empty tag name means to just use the 126field name. A "-" tag name means that the datastore will ignore that field. 127 128The only valid options are "omitempty", "noindex" and "flatten". 129 130If the options include "omitempty" and the value of the field is a zero value, 131then the field will be omitted on Save. Zero values are best defined in the 132golang spec (https://golang.org/ref/spec#The_zero_value). Struct field values 133will never be empty, except for nil pointers. 134 135If options include "noindex" then the field will not be indexed. All fields 136are indexed by default. Strings or byte slices longer than 1500 bytes cannot 137be indexed; fields used to store long strings and byte slices must be tagged 138with "noindex" or they will cause Put operations to fail. 139 140For a nested struct field, the options may also include "flatten". This 141indicates that the immediate fields and any nested substruct fields of the 142nested struct should be flattened. See below for examples. 143 144To use multiple options together, separate them by a comma. 145The order does not matter. 146 147If the options is "" then the comma may be omitted. 148 149Example code: 150 151 // A and B are renamed to a and b. 152 // A, C and J are not indexed. 153 // D's tag is equivalent to having no tag at all (E). 154 // I is ignored entirely by the datastore. 155 // J has tag information for both the datastore and json packages. 156 type TaggedStruct struct { 157 A int `datastore:"a,noindex"` 158 B int `datastore:"b"` 159 C int `datastore:",noindex"` 160 D int `datastore:""` 161 E int 162 I int `datastore:"-"` 163 J int `datastore:",noindex" json:"j"` 164 } 165 166 167Slice Fields 168 169A field of slice type corresponds to a Datastore array property, except for []byte, which corresponds 170to a Datastore blob. 171 172Zero-length slice fields are not saved. Slice fields of length 1 or greater are saved 173as Datastore arrays. When a zero-length Datastore array is loaded into a slice field, 174the slice field remains unchanged. 175 176If a non-array value is loaded into a slice field, the result will be a slice with 177one element, containing the value. 178 179Loading Nulls 180 181Loading a Datastore Null into a basic type (int, float, etc.) results in a zero value. 182Loading a Null into a slice of basic type results in a slice of size 1 containing the zero value. 183Loading a Null into a pointer field results in nil. 184Loading a Null into a field of struct type is an error. 185 186Pointer Fields 187 188A struct field can be a pointer to a signed integer, floating-point number, string or 189bool. Putting a non-nil pointer will store its dereferenced value. Putting a nil 190pointer will store a Datastore Null property, unless the field is marked omitempty, 191in which case no property will be stored. 192 193Loading a Null into a pointer field sets the pointer to nil. Loading any other value 194allocates new storage with the value, and sets the field to point to it. 195 196 197Key Field 198 199If the struct contains a *datastore.Key field tagged with the name "__key__", 200its value will be ignored on Put. When reading the Entity back into the Go struct, 201the field will be populated with the *datastore.Key value used to query for 202the Entity. 203 204Example code: 205 206 type MyEntity struct { 207 A int 208 K *datastore.Key `datastore:"__key__"` 209 } 210 211 func main() { 212 ctx := context.Background() 213 dsClient, err := datastore.NewClient(ctx, "my-project") 214 if err != nil { 215 // Handle error. 216 } 217 defer dsClient.Close() 218 219 k := datastore.NameKey("Entity", "stringID", nil) 220 e := MyEntity{A: 12} 221 if _, err := dsClient.Put(ctx, k, &e); err != nil { 222 // Handle error. 223 } 224 225 var entities []MyEntity 226 q := datastore.NewQuery("Entity").Filter("A =", 12).Limit(1) 227 if _, err := dsClient.GetAll(ctx, q, &entities); err != nil { 228 // Handle error 229 } 230 231 log.Println(entities[0]) 232 // Prints {12 /Entity,stringID} 233 } 234 235 236 237Structured Properties 238 239If the struct pointed to contains other structs, then the nested or embedded 240structs are themselves saved as Entity values. For example, given these definitions: 241 242 type Inner struct { 243 W int32 244 X string 245 } 246 247 type Outer struct { 248 I Inner 249 } 250 251then an Outer would have one property, Inner, encoded as an Entity value. 252 253Note: embedded struct fields must be named to be encoded as an Entity. For 254example, in case of a type Outer with an embedded field Inner: 255 256 type Outer struct { 257 Inner 258 } 259 260all the Inner struct fields will be treated as fields of Outer itself. 261 262If an outer struct is tagged "noindex" then all of its implicit flattened 263fields are effectively "noindex". 264 265If the Inner struct contains a *Key field with the name "__key__", like so: 266 267 type Inner struct { 268 W int32 269 X string 270 K *datastore.Key `datastore:"__key__"` 271 } 272 273 type Outer struct { 274 I Inner 275 } 276 277then the value of K will be used as the Key for Inner, represented 278as an Entity value in datastore. 279 280If any nested struct fields should be flattened, instead of encoded as 281Entity values, the nested struct field should be tagged with the "flatten" 282option. For example, given the following: 283 284 type Inner1 struct { 285 W int32 286 X string 287 } 288 289 type Inner2 struct { 290 Y float64 291 } 292 293 type Inner3 struct { 294 Z bool 295 } 296 297 type Inner4 struct { 298 WW int 299 } 300 301 type Inner5 struct { 302 X Inner4 303 } 304 305 type Outer struct { 306 A int16 307 I []Inner1 `datastore:",flatten"` 308 J Inner2 `datastore:",flatten"` 309 K Inner5 `datastore:",flatten"` 310 Inner3 `datastore:",flatten"` 311 } 312 313an Outer's properties would be equivalent to those of: 314 315 type OuterEquivalent struct { 316 A int16 317 IDotW []int32 `datastore:"I.W"` 318 IDotX []string `datastore:"I.X"` 319 JDotY float64 `datastore:"J.Y"` 320 KDotXDotWW int `datastore:"K.X.WW"` 321 Z bool 322 } 323 324Note that the "flatten" option cannot be used for Entity value fields or 325PropertyLoadSaver implementers. The server will reject any dotted field names 326for an Entity value. 327 328 329The PropertyLoadSaver Interface 330 331An entity's contents can also be represented by any type that implements the 332PropertyLoadSaver interface. This type may be a struct pointer, but it does 333not have to be. The datastore package will call Load when getting the entity's 334contents, and Save when putting the entity's contents. 335Possible uses include deriving non-stored fields, verifying fields, or indexing 336a field only if its value is positive. 337 338Example code: 339 340 type CustomPropsExample struct { 341 I, J int 342 // Sum is not stored, but should always be equal to I + J. 343 Sum int `datastore:"-"` 344 } 345 346 func (x *CustomPropsExample) Load(ps []datastore.Property) error { 347 // Load I and J as usual. 348 if err := datastore.LoadStruct(x, ps); err != nil { 349 return err 350 } 351 // Derive the Sum field. 352 x.Sum = x.I + x.J 353 return nil 354 } 355 356 func (x *CustomPropsExample) Save() ([]datastore.Property, error) { 357 // Validate the Sum field. 358 if x.Sum != x.I + x.J { 359 return nil, errors.New("CustomPropsExample has inconsistent sum") 360 } 361 // Save I and J as usual. The code below is equivalent to calling 362 // "return datastore.SaveStruct(x)", but is done manually for 363 // demonstration purposes. 364 return []datastore.Property{ 365 { 366 Name: "I", 367 Value: int64(x.I), 368 }, 369 { 370 Name: "J", 371 Value: int64(x.J), 372 }, 373 }, nil 374 } 375 376The *PropertyList type implements PropertyLoadSaver, and can therefore hold an 377arbitrary entity's contents. 378 379The KeyLoader Interface 380 381If a type implements the PropertyLoadSaver interface, it may 382also want to implement the KeyLoader interface. 383The KeyLoader interface exists to allow implementations of PropertyLoadSaver 384to also load an Entity's Key into the Go type. This type may be a struct 385pointer, but it does not have to be. The datastore package will call LoadKey 386when getting the entity's contents, after calling Load. 387 388Example code: 389 390 type WithKeyExample struct { 391 I int 392 Key *datastore.Key 393 } 394 395 func (x *WithKeyExample) LoadKey(k *datastore.Key) error { 396 x.Key = k 397 return nil 398 } 399 400 func (x *WithKeyExample) Load(ps []datastore.Property) error { 401 // Load I as usual. 402 return datastore.LoadStruct(x, ps) 403 } 404 405 func (x *WithKeyExample) Save() ([]datastore.Property, error) { 406 // Save I as usual. 407 return datastore.SaveStruct(x) 408 } 409 410To load a Key into a struct which does not implement the PropertyLoadSaver 411interface, see the "Key Field" section above. 412 413 414Queries 415 416Queries retrieve entities based on their properties or key's ancestry. Running 417a query yields an iterator of results: either keys or (key, entity) pairs. 418Queries are re-usable and it is safe to call Query.Run from concurrent 419goroutines. Iterators are not safe for concurrent use. 420 421Queries are immutable, and are either created by calling NewQuery, or derived 422from an existing query by calling a method like Filter or Order that returns a 423new query value. A query is typically constructed by calling NewQuery followed 424by a chain of zero or more such methods. These methods are: 425 - Ancestor and Filter constrain the entities returned by running a query. 426 - Order affects the order in which they are returned. 427 - Project constrains the fields returned. 428 - Distinct de-duplicates projected entities. 429 - KeysOnly makes the iterator return only keys, not (key, entity) pairs. 430 - Start, End, Offset and Limit define which sub-sequence of matching entities 431 to return. Start and End take cursors, Offset and Limit take integers. Start 432 and Offset affect the first result, End and Limit affect the last result. 433 If both Start and Offset are set, then the offset is relative to Start. 434 If both End and Limit are set, then the earliest constraint wins. Limit is 435 relative to Start+Offset, not relative to End. As a special case, a 436 negative limit means unlimited. 437 438Example code: 439 440 type Widget struct { 441 Description string 442 Price int 443 } 444 445 func printWidgets(ctx context.Context, client *datastore.Client) { 446 q := datastore.NewQuery("Widget"). 447 Filter("Price <", 1000). 448 Order("-Price") 449 450 t := client.Run(ctx, q) 451 for { 452 var x Widget 453 key, err := t.Next(&x) 454 if err == iterator.Done { 455 break 456 } 457 if err != nil { 458 // Handle error. 459 } 460 fmt.Printf("Key=%v\nWidget=%#v\n\n", key, x) 461 } 462 } 463 464 465Transactions 466 467Client.RunInTransaction runs a function in a transaction. 468 469Example code: 470 471 type Counter struct { 472 Count int 473 } 474 475 func incCount(ctx context.Context, client *datastore.Client) { 476 var count int 477 key := datastore.NameKey("Counter", "singleton", nil) 478 _, err := client.RunInTransaction(ctx, func(tx *datastore.Transaction) error { 479 var x Counter 480 if err := tx.Get(key, &x); err != nil && err != datastore.ErrNoSuchEntity { 481 return err 482 } 483 x.Count++ 484 if _, err := tx.Put(key, &x); err != nil { 485 return err 486 } 487 count = x.Count 488 return nil 489 }) 490 if err != nil { 491 // Handle error. 492 } 493 // The value of count is only valid once the transaction is successful 494 // (RunInTransaction has returned nil). 495 fmt.Printf("Count=%d\n", count) 496 } 497 498Pass the ReadOnly option to RunInTransaction if your transaction is used only for Get, 499GetMulti or queries. Read-only transactions are more efficient. 500 501Google Cloud Datastore Emulator 502 503This package supports the Cloud Datastore emulator, which is useful for testing and 504development. Environment variables are used to indicate that datastore traffic should be 505directed to the emulator instead of the production Datastore service. 506 507To install and set up the emulator and its environment variables, see the documentation 508at https://cloud.google.com/datastore/docs/tools/datastore-emulator. 509*/ 510package datastore // import "cloud.google.com/go/datastore" 511