1/* 2Copyright 2019 Google LLC 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17/* 18This file holds tests for the in-memory fake for comparing it against a real Cloud Spanner. 19 20By default it uses the Spanner client against the in-memory fake; set the 21-test_db flag to "projects/P/instances/I/databases/D" to make it use a real 22Cloud Spanner database instead. You may need to provide -timeout=5m too. 23*/ 24 25package spannertest 26 27import ( 28 "context" 29 "flag" 30 "reflect" 31 "sort" 32 "testing" 33 "time" 34 35 "cloud.google.com/go/civil" 36 "cloud.google.com/go/spanner" 37 dbadmin "cloud.google.com/go/spanner/admin/database/apiv1" 38 "google.golang.org/api/iterator" 39 "google.golang.org/api/option" 40 "google.golang.org/grpc" 41 "google.golang.org/grpc/codes" 42 "google.golang.org/grpc/status" 43 44 dbadminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" 45 spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 46 structpb "google.golang.org/protobuf/types/known/structpb" 47) 48 49var testDBFlag = flag.String("test_db", "", "Fully-qualified database name to test against; empty means use an in-memory fake.") 50 51func dbName() string { 52 if *testDBFlag != "" { 53 return *testDBFlag 54 } 55 return "projects/fake-proj/instances/fake-instance/databases/fake-db" 56} 57 58func makeClient(t *testing.T) (*spanner.Client, *dbadmin.DatabaseAdminClient, func()) { 59 // Despite the docs, this context is also used for auth, 60 // so it needs to be long-lived. 61 ctx := context.Background() 62 63 if *testDBFlag != "" { 64 t.Logf("Using real Spanner DB %s", *testDBFlag) 65 dialOpt := option.WithGRPCDialOption(grpc.WithTimeout(5 * time.Second)) 66 client, err := spanner.NewClient(ctx, *testDBFlag, dialOpt) 67 if err != nil { 68 t.Fatalf("Connecting to %s: %v", *testDBFlag, err) 69 } 70 adminClient, err := dbadmin.NewDatabaseAdminClient(ctx, dialOpt) 71 if err != nil { 72 client.Close() 73 t.Fatalf("Connecting DB admin client: %v", err) 74 } 75 return client, adminClient, func() { client.Close(); adminClient.Close() } 76 } 77 78 // Don't use SPANNER_EMULATOR_HOST because we need the raw connection for 79 // the database admin client anyway. 80 81 t.Logf("Using in-memory fake Spanner DB") 82 srv, err := NewServer("localhost:0") 83 if err != nil { 84 t.Fatalf("Starting in-memory fake: %v", err) 85 } 86 srv.SetLogger(t.Logf) 87 dialCtx, cancel := context.WithTimeout(ctx, 1*time.Second) 88 defer cancel() 89 conn, err := grpc.DialContext(dialCtx, srv.Addr, grpc.WithInsecure()) 90 if err != nil { 91 srv.Close() 92 t.Fatalf("Dialing in-memory fake: %v", err) 93 } 94 client, err := spanner.NewClient(ctx, dbName(), option.WithGRPCConn(conn)) 95 if err != nil { 96 srv.Close() 97 t.Fatalf("Connecting to in-memory fake: %v", err) 98 } 99 adminClient, err := dbadmin.NewDatabaseAdminClient(ctx, option.WithGRPCConn(conn)) 100 if err != nil { 101 srv.Close() 102 t.Fatalf("Connecting to in-memory fake DB admin: %v", err) 103 } 104 return client, adminClient, func() { 105 client.Close() 106 adminClient.Close() 107 conn.Close() 108 srv.Close() 109 } 110} 111 112func TestIntegration_SpannerBasics(t *testing.T) { 113 client, adminClient, cleanup := makeClient(t) 114 defer cleanup() 115 116 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 117 defer cancel() 118 119 // Do a trivial query to verify the connection works. 120 it := client.Single().Query(ctx, spanner.NewStatement("SELECT 1")) 121 row, err := it.Next() 122 if err != nil { 123 t.Fatalf("Getting first row of trivial query: %v", err) 124 } 125 var value int64 126 if err := row.Column(0, &value); err != nil { 127 t.Fatalf("Decoding row data from trivial query: %v", err) 128 } 129 if value != 1 { 130 t.Errorf("Trivial query gave %d, want 1", value) 131 } 132 // There shouldn't be a next row. 133 _, err = it.Next() 134 if err != iterator.Done { 135 t.Errorf("Reading second row of trivial query gave %v, want iterator.Done", err) 136 } 137 it.Stop() 138 139 // Drop any previous test table/index, and make a fresh one in a few stages. 140 const tableName = "Characters" 141 err = updateDDL(t, adminClient, "DROP INDEX AgeIndex") 142 // NotFound is an acceptable failure mode here. 143 if st, _ := status.FromError(err); st.Code() == codes.NotFound { 144 err = nil 145 } 146 if err != nil { 147 t.Fatalf("Dropping old index: %v", err) 148 } 149 dropTable(t, adminClient, tableName) 150 err = updateDDL(t, adminClient, 151 `CREATE TABLE `+tableName+` ( 152 FirstName STRING(20) NOT NULL, 153 LastName STRING(20) NOT NULL, 154 Alias STRING(MAX), 155 ) PRIMARY KEY (FirstName, LastName)`) 156 if err != nil { 157 t.Fatalf("Setting up fresh table: %v", err) 158 } 159 err = updateDDL(t, adminClient, 160 `ALTER TABLE `+tableName+` ADD COLUMN Age INT64`, 161 `CREATE INDEX AgeIndex ON `+tableName+` (Age DESC)`) 162 if err != nil { 163 t.Fatalf("Adding new column: %v", err) 164 } 165 166 // Insert some data. 167 _, err = client.Apply(ctx, []*spanner.Mutation{ 168 spanner.Insert(tableName, 169 []string{"FirstName", "LastName", "Alias", "Age"}, 170 []interface{}{"Steve", "Rogers", "Captain America", 101}), 171 spanner.Insert(tableName, 172 []string{"LastName", "FirstName", "Age", "Alias"}, 173 []interface{}{"Romanoff", "Natasha", 35, "Black Widow"}), 174 spanner.Insert(tableName, 175 []string{"Age", "Alias", "FirstName", "LastName"}, 176 []interface{}{49, "Iron Man", "Tony", "Stark"}), 177 spanner.Insert(tableName, 178 []string{"FirstName", "Alias", "LastName"}, // no Age 179 []interface{}{"Clark", "Superman", "Kent"}), 180 // Two rows with the same value in one column, 181 // but with distinct primary keys. 182 spanner.Insert(tableName, 183 []string{"FirstName", "LastName", "Alias"}, 184 []interface{}{"Peter", "Parker", "Spider-Man"}), 185 spanner.Insert(tableName, 186 []string{"FirstName", "LastName", "Alias"}, 187 []interface{}{"Peter", "Quill", "Star-Lord"}), 188 }) 189 if err != nil { 190 t.Fatalf("Applying mutations: %v", err) 191 } 192 193 // Delete some data. 194 _, err = client.Apply(ctx, []*spanner.Mutation{ 195 // Whoops. DC, not MCU. 196 spanner.Delete(tableName, spanner.Key{"Clark", "Kent"}), 197 }) 198 if err != nil { 199 t.Fatalf("Applying mutations: %v", err) 200 } 201 202 // Read a single row. 203 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Alias", "Age"}) 204 if err != nil { 205 t.Fatalf("Reading single row: %v", err) 206 } 207 var alias string 208 var age int64 209 if err := row.Columns(&alias, &age); err != nil { 210 t.Fatalf("Decoding single row: %v", err) 211 } 212 if alias != "Iron Man" || age != 49 { 213 t.Errorf(`Single row read gave (%q, %d), want ("Iron Man", 49)`, alias, age) 214 } 215 216 // Read all rows, and do a local age sum. 217 rows := client.Single().Read(ctx, tableName, spanner.AllKeys(), []string{"Age"}) 218 var ageSum int64 219 err = rows.Do(func(row *spanner.Row) error { 220 var age spanner.NullInt64 221 if err := row.Columns(&age); err != nil { 222 return err 223 } 224 if age.Valid { 225 ageSum += age.Int64 226 } 227 return nil 228 }) 229 if err != nil { 230 t.Fatalf("Iterating over all row read: %v", err) 231 } 232 if want := int64(101 + 35 + 49); ageSum != want { 233 t.Errorf("Age sum after iterating over all rows = %d, want %d", ageSum, want) 234 } 235 236 // Do a more complex query to find the aliases of the two oldest non-centenarian characters. 237 stmt := spanner.NewStatement(`SELECT Alias FROM ` + tableName + ` WHERE Age < @ageLimit AND Alias IS NOT NULL ORDER BY Age DESC LIMIT @limit`) 238 stmt.Params = map[string]interface{}{ 239 "ageLimit": 100, 240 "limit": 2, 241 } 242 rows = client.Single().Query(ctx, stmt) 243 var oldFolk []string 244 err = rows.Do(func(row *spanner.Row) error { 245 var alias string 246 if err := row.Columns(&alias); err != nil { 247 return err 248 } 249 oldFolk = append(oldFolk, alias) 250 return nil 251 }) 252 if err != nil { 253 t.Fatalf("Iterating over complex query: %v", err) 254 } 255 if want := []string{"Iron Man", "Black Widow"}; !reflect.DeepEqual(oldFolk, want) { 256 t.Errorf("Complex query results = %v, want %v", oldFolk, want) 257 } 258 259 // Apply an update. 260 _, err = client.Apply(ctx, []*spanner.Mutation{ 261 spanner.Update(tableName, 262 []string{"FirstName", "LastName", "Age"}, 263 []interface{}{"Steve", "Rogers", 102}), 264 }) 265 if err != nil { 266 t.Fatalf("Applying mutations: %v", err) 267 } 268 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Steve", "Rogers"}, []string{"Age"}) 269 if err != nil { 270 t.Fatalf("Reading single row: %v", err) 271 } 272 if err := row.Columns(&age); err != nil { 273 t.Fatalf("Decoding single row: %v", err) 274 } 275 if age != 102 { 276 t.Errorf("After updating Captain America, age = %d, want 102", age) 277 } 278 279 // Do a query where the result type isn't deducible from the first row. 280 stmt = spanner.NewStatement(`SELECT Age FROM ` + tableName + ` WHERE FirstName = "Peter"`) 281 rows = client.Single().Query(ctx, stmt) 282 var nullPeters int 283 err = rows.Do(func(row *spanner.Row) error { 284 var age spanner.NullInt64 285 if err := row.Column(0, &age); err != nil { 286 return err 287 } 288 if age.Valid { 289 t.Errorf("Got non-NULL Age %d for a Peter", age.Int64) 290 } else { 291 nullPeters++ 292 } 293 return nil 294 }) 295 if err != nil { 296 t.Fatalf("Counting Peters with NULL Ages: %v", err) 297 } 298 if nullPeters != 2 { 299 t.Errorf("Found %d Peters with NULL Ages, want 2", nullPeters) 300 } 301 302 // Check handling of array types. 303 err = updateDDL(t, adminClient, `ALTER TABLE `+tableName+` ADD COLUMN Allies ARRAY<STRING(20)>`) 304 if err != nil { 305 t.Fatalf("Adding new array-typed column: %v", err) 306 } 307 _, err = client.Apply(ctx, []*spanner.Mutation{ 308 spanner.Update(tableName, 309 []string{"FirstName", "LastName", "Allies"}, 310 []interface{}{"Steve", "Rogers", []string{}}), 311 spanner.Update(tableName, 312 []string{"FirstName", "LastName", "Allies"}, 313 []interface{}{"Tony", "Stark", []string{"Black Widow", "Spider-Man"}}), 314 }) 315 if err != nil { 316 t.Fatalf("Applying mutations: %v", err) 317 } 318 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Allies"}) 319 if err != nil { 320 t.Fatalf("Reading row with array value: %v", err) 321 } 322 var names []string 323 if err := row.Column(0, &names); err != nil { 324 t.Fatalf("Unpacking array value: %v", err) 325 } 326 if want := []string{"Black Widow", "Spider-Man"}; !reflect.DeepEqual(names, want) { 327 t.Errorf("Read array value: got %q, want %q", names, want) 328 } 329 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Steve", "Rogers"}, []string{"Allies"}) 330 if err != nil { 331 t.Fatalf("Reading row with empty array value: %v", err) 332 } 333 if err := row.Column(0, &names); err != nil { 334 t.Fatalf("Unpacking empty array value: %v", err) 335 } 336 if len(names) > 0 { 337 t.Errorf("Read empty array value: got %q", names) 338 } 339 340 // Exercise commit timestamp. 341 err = updateDDL(t, adminClient, `ALTER TABLE `+tableName+` ADD COLUMN Updated TIMESTAMP OPTIONS (allow_commit_timestamp=true)`) 342 if err != nil { 343 t.Fatalf("Adding new timestamp column: %v", err) 344 } 345 cts, err := client.Apply(ctx, []*spanner.Mutation{ 346 // Update one row in place. 347 spanner.Update(tableName, 348 []string{"FirstName", "LastName", "Allies", "Updated"}, 349 []interface{}{"Tony", "Stark", []string{"Spider-Man", "Professor Hulk"}, spanner.CommitTimestamp}), 350 }) 351 if err != nil { 352 t.Fatalf("Applying mutations: %v", err) 353 } 354 cts = cts.In(time.UTC) 355 if d := time.Since(cts); d < 0 || d > 10*time.Second { 356 t.Errorf("Commit timestamp %v not in the last 10s", cts) 357 } 358 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Allies", "Updated"}) 359 if err != nil { 360 t.Fatalf("Reading single row: %v", err) 361 } 362 var gotAllies []string 363 var gotUpdated time.Time 364 if err := row.Columns(&gotAllies, &gotUpdated); err != nil { 365 t.Fatalf("Decoding single row: %v", err) 366 } 367 if want := []string{"Spider-Man", "Professor Hulk"}; !reflect.DeepEqual(gotAllies, want) { 368 t.Errorf("After updating Iron Man, allies = %+v, want %+v", gotAllies, want) 369 } 370 if !gotUpdated.Equal(cts) { 371 t.Errorf("After updating Iron Man, updated = %v, want %v", gotUpdated, cts) 372 } 373 374 // Check if IN UNNEST works. 375 stmt = spanner.NewStatement(`SELECT Age FROM ` + tableName + ` WHERE FirstName IN UNNEST(@list)`) 376 stmt.Params = map[string]interface{}{ 377 "list": []string{"Peter", "Steve"}, 378 } 379 rows = client.Single().Query(ctx, stmt) 380 var ages []int64 381 err = rows.Do(func(row *spanner.Row) error { 382 var age spanner.NullInt64 383 if err := row.Column(0, &age); err != nil { 384 return err 385 } 386 ages = append(ages, age.Int64) // zero for NULL 387 return nil 388 }) 389 if err != nil { 390 t.Fatalf("Getting ages using IN UNNEST: %v", err) 391 } 392 sort.Slice(ages, func(i, j int) bool { return ages[i] < ages[j] }) 393 wantAges := []int64{0, 0, 102} // Peter Parker, Peter Quill, Steve Rogers (modified) 394 if !reflect.DeepEqual(ages, wantAges) { 395 t.Errorf("Query with IN UNNEST gave wrong ages: got %+v, want %+v", ages, wantAges) 396 } 397} 398 399func TestIntegration_ReadsAndQueries(t *testing.T) { 400 client, adminClient, cleanup := makeClient(t) 401 defer cleanup() 402 403 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 404 defer cancel() 405 406 // Drop any old tables. 407 allTables := []string{ 408 "Staff", 409 "PlayerStats", 410 "JoinA", "JoinB", "JoinC", "JoinD", "JoinE", "JoinF", 411 "SomeStrings", 412 } 413 for _, table := range allTables { 414 dropTable(t, adminClient, table) 415 } 416 417 err := updateDDL(t, adminClient, 418 `CREATE TABLE Staff ( 419 Tenure INT64, 420 ID INT64, 421 Name STRING(MAX), 422 Cool BOOL, 423 Height FLOAT64, 424 ) PRIMARY KEY (Name, ID)`) 425 if err != nil { 426 t.Fatal(err) 427 } 428 429 // Insert a subset of columns. 430 _, err = client.Apply(ctx, []*spanner.Mutation{ 431 spanner.Insert("Staff", []string{"ID", "Name", "Tenure", "Height"}, []interface{}{1, "Jack", 10, 1.85}), 432 spanner.Insert("Staff", []string{"ID", "Name", "Tenure", "Height"}, []interface{}{2, "Daniel", 11, 1.83}), 433 }) 434 if err != nil { 435 t.Fatalf("Inserting data: %v", err) 436 } 437 // Insert a different set of columns. 438 _, err = client.Apply(ctx, []*spanner.Mutation{ 439 spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Sam", 3, false, 9, 1.75}), 440 spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Teal'c", 4, true, 8, 1.91}), 441 spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"George", 5, nil, 6, 1.73}), 442 spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Harry", 6, true, nil, nil}), 443 }) 444 if err != nil { 445 t.Fatalf("Inserting more data: %v", err) 446 } 447 // Delete that last one. 448 _, err = client.Apply(ctx, []*spanner.Mutation{ 449 spanner.Delete("Staff", spanner.Key{"Harry", 6}), 450 }) 451 if err != nil { 452 t.Fatalf("Deleting a row: %v", err) 453 } 454 // Turns out this guy isn't cool after all. 455 _, err = client.Apply(ctx, []*spanner.Mutation{ 456 // Missing columns should be left alone. 457 spanner.Update("Staff", []string{"Name", "ID", "Cool"}, []interface{}{"Daniel", 2, false}), 458 }) 459 if err != nil { 460 t.Fatalf("Updating a row: %v", err) 461 } 462 463 // Read some specific keys. 464 ri := client.Single().Read(ctx, "Staff", spanner.KeySets( 465 spanner.Key{"George", 5}, 466 spanner.Key{"Harry", 6}, // Missing key should be silently ignored. 467 spanner.Key{"Sam", 3}, 468 spanner.Key{"George", 5}, // Duplicate key should be silently ignored. 469 ), []string{"Name", "Tenure"}) 470 if err != nil { 471 t.Fatalf("Reading keys: %v", err) 472 } 473 all := mustSlurpRows(t, ri) 474 wantAll := [][]interface{}{ 475 {"George", int64(6)}, 476 {"Sam", int64(9)}, 477 } 478 if !reflect.DeepEqual(all, wantAll) { 479 t.Errorf("Read data by keys wrong.\n got %v\nwant %v", all, wantAll) 480 } 481 // Read the same, but by key range. 482 ri = client.Single().Read(ctx, "Staff", spanner.KeySets( 483 spanner.KeyRange{Start: spanner.Key{"Gabriel"}, End: spanner.Key{"Harpo"}, Kind: spanner.OpenOpen}, 484 spanner.KeyRange{Start: spanner.Key{"Sam", 3}, End: spanner.Key{"Teal'c", 4}, Kind: spanner.ClosedOpen}, 485 ), []string{"Name", "Tenure"}) 486 all = mustSlurpRows(t, ri) 487 if !reflect.DeepEqual(all, wantAll) { 488 t.Errorf("Read data by key ranges wrong.\n got %v\nwant %v", all, wantAll) 489 } 490 // Read a subset of all rows, with a limit. 491 ri = client.Single().ReadWithOptions(ctx, "Staff", spanner.AllKeys(), []string{"Tenure", "Name", "Height"}, 492 &spanner.ReadOptions{Limit: 4}) 493 all = mustSlurpRows(t, ri) 494 wantAll = [][]interface{}{ 495 // Primary key is (Name, ID), so results should come back sorted by Name then ID. 496 {int64(11), "Daniel", 1.83}, 497 {int64(6), "George", 1.73}, 498 {int64(10), "Jack", 1.85}, 499 {int64(9), "Sam", 1.75}, 500 } 501 if !reflect.DeepEqual(all, wantAll) { 502 t.Errorf("ReadAll data wrong.\n got %v\nwant %v", all, wantAll) 503 } 504 505 // Add DATE and TIMESTAMP columns, and populate them with some data. 506 err = updateDDL(t, adminClient, 507 `ALTER TABLE Staff ADD COLUMN FirstSeen DATE`, 508 "ALTER TABLE Staff ADD COLUMN `To` TIMESTAMP", // "TO" is a keyword; needs quoting 509 ) 510 if err != nil { 511 t.Fatalf("Adding columns: %v", err) 512 } 513 _, err = client.Apply(ctx, []*spanner.Mutation{ 514 spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"Jack", 1, "1994-10-28", nil}), 515 spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"Daniel", 2, "1994-10-28", nil}), 516 spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"George", 5, "1997-07-27", "2008-07-29T11:22:43Z"}), 517 }) 518 if err != nil { 519 t.Fatalf("Updating rows: %v", err) 520 } 521 522 // Add some more data, then delete it with a KeyRange. 523 // The queries below ensure that this was all deleted. 524 _, err = client.Apply(ctx, []*spanner.Mutation{ 525 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"01", 1}), 526 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"03", 3}), 527 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"06", 6}), 528 }) 529 if err != nil { 530 t.Fatalf("Inserting data: %v", err) 531 } 532 _, err = client.Apply(ctx, []*spanner.Mutation{ 533 spanner.Delete("Staff", spanner.KeyRange{ 534 /* This should work: 535 Start: spanner.Key{"01", 1}, 536 End: spanner.Key{"9"}, 537 However, that is unimplemented in the production Cloud Spanner, which rejects 538 that: ""For delete ranges, start and limit keys may only differ in the final key part" 539 */ 540 Start: spanner.Key{"01"}, 541 End: spanner.Key{"9"}, 542 Kind: spanner.ClosedOpen, 543 }), 544 }) 545 if err != nil { 546 t.Fatalf("Deleting key range: %v", err) 547 } 548 // Re-add the data and delete with DML. 549 _, err = client.Apply(ctx, []*spanner.Mutation{ 550 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"01", 1}), 551 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"03", 3}), 552 spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"06", 6}), 553 }) 554 if err != nil { 555 t.Fatalf("Inserting data: %v", err) 556 } 557 var n int64 558 _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error { 559 stmt := spanner.NewStatement("DELETE FROM Staff WHERE Name >= @min AND Name < @max") 560 stmt.Params["min"] = "01" 561 stmt.Params["max"] = "07" 562 n, err = tx.Update(ctx, stmt) 563 return err 564 }) 565 if err != nil { 566 t.Fatalf("Deleting with DML: %v", err) 567 } 568 if n != 3 { 569 t.Errorf("Deleting with DML affected %d rows, want 3", n) 570 } 571 572 // Add a BYTES column, and populate it with some data. 573 err = updateDDL(t, adminClient, `ALTER TABLE Staff ADD COLUMN RawBytes BYTES(MAX)`) 574 if err != nil { 575 t.Fatalf("Adding column: %v", err) 576 } 577 _, err = client.Apply(ctx, []*spanner.Mutation{ 578 // bytes {0x01 0x00 0x01} encode as base-64 AQAB. 579 spanner.Update("Staff", []string{"Name", "ID", "RawBytes"}, []interface{}{"Jack", 1, []byte{0x01, 0x00, 0x01}}), 580 }) 581 if err != nil { 582 t.Fatalf("Updating rows: %v", err) 583 } 584 585 // Prepare the sample tables from the Cloud Spanner docs. 586 // https://cloud.google.com/spanner/docs/query-syntax#appendix-a-examples-with-sample-data 587 err = updateDDL(t, adminClient, 588 // TODO: Roster, TeamMascot when we implement JOINs. 589 `CREATE TABLE PlayerStats ( 590 LastName STRING(MAX), 591 OpponentID INT64, 592 PointsScored INT64, 593 ) PRIMARY KEY (LastName, OpponentID)`, // TODO: is this right? 594 // JoinFoo are from https://cloud.google.com/spanner/docs/query-syntax#join_types. 595 // They aren't consistently named in the docs. 596 `CREATE TABLE JoinA ( w INT64, x STRING(MAX) ) PRIMARY KEY (w, x)`, 597 `CREATE TABLE JoinB ( y INT64, z STRING(MAX) ) PRIMARY KEY (y, z)`, 598 `CREATE TABLE JoinC ( x INT64, y STRING(MAX) ) PRIMARY KEY (x, y)`, 599 `CREATE TABLE JoinD ( x INT64, z STRING(MAX) ) PRIMARY KEY (x, z)`, 600 `CREATE TABLE JoinE ( w INT64, x STRING(MAX) ) PRIMARY KEY (w, x)`, 601 `CREATE TABLE JoinF ( y INT64, z STRING(MAX) ) PRIMARY KEY (y, z)`, 602 // Some other test tables. 603 `CREATE TABLE SomeStrings ( i INT64, str STRING(MAX) ) PRIMARY KEY (i)`, 604 ) 605 if err != nil { 606 t.Fatalf("Creating sample tables: %v", err) 607 } 608 _, err = client.Apply(ctx, []*spanner.Mutation{ 609 spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Adams", 51, 3}), 610 spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Buchanan", 77, 0}), 611 spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Coolidge", 77, 1}), 612 spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Adams", 52, 4}), 613 spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Buchanan", 50, 13}), 614 615 spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{1, "a"}), 616 spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{2, "b"}), 617 spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{3, "c"}), 618 spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{3, "d"}), 619 620 spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{2, "k"}), 621 spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{3, "m"}), 622 spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{3, "n"}), 623 spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{4, "p"}), 624 625 // JoinC and JoinD have the same contents as JoinA and JoinB; they have different column names. 626 spanner.Insert("JoinC", []string{"x", "y"}, []interface{}{1, "a"}), 627 spanner.Insert("JoinC", []string{"x", "y"}, []interface{}{2, "b"}), 628 spanner.Insert("JoinC", []string{"x", "y"}, []interface{}{3, "c"}), 629 spanner.Insert("JoinC", []string{"x", "y"}, []interface{}{3, "d"}), 630 631 spanner.Insert("JoinD", []string{"x", "z"}, []interface{}{2, "k"}), 632 spanner.Insert("JoinD", []string{"x", "z"}, []interface{}{3, "m"}), 633 spanner.Insert("JoinD", []string{"x", "z"}, []interface{}{3, "n"}), 634 spanner.Insert("JoinD", []string{"x", "z"}, []interface{}{4, "p"}), 635 636 // JoinE and JoinF are used in the CROSS JOIN test. 637 spanner.Insert("JoinE", []string{"w", "x"}, []interface{}{1, "a"}), 638 spanner.Insert("JoinE", []string{"w", "x"}, []interface{}{2, "b"}), 639 640 spanner.Insert("JoinF", []string{"y", "z"}, []interface{}{2, "c"}), 641 spanner.Insert("JoinF", []string{"y", "z"}, []interface{}{3, "d"}), 642 643 spanner.Insert("SomeStrings", []string{"i", "str"}, []interface{}{0, "afoo"}), 644 spanner.Insert("SomeStrings", []string{"i", "str"}, []interface{}{1, "abar"}), 645 spanner.Insert("SomeStrings", []string{"i", "str"}, []interface{}{2, nil}), 646 spanner.Insert("SomeStrings", []string{"i", "str"}, []interface{}{3, "bbar"}), 647 }) 648 if err != nil { 649 t.Fatalf("Inserting sample data: %v", err) 650 } 651 652 // Do some complex queries. 653 tests := []struct { 654 q string 655 params map[string]interface{} 656 want [][]interface{} 657 }{ 658 { 659 `SELECT 17, "sweet", TRUE AND FALSE, NULL, B"hello"`, 660 nil, 661 [][]interface{}{{int64(17), "sweet", false, nil, []byte("hello")}}, 662 }, 663 // Check handling of NULL values for the IS operator. 664 // There was a bug that returned errors for some of these cases. 665 { 666 `SELECT @x IS TRUE, @x IS NOT TRUE, @x IS FALSE, @x IS NOT FALSE, @x IS NULL, @x IS NOT NULL`, 667 map[string]interface{}{"x": (*bool)(nil)}, 668 [][]interface{}{ 669 {false, true, false, true, true, false}, 670 }, 671 }, 672 // Check handling of bools that might be NULL. 673 // There was a bug where logical operators always returned true/false. 674 { 675 `SELECT @x, NOT @x, @x AND TRUE, @x AND FALSE, @x OR TRUE, @x OR FALSE`, 676 map[string]interface{}{"x": (*bool)(nil)}, 677 [][]interface{}{ 678 // At the time of writing (9 Oct 2020), the docs are wrong for `NULL AND FALSE`; 679 // the production Spanner returns FALSE, which is what we match. 680 {nil, nil, nil, false, true, nil}, 681 }, 682 }, 683 { 684 `SELECT Name FROM Staff WHERE Cool`, 685 nil, 686 [][]interface{}{{"Teal'c"}}, 687 }, 688 { 689 `SELECT ID FROM Staff WHERE Cool IS NOT NULL ORDER BY ID DESC`, 690 nil, 691 [][]interface{}{{int64(4)}, {int64(3)}, {int64(2)}}, 692 }, 693 { 694 `SELECT Name, Tenure FROM Staff WHERE Cool IS NULL OR Cool ORDER BY Name LIMIT 2`, 695 nil, 696 [][]interface{}{ 697 {"George", int64(6)}, 698 {"Jack", int64(10)}, 699 }, 700 }, 701 { 702 `SELECT Name, ID + 100 FROM Staff WHERE @min <= Tenure AND Tenure < @lim ORDER BY Cool, Name DESC LIMIT @numResults`, 703 map[string]interface{}{"min": 9, "lim": 11, "numResults": 100}, 704 [][]interface{}{ 705 {"Jack", int64(101)}, 706 {"Sam", int64(103)}, 707 }, 708 }, 709 { 710 // Expression in SELECT list. 711 `SELECT Name, Cool IS NOT NULL FROM Staff WHERE Tenure/2 > 4 ORDER BY NOT Cool, Name`, 712 nil, 713 [][]interface{}{ 714 {"Jack", false}, // Jack has NULL Cool (NULLs always come first in orderings) 715 {"Daniel", true}, // Daniel has Cool==true 716 {"Sam", true}, // Sam has Cool==false 717 }, 718 }, 719 { 720 `SELECT Name, Height FROM Staff ORDER BY Height DESC LIMIT 2`, 721 nil, 722 [][]interface{}{ 723 {"Teal'c", 1.91}, 724 {"Jack", 1.85}, 725 }, 726 }, 727 { 728 `SELECT Name FROM Staff WHERE Name LIKE "J%k" OR Name LIKE "_am"`, 729 nil, 730 [][]interface{}{ 731 {"Jack"}, 732 {"Sam"}, 733 }, 734 }, 735 { 736 `SELECT Name, Height FROM Staff WHERE Height BETWEEN @min AND @max ORDER BY Height DESC`, 737 map[string]interface{}{"min": 1.75, "max": 1.85}, 738 [][]interface{}{ 739 {"Jack", 1.85}, 740 {"Daniel", 1.83}, 741 {"Sam", 1.75}, 742 }, 743 }, 744 { 745 `SELECT COUNT(*) FROM Staff WHERE Name < "T"`, 746 nil, 747 [][]interface{}{ 748 {int64(4)}, 749 }, 750 }, 751 { 752 // Check that aggregation still works for the empty set. 753 `SELECT COUNT(*) FROM Staff WHERE Name = "Nobody"`, 754 nil, 755 [][]interface{}{ 756 {int64(0)}, 757 }, 758 }, 759 { 760 `SELECT * FROM Staff WHERE Name LIKE "S%"`, 761 nil, 762 [][]interface{}{ 763 // These are returned in table column order, based on the appearance in the DDL. 764 // Our internal implementation sorts the primary key columns first, 765 // but that should not become visible via SELECT *. 766 {int64(9), int64(3), "Sam", false, 1.75, nil, nil, nil}, 767 }, 768 }, 769 { 770 // Exactly the same as the previous, except with a redundant ORDER BY clause. 771 `SELECT * FROM Staff WHERE Name LIKE "S%" ORDER BY Name`, 772 nil, 773 [][]interface{}{ 774 {int64(9), int64(3), "Sam", false, 1.75, nil, nil, nil}, 775 }, 776 }, 777 { 778 `SELECT Name FROM Staff WHERE FirstSeen >= @min`, 779 map[string]interface{}{"min": civil.Date{Year: 1996, Month: 1, Day: 1}}, 780 [][]interface{}{ 781 {"George"}, 782 }, 783 }, 784 { 785 `SELECT RawBytes FROM Staff WHERE RawBytes IS NOT NULL`, 786 nil, 787 [][]interface{}{ 788 {[]byte("\x01\x00\x01")}, 789 }, 790 }, 791 { 792 // The keyword "To" needs quoting in queries. 793 // Check coercion of comparison operator literal args too. 794 "SELECT COUNT(*) FROM Staff WHERE `To` > '2000-01-01T00:00:00Z'", 795 nil, 796 [][]interface{}{ 797 {int64(1)}, 798 }, 799 }, 800 { 801 `SELECT DISTINCT Cool, Tenure > 8 FROM Staff ORDER BY Cool`, 802 nil, 803 [][]interface{}{ 804 // The non-distinct results are 805 // [[false true] [<nil> false] [<nil> true] [false true] [true false]] 806 {nil, false}, 807 {nil, true}, 808 {false, true}, 809 {true, false}, 810 }, 811 }, 812 { 813 `SELECT Name FROM Staff WHERE ID IN UNNEST(@ids)`, 814 map[string]interface{}{"ids": []int64{3, 1}}, 815 [][]interface{}{ 816 {"Jack"}, 817 {"Sam"}, 818 }, 819 }, 820 // From https://cloud.google.com/spanner/docs/query-syntax#group-by-clause_1: 821 { 822 // TODO: Ordering matters? Our implementation sorts by the GROUP BY key, 823 // but nothing documented seems to guarantee that. 824 `SELECT LastName, SUM(PointsScored) FROM PlayerStats GROUP BY LastName`, 825 nil, 826 [][]interface{}{ 827 {"Adams", int64(7)}, 828 {"Buchanan", int64(13)}, 829 {"Coolidge", int64(1)}, 830 }, 831 }, 832 { 833 // Another GROUP BY, but referring to an alias. 834 // Group by ID oddness, SUM over Tenure. 835 `SELECT ID&0x01 AS odd, SUM(Tenure) FROM Staff GROUP BY odd`, 836 nil, 837 [][]interface{}{ 838 {int64(0), int64(19)}, // Daniel(ID=2, Tenure=11), Teal'c(ID=4, Tenure=8) 839 {int64(1), int64(25)}, // Jack(ID=1, Tenure=10), Sam(ID=3, Tenure=9), George(ID=5, Tenure=6) 840 }, 841 }, 842 { 843 `SELECT MAX(Name) FROM Staff WHERE Name < @lim`, 844 map[string]interface{}{"lim": "Teal'c"}, 845 [][]interface{}{ 846 {"Sam"}, 847 }, 848 }, 849 { 850 `SELECT MAX(Name) FROM Staff WHERE Cool = @p1 LIMIT 1`, 851 map[string]interface{}{"p1": true}, 852 [][]interface{}{ 853 {"Teal'c"}, 854 }, 855 }, 856 { 857 `SELECT MIN(Name) FROM Staff`, 858 nil, 859 [][]interface{}{ 860 {"Daniel"}, 861 }, 862 }, 863 { 864 // TODO: This is broken against production; does not permit ORDER BY on something not grouped/aggregated. 865 `SELECT ARRAY_AGG(Cool) FROM Staff ORDER BY Name`, 866 nil, 867 [][]interface{}{ 868 // Daniel, George (NULL), Jack (NULL), Sam, Teal'c 869 {[]interface{}{false, nil, nil, false, true}}, 870 }, 871 }, 872 // SELECT with aliases. 873 { 874 `SELECT s.Name FROM Staff AS s WHERE s.ID = 3 ORDER BY s.Tenure`, 875 nil, 876 [][]interface{}{ 877 {"Sam"}, 878 }, 879 }, 880 // Joins. 881 { 882 `SELECT * FROM JoinA INNER JOIN JoinB ON JoinA.w = JoinB.y ORDER BY w, x, y, z`, 883 nil, 884 [][]interface{}{ 885 {int64(2), "b", int64(2), "k"}, 886 {int64(3), "c", int64(3), "m"}, 887 {int64(3), "c", int64(3), "n"}, 888 {int64(3), "d", int64(3), "m"}, 889 {int64(3), "d", int64(3), "n"}, 890 }, 891 }, 892 { 893 `SELECT * FROM JoinE CROSS JOIN JoinF ORDER BY w, x, y, z`, 894 nil, 895 [][]interface{}{ 896 {int64(1), "a", int64(2), "c"}, 897 {int64(1), "a", int64(3), "d"}, 898 {int64(2), "b", int64(2), "c"}, 899 {int64(2), "b", int64(3), "d"}, 900 }, 901 }, 902 { 903 `SELECT * FROM JoinA LEFT OUTER JOIN JoinB AS B ON JoinA.w = B.y ORDER BY w, x, y, z`, 904 nil, 905 [][]interface{}{ 906 {int64(1), "a", nil, nil}, 907 {int64(2), "b", int64(2), "k"}, 908 {int64(3), "c", int64(3), "m"}, 909 {int64(3), "c", int64(3), "n"}, 910 {int64(3), "d", int64(3), "m"}, 911 {int64(3), "d", int64(3), "n"}, 912 }, 913 }, 914 { 915 // Same as the previous, but using a USING clause instead of an ON clause. 916 `SELECT * FROM JoinC LEFT OUTER JOIN JoinD USING (x) ORDER BY x, y, z`, 917 nil, 918 [][]interface{}{ 919 {int64(1), "a", nil}, 920 {int64(2), "b", "k"}, 921 {int64(3), "c", "m"}, 922 {int64(3), "c", "n"}, 923 {int64(3), "d", "m"}, 924 {int64(3), "d", "n"}, 925 }, 926 }, 927 { 928 // Same as in docs, but with a weird ORDER BY clause to match the row ordering. 929 `SELECT * FROM JoinA RIGHT OUTER JOIN JoinB AS B ON JoinA.w = B.y ORDER BY w IS NULL, w, x, y, z`, 930 nil, 931 [][]interface{}{ 932 {int64(2), "b", int64(2), "k"}, 933 {int64(3), "c", int64(3), "m"}, 934 {int64(3), "c", int64(3), "n"}, 935 {int64(3), "d", int64(3), "m"}, 936 {int64(3), "d", int64(3), "n"}, 937 {nil, nil, int64(4), "p"}, 938 }, 939 }, 940 { 941 `SELECT * FROM JoinC RIGHT OUTER JOIN JoinD USING (x) ORDER BY x, y, z`, 942 nil, 943 [][]interface{}{ 944 {int64(2), "b", "k"}, 945 {int64(3), "c", "m"}, 946 {int64(3), "c", "n"}, 947 {int64(3), "d", "m"}, 948 {int64(3), "d", "n"}, 949 {int64(4), nil, "p"}, 950 }, 951 }, 952 // Regression test for aggregating no rows; it used to return an empty row. 953 // https://github.com/googleapis/google-cloud-go/issues/2793 954 { 955 `SELECT Cool, ARRAY_AGG(Name) FROM Staff WHERE Name > "zzz" GROUP BY Cool`, 956 nil, 957 nil, 958 }, 959 // Regression test for evaluating `IN` incorrectly using ==. 960 // https://github.com/googleapis/google-cloud-go/issues/2458 961 { 962 `SELECT COUNT(*) FROM Staff WHERE RawBytes IN UNNEST(@arg)`, 963 map[string]interface{}{ 964 "arg": [][]byte{ 965 {0x02}, 966 {0x01, 0x00, 0x01}, // only one present 967 }, 968 }, 969 [][]interface{}{ 970 {int64(1)}, 971 }, 972 }, 973 // Regression test for mishandling NULLs with LIKE operator. 974 { 975 `SELECT i, str FROM SomeStrings WHERE str LIKE "%bar"`, 976 nil, 977 [][]interface{}{ 978 // Does not include [0, "afoo"] or [2, nil]. 979 {int64(1), "abar"}, 980 {int64(3), "bbar"}, 981 }, 982 }, 983 { 984 `SELECT i, str FROM SomeStrings WHERE str NOT LIKE "%bar"`, 985 nil, 986 [][]interface{}{ 987 // Does not include [1, "abar"], [2, nil] or [3, "bbar"]. 988 {int64(0), "afoo"}, 989 }, 990 }, 991 // Regression test for ORDER BY combined with SELECT aliases. 992 { 993 `SELECT Name AS nom FROM Staff ORDER BY ID LIMIT 2`, 994 nil, 995 [][]interface{}{ 996 {"Jack"}, 997 {"Daniel"}, 998 }, 999 }, 1000 } 1001 var failures int 1002 for _, test := range tests { 1003 stmt := spanner.NewStatement(test.q) 1004 stmt.Params = test.params 1005 1006 ri = client.Single().Query(ctx, stmt) 1007 all, err := slurpRows(t, ri) 1008 if err != nil { 1009 t.Errorf("Query(%q, %v): %v", test.q, test.params, err) 1010 failures++ 1011 continue 1012 } 1013 if !reflect.DeepEqual(all, test.want) { 1014 t.Errorf("Results from Query(%q, %v) are wrong.\n got %v\nwant %v", test.q, test.params, all, test.want) 1015 failures++ 1016 } 1017 } 1018 if failures > 0 { 1019 t.Logf("%d queries failed", failures) 1020 } 1021} 1022 1023func dropTable(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, table string) { 1024 t.Helper() 1025 err := updateDDL(t, adminClient, "DROP TABLE "+table) 1026 // NotFound is an acceptable failure mode here. 1027 if st, _ := status.FromError(err); st.Code() == codes.NotFound { 1028 err = nil 1029 } 1030 if err != nil { 1031 t.Fatalf("Dropping old table %q: %v", table, err) 1032 } 1033} 1034 1035func updateDDL(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, statements ...string) error { 1036 t.Helper() 1037 ctx := context.Background() 1038 t.Logf("DDL update: %q", statements) 1039 op, err := adminClient.UpdateDatabaseDdl(ctx, &dbadminpb.UpdateDatabaseDdlRequest{ 1040 Database: dbName(), 1041 Statements: statements, 1042 }) 1043 if err != nil { 1044 t.Fatalf("Starting DDL update: %v", err) 1045 } 1046 return op.Wait(ctx) 1047} 1048 1049func mustSlurpRows(t *testing.T, ri *spanner.RowIterator) [][]interface{} { 1050 t.Helper() 1051 all, err := slurpRows(t, ri) 1052 if err != nil { 1053 t.Fatalf("Reading rows: %v", err) 1054 } 1055 return all 1056} 1057 1058func slurpRows(t *testing.T, ri *spanner.RowIterator) (all [][]interface{}, err error) { 1059 t.Helper() 1060 err = ri.Do(func(r *spanner.Row) error { 1061 var data []interface{} 1062 for i := 0; i < r.Size(); i++ { 1063 var gcv spanner.GenericColumnValue 1064 if err := r.Column(i, &gcv); err != nil { 1065 return err 1066 } 1067 data = append(data, genericValue(t, gcv)) 1068 } 1069 all = append(all, data) 1070 return nil 1071 }) 1072 return 1073} 1074 1075func genericValue(t *testing.T, gcv spanner.GenericColumnValue) interface{} { 1076 t.Helper() 1077 1078 if _, ok := gcv.Value.Kind.(*structpb.Value_NullValue); ok { 1079 return nil 1080 } 1081 if gcv.Type.Code == spannerpb.TypeCode_ARRAY { 1082 var arr []interface{} 1083 for _, v := range gcv.Value.GetListValue().Values { 1084 arr = append(arr, genericValue(t, spanner.GenericColumnValue{ 1085 Type: &spannerpb.Type{Code: gcv.Type.ArrayElementType.Code}, 1086 Value: v, 1087 })) 1088 } 1089 return arr 1090 } 1091 1092 var dst interface{} 1093 switch gcv.Type.Code { 1094 case spannerpb.TypeCode_BOOL: 1095 dst = new(bool) 1096 case spannerpb.TypeCode_INT64: 1097 dst = new(int64) 1098 case spannerpb.TypeCode_FLOAT64: 1099 dst = new(float64) 1100 case spannerpb.TypeCode_TIMESTAMP: 1101 dst = new(time.Time) // TODO: do we need to force to UTC? 1102 case spannerpb.TypeCode_DATE: 1103 dst = new(civil.Date) 1104 case spannerpb.TypeCode_STRING: 1105 dst = new(string) 1106 case spannerpb.TypeCode_BYTES: 1107 dst = new([]byte) 1108 } 1109 if dst == nil { 1110 t.Fatalf("Can't decode Spanner generic column value: %v", gcv.Type) 1111 } 1112 if err := gcv.Decode(dst); err != nil { 1113 t.Fatalf("Decoding %v into %T: %v", gcv, dst, err) 1114 } 1115 return reflect.ValueOf(dst).Elem().Interface() 1116} 1117