1/* 2Copyright 2019 Google LLC 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17/* 18This file holds tests for the in-memory fake for comparing it against a real Cloud Spanner. 19 20By default it uses the Spanner client against the in-memory fake; set the 21-test_db flag to "projects/P/instances/I/databases/D" to make it use a real 22Cloud Spanner database instead. You may need to provide -timeout=5m too. 23*/ 24 25package spannertest 26 27import ( 28 "context" 29 "flag" 30 "reflect" 31 "sort" 32 "testing" 33 "time" 34 35 "cloud.google.com/go/spanner" 36 dbadmin "cloud.google.com/go/spanner/admin/database/apiv1" 37 "google.golang.org/api/iterator" 38 "google.golang.org/api/option" 39 "google.golang.org/grpc" 40 "google.golang.org/grpc/codes" 41 "google.golang.org/grpc/status" 42 43 dbadminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" 44) 45 46var testDBFlag = flag.String("test_db", "", "Fully-qualified database name to test against; empty means use an in-memory fake.") 47 48func dbName() string { 49 if *testDBFlag != "" { 50 return *testDBFlag 51 } 52 return "projects/fake-proj/instances/fake-instance/databases/fake-db" 53} 54 55func makeClient(t *testing.T) (*spanner.Client, *dbadmin.DatabaseAdminClient, func()) { 56 // Despite the docs, this context is also used for auth, 57 // so it needs to be long-lived. 58 ctx := context.Background() 59 60 if *testDBFlag != "" { 61 t.Logf("Using real Spanner DB %s", *testDBFlag) 62 dialOpt := option.WithGRPCDialOption(grpc.WithTimeout(5 * time.Second)) 63 client, err := spanner.NewClient(ctx, *testDBFlag, dialOpt) 64 if err != nil { 65 t.Fatalf("Connecting to %s: %v", *testDBFlag, err) 66 } 67 adminClient, err := dbadmin.NewDatabaseAdminClient(ctx, dialOpt) 68 if err != nil { 69 client.Close() 70 t.Fatalf("Connecting DB admin client: %v", err) 71 } 72 return client, adminClient, func() { client.Close(); adminClient.Close() } 73 } 74 75 // Don't use SPANNER_EMULATOR_HOST because we need the raw connection for 76 // the database admin client anyway. 77 78 t.Logf("Using in-memory fake Spanner DB") 79 srv, err := NewServer("localhost:0") 80 if err != nil { 81 t.Fatalf("Starting in-memory fake: %v", err) 82 } 83 srv.SetLogger(t.Logf) 84 dialCtx, cancel := context.WithTimeout(ctx, 1*time.Second) 85 defer cancel() 86 conn, err := grpc.DialContext(dialCtx, srv.Addr, grpc.WithInsecure()) 87 if err != nil { 88 srv.Close() 89 t.Fatalf("Dialing in-memory fake: %v", err) 90 } 91 client, err := spanner.NewClient(ctx, dbName(), option.WithGRPCConn(conn)) 92 if err != nil { 93 srv.Close() 94 t.Fatalf("Connecting to in-memory fake: %v", err) 95 } 96 adminClient, err := dbadmin.NewDatabaseAdminClient(ctx, option.WithGRPCConn(conn)) 97 if err != nil { 98 srv.Close() 99 t.Fatalf("Connecting to in-memory fake DB admin: %v", err) 100 } 101 return client, adminClient, func() { 102 client.Close() 103 adminClient.Close() 104 conn.Close() 105 srv.Close() 106 } 107} 108 109func TestIntegration_SpannerBasics(t *testing.T) { 110 client, adminClient, cleanup := makeClient(t) 111 defer cleanup() 112 113 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 114 defer cancel() 115 116 // Do a trivial query to verify the connection works. 117 it := client.Single().Query(ctx, spanner.NewStatement("SELECT 1")) 118 row, err := it.Next() 119 if err != nil { 120 t.Fatalf("Getting first row of trivial query: %v", err) 121 } 122 var value int64 123 if err := row.Column(0, &value); err != nil { 124 t.Fatalf("Decoding row data from trivial query: %v", err) 125 } 126 if value != 1 { 127 t.Errorf("Trivial query gave %d, want 1", value) 128 } 129 // There shouldn't be a next row. 130 _, err = it.Next() 131 if err != iterator.Done { 132 t.Errorf("Reading second row of trivial query gave %v, want iterator.Done", err) 133 } 134 it.Stop() 135 136 // Drop any previous test table/index, and make a fresh one in a few stages. 137 const tableName = "Characters" 138 err = updateDDL(t, adminClient, "DROP INDEX AgeIndex") 139 // NotFound is an acceptable failure mode here. 140 if st, _ := status.FromError(err); st.Code() == codes.NotFound { 141 err = nil 142 } 143 if err != nil { 144 t.Fatalf("Dropping old index: %v", err) 145 } 146 err = updateDDL(t, adminClient, "DROP TABLE "+tableName) 147 // NotFound is an acceptable failure mode here. 148 if st, _ := status.FromError(err); st.Code() == codes.NotFound { 149 err = nil 150 } 151 if err != nil { 152 t.Fatalf("Dropping old table: %v", err) 153 } 154 err = updateDDL(t, adminClient, 155 `CREATE TABLE `+tableName+` ( 156 FirstName STRING(20) NOT NULL, 157 LastName STRING(20) NOT NULL, 158 Alias STRING(MAX), 159 ) PRIMARY KEY (FirstName, LastName)`) 160 if err != nil { 161 t.Fatalf("Setting up fresh table: %v", err) 162 } 163 err = updateDDL(t, adminClient, 164 `ALTER TABLE `+tableName+` ADD COLUMN Age INT64`, 165 `CREATE INDEX AgeIndex ON `+tableName+` (Age DESC)`) 166 if err != nil { 167 t.Fatalf("Adding new column: %v", err) 168 } 169 170 // Insert some data. 171 _, err = client.Apply(ctx, []*spanner.Mutation{ 172 spanner.Insert(tableName, 173 []string{"FirstName", "LastName", "Alias", "Age"}, 174 []interface{}{"Steve", "Rogers", "Captain America", 101}), 175 spanner.Insert(tableName, 176 []string{"LastName", "FirstName", "Age", "Alias"}, 177 []interface{}{"Romanoff", "Natasha", 35, "Black Widow"}), 178 spanner.Insert(tableName, 179 []string{"Age", "Alias", "FirstName", "LastName"}, 180 []interface{}{49, "Iron Man", "Tony", "Stark"}), 181 spanner.Insert(tableName, 182 []string{"FirstName", "Alias", "LastName"}, // no Age 183 []interface{}{"Clark", "Superman", "Kent"}), 184 // Two rows with the same value in one column, 185 // but with distinct primary keys. 186 spanner.Insert(tableName, 187 []string{"FirstName", "LastName", "Alias"}, 188 []interface{}{"Peter", "Parker", "Spider-Man"}), 189 spanner.Insert(tableName, 190 []string{"FirstName", "LastName", "Alias"}, 191 []interface{}{"Peter", "Quill", "Star-Lord"}), 192 }) 193 if err != nil { 194 t.Fatalf("Applying mutations: %v", err) 195 } 196 197 // Delete some data. 198 _, err = client.Apply(ctx, []*spanner.Mutation{ 199 // Whoops. DC, not MCU. 200 spanner.Delete(tableName, spanner.Key{"Clark", "Kent"}), 201 }) 202 if err != nil { 203 t.Fatalf("Applying mutations: %v", err) 204 } 205 206 // Read a single row. 207 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Alias", "Age"}) 208 if err != nil { 209 t.Fatalf("Reading single row: %v", err) 210 } 211 var alias string 212 var age int64 213 if err := row.Columns(&alias, &age); err != nil { 214 t.Fatalf("Decoding single row: %v", err) 215 } 216 if alias != "Iron Man" || age != 49 { 217 t.Errorf(`Single row read gave (%q, %d), want ("Iron Man", 49)`, alias, age) 218 } 219 220 // Read all rows, and do a local age sum. 221 rows := client.Single().Read(ctx, tableName, spanner.AllKeys(), []string{"Age"}) 222 var ageSum int64 223 err = rows.Do(func(row *spanner.Row) error { 224 var age spanner.NullInt64 225 if err := row.Columns(&age); err != nil { 226 return err 227 } 228 if age.Valid { 229 ageSum += age.Int64 230 } 231 return nil 232 }) 233 if err != nil { 234 t.Fatalf("Iterating over all row read: %v", err) 235 } 236 if want := int64(101 + 35 + 49); ageSum != want { 237 t.Errorf("Age sum after iterating over all rows = %d, want %d", ageSum, want) 238 } 239 240 // Do a more complex query to find the aliases of the two oldest non-centenarian characters. 241 stmt := spanner.NewStatement(`SELECT Alias FROM ` + tableName + ` WHERE Age < @ageLimit AND Alias IS NOT NULL ORDER BY Age DESC LIMIT @limit`) 242 stmt.Params = map[string]interface{}{ 243 "ageLimit": 100, 244 "limit": 2, 245 } 246 rows = client.Single().Query(ctx, stmt) 247 var oldFolk []string 248 err = rows.Do(func(row *spanner.Row) error { 249 var alias string 250 if err := row.Columns(&alias); err != nil { 251 return err 252 } 253 oldFolk = append(oldFolk, alias) 254 return nil 255 }) 256 if err != nil { 257 t.Fatalf("Iterating over complex query: %v", err) 258 } 259 if want := []string{"Iron Man", "Black Widow"}; !reflect.DeepEqual(oldFolk, want) { 260 t.Errorf("Complex query results = %v, want %v", oldFolk, want) 261 } 262 263 // Apply an update. 264 _, err = client.Apply(ctx, []*spanner.Mutation{ 265 spanner.Update(tableName, 266 []string{"FirstName", "LastName", "Age"}, 267 []interface{}{"Steve", "Rogers", 102}), 268 }) 269 if err != nil { 270 t.Fatalf("Applying mutations: %v", err) 271 } 272 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Steve", "Rogers"}, []string{"Age"}) 273 if err != nil { 274 t.Fatalf("Reading single row: %v", err) 275 } 276 if err := row.Columns(&age); err != nil { 277 t.Fatalf("Decoding single row: %v", err) 278 } 279 if age != 102 { 280 t.Errorf("After updating Captain America, age = %d, want 102", age) 281 } 282 283 // Do a query where the result type isn't deducible from the first row. 284 stmt = spanner.NewStatement(`SELECT Age FROM ` + tableName + ` WHERE FirstName = "Peter"`) 285 rows = client.Single().Query(ctx, stmt) 286 var nullPeters int 287 err = rows.Do(func(row *spanner.Row) error { 288 var age spanner.NullInt64 289 if err := row.Column(0, &age); err != nil { 290 return err 291 } 292 if age.Valid { 293 t.Errorf("Got non-NULL Age %d for a Peter", age.Int64) 294 } else { 295 nullPeters++ 296 } 297 return nil 298 }) 299 if err != nil { 300 t.Fatalf("Counting Peters with NULL Ages: %v", err) 301 } 302 if nullPeters != 2 { 303 t.Errorf("Found %d Peters with NULL Ages, want 2", nullPeters) 304 } 305 306 // Check handling of array types. 307 err = updateDDL(t, adminClient, `ALTER TABLE `+tableName+` ADD COLUMN Allies ARRAY<STRING(20)>`) 308 if err != nil { 309 t.Fatalf("Adding new array-typed column: %v", err) 310 } 311 _, err = client.Apply(ctx, []*spanner.Mutation{ 312 spanner.Update(tableName, 313 []string{"FirstName", "LastName", "Allies"}, 314 []interface{}{"Steve", "Rogers", []string{}}), 315 spanner.Update(tableName, 316 []string{"FirstName", "LastName", "Allies"}, 317 []interface{}{"Tony", "Stark", []string{"Black Widow", "Spider-Man"}}), 318 }) 319 if err != nil { 320 t.Fatalf("Applying mutations: %v", err) 321 } 322 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Allies"}) 323 if err != nil { 324 t.Fatalf("Reading row with array value: %v", err) 325 } 326 var names []string 327 if err := row.Column(0, &names); err != nil { 328 t.Fatalf("Unpacking array value: %v", err) 329 } 330 if want := []string{"Black Widow", "Spider-Man"}; !reflect.DeepEqual(names, want) { 331 t.Errorf("Read array value: got %q, want %q", names, want) 332 } 333 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Steve", "Rogers"}, []string{"Allies"}) 334 if err != nil { 335 t.Fatalf("Reading row with empty array value: %v", err) 336 } 337 if err := row.Column(0, &names); err != nil { 338 t.Fatalf("Unpacking empty array value: %v", err) 339 } 340 if len(names) > 0 { 341 t.Errorf("Read empty array value: got %q", names) 342 } 343 344 // Exercise commit timestamp. 345 err = updateDDL(t, adminClient, `ALTER TABLE `+tableName+` ADD COLUMN Updated TIMESTAMP OPTIONS (allow_commit_timestamp=true)`) 346 if err != nil { 347 t.Fatalf("Adding new timestamp column: %v", err) 348 } 349 cts, err := client.Apply(ctx, []*spanner.Mutation{ 350 // Update one row in place. 351 spanner.Update(tableName, 352 []string{"FirstName", "LastName", "Allies", "Updated"}, 353 []interface{}{"Tony", "Stark", []string{"Spider-Man", "Professor Hulk"}, spanner.CommitTimestamp}), 354 }) 355 if err != nil { 356 t.Fatalf("Applying mutations: %v", err) 357 } 358 cts = cts.In(time.UTC) 359 if d := time.Since(cts); d < 0 || d > 10*time.Second { 360 t.Errorf("Commit timestamp %v not in the last 10s", cts) 361 } 362 row, err = client.Single().ReadRow(ctx, tableName, spanner.Key{"Tony", "Stark"}, []string{"Allies", "Updated"}) 363 if err != nil { 364 t.Fatalf("Reading single row: %v", err) 365 } 366 var gotAllies []string 367 var gotUpdated time.Time 368 if err := row.Columns(&gotAllies, &gotUpdated); err != nil { 369 t.Fatalf("Decoding single row: %v", err) 370 } 371 if want := []string{"Spider-Man", "Professor Hulk"}; !reflect.DeepEqual(gotAllies, want) { 372 t.Errorf("After updating Iron Man, allies = %+v, want %+v", gotAllies, want) 373 } 374 if !gotUpdated.Equal(cts) { 375 t.Errorf("After updating Iron Man, updated = %v, want %v", gotUpdated, cts) 376 } 377 378 // Check if IN UNNEST works. 379 stmt = spanner.NewStatement(`SELECT Age FROM ` + tableName + ` WHERE FirstName IN UNNEST(@list)`) 380 stmt.Params = map[string]interface{}{ 381 "list": []string{"Peter", "Steve"}, 382 } 383 rows = client.Single().Query(ctx, stmt) 384 var ages []int64 385 err = rows.Do(func(row *spanner.Row) error { 386 var age spanner.NullInt64 387 if err := row.Column(0, &age); err != nil { 388 return err 389 } 390 ages = append(ages, age.Int64) // zero for NULL 391 return nil 392 }) 393 if err != nil { 394 t.Fatalf("Getting ages using IN UNNEST: %v", err) 395 } 396 sort.Slice(ages, func(i, j int) bool { return ages[i] < ages[j] }) 397 wantAges := []int64{0, 0, 102} // Peter Parker, Peter Quill, Steve Rogers (modified) 398 if !reflect.DeepEqual(ages, wantAges) { 399 t.Errorf("Query with IN UNNEST gave wrong ages: got %+v, want %+v", ages, wantAges) 400 } 401} 402 403func updateDDL(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, statements ...string) error { 404 t.Helper() 405 ctx := context.Background() 406 t.Logf("DDL update: %q", statements) 407 op, err := adminClient.UpdateDatabaseDdl(ctx, &dbadminpb.UpdateDatabaseDdlRequest{ 408 Database: dbName(), 409 Statements: statements, 410 }) 411 if err != nil { 412 t.Fatalf("Starting DDL update: %v", err) 413 } 414 return op.Wait(ctx) 415} 416