1// Copyright 2017 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "testing" 19 "time" 20 21 "cloud.google.com/go/internal/testutil" 22 bq "google.golang.org/api/bigquery/v2" 23) 24 25func TestBQToTableMetadata(t *testing.T) { 26 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local) 27 aTimeMillis := aTime.UnixNano() / 1e6 28 aDurationMillis := int64(1800000) 29 aDuration := time.Duration(aDurationMillis) * time.Millisecond 30 for _, test := range []struct { 31 in *bq.Table 32 want *TableMetadata 33 }{ 34 {&bq.Table{}, &TableMetadata{}}, // test minimal case 35 { 36 &bq.Table{ 37 CreationTime: aTimeMillis, 38 Description: "desc", 39 Etag: "etag", 40 ExpirationTime: aTimeMillis, 41 FriendlyName: "fname", 42 Id: "id", 43 LastModifiedTime: uint64(aTimeMillis), 44 Location: "loc", 45 NumBytes: 123, 46 NumLongTermBytes: 23, 47 NumRows: 7, 48 StreamingBuffer: &bq.Streamingbuffer{ 49 EstimatedBytes: 11, 50 EstimatedRows: 3, 51 OldestEntryTime: uint64(aTimeMillis), 52 }, 53 MaterializedView: &bq.MaterializedViewDefinition{ 54 EnableRefresh: true, 55 Query: "mat view query", 56 LastRefreshTime: aTimeMillis, 57 RefreshIntervalMs: aDurationMillis, 58 }, 59 TimePartitioning: &bq.TimePartitioning{ 60 ExpirationMs: 7890, 61 Type: "DAY", 62 Field: "pfield", 63 }, 64 Clustering: &bq.Clustering{ 65 Fields: []string{"cfield1", "cfield2"}, 66 }, 67 RequirePartitionFilter: true, 68 EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"}, 69 Type: "EXTERNAL", 70 View: &bq.ViewDefinition{Query: "view-query"}, 71 Labels: map[string]string{"a": "b"}, 72 ExternalDataConfiguration: &bq.ExternalDataConfiguration{ 73 SourceFormat: "GOOGLE_SHEETS", 74 }, 75 }, 76 &TableMetadata{ 77 Description: "desc", 78 Name: "fname", 79 Location: "loc", 80 ViewQuery: "view-query", 81 FullID: "id", 82 Type: ExternalTable, 83 Labels: map[string]string{"a": "b"}, 84 ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets}, 85 ExpirationTime: aTime.Truncate(time.Millisecond), 86 CreationTime: aTime.Truncate(time.Millisecond), 87 LastModifiedTime: aTime.Truncate(time.Millisecond), 88 NumBytes: 123, 89 NumLongTermBytes: 23, 90 NumRows: 7, 91 MaterializedView: &MaterializedViewDefinition{ 92 EnableRefresh: true, 93 Query: "mat view query", 94 LastRefreshTime: aTime, 95 RefreshInterval: aDuration, 96 }, 97 TimePartitioning: &TimePartitioning{ 98 Type: DayPartitioningType, 99 Expiration: 7890 * time.Millisecond, 100 Field: "pfield", 101 }, 102 Clustering: &Clustering{ 103 Fields: []string{"cfield1", "cfield2"}, 104 }, 105 RequirePartitionFilter: true, 106 StreamingBuffer: &StreamingBuffer{ 107 EstimatedBytes: 11, 108 EstimatedRows: 3, 109 OldestEntryTime: aTime, 110 }, 111 EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"}, 112 ETag: "etag", 113 }, 114 }, 115 } { 116 got, err := bqToTableMetadata(test.in) 117 if err != nil { 118 t.Fatal(err) 119 } 120 if diff := testutil.Diff(got, test.want); diff != "" { 121 t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff) 122 } 123 } 124} 125 126func TestTableMetadataToBQ(t *testing.T) { 127 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local) 128 aTimeMillis := aTime.UnixNano() / 1e6 129 sc := Schema{fieldSchema("desc", "name", "STRING", false, true, nil)} 130 131 for _, test := range []struct { 132 in *TableMetadata 133 want *bq.Table 134 }{ 135 {nil, &bq.Table{}}, 136 {&TableMetadata{}, &bq.Table{}}, 137 { 138 &TableMetadata{ 139 Name: "n", 140 Description: "d", 141 Schema: sc, 142 ExpirationTime: aTime, 143 Labels: map[string]string{"a": "b"}, 144 ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable}, 145 EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"}, 146 }, 147 &bq.Table{ 148 FriendlyName: "n", 149 Description: "d", 150 Schema: &bq.TableSchema{ 151 Fields: []*bq.TableFieldSchema{ 152 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil), 153 }, 154 }, 155 ExpirationTime: aTimeMillis, 156 Labels: map[string]string{"a": "b"}, 157 ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"}, 158 EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"}, 159 }, 160 }, 161 { 162 &TableMetadata{ViewQuery: "q"}, 163 &bq.Table{ 164 View: &bq.ViewDefinition{ 165 Query: "q", 166 UseLegacySql: false, 167 ForceSendFields: []string{"UseLegacySql"}, 168 }, 169 }, 170 }, 171 { 172 &TableMetadata{ 173 ViewQuery: "q", 174 UseLegacySQL: true, 175 TimePartitioning: &TimePartitioning{}, 176 RequirePartitionFilter: true, 177 }, 178 &bq.Table{ 179 View: &bq.ViewDefinition{ 180 Query: "q", 181 UseLegacySql: true, 182 }, 183 TimePartitioning: &bq.TimePartitioning{ 184 Type: "DAY", 185 ExpirationMs: 0, 186 }, 187 RequirePartitionFilter: true, 188 }, 189 }, 190 { 191 &TableMetadata{ 192 ViewQuery: "q", 193 UseStandardSQL: true, 194 TimePartitioning: &TimePartitioning{ 195 Type: HourPartitioningType, 196 Expiration: time.Second, 197 Field: "ofDreams", 198 }, 199 Clustering: &Clustering{ 200 Fields: []string{"cfield1"}, 201 }, 202 }, 203 &bq.Table{ 204 View: &bq.ViewDefinition{ 205 Query: "q", 206 UseLegacySql: false, 207 ForceSendFields: []string{"UseLegacySql"}, 208 }, 209 TimePartitioning: &bq.TimePartitioning{ 210 Type: "HOUR", 211 ExpirationMs: 1000, 212 Field: "ofDreams", 213 }, 214 Clustering: &bq.Clustering{ 215 Fields: []string{"cfield1"}, 216 }, 217 }, 218 }, 219 { 220 &TableMetadata{ 221 RangePartitioning: &RangePartitioning{ 222 Field: "ofNumbers", 223 Range: &RangePartitioningRange{ 224 Start: 1, 225 End: 100, 226 Interval: 5, 227 }, 228 }, 229 Clustering: &Clustering{ 230 Fields: []string{"cfield1"}, 231 }, 232 }, 233 &bq.Table{ 234 235 RangePartitioning: &bq.RangePartitioning{ 236 Field: "ofNumbers", 237 Range: &bq.RangePartitioningRange{ 238 Start: 1, 239 End: 100, 240 Interval: 5, 241 ForceSendFields: []string{"Start", "End", "Interval"}, 242 }, 243 }, 244 Clustering: &bq.Clustering{ 245 Fields: []string{"cfield1"}, 246 }, 247 }, 248 }, 249 { 250 &TableMetadata{ExpirationTime: NeverExpire}, 251 &bq.Table{ExpirationTime: 0}, 252 }, 253 } { 254 got, err := test.in.toBQ() 255 if err != nil { 256 t.Fatalf("%+v: %v", test.in, err) 257 } 258 if diff := testutil.Diff(got, test.want); diff != "" { 259 t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff) 260 } 261 } 262 263 // Errors 264 for _, in := range []*TableMetadata{ 265 {Schema: sc, ViewQuery: "q"}, // can't have both schema and query 266 {UseLegacySQL: true}, // UseLegacySQL without query 267 {UseStandardSQL: true}, // UseStandardSQL without query 268 // read-only fields 269 {FullID: "x"}, 270 {Type: "x"}, 271 {CreationTime: aTime}, 272 {LastModifiedTime: aTime}, 273 {NumBytes: 1}, 274 {NumLongTermBytes: 1}, 275 {NumRows: 1}, 276 {StreamingBuffer: &StreamingBuffer{}}, 277 {ETag: "x"}, 278 // expiration time outside allowable range is invalid 279 // See https://godoc.org/time#Time.UnixNano 280 {ExpirationTime: time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC).Add(-1)}, 281 {ExpirationTime: time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC).Add(1)}, 282 } { 283 _, err := in.toBQ() 284 if err == nil { 285 t.Errorf("%+v: got nil, want error", in) 286 } 287 } 288} 289 290func TestTableMetadataToUpdateToBQ(t *testing.T) { 291 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local) 292 for _, test := range []struct { 293 tm TableMetadataToUpdate 294 want *bq.Table 295 }{ 296 { 297 tm: TableMetadataToUpdate{}, 298 want: &bq.Table{}, 299 }, 300 { 301 tm: TableMetadataToUpdate{ 302 Description: "d", 303 Name: "n", 304 }, 305 want: &bq.Table{ 306 Description: "d", 307 FriendlyName: "n", 308 ForceSendFields: []string{"Description", "FriendlyName"}, 309 }, 310 }, 311 { 312 tm: TableMetadataToUpdate{ 313 Schema: Schema{fieldSchema("desc", "name", "STRING", false, true, nil)}, 314 ExpirationTime: aTime, 315 }, 316 want: &bq.Table{ 317 Schema: &bq.TableSchema{ 318 Fields: []*bq.TableFieldSchema{ 319 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil), 320 }, 321 }, 322 ExpirationTime: aTime.UnixNano() / 1e6, 323 ForceSendFields: []string{"Schema", "ExpirationTime"}, 324 }, 325 }, 326 { 327 tm: TableMetadataToUpdate{ViewQuery: "q"}, 328 want: &bq.Table{ 329 View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}}, 330 }, 331 }, 332 { 333 tm: TableMetadataToUpdate{UseLegacySQL: false}, 334 want: &bq.Table{ 335 View: &bq.ViewDefinition{ 336 UseLegacySql: false, 337 ForceSendFields: []string{"UseLegacySql"}, 338 }, 339 }, 340 }, 341 { 342 tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true}, 343 want: &bq.Table{ 344 View: &bq.ViewDefinition{ 345 Query: "q", 346 UseLegacySql: true, 347 ForceSendFields: []string{"Query", "UseLegacySql"}, 348 }, 349 }, 350 }, 351 { 352 tm: func() (tm TableMetadataToUpdate) { 353 tm.SetLabel("L", "V") 354 tm.DeleteLabel("D") 355 return tm 356 }(), 357 want: &bq.Table{ 358 Labels: map[string]string{"L": "V"}, 359 NullFields: []string{"Labels.D"}, 360 }, 361 }, 362 { 363 tm: TableMetadataToUpdate{ExpirationTime: NeverExpire}, 364 want: &bq.Table{ 365 NullFields: []string{"ExpirationTime"}, 366 }, 367 }, 368 { 369 tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: 0}}, 370 want: &bq.Table{ 371 TimePartitioning: &bq.TimePartitioning{ 372 Type: "DAY", 373 ForceSendFields: []string{"RequirePartitionFilter"}, 374 NullFields: []string{"ExpirationMs"}, 375 }, 376 }, 377 }, 378 { 379 tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: time.Duration(time.Hour)}}, 380 want: &bq.Table{ 381 TimePartitioning: &bq.TimePartitioning{ 382 ExpirationMs: 3600000, 383 Type: "DAY", 384 ForceSendFields: []string{"RequirePartitionFilter"}, 385 }, 386 }, 387 }, 388 { 389 tm: TableMetadataToUpdate{RequirePartitionFilter: false}, 390 want: &bq.Table{ 391 RequirePartitionFilter: false, 392 ForceSendFields: []string{"RequirePartitionFilter"}, 393 }, 394 }, 395 { 396 tm: TableMetadataToUpdate{RequirePartitionFilter: true}, 397 want: &bq.Table{ 398 RequirePartitionFilter: true, 399 ForceSendFields: []string{"RequirePartitionFilter"}, 400 }, 401 }, 402 } { 403 got, _ := test.tm.toBQ() 404 if !testutil.Equal(got, test.want) { 405 t.Errorf("%+v:\ngot %+v\nwant %+v", test.tm, got, test.want) 406 } 407 } 408} 409 410func TestTableMetadataToUpdateToBQErrors(t *testing.T) { 411 // See https://godoc.org/time#Time.UnixNano 412 start := time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC) 413 end := time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC) 414 415 for _, test := range []struct { 416 desc string 417 aTime time.Time 418 wantErr bool 419 }{ 420 {desc: "ignored zero value", aTime: time.Time{}, wantErr: false}, 421 {desc: "earliest valid time", aTime: start, wantErr: false}, 422 {desc: "latested valid time", aTime: end, wantErr: false}, 423 {desc: "invalid times before 1678", aTime: start.Add(-1), wantErr: true}, 424 {desc: "invalid times after 2262", aTime: end.Add(1), wantErr: true}, 425 {desc: "valid times after 1678", aTime: start.Add(1), wantErr: false}, 426 {desc: "valid times before 2262", aTime: end.Add(-1), wantErr: false}, 427 } { 428 tm := &TableMetadataToUpdate{ExpirationTime: test.aTime} 429 _, err := tm.toBQ() 430 if test.wantErr && err == nil { 431 t.Errorf("[%s] got no error, want error", test.desc) 432 } 433 if !test.wantErr && err != nil { 434 t.Errorf("[%s] got error, want no error", test.desc) 435 } 436 } 437} 438