1// Copyright 2015 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "fmt" 19 "math/big" 20 "reflect" 21 "testing" 22 "time" 23 24 "cloud.google.com/go/civil" 25 "cloud.google.com/go/internal/pretty" 26 "cloud.google.com/go/internal/testutil" 27 bq "google.golang.org/api/bigquery/v2" 28) 29 30func (fs *FieldSchema) GoString() string { 31 if fs == nil { 32 return "<nil>" 33 } 34 35 return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}", 36 fs.Name, 37 fs.Description, 38 fs.Repeated, 39 fs.Required, 40 fs.Type, 41 fmt.Sprintf("%#v", fs.Schema), 42 ) 43} 44 45func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema { 46 return &bq.TableFieldSchema{ 47 Description: desc, 48 Name: name, 49 Mode: mode, 50 Type: typ, 51 } 52} 53 54func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema { 55 return &FieldSchema{ 56 Description: desc, 57 Name: name, 58 Repeated: repeated, 59 Required: required, 60 Type: FieldType(typ), 61 } 62} 63 64func TestSchemaConversion(t *testing.T) { 65 testCases := []struct { 66 schema Schema 67 bqSchema *bq.TableSchema 68 }{ 69 { 70 // required 71 bqSchema: &bq.TableSchema{ 72 Fields: []*bq.TableFieldSchema{ 73 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"), 74 }, 75 }, 76 schema: Schema{ 77 fieldSchema("desc", "name", "STRING", false, true), 78 }, 79 }, 80 { 81 // repeated 82 bqSchema: &bq.TableSchema{ 83 Fields: []*bq.TableFieldSchema{ 84 bqTableFieldSchema("desc", "name", "STRING", "REPEATED"), 85 }, 86 }, 87 schema: Schema{ 88 fieldSchema("desc", "name", "STRING", true, false), 89 }, 90 }, 91 { 92 // nullable, string 93 bqSchema: &bq.TableSchema{ 94 Fields: []*bq.TableFieldSchema{ 95 bqTableFieldSchema("desc", "name", "STRING", ""), 96 }, 97 }, 98 schema: Schema{ 99 fieldSchema("desc", "name", "STRING", false, false), 100 }, 101 }, 102 { 103 // integer 104 bqSchema: &bq.TableSchema{ 105 Fields: []*bq.TableFieldSchema{ 106 bqTableFieldSchema("desc", "name", "INTEGER", ""), 107 }, 108 }, 109 schema: Schema{ 110 fieldSchema("desc", "name", "INTEGER", false, false), 111 }, 112 }, 113 { 114 // float 115 bqSchema: &bq.TableSchema{ 116 Fields: []*bq.TableFieldSchema{ 117 bqTableFieldSchema("desc", "name", "FLOAT", ""), 118 }, 119 }, 120 schema: Schema{ 121 fieldSchema("desc", "name", "FLOAT", false, false), 122 }, 123 }, 124 { 125 // boolean 126 bqSchema: &bq.TableSchema{ 127 Fields: []*bq.TableFieldSchema{ 128 bqTableFieldSchema("desc", "name", "BOOLEAN", ""), 129 }, 130 }, 131 schema: Schema{ 132 fieldSchema("desc", "name", "BOOLEAN", false, false), 133 }, 134 }, 135 { 136 // timestamp 137 bqSchema: &bq.TableSchema{ 138 Fields: []*bq.TableFieldSchema{ 139 bqTableFieldSchema("desc", "name", "TIMESTAMP", ""), 140 }, 141 }, 142 schema: Schema{ 143 fieldSchema("desc", "name", "TIMESTAMP", false, false), 144 }, 145 }, 146 { 147 // civil times 148 bqSchema: &bq.TableSchema{ 149 Fields: []*bq.TableFieldSchema{ 150 bqTableFieldSchema("desc", "f1", "TIME", ""), 151 bqTableFieldSchema("desc", "f2", "DATE", ""), 152 bqTableFieldSchema("desc", "f3", "DATETIME", ""), 153 }, 154 }, 155 schema: Schema{ 156 fieldSchema("desc", "f1", "TIME", false, false), 157 fieldSchema("desc", "f2", "DATE", false, false), 158 fieldSchema("desc", "f3", "DATETIME", false, false), 159 }, 160 }, 161 { 162 // numeric 163 bqSchema: &bq.TableSchema{ 164 Fields: []*bq.TableFieldSchema{ 165 bqTableFieldSchema("desc", "n", "NUMERIC", ""), 166 }, 167 }, 168 schema: Schema{ 169 fieldSchema("desc", "n", "NUMERIC", false, false), 170 }, 171 }, 172 { 173 bqSchema: &bq.TableSchema{ 174 Fields: []*bq.TableFieldSchema{ 175 bqTableFieldSchema("geo", "g", "GEOGRAPHY", ""), 176 }, 177 }, 178 schema: Schema{ 179 fieldSchema("geo", "g", "GEOGRAPHY", false, false), 180 }, 181 }, 182 { 183 // nested 184 bqSchema: &bq.TableSchema{ 185 Fields: []*bq.TableFieldSchema{ 186 { 187 Description: "An outer schema wrapping a nested schema", 188 Name: "outer", 189 Mode: "REQUIRED", 190 Type: "RECORD", 191 Fields: []*bq.TableFieldSchema{ 192 bqTableFieldSchema("inner field", "inner", "STRING", ""), 193 }, 194 }, 195 }, 196 }, 197 schema: Schema{ 198 &FieldSchema{ 199 Description: "An outer schema wrapping a nested schema", 200 Name: "outer", 201 Required: true, 202 Type: "RECORD", 203 Schema: Schema{ 204 { 205 Description: "inner field", 206 Name: "inner", 207 Type: "STRING", 208 }, 209 }, 210 }, 211 }, 212 }, 213 } 214 for _, tc := range testCases { 215 bqSchema := tc.schema.toBQ() 216 if !testutil.Equal(bqSchema, tc.bqSchema) { 217 t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v", 218 pretty.Value(bqSchema), pretty.Value(tc.bqSchema)) 219 } 220 schema := bqToSchema(tc.bqSchema) 221 if !testutil.Equal(schema, tc.schema) { 222 t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema) 223 } 224 } 225} 226 227type allStrings struct { 228 String string 229 ByteSlice []byte 230} 231 232type allSignedIntegers struct { 233 Int64 int64 234 Int32 int32 235 Int16 int16 236 Int8 int8 237 Int int 238} 239 240type allUnsignedIntegers struct { 241 Uint32 uint32 242 Uint16 uint16 243 Uint8 uint8 244} 245 246type allFloat struct { 247 Float64 float64 248 Float32 float32 249 // NOTE: Complex32 and Complex64 are unsupported by BigQuery 250} 251 252type allBoolean struct { 253 Bool bool 254} 255 256type allTime struct { 257 Timestamp time.Time 258 Time civil.Time 259 Date civil.Date 260 DateTime civil.DateTime 261} 262 263type allNumeric struct { 264 Numeric *big.Rat 265} 266 267func reqField(name, typ string) *FieldSchema { 268 return &FieldSchema{ 269 Name: name, 270 Type: FieldType(typ), 271 Required: true, 272 } 273} 274 275func optField(name, typ string) *FieldSchema { 276 return &FieldSchema{ 277 Name: name, 278 Type: FieldType(typ), 279 Required: false, 280 } 281} 282 283func TestSimpleInference(t *testing.T) { 284 testCases := []struct { 285 in interface{} 286 want Schema 287 }{ 288 { 289 in: allSignedIntegers{}, 290 want: Schema{ 291 reqField("Int64", "INTEGER"), 292 reqField("Int32", "INTEGER"), 293 reqField("Int16", "INTEGER"), 294 reqField("Int8", "INTEGER"), 295 reqField("Int", "INTEGER"), 296 }, 297 }, 298 { 299 in: allUnsignedIntegers{}, 300 want: Schema{ 301 reqField("Uint32", "INTEGER"), 302 reqField("Uint16", "INTEGER"), 303 reqField("Uint8", "INTEGER"), 304 }, 305 }, 306 { 307 in: allFloat{}, 308 want: Schema{ 309 reqField("Float64", "FLOAT"), 310 reqField("Float32", "FLOAT"), 311 }, 312 }, 313 { 314 in: allBoolean{}, 315 want: Schema{ 316 reqField("Bool", "BOOLEAN"), 317 }, 318 }, 319 { 320 in: &allBoolean{}, 321 want: Schema{ 322 reqField("Bool", "BOOLEAN"), 323 }, 324 }, 325 { 326 in: allTime{}, 327 want: Schema{ 328 reqField("Timestamp", "TIMESTAMP"), 329 reqField("Time", "TIME"), 330 reqField("Date", "DATE"), 331 reqField("DateTime", "DATETIME"), 332 }, 333 }, 334 { 335 in: &allNumeric{}, 336 want: Schema{ 337 reqField("Numeric", "NUMERIC"), 338 }, 339 }, 340 { 341 in: allStrings{}, 342 want: Schema{ 343 reqField("String", "STRING"), 344 reqField("ByteSlice", "BYTES"), 345 }, 346 }, 347 } 348 for _, tc := range testCases { 349 got, err := InferSchema(tc.in) 350 if err != nil { 351 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 352 } 353 if !testutil.Equal(got, tc.want) { 354 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 355 pretty.Value(got), pretty.Value(tc.want)) 356 } 357 } 358} 359 360type containsNested struct { 361 NotNested int 362 Nested struct { 363 Inside int 364 } 365} 366 367type containsDoubleNested struct { 368 NotNested int 369 Nested struct { 370 InsideNested struct { 371 Inside int 372 } 373 } 374} 375 376type ptrNested struct { 377 Ptr *struct{ Inside int } 378} 379 380type dup struct { // more than one field of the same struct type 381 A, B allBoolean 382} 383 384func TestNestedInference(t *testing.T) { 385 testCases := []struct { 386 in interface{} 387 want Schema 388 }{ 389 { 390 in: containsNested{}, 391 want: Schema{ 392 reqField("NotNested", "INTEGER"), 393 &FieldSchema{ 394 Name: "Nested", 395 Required: true, 396 Type: "RECORD", 397 Schema: Schema{reqField("Inside", "INTEGER")}, 398 }, 399 }, 400 }, 401 { 402 in: containsDoubleNested{}, 403 want: Schema{ 404 reqField("NotNested", "INTEGER"), 405 &FieldSchema{ 406 Name: "Nested", 407 Required: true, 408 Type: "RECORD", 409 Schema: Schema{ 410 { 411 Name: "InsideNested", 412 Required: true, 413 Type: "RECORD", 414 Schema: Schema{reqField("Inside", "INTEGER")}, 415 }, 416 }, 417 }, 418 }, 419 }, 420 { 421 in: ptrNested{}, 422 want: Schema{ 423 &FieldSchema{ 424 Name: "Ptr", 425 Required: true, 426 Type: "RECORD", 427 Schema: Schema{reqField("Inside", "INTEGER")}, 428 }, 429 }, 430 }, 431 { 432 in: dup{}, 433 want: Schema{ 434 &FieldSchema{ 435 Name: "A", 436 Required: true, 437 Type: "RECORD", 438 Schema: Schema{reqField("Bool", "BOOLEAN")}, 439 }, 440 &FieldSchema{ 441 Name: "B", 442 Required: true, 443 Type: "RECORD", 444 Schema: Schema{reqField("Bool", "BOOLEAN")}, 445 }, 446 }, 447 }, 448 } 449 450 for _, tc := range testCases { 451 got, err := InferSchema(tc.in) 452 if err != nil { 453 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 454 } 455 if !testutil.Equal(got, tc.want) { 456 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 457 pretty.Value(got), pretty.Value(tc.want)) 458 } 459 } 460} 461 462type repeated struct { 463 NotRepeated []byte 464 RepeatedByteSlice [][]byte 465 Slice []int 466 Array [5]bool 467} 468 469type nestedRepeated struct { 470 NotRepeated int 471 Repeated []struct { 472 Inside int 473 } 474 RepeatedPtr []*struct{ Inside int } 475} 476 477func repField(name, typ string) *FieldSchema { 478 return &FieldSchema{ 479 Name: name, 480 Type: FieldType(typ), 481 Repeated: true, 482 } 483} 484 485func TestRepeatedInference(t *testing.T) { 486 testCases := []struct { 487 in interface{} 488 want Schema 489 }{ 490 { 491 in: repeated{}, 492 want: Schema{ 493 reqField("NotRepeated", "BYTES"), 494 repField("RepeatedByteSlice", "BYTES"), 495 repField("Slice", "INTEGER"), 496 repField("Array", "BOOLEAN"), 497 }, 498 }, 499 { 500 in: nestedRepeated{}, 501 want: Schema{ 502 reqField("NotRepeated", "INTEGER"), 503 { 504 Name: "Repeated", 505 Repeated: true, 506 Type: "RECORD", 507 Schema: Schema{reqField("Inside", "INTEGER")}, 508 }, 509 { 510 Name: "RepeatedPtr", 511 Repeated: true, 512 Type: "RECORD", 513 Schema: Schema{reqField("Inside", "INTEGER")}, 514 }, 515 }, 516 }, 517 } 518 519 for i, tc := range testCases { 520 got, err := InferSchema(tc.in) 521 if err != nil { 522 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 523 } 524 if !testutil.Equal(got, tc.want) { 525 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 526 pretty.Value(got), pretty.Value(tc.want)) 527 } 528 } 529} 530 531type allNulls struct { 532 A NullInt64 533 B NullFloat64 534 C NullBool 535 D NullString 536 E NullTimestamp 537 F NullTime 538 G NullDate 539 H NullDateTime 540 I NullGeography 541} 542 543func TestNullInference(t *testing.T) { 544 got, err := InferSchema(allNulls{}) 545 if err != nil { 546 t.Fatal(err) 547 } 548 want := Schema{ 549 optField("A", "INTEGER"), 550 optField("B", "FLOAT"), 551 optField("C", "BOOLEAN"), 552 optField("D", "STRING"), 553 optField("E", "TIMESTAMP"), 554 optField("F", "TIME"), 555 optField("G", "DATE"), 556 optField("H", "DATETIME"), 557 optField("I", "GEOGRAPHY"), 558 } 559 if diff := testutil.Diff(got, want); diff != "" { 560 t.Error(diff) 561 } 562} 563 564type Embedded struct { 565 Embedded int 566} 567 568type embedded struct { 569 Embedded2 int 570} 571 572type nestedEmbedded struct { 573 Embedded 574 embedded 575} 576 577func TestEmbeddedInference(t *testing.T) { 578 got, err := InferSchema(nestedEmbedded{}) 579 if err != nil { 580 t.Fatal(err) 581 } 582 want := Schema{ 583 reqField("Embedded", "INTEGER"), 584 reqField("Embedded2", "INTEGER"), 585 } 586 if !testutil.Equal(got, want) { 587 t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want)) 588 } 589} 590 591func TestRecursiveInference(t *testing.T) { 592 type List struct { 593 Val int 594 Next *List 595 } 596 597 _, err := InferSchema(List{}) 598 if err == nil { 599 t.Fatal("got nil, want error") 600 } 601} 602 603type withTags struct { 604 NoTag int 605 ExcludeTag int `bigquery:"-"` 606 SimpleTag int `bigquery:"simple_tag"` 607 UnderscoreTag int `bigquery:"_id"` 608 MixedCase int `bigquery:"MIXEDcase"` 609 Nullable []byte `bigquery:",nullable"` 610 NullNumeric *big.Rat `bigquery:",nullable"` 611} 612 613type withTagsNested struct { 614 Nested withTags `bigquery:"nested"` 615 NestedAnonymous struct { 616 ExcludeTag int `bigquery:"-"` 617 Inside int `bigquery:"inside"` 618 } `bigquery:"anon"` 619 PNested *struct{ X int } // not nullable, for backwards compatibility 620 PNestedNullable *struct{ X int } `bigquery:",nullable"` 621} 622 623type withTagsRepeated struct { 624 Repeated []withTags `bigquery:"repeated"` 625 RepeatedAnonymous []struct { 626 ExcludeTag int `bigquery:"-"` 627 Inside int `bigquery:"inside"` 628 } `bigquery:"anon"` 629} 630 631type withTagsEmbedded struct { 632 withTags 633} 634 635var withTagsSchema = Schema{ 636 reqField("NoTag", "INTEGER"), 637 reqField("simple_tag", "INTEGER"), 638 reqField("_id", "INTEGER"), 639 reqField("MIXEDcase", "INTEGER"), 640 optField("Nullable", "BYTES"), 641 optField("NullNumeric", "NUMERIC"), 642} 643 644func TestTagInference(t *testing.T) { 645 testCases := []struct { 646 in interface{} 647 want Schema 648 }{ 649 { 650 in: withTags{}, 651 want: withTagsSchema, 652 }, 653 { 654 in: withTagsNested{}, 655 want: Schema{ 656 &FieldSchema{ 657 Name: "nested", 658 Required: true, 659 Type: "RECORD", 660 Schema: withTagsSchema, 661 }, 662 &FieldSchema{ 663 Name: "anon", 664 Required: true, 665 Type: "RECORD", 666 Schema: Schema{reqField("inside", "INTEGER")}, 667 }, 668 &FieldSchema{ 669 Name: "PNested", 670 Required: true, 671 Type: "RECORD", 672 Schema: Schema{reqField("X", "INTEGER")}, 673 }, 674 &FieldSchema{ 675 Name: "PNestedNullable", 676 Required: false, 677 Type: "RECORD", 678 Schema: Schema{reqField("X", "INTEGER")}, 679 }, 680 }, 681 }, 682 { 683 in: withTagsRepeated{}, 684 want: Schema{ 685 &FieldSchema{ 686 Name: "repeated", 687 Repeated: true, 688 Type: "RECORD", 689 Schema: withTagsSchema, 690 }, 691 &FieldSchema{ 692 Name: "anon", 693 Repeated: true, 694 Type: "RECORD", 695 Schema: Schema{reqField("inside", "INTEGER")}, 696 }, 697 }, 698 }, 699 { 700 in: withTagsEmbedded{}, 701 want: withTagsSchema, 702 }, 703 } 704 for i, tc := range testCases { 705 got, err := InferSchema(tc.in) 706 if err != nil { 707 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 708 } 709 if !testutil.Equal(got, tc.want) { 710 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 711 pretty.Value(got), pretty.Value(tc.want)) 712 } 713 } 714} 715 716func TestTagInferenceErrors(t *testing.T) { 717 testCases := []interface{}{ 718 struct { 719 LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"` 720 }{}, 721 struct { 722 UnsupporedStartChar int `bigquery:"øab"` 723 }{}, 724 struct { 725 UnsupportedEndChar int `bigquery:"abø"` 726 }{}, 727 struct { 728 UnsupportedMiddleChar int `bigquery:"aøb"` 729 }{}, 730 struct { 731 StartInt int `bigquery:"1abc"` 732 }{}, 733 struct { 734 Hyphens int `bigquery:"a-b"` 735 }{}, 736 } 737 for i, tc := range testCases { 738 739 _, got := InferSchema(tc) 740 if _, ok := got.(invalidFieldNameError); !ok { 741 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant invalidFieldNameError", i, got) 742 } 743 } 744 745 _, err := InferSchema(struct { 746 X int `bigquery:",optional"` 747 }{}) 748 if err == nil { 749 t.Error("got nil, want error") 750 } 751} 752 753func TestSchemaErrors(t *testing.T) { 754 testCases := []struct { 755 in interface{} 756 want interface{} 757 }{ 758 { 759 in: []byte{}, 760 want: noStructError{}, 761 }, 762 { 763 in: new(int), 764 want: noStructError{}, 765 }, 766 { 767 in: struct{ Uint uint }{}, 768 want: unsupportedFieldTypeError{}, 769 }, 770 { 771 in: struct{ Uint64 uint64 }{}, 772 want: unsupportedFieldTypeError{}, 773 }, 774 { 775 in: struct{ Uintptr uintptr }{}, 776 want: unsupportedFieldTypeError{}, 777 }, 778 { 779 in: struct{ Complex complex64 }{}, 780 want: unsupportedFieldTypeError{}, 781 }, 782 { 783 in: struct{ Map map[string]int }{}, 784 want: unsupportedFieldTypeError{}, 785 }, 786 { 787 in: struct{ Chan chan bool }{}, 788 want: unsupportedFieldTypeError{}, 789 }, 790 { 791 in: struct{ Ptr *int }{}, 792 want: unsupportedFieldTypeError{}, 793 }, 794 { 795 in: struct{ Interface interface{} }{}, 796 want: unsupportedFieldTypeError{}, 797 }, 798 { 799 in: struct{ MultiDimensional [][]int }{}, 800 want: unsupportedFieldTypeError{}, 801 }, 802 { 803 in: struct{ MultiDimensional [][][]byte }{}, 804 want: unsupportedFieldTypeError{}, 805 }, 806 { 807 in: struct{ SliceOfPointer []*int }{}, 808 want: unsupportedFieldTypeError{}, 809 }, 810 { 811 in: struct{ SliceOfNull []NullInt64 }{}, 812 want: unsupportedFieldTypeError{}, 813 }, 814 { 815 in: struct{ ChanSlice []chan bool }{}, 816 want: unsupportedFieldTypeError{}, 817 }, 818 { 819 in: struct{ NestedChan struct{ Chan []chan bool } }{}, 820 want: unsupportedFieldTypeError{}, 821 }, 822 { 823 in: struct { 824 X int `bigquery:",nullable"` 825 }{}, 826 want: badNullableError{}, 827 }, 828 { 829 in: struct { 830 X bool `bigquery:",nullable"` 831 }{}, 832 want: badNullableError{}, 833 }, 834 { 835 in: struct { 836 X struct{ N int } `bigquery:",nullable"` 837 }{}, 838 want: badNullableError{}, 839 }, 840 { 841 in: struct { 842 X []int `bigquery:",nullable"` 843 }{}, 844 want: badNullableError{}, 845 }, 846 { 847 in: struct{ X *[]byte }{}, 848 want: unsupportedFieldTypeError{}, 849 }, 850 { 851 in: struct{ X *[]int }{}, 852 want: unsupportedFieldTypeError{}, 853 }, 854 { 855 in: struct{ X *int }{}, 856 want: unsupportedFieldTypeError{}, 857 }, 858 } 859 for _, tc := range testCases { 860 _, got := InferSchema(tc.in) 861 if reflect.TypeOf(got) != reflect.TypeOf(tc.want) { 862 t.Errorf("%#v: got:\n%#v\nwant type %T", tc.in, got, tc.want) 863 } 864 } 865} 866 867func TestHasRecursiveType(t *testing.T) { 868 type ( 869 nonStruct int 870 nonRec struct{ A string } 871 dup struct{ A, B nonRec } 872 rec struct { 873 A int 874 B *rec 875 } 876 recUnexported struct { 877 A int 878 } 879 hasRec struct { 880 A int 881 R *rec 882 } 883 recSlicePointer struct { 884 A []*recSlicePointer 885 } 886 ) 887 for _, test := range []struct { 888 in interface{} 889 want bool 890 }{ 891 {nonStruct(0), false}, 892 {nonRec{}, false}, 893 {dup{}, false}, 894 {rec{}, true}, 895 {recUnexported{}, false}, 896 {hasRec{}, true}, 897 {&recSlicePointer{}, true}, 898 } { 899 got, err := hasRecursiveType(reflect.TypeOf(test.in), nil) 900 if err != nil { 901 t.Fatal(err) 902 } 903 if got != test.want { 904 t.Errorf("%T: got %t, want %t", test.in, got, test.want) 905 } 906 } 907} 908 909func TestSchemaFromJSON(t *testing.T) { 910 testCasesExpectingSuccess := []struct { 911 bqSchemaJSON []byte 912 description string 913 expectedSchema Schema 914 }{ 915 { 916 description: "Flat table with a mixture of NULLABLE and REQUIRED fields", 917 bqSchemaJSON: []byte(` 918[ 919 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 920 {"name":"flat_bytes","type":"BYTES","mode":"REQUIRED","description":"Flat required BYTES"}, 921 {"name":"flat_integer","type":"INTEGER","mode":"NULLABLE","description":"Flat nullable INTEGER"}, 922 {"name":"flat_float","type":"FLOAT","mode":"REQUIRED","description":"Flat required FLOAT"}, 923 {"name":"flat_boolean","type":"BOOLEAN","mode":"NULLABLE","description":"Flat nullable BOOLEAN"}, 924 {"name":"flat_timestamp","type":"TIMESTAMP","mode":"REQUIRED","description":"Flat required TIMESTAMP"}, 925 {"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"}, 926 {"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"}, 927 {"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"}, 928 {"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat nullable NUMERIC"}, 929 {"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"} 930]`), 931 expectedSchema: Schema{ 932 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false), 933 fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true), 934 fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false), 935 fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true), 936 fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false), 937 fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true), 938 fieldSchema("Flat required DATE", "flat_date", "DATE", false, false), 939 fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true), 940 fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false), 941 fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true), 942 fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true), 943 }, 944 }, 945 { 946 description: "Table with a nested RECORD", 947 bqSchemaJSON: []byte(` 948[ 949 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 950 {"name":"nested_record","type":"RECORD","mode":"NULLABLE","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 951]`), 952 expectedSchema: Schema{ 953 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false), 954 &FieldSchema{ 955 Description: "Nested nullable RECORD", 956 Name: "nested_record", 957 Required: false, 958 Type: "RECORD", 959 Schema: Schema{ 960 { 961 Description: "First nested record field", 962 Name: "record_field_1", 963 Required: false, 964 Type: "STRING", 965 }, 966 { 967 Description: "Second nested record field", 968 Name: "record_field_2", 969 Required: true, 970 Type: "INTEGER", 971 }, 972 }, 973 }, 974 }, 975 }, 976 { 977 description: "Table with a repeated RECORD", 978 bqSchemaJSON: []byte(` 979[ 980 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 981 {"name":"nested_record","type":"RECORD","mode":"REPEATED","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 982]`), 983 expectedSchema: Schema{ 984 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false), 985 &FieldSchema{ 986 Description: "Nested nullable RECORD", 987 Name: "nested_record", 988 Repeated: true, 989 Required: false, 990 Type: "RECORD", 991 Schema: Schema{ 992 { 993 Description: "First nested record field", 994 Name: "record_field_1", 995 Required: false, 996 Type: "STRING", 997 }, 998 { 999 Description: "Second nested record field", 1000 Name: "record_field_2", 1001 Required: true, 1002 Type: "INTEGER", 1003 }, 1004 }, 1005 }, 1006 }, 1007 }, 1008 } 1009 for _, tc := range testCasesExpectingSuccess { 1010 convertedSchema, err := SchemaFromJSON(tc.bqSchemaJSON) 1011 if err != nil { 1012 t.Errorf("encountered an error when converting JSON table schema (%s): %v", tc.description, err) 1013 continue 1014 } 1015 if !testutil.Equal(convertedSchema, tc.expectedSchema) { 1016 t.Errorf("generated JSON table schema (%s) differs from the expected schema", tc.description) 1017 } 1018 } 1019 1020 testCasesExpectingFailure := []struct { 1021 bqSchemaJSON []byte 1022 description string 1023 }{ 1024 { 1025 description: "Schema with invalid JSON", 1026 bqSchemaJSON: []byte(`This is not JSON`), 1027 }, 1028 { 1029 description: "Schema with unknown field type", 1030 bqSchemaJSON: []byte(`[{"name":"strange_type","type":"STRANGE","description":"This type should not exist"}]`), 1031 }, 1032 { 1033 description: "Schema with zero length", 1034 bqSchemaJSON: []byte(``), 1035 }, 1036 } 1037 for _, tc := range testCasesExpectingFailure { 1038 _, err := SchemaFromJSON(tc.bqSchemaJSON) 1039 if err == nil { 1040 t.Errorf("converting this schema should have returned an error (%s): %v", tc.description, err) 1041 continue 1042 } 1043 } 1044} 1045