1// Copyright 2015 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "fmt" 19 "math/big" 20 "reflect" 21 "testing" 22 "time" 23 24 "cloud.google.com/go/civil" 25 "cloud.google.com/go/internal/pretty" 26 "cloud.google.com/go/internal/testutil" 27 bq "google.golang.org/api/bigquery/v2" 28) 29 30func (fs *FieldSchema) GoString() string { 31 if fs == nil { 32 return "<nil>" 33 } 34 35 return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}", 36 fs.Name, 37 fs.Description, 38 fs.Repeated, 39 fs.Required, 40 fs.Type, 41 fmt.Sprintf("%#v", fs.Schema), 42 ) 43} 44 45func bqTableFieldSchema(desc, name, typ, mode string, tags []string) *bq.TableFieldSchema { 46 var policy *bq.TableFieldSchemaPolicyTags 47 if tags != nil { 48 policy = &bq.TableFieldSchemaPolicyTags{ 49 Names: tags, 50 } 51 } 52 return &bq.TableFieldSchema{ 53 Description: desc, 54 Name: name, 55 Mode: mode, 56 Type: typ, 57 PolicyTags: policy, 58 } 59} 60 61func fieldSchema(desc, name, typ string, repeated, required bool, tags []string) *FieldSchema { 62 var policy *PolicyTagList 63 if tags != nil { 64 policy = &PolicyTagList{ 65 Names: tags, 66 } 67 } 68 return &FieldSchema{ 69 Description: desc, 70 Name: name, 71 Repeated: repeated, 72 Required: required, 73 Type: FieldType(typ), 74 PolicyTags: policy, 75 } 76} 77 78func TestRelaxSchema(t *testing.T) { 79 testCases := []struct { 80 in Schema 81 expected Schema 82 }{ 83 { 84 Schema{ 85 &FieldSchema{ 86 Description: "a relaxed schema", 87 Required: false, 88 Type: StringFieldType, 89 }, 90 }, 91 Schema{ 92 &FieldSchema{ 93 Description: "a relaxed schema", 94 Required: false, 95 Type: StringFieldType, 96 }, 97 }, 98 }, 99 { 100 Schema{ 101 &FieldSchema{ 102 Description: "a required string", 103 Required: true, 104 Type: StringFieldType, 105 }, 106 &FieldSchema{ 107 Description: "a required integer", 108 Required: true, 109 Type: IntegerFieldType, 110 }, 111 }, 112 Schema{ 113 &FieldSchema{ 114 Description: "a required string", 115 Required: false, 116 Type: StringFieldType, 117 }, 118 &FieldSchema{ 119 Description: "a required integer", 120 Required: false, 121 Type: IntegerFieldType, 122 }, 123 }, 124 }, 125 { 126 Schema{ 127 &FieldSchema{ 128 Description: "An outer schema wrapping a nested schema", 129 Name: "outer", 130 Required: true, 131 Type: RecordFieldType, 132 Schema: Schema{ 133 { 134 Description: "inner field", 135 Name: "inner", 136 Type: StringFieldType, 137 Required: true, 138 }, 139 }, 140 }, 141 }, 142 Schema{ 143 &FieldSchema{ 144 Description: "An outer schema wrapping a nested schema", 145 Name: "outer", 146 Required: false, 147 Type: "RECORD", 148 Schema: Schema{ 149 { 150 Description: "inner field", 151 Name: "inner", 152 Type: "STRING", 153 Required: false, 154 }, 155 }, 156 }, 157 }, 158 }, 159 } 160 for _, tc := range testCases { 161 converted := tc.in.Relax() 162 if !testutil.Equal(converted, tc.expected) { 163 t.Errorf("relaxing schema: got:\n%v\nwant:\n%v", 164 pretty.Value(converted), pretty.Value(tc.expected)) 165 } 166 } 167} 168func TestSchemaConversion(t *testing.T) { 169 testCases := []struct { 170 schema Schema 171 bqSchema *bq.TableSchema 172 }{ 173 { 174 // required 175 bqSchema: &bq.TableSchema{ 176 Fields: []*bq.TableFieldSchema{ 177 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil), 178 }, 179 }, 180 schema: Schema{ 181 fieldSchema("desc", "name", "STRING", false, true, nil), 182 }, 183 }, 184 { 185 // repeated 186 bqSchema: &bq.TableSchema{ 187 Fields: []*bq.TableFieldSchema{ 188 bqTableFieldSchema("desc", "name", "STRING", "REPEATED", nil), 189 }, 190 }, 191 schema: Schema{ 192 fieldSchema("desc", "name", "STRING", true, false, nil), 193 }, 194 }, 195 { 196 // nullable, string 197 bqSchema: &bq.TableSchema{ 198 Fields: []*bq.TableFieldSchema{ 199 bqTableFieldSchema("desc", "name", "STRING", "", nil), 200 }, 201 }, 202 schema: Schema{ 203 fieldSchema("desc", "name", "STRING", false, false, nil), 204 }, 205 }, 206 { 207 // integer 208 bqSchema: &bq.TableSchema{ 209 Fields: []*bq.TableFieldSchema{ 210 bqTableFieldSchema("desc", "name", "INTEGER", "", nil), 211 }, 212 }, 213 schema: Schema{ 214 fieldSchema("desc", "name", "INTEGER", false, false, nil), 215 }, 216 }, 217 { 218 // float 219 bqSchema: &bq.TableSchema{ 220 Fields: []*bq.TableFieldSchema{ 221 bqTableFieldSchema("desc", "name", "FLOAT", "", nil), 222 }, 223 }, 224 schema: Schema{ 225 fieldSchema("desc", "name", "FLOAT", false, false, nil), 226 }, 227 }, 228 { 229 // boolean 230 bqSchema: &bq.TableSchema{ 231 Fields: []*bq.TableFieldSchema{ 232 bqTableFieldSchema("desc", "name", "BOOLEAN", "", nil), 233 }, 234 }, 235 schema: Schema{ 236 fieldSchema("desc", "name", "BOOLEAN", false, false, nil), 237 }, 238 }, 239 { 240 // timestamp 241 bqSchema: &bq.TableSchema{ 242 Fields: []*bq.TableFieldSchema{ 243 bqTableFieldSchema("desc", "name", "TIMESTAMP", "", nil), 244 }, 245 }, 246 schema: Schema{ 247 fieldSchema("desc", "name", "TIMESTAMP", false, false, nil), 248 }, 249 }, 250 { 251 // civil times 252 bqSchema: &bq.TableSchema{ 253 Fields: []*bq.TableFieldSchema{ 254 bqTableFieldSchema("desc", "f1", "TIME", "", nil), 255 bqTableFieldSchema("desc", "f2", "DATE", "", nil), 256 bqTableFieldSchema("desc", "f3", "DATETIME", "", nil), 257 }, 258 }, 259 schema: Schema{ 260 fieldSchema("desc", "f1", "TIME", false, false, nil), 261 fieldSchema("desc", "f2", "DATE", false, false, nil), 262 fieldSchema("desc", "f3", "DATETIME", false, false, nil), 263 }, 264 }, 265 { 266 // numeric 267 bqSchema: &bq.TableSchema{ 268 Fields: []*bq.TableFieldSchema{ 269 bqTableFieldSchema("desc", "n", "NUMERIC", "", nil), 270 }, 271 }, 272 schema: Schema{ 273 fieldSchema("desc", "n", "NUMERIC", false, false, nil), 274 }, 275 }, 276 { 277 // geography 278 bqSchema: &bq.TableSchema{ 279 Fields: []*bq.TableFieldSchema{ 280 bqTableFieldSchema("geo", "g", "GEOGRAPHY", "", nil), 281 }, 282 }, 283 schema: Schema{ 284 fieldSchema("geo", "g", "GEOGRAPHY", false, false, nil), 285 }, 286 }, 287 { 288 // policy tags 289 bqSchema: &bq.TableSchema{ 290 Fields: []*bq.TableFieldSchema{ 291 bqTableFieldSchema("some pii", "restrictedfield", "STRING", "", []string{"tag1"}), 292 }, 293 }, 294 schema: Schema{ 295 fieldSchema("some pii", "restrictedfield", "STRING", false, false, []string{"tag1"}), 296 }, 297 }, 298 { 299 // nested 300 bqSchema: &bq.TableSchema{ 301 Fields: []*bq.TableFieldSchema{ 302 { 303 Description: "An outer schema wrapping a nested schema", 304 Name: "outer", 305 Mode: "REQUIRED", 306 Type: "RECORD", 307 Fields: []*bq.TableFieldSchema{ 308 bqTableFieldSchema("inner field", "inner", "STRING", "", nil), 309 }, 310 }, 311 }, 312 }, 313 schema: Schema{ 314 &FieldSchema{ 315 Description: "An outer schema wrapping a nested schema", 316 Name: "outer", 317 Required: true, 318 Type: "RECORD", 319 Schema: Schema{ 320 { 321 Description: "inner field", 322 Name: "inner", 323 Type: "STRING", 324 }, 325 }, 326 }, 327 }, 328 }, 329 } 330 for _, tc := range testCases { 331 bqSchema := tc.schema.toBQ() 332 if !testutil.Equal(bqSchema, tc.bqSchema) { 333 t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v", 334 pretty.Value(bqSchema), pretty.Value(tc.bqSchema)) 335 } 336 schema := bqToSchema(tc.bqSchema) 337 if !testutil.Equal(schema, tc.schema) { 338 t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema) 339 } 340 } 341} 342 343type allStrings struct { 344 String string 345 ByteSlice []byte 346} 347 348type allSignedIntegers struct { 349 Int64 int64 350 Int32 int32 351 Int16 int16 352 Int8 int8 353 Int int 354} 355 356type allUnsignedIntegers struct { 357 Uint32 uint32 358 Uint16 uint16 359 Uint8 uint8 360} 361 362type allFloat struct { 363 Float64 float64 364 Float32 float32 365 // NOTE: Complex32 and Complex64 are unsupported by BigQuery 366} 367 368type allBoolean struct { 369 Bool bool 370} 371 372type allTime struct { 373 Timestamp time.Time 374 Time civil.Time 375 Date civil.Date 376 DateTime civil.DateTime 377} 378 379type allNumeric struct { 380 Numeric *big.Rat 381} 382 383func reqField(name, typ string) *FieldSchema { 384 return &FieldSchema{ 385 Name: name, 386 Type: FieldType(typ), 387 Required: true, 388 } 389} 390 391func optField(name, typ string) *FieldSchema { 392 return &FieldSchema{ 393 Name: name, 394 Type: FieldType(typ), 395 Required: false, 396 } 397} 398 399func TestSimpleInference(t *testing.T) { 400 testCases := []struct { 401 in interface{} 402 want Schema 403 }{ 404 { 405 in: allSignedIntegers{}, 406 want: Schema{ 407 reqField("Int64", "INTEGER"), 408 reqField("Int32", "INTEGER"), 409 reqField("Int16", "INTEGER"), 410 reqField("Int8", "INTEGER"), 411 reqField("Int", "INTEGER"), 412 }, 413 }, 414 { 415 in: allUnsignedIntegers{}, 416 want: Schema{ 417 reqField("Uint32", "INTEGER"), 418 reqField("Uint16", "INTEGER"), 419 reqField("Uint8", "INTEGER"), 420 }, 421 }, 422 { 423 in: allFloat{}, 424 want: Schema{ 425 reqField("Float64", "FLOAT"), 426 reqField("Float32", "FLOAT"), 427 }, 428 }, 429 { 430 in: allBoolean{}, 431 want: Schema{ 432 reqField("Bool", "BOOLEAN"), 433 }, 434 }, 435 { 436 in: &allBoolean{}, 437 want: Schema{ 438 reqField("Bool", "BOOLEAN"), 439 }, 440 }, 441 { 442 in: allTime{}, 443 want: Schema{ 444 reqField("Timestamp", "TIMESTAMP"), 445 reqField("Time", "TIME"), 446 reqField("Date", "DATE"), 447 reqField("DateTime", "DATETIME"), 448 }, 449 }, 450 { 451 in: &allNumeric{}, 452 want: Schema{ 453 reqField("Numeric", "NUMERIC"), 454 }, 455 }, 456 { 457 in: allStrings{}, 458 want: Schema{ 459 reqField("String", "STRING"), 460 reqField("ByteSlice", "BYTES"), 461 }, 462 }, 463 } 464 for _, tc := range testCases { 465 got, err := InferSchema(tc.in) 466 if err != nil { 467 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 468 } 469 if !testutil.Equal(got, tc.want) { 470 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 471 pretty.Value(got), pretty.Value(tc.want)) 472 } 473 } 474} 475 476type containsNested struct { 477 NotNested int 478 Nested struct { 479 Inside int 480 } 481} 482 483type containsDoubleNested struct { 484 NotNested int 485 Nested struct { 486 InsideNested struct { 487 Inside int 488 } 489 } 490} 491 492type ptrNested struct { 493 Ptr *struct{ Inside int } 494} 495 496type dup struct { // more than one field of the same struct type 497 A, B allBoolean 498} 499 500func TestNestedInference(t *testing.T) { 501 testCases := []struct { 502 in interface{} 503 want Schema 504 }{ 505 { 506 in: containsNested{}, 507 want: Schema{ 508 reqField("NotNested", "INTEGER"), 509 &FieldSchema{ 510 Name: "Nested", 511 Required: true, 512 Type: "RECORD", 513 Schema: Schema{reqField("Inside", "INTEGER")}, 514 }, 515 }, 516 }, 517 { 518 in: containsDoubleNested{}, 519 want: Schema{ 520 reqField("NotNested", "INTEGER"), 521 &FieldSchema{ 522 Name: "Nested", 523 Required: true, 524 Type: "RECORD", 525 Schema: Schema{ 526 { 527 Name: "InsideNested", 528 Required: true, 529 Type: "RECORD", 530 Schema: Schema{reqField("Inside", "INTEGER")}, 531 }, 532 }, 533 }, 534 }, 535 }, 536 { 537 in: ptrNested{}, 538 want: Schema{ 539 &FieldSchema{ 540 Name: "Ptr", 541 Required: true, 542 Type: "RECORD", 543 Schema: Schema{reqField("Inside", "INTEGER")}, 544 }, 545 }, 546 }, 547 { 548 in: dup{}, 549 want: Schema{ 550 &FieldSchema{ 551 Name: "A", 552 Required: true, 553 Type: "RECORD", 554 Schema: Schema{reqField("Bool", "BOOLEAN")}, 555 }, 556 &FieldSchema{ 557 Name: "B", 558 Required: true, 559 Type: "RECORD", 560 Schema: Schema{reqField("Bool", "BOOLEAN")}, 561 }, 562 }, 563 }, 564 } 565 566 for _, tc := range testCases { 567 got, err := InferSchema(tc.in) 568 if err != nil { 569 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 570 } 571 if !testutil.Equal(got, tc.want) { 572 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 573 pretty.Value(got), pretty.Value(tc.want)) 574 } 575 } 576} 577 578type repeated struct { 579 NotRepeated []byte 580 RepeatedByteSlice [][]byte 581 Slice []int 582 Array [5]bool 583} 584 585type nestedRepeated struct { 586 NotRepeated int 587 Repeated []struct { 588 Inside int 589 } 590 RepeatedPtr []*struct{ Inside int } 591} 592 593func repField(name, typ string) *FieldSchema { 594 return &FieldSchema{ 595 Name: name, 596 Type: FieldType(typ), 597 Repeated: true, 598 } 599} 600 601func TestRepeatedInference(t *testing.T) { 602 testCases := []struct { 603 in interface{} 604 want Schema 605 }{ 606 { 607 in: repeated{}, 608 want: Schema{ 609 reqField("NotRepeated", "BYTES"), 610 repField("RepeatedByteSlice", "BYTES"), 611 repField("Slice", "INTEGER"), 612 repField("Array", "BOOLEAN"), 613 }, 614 }, 615 { 616 in: nestedRepeated{}, 617 want: Schema{ 618 reqField("NotRepeated", "INTEGER"), 619 { 620 Name: "Repeated", 621 Repeated: true, 622 Type: "RECORD", 623 Schema: Schema{reqField("Inside", "INTEGER")}, 624 }, 625 { 626 Name: "RepeatedPtr", 627 Repeated: true, 628 Type: "RECORD", 629 Schema: Schema{reqField("Inside", "INTEGER")}, 630 }, 631 }, 632 }, 633 } 634 635 for i, tc := range testCases { 636 got, err := InferSchema(tc.in) 637 if err != nil { 638 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 639 } 640 if !testutil.Equal(got, tc.want) { 641 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 642 pretty.Value(got), pretty.Value(tc.want)) 643 } 644 } 645} 646 647type allNulls struct { 648 A NullInt64 649 B NullFloat64 650 C NullBool 651 D NullString 652 E NullTimestamp 653 F NullTime 654 G NullDate 655 H NullDateTime 656 I NullGeography 657} 658 659func TestNullInference(t *testing.T) { 660 got, err := InferSchema(allNulls{}) 661 if err != nil { 662 t.Fatal(err) 663 } 664 want := Schema{ 665 optField("A", "INTEGER"), 666 optField("B", "FLOAT"), 667 optField("C", "BOOLEAN"), 668 optField("D", "STRING"), 669 optField("E", "TIMESTAMP"), 670 optField("F", "TIME"), 671 optField("G", "DATE"), 672 optField("H", "DATETIME"), 673 optField("I", "GEOGRAPHY"), 674 } 675 if diff := testutil.Diff(got, want); diff != "" { 676 t.Error(diff) 677 } 678} 679 680type Embedded struct { 681 Embedded int 682} 683 684type embedded struct { 685 Embedded2 int 686} 687 688type nestedEmbedded struct { 689 Embedded 690 embedded 691} 692 693func TestEmbeddedInference(t *testing.T) { 694 got, err := InferSchema(nestedEmbedded{}) 695 if err != nil { 696 t.Fatal(err) 697 } 698 want := Schema{ 699 reqField("Embedded", "INTEGER"), 700 reqField("Embedded2", "INTEGER"), 701 } 702 if !testutil.Equal(got, want) { 703 t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want)) 704 } 705} 706 707func TestRecursiveInference(t *testing.T) { 708 type List struct { 709 Val int 710 Next *List 711 } 712 713 _, err := InferSchema(List{}) 714 if err == nil { 715 t.Fatal("got nil, want error") 716 } 717} 718 719type withTags struct { 720 NoTag int 721 ExcludeTag int `bigquery:"-"` 722 SimpleTag int `bigquery:"simple_tag"` 723 UnderscoreTag int `bigquery:"_id"` 724 MixedCase int `bigquery:"MIXEDcase"` 725 Nullable []byte `bigquery:",nullable"` 726 NullNumeric *big.Rat `bigquery:",nullable"` 727} 728 729type withTagsNested struct { 730 Nested withTags `bigquery:"nested"` 731 NestedAnonymous struct { 732 ExcludeTag int `bigquery:"-"` 733 Inside int `bigquery:"inside"` 734 } `bigquery:"anon"` 735 PNested *struct{ X int } // not nullable, for backwards compatibility 736 PNestedNullable *struct{ X int } `bigquery:",nullable"` 737} 738 739type withTagsRepeated struct { 740 Repeated []withTags `bigquery:"repeated"` 741 RepeatedAnonymous []struct { 742 ExcludeTag int `bigquery:"-"` 743 Inside int `bigquery:"inside"` 744 } `bigquery:"anon"` 745} 746 747type withTagsEmbedded struct { 748 withTags 749} 750 751var withTagsSchema = Schema{ 752 reqField("NoTag", "INTEGER"), 753 reqField("simple_tag", "INTEGER"), 754 reqField("_id", "INTEGER"), 755 reqField("MIXEDcase", "INTEGER"), 756 optField("Nullable", "BYTES"), 757 optField("NullNumeric", "NUMERIC"), 758} 759 760func TestTagInference(t *testing.T) { 761 testCases := []struct { 762 in interface{} 763 want Schema 764 }{ 765 { 766 in: withTags{}, 767 want: withTagsSchema, 768 }, 769 { 770 in: withTagsNested{}, 771 want: Schema{ 772 &FieldSchema{ 773 Name: "nested", 774 Required: true, 775 Type: "RECORD", 776 Schema: withTagsSchema, 777 }, 778 &FieldSchema{ 779 Name: "anon", 780 Required: true, 781 Type: "RECORD", 782 Schema: Schema{reqField("inside", "INTEGER")}, 783 }, 784 &FieldSchema{ 785 Name: "PNested", 786 Required: true, 787 Type: "RECORD", 788 Schema: Schema{reqField("X", "INTEGER")}, 789 }, 790 &FieldSchema{ 791 Name: "PNestedNullable", 792 Required: false, 793 Type: "RECORD", 794 Schema: Schema{reqField("X", "INTEGER")}, 795 }, 796 }, 797 }, 798 { 799 in: withTagsRepeated{}, 800 want: Schema{ 801 &FieldSchema{ 802 Name: "repeated", 803 Repeated: true, 804 Type: "RECORD", 805 Schema: withTagsSchema, 806 }, 807 &FieldSchema{ 808 Name: "anon", 809 Repeated: true, 810 Type: "RECORD", 811 Schema: Schema{reqField("inside", "INTEGER")}, 812 }, 813 }, 814 }, 815 { 816 in: withTagsEmbedded{}, 817 want: withTagsSchema, 818 }, 819 } 820 for i, tc := range testCases { 821 got, err := InferSchema(tc.in) 822 if err != nil { 823 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 824 } 825 if !testutil.Equal(got, tc.want) { 826 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 827 pretty.Value(got), pretty.Value(tc.want)) 828 } 829 } 830} 831 832func TestTagInferenceErrors(t *testing.T) { 833 testCases := []interface{}{ 834 struct { 835 LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"` 836 }{}, 837 struct { 838 UnsupporedStartChar int `bigquery:"øab"` 839 }{}, 840 struct { 841 UnsupportedEndChar int `bigquery:"abø"` 842 }{}, 843 struct { 844 UnsupportedMiddleChar int `bigquery:"aøb"` 845 }{}, 846 struct { 847 StartInt int `bigquery:"1abc"` 848 }{}, 849 struct { 850 Hyphens int `bigquery:"a-b"` 851 }{}, 852 } 853 for i, tc := range testCases { 854 855 _, got := InferSchema(tc) 856 if _, ok := got.(invalidFieldNameError); !ok { 857 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant invalidFieldNameError", i, got) 858 } 859 } 860 861 _, err := InferSchema(struct { 862 X int `bigquery:",optional"` 863 }{}) 864 if err == nil { 865 t.Error("got nil, want error") 866 } 867} 868 869func TestSchemaErrors(t *testing.T) { 870 testCases := []struct { 871 in interface{} 872 want interface{} 873 }{ 874 { 875 in: []byte{}, 876 want: noStructError{}, 877 }, 878 { 879 in: new(int), 880 want: noStructError{}, 881 }, 882 { 883 in: struct{ Uint uint }{}, 884 want: unsupportedFieldTypeError{}, 885 }, 886 { 887 in: struct{ Uint64 uint64 }{}, 888 want: unsupportedFieldTypeError{}, 889 }, 890 { 891 in: struct{ Uintptr uintptr }{}, 892 want: unsupportedFieldTypeError{}, 893 }, 894 { 895 in: struct{ Complex complex64 }{}, 896 want: unsupportedFieldTypeError{}, 897 }, 898 { 899 in: struct{ Map map[string]int }{}, 900 want: unsupportedFieldTypeError{}, 901 }, 902 { 903 in: struct{ Chan chan bool }{}, 904 want: unsupportedFieldTypeError{}, 905 }, 906 { 907 in: struct{ Ptr *int }{}, 908 want: unsupportedFieldTypeError{}, 909 }, 910 { 911 in: struct{ Interface interface{} }{}, 912 want: unsupportedFieldTypeError{}, 913 }, 914 { 915 in: struct{ MultiDimensional [][]int }{}, 916 want: unsupportedFieldTypeError{}, 917 }, 918 { 919 in: struct{ MultiDimensional [][][]byte }{}, 920 want: unsupportedFieldTypeError{}, 921 }, 922 { 923 in: struct{ SliceOfPointer []*int }{}, 924 want: unsupportedFieldTypeError{}, 925 }, 926 { 927 in: struct{ SliceOfNull []NullInt64 }{}, 928 want: unsupportedFieldTypeError{}, 929 }, 930 { 931 in: struct{ ChanSlice []chan bool }{}, 932 want: unsupportedFieldTypeError{}, 933 }, 934 { 935 in: struct{ NestedChan struct{ Chan []chan bool } }{}, 936 want: unsupportedFieldTypeError{}, 937 }, 938 { 939 in: struct { 940 X int `bigquery:",nullable"` 941 }{}, 942 want: badNullableError{}, 943 }, 944 { 945 in: struct { 946 X bool `bigquery:",nullable"` 947 }{}, 948 want: badNullableError{}, 949 }, 950 { 951 in: struct { 952 X struct{ N int } `bigquery:",nullable"` 953 }{}, 954 want: badNullableError{}, 955 }, 956 { 957 in: struct { 958 X []int `bigquery:",nullable"` 959 }{}, 960 want: badNullableError{}, 961 }, 962 { 963 in: struct{ X *[]byte }{}, 964 want: unsupportedFieldTypeError{}, 965 }, 966 { 967 in: struct{ X *[]int }{}, 968 want: unsupportedFieldTypeError{}, 969 }, 970 { 971 in: struct{ X *int }{}, 972 want: unsupportedFieldTypeError{}, 973 }, 974 } 975 for _, tc := range testCases { 976 _, got := InferSchema(tc.in) 977 if reflect.TypeOf(got) != reflect.TypeOf(tc.want) { 978 t.Errorf("%#v: got:\n%#v\nwant type %T", tc.in, got, tc.want) 979 } 980 } 981} 982 983func TestHasRecursiveType(t *testing.T) { 984 type ( 985 nonStruct int 986 nonRec struct{ A string } 987 dup struct{ A, B nonRec } 988 rec struct { 989 A int 990 B *rec 991 } 992 recUnexported struct { 993 A int 994 } 995 hasRec struct { 996 A int 997 R *rec 998 } 999 recSlicePointer struct { 1000 A []*recSlicePointer 1001 } 1002 ) 1003 for _, test := range []struct { 1004 in interface{} 1005 want bool 1006 }{ 1007 {nonStruct(0), false}, 1008 {nonRec{}, false}, 1009 {dup{}, false}, 1010 {rec{}, true}, 1011 {recUnexported{}, false}, 1012 {hasRec{}, true}, 1013 {&recSlicePointer{}, true}, 1014 } { 1015 got, err := hasRecursiveType(reflect.TypeOf(test.in), nil) 1016 if err != nil { 1017 t.Fatal(err) 1018 } 1019 if got != test.want { 1020 t.Errorf("%T: got %t, want %t", test.in, got, test.want) 1021 } 1022 } 1023} 1024 1025func TestSchemaFromJSON(t *testing.T) { 1026 testCasesExpectingSuccess := []struct { 1027 bqSchemaJSON []byte 1028 description string 1029 expectedSchema Schema 1030 }{ 1031 { 1032 description: "Flat table with a mixture of NULLABLE and REQUIRED fields", 1033 bqSchemaJSON: []byte(` 1034[ 1035 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1036 {"name":"flat_bytes","type":"BYTES","mode":"REQUIRED","description":"Flat required BYTES"}, 1037 {"name":"flat_integer","type":"INTEGER","mode":"NULLABLE","description":"Flat nullable INTEGER"}, 1038 {"name":"flat_float","type":"FLOAT","mode":"REQUIRED","description":"Flat required FLOAT"}, 1039 {"name":"flat_boolean","type":"BOOLEAN","mode":"NULLABLE","description":"Flat nullable BOOLEAN"}, 1040 {"name":"flat_timestamp","type":"TIMESTAMP","mode":"REQUIRED","description":"Flat required TIMESTAMP"}, 1041 {"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"}, 1042 {"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"}, 1043 {"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"}, 1044 {"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat nullable NUMERIC"}, 1045 {"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"}, 1046 {"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"}, 1047 {"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"}, 1048 {"name":"aliased_float","type":"FLOAT64","mode":"REQUIRED","description":"Aliased required float"}, 1049 {"name":"aliased_record","type":"STRUCT","mode":"NULLABLE","description":"Aliased nullable record"} 1050]`), 1051 expectedSchema: Schema{ 1052 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1053 fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true, nil), 1054 fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false, nil), 1055 fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true, nil), 1056 fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false, nil), 1057 fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true, nil), 1058 fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil), 1059 fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil), 1060 fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil), 1061 fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true, nil), 1062 fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil), 1063 fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil), 1064 fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil), 1065 fieldSchema("Aliased required float", "aliased_float", "FLOAT", false, true, nil), 1066 fieldSchema("Aliased nullable record", "aliased_record", "RECORD", false, false, nil), 1067 }, 1068 }, 1069 { 1070 description: "Table with a nested RECORD", 1071 bqSchemaJSON: []byte(` 1072[ 1073 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1074 {"name":"nested_record","type":"RECORD","mode":"NULLABLE","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 1075]`), 1076 expectedSchema: Schema{ 1077 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1078 &FieldSchema{ 1079 Description: "Nested nullable RECORD", 1080 Name: "nested_record", 1081 Required: false, 1082 Type: "RECORD", 1083 Schema: Schema{ 1084 { 1085 Description: "First nested record field", 1086 Name: "record_field_1", 1087 Required: false, 1088 Type: "STRING", 1089 }, 1090 { 1091 Description: "Second nested record field", 1092 Name: "record_field_2", 1093 Required: true, 1094 Type: "INTEGER", 1095 }, 1096 }, 1097 }, 1098 }, 1099 }, 1100 { 1101 description: "Table with a repeated RECORD", 1102 bqSchemaJSON: []byte(` 1103[ 1104 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1105 {"name":"nested_record","type":"RECORD","mode":"REPEATED","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 1106]`), 1107 expectedSchema: Schema{ 1108 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1109 &FieldSchema{ 1110 Description: "Nested nullable RECORD", 1111 Name: "nested_record", 1112 Repeated: true, 1113 Required: false, 1114 Type: "RECORD", 1115 Schema: Schema{ 1116 { 1117 Description: "First nested record field", 1118 Name: "record_field_1", 1119 Required: false, 1120 Type: "STRING", 1121 }, 1122 { 1123 Description: "Second nested record field", 1124 Name: "record_field_2", 1125 Required: true, 1126 Type: "INTEGER", 1127 }, 1128 }, 1129 }, 1130 }, 1131 }, 1132 } 1133 for _, tc := range testCasesExpectingSuccess { 1134 convertedSchema, err := SchemaFromJSON(tc.bqSchemaJSON) 1135 if err != nil { 1136 t.Errorf("encountered an error when converting JSON table schema (%s): %v", tc.description, err) 1137 continue 1138 } 1139 if !testutil.Equal(convertedSchema, tc.expectedSchema) { 1140 t.Errorf("generated JSON table schema (%s) differs from the expected schema", tc.description) 1141 } 1142 } 1143 1144 testCasesExpectingFailure := []struct { 1145 bqSchemaJSON []byte 1146 description string 1147 }{ 1148 { 1149 description: "Schema with invalid JSON", 1150 bqSchemaJSON: []byte(`This is not JSON`), 1151 }, 1152 { 1153 description: "Schema with unknown field type", 1154 bqSchemaJSON: []byte(`[{"name":"strange_type","type":"STRANGE","description":"This type should not exist"}]`), 1155 }, 1156 { 1157 description: "Schema with zero length", 1158 bqSchemaJSON: []byte(``), 1159 }, 1160 } 1161 for _, tc := range testCasesExpectingFailure { 1162 _, err := SchemaFromJSON(tc.bqSchemaJSON) 1163 if err == nil { 1164 t.Errorf("converting this schema should have returned an error (%s): %v", tc.description, err) 1165 continue 1166 } 1167 } 1168} 1169