1// Copyright 2015 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "fmt" 19 "math/big" 20 "reflect" 21 "testing" 22 "time" 23 24 "cloud.google.com/go/civil" 25 "cloud.google.com/go/internal/pretty" 26 "cloud.google.com/go/internal/testutil" 27 bq "google.golang.org/api/bigquery/v2" 28) 29 30func (fs *FieldSchema) GoString() string { 31 if fs == nil { 32 return "<nil>" 33 } 34 35 return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}", 36 fs.Name, 37 fs.Description, 38 fs.Repeated, 39 fs.Required, 40 fs.Type, 41 fmt.Sprintf("%#v", fs.Schema), 42 ) 43} 44 45func bqTableFieldSchema(desc, name, typ, mode string, tags []string) *bq.TableFieldSchema { 46 var policy *bq.TableFieldSchemaPolicyTags 47 if tags != nil { 48 policy = &bq.TableFieldSchemaPolicyTags{ 49 Names: tags, 50 } 51 } 52 return &bq.TableFieldSchema{ 53 Description: desc, 54 Name: name, 55 Mode: mode, 56 Type: typ, 57 PolicyTags: policy, 58 } 59} 60 61func fieldSchema(desc, name, typ string, repeated, required bool, tags []string) *FieldSchema { 62 var policy *PolicyTagList 63 if tags != nil { 64 policy = &PolicyTagList{ 65 Names: tags, 66 } 67 } 68 return &FieldSchema{ 69 Description: desc, 70 Name: name, 71 Repeated: repeated, 72 Required: required, 73 Type: FieldType(typ), 74 PolicyTags: policy, 75 } 76} 77 78func TestRelaxSchema(t *testing.T) { 79 testCases := []struct { 80 in Schema 81 expected Schema 82 }{ 83 { 84 Schema{ 85 &FieldSchema{ 86 Description: "a relaxed schema", 87 Required: false, 88 Type: StringFieldType, 89 }, 90 }, 91 Schema{ 92 &FieldSchema{ 93 Description: "a relaxed schema", 94 Required: false, 95 Type: StringFieldType, 96 }, 97 }, 98 }, 99 { 100 Schema{ 101 &FieldSchema{ 102 Description: "a required string", 103 Required: true, 104 Type: StringFieldType, 105 }, 106 &FieldSchema{ 107 Description: "a required integer", 108 Required: true, 109 Type: IntegerFieldType, 110 }, 111 }, 112 Schema{ 113 &FieldSchema{ 114 Description: "a required string", 115 Required: false, 116 Type: StringFieldType, 117 }, 118 &FieldSchema{ 119 Description: "a required integer", 120 Required: false, 121 Type: IntegerFieldType, 122 }, 123 }, 124 }, 125 { 126 Schema{ 127 &FieldSchema{ 128 Description: "An outer schema wrapping a nested schema", 129 Name: "outer", 130 Required: true, 131 Type: RecordFieldType, 132 Schema: Schema{ 133 { 134 Description: "inner field", 135 Name: "inner", 136 Type: StringFieldType, 137 Required: true, 138 }, 139 }, 140 }, 141 }, 142 Schema{ 143 &FieldSchema{ 144 Description: "An outer schema wrapping a nested schema", 145 Name: "outer", 146 Required: false, 147 Type: "RECORD", 148 Schema: Schema{ 149 { 150 Description: "inner field", 151 Name: "inner", 152 Type: "STRING", 153 Required: false, 154 }, 155 }, 156 }, 157 }, 158 }, 159 } 160 for _, tc := range testCases { 161 converted := tc.in.Relax() 162 if !testutil.Equal(converted, tc.expected) { 163 t.Errorf("relaxing schema: got:\n%v\nwant:\n%v", 164 pretty.Value(converted), pretty.Value(tc.expected)) 165 } 166 } 167} 168func TestSchemaConversion(t *testing.T) { 169 testCases := []struct { 170 schema Schema 171 bqSchema *bq.TableSchema 172 }{ 173 { 174 // required 175 bqSchema: &bq.TableSchema{ 176 Fields: []*bq.TableFieldSchema{ 177 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil), 178 }, 179 }, 180 schema: Schema{ 181 fieldSchema("desc", "name", "STRING", false, true, nil), 182 }, 183 }, 184 { 185 // repeated 186 bqSchema: &bq.TableSchema{ 187 Fields: []*bq.TableFieldSchema{ 188 bqTableFieldSchema("desc", "name", "STRING", "REPEATED", nil), 189 }, 190 }, 191 schema: Schema{ 192 fieldSchema("desc", "name", "STRING", true, false, nil), 193 }, 194 }, 195 { 196 // nullable, string 197 bqSchema: &bq.TableSchema{ 198 Fields: []*bq.TableFieldSchema{ 199 bqTableFieldSchema("desc", "name", "STRING", "", nil), 200 }, 201 }, 202 schema: Schema{ 203 fieldSchema("desc", "name", "STRING", false, false, nil), 204 }, 205 }, 206 { 207 // integer 208 bqSchema: &bq.TableSchema{ 209 Fields: []*bq.TableFieldSchema{ 210 bqTableFieldSchema("desc", "name", "INTEGER", "", nil), 211 }, 212 }, 213 schema: Schema{ 214 fieldSchema("desc", "name", "INTEGER", false, false, nil), 215 }, 216 }, 217 { 218 // float 219 bqSchema: &bq.TableSchema{ 220 Fields: []*bq.TableFieldSchema{ 221 bqTableFieldSchema("desc", "name", "FLOAT", "", nil), 222 }, 223 }, 224 schema: Schema{ 225 fieldSchema("desc", "name", "FLOAT", false, false, nil), 226 }, 227 }, 228 { 229 // boolean 230 bqSchema: &bq.TableSchema{ 231 Fields: []*bq.TableFieldSchema{ 232 bqTableFieldSchema("desc", "name", "BOOLEAN", "", nil), 233 }, 234 }, 235 schema: Schema{ 236 fieldSchema("desc", "name", "BOOLEAN", false, false, nil), 237 }, 238 }, 239 { 240 // timestamp 241 bqSchema: &bq.TableSchema{ 242 Fields: []*bq.TableFieldSchema{ 243 bqTableFieldSchema("desc", "name", "TIMESTAMP", "", nil), 244 }, 245 }, 246 schema: Schema{ 247 fieldSchema("desc", "name", "TIMESTAMP", false, false, nil), 248 }, 249 }, 250 { 251 // civil times 252 bqSchema: &bq.TableSchema{ 253 Fields: []*bq.TableFieldSchema{ 254 bqTableFieldSchema("desc", "f1", "TIME", "", nil), 255 bqTableFieldSchema("desc", "f2", "DATE", "", nil), 256 bqTableFieldSchema("desc", "f3", "DATETIME", "", nil), 257 }, 258 }, 259 schema: Schema{ 260 fieldSchema("desc", "f1", "TIME", false, false, nil), 261 fieldSchema("desc", "f2", "DATE", false, false, nil), 262 fieldSchema("desc", "f3", "DATETIME", false, false, nil), 263 }, 264 }, 265 { 266 // numeric 267 bqSchema: &bq.TableSchema{ 268 Fields: []*bq.TableFieldSchema{ 269 bqTableFieldSchema("desc", "n", "NUMERIC", "", nil), 270 }, 271 }, 272 schema: Schema{ 273 fieldSchema("desc", "n", "NUMERIC", false, false, nil), 274 }, 275 }, 276 { 277 // geography 278 bqSchema: &bq.TableSchema{ 279 Fields: []*bq.TableFieldSchema{ 280 bqTableFieldSchema("geo", "g", "GEOGRAPHY", "", nil), 281 }, 282 }, 283 schema: Schema{ 284 fieldSchema("geo", "g", "GEOGRAPHY", false, false, nil), 285 }, 286 }, 287 { 288 // constrained 289 bqSchema: &bq.TableSchema{ 290 Fields: []*bq.TableFieldSchema{ 291 { 292 Name: "foo", 293 Type: "STRING", 294 MaxLength: 0, 295 Precision: 0, 296 Scale: 0, 297 }, 298 { 299 Name: "bar", 300 Type: "STRING", 301 MaxLength: 1, 302 Precision: 2, 303 Scale: 3, 304 }, 305 }}, 306 schema: Schema{ 307 {Name: "foo", 308 Type: StringFieldType, 309 MaxLength: 0, 310 Precision: 0, 311 Scale: 0, 312 }, 313 {Name: "bar", 314 Type: StringFieldType, 315 MaxLength: 1, 316 Precision: 2, 317 Scale: 3, 318 }, 319 }, 320 }, 321 { 322 // policy tags 323 bqSchema: &bq.TableSchema{ 324 Fields: []*bq.TableFieldSchema{ 325 bqTableFieldSchema("some pii", "restrictedfield", "STRING", "", []string{"tag1"}), 326 }, 327 }, 328 schema: Schema{ 329 fieldSchema("some pii", "restrictedfield", "STRING", false, false, []string{"tag1"}), 330 }, 331 }, 332 { 333 // nested 334 bqSchema: &bq.TableSchema{ 335 Fields: []*bq.TableFieldSchema{ 336 { 337 Description: "An outer schema wrapping a nested schema", 338 Name: "outer", 339 Mode: "REQUIRED", 340 Type: "RECORD", 341 Fields: []*bq.TableFieldSchema{ 342 bqTableFieldSchema("inner field", "inner", "STRING", "", nil), 343 }, 344 }, 345 }, 346 }, 347 schema: Schema{ 348 &FieldSchema{ 349 Description: "An outer schema wrapping a nested schema", 350 Name: "outer", 351 Required: true, 352 Type: "RECORD", 353 Schema: Schema{ 354 { 355 Description: "inner field", 356 Name: "inner", 357 Type: "STRING", 358 }, 359 }, 360 }, 361 }, 362 }, 363 } 364 for _, tc := range testCases { 365 bqSchema := tc.schema.toBQ() 366 if !testutil.Equal(bqSchema, tc.bqSchema) { 367 t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v", 368 pretty.Value(bqSchema), pretty.Value(tc.bqSchema)) 369 } 370 schema := bqToSchema(tc.bqSchema) 371 if !testutil.Equal(schema, tc.schema) { 372 t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema) 373 } 374 } 375} 376 377type allStrings struct { 378 String string 379 ByteSlice []byte 380} 381 382type allSignedIntegers struct { 383 Int64 int64 384 Int32 int32 385 Int16 int16 386 Int8 int8 387 Int int 388} 389 390type allUnsignedIntegers struct { 391 Uint32 uint32 392 Uint16 uint16 393 Uint8 uint8 394} 395 396type allFloat struct { 397 Float64 float64 398 Float32 float32 399 // NOTE: Complex32 and Complex64 are unsupported by BigQuery 400} 401 402type allBoolean struct { 403 Bool bool 404} 405 406type allTime struct { 407 Timestamp time.Time 408 Time civil.Time 409 Date civil.Date 410 DateTime civil.DateTime 411} 412 413type allNumeric struct { 414 Numeric *big.Rat 415} 416 417func reqField(name, typ string) *FieldSchema { 418 return &FieldSchema{ 419 Name: name, 420 Type: FieldType(typ), 421 Required: true, 422 } 423} 424 425func optField(name, typ string) *FieldSchema { 426 return &FieldSchema{ 427 Name: name, 428 Type: FieldType(typ), 429 Required: false, 430 } 431} 432 433func TestSimpleInference(t *testing.T) { 434 testCases := []struct { 435 in interface{} 436 want Schema 437 }{ 438 { 439 in: allSignedIntegers{}, 440 want: Schema{ 441 reqField("Int64", "INTEGER"), 442 reqField("Int32", "INTEGER"), 443 reqField("Int16", "INTEGER"), 444 reqField("Int8", "INTEGER"), 445 reqField("Int", "INTEGER"), 446 }, 447 }, 448 { 449 in: allUnsignedIntegers{}, 450 want: Schema{ 451 reqField("Uint32", "INTEGER"), 452 reqField("Uint16", "INTEGER"), 453 reqField("Uint8", "INTEGER"), 454 }, 455 }, 456 { 457 in: allFloat{}, 458 want: Schema{ 459 reqField("Float64", "FLOAT"), 460 reqField("Float32", "FLOAT"), 461 }, 462 }, 463 { 464 in: allBoolean{}, 465 want: Schema{ 466 reqField("Bool", "BOOLEAN"), 467 }, 468 }, 469 { 470 in: &allBoolean{}, 471 want: Schema{ 472 reqField("Bool", "BOOLEAN"), 473 }, 474 }, 475 { 476 in: allTime{}, 477 want: Schema{ 478 reqField("Timestamp", "TIMESTAMP"), 479 reqField("Time", "TIME"), 480 reqField("Date", "DATE"), 481 reqField("DateTime", "DATETIME"), 482 }, 483 }, 484 { 485 in: &allNumeric{}, 486 want: Schema{ 487 reqField("Numeric", "NUMERIC"), 488 }, 489 }, 490 { 491 in: allStrings{}, 492 want: Schema{ 493 reqField("String", "STRING"), 494 reqField("ByteSlice", "BYTES"), 495 }, 496 }, 497 } 498 for _, tc := range testCases { 499 got, err := InferSchema(tc.in) 500 if err != nil { 501 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 502 } 503 if !testutil.Equal(got, tc.want) { 504 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 505 pretty.Value(got), pretty.Value(tc.want)) 506 } 507 } 508} 509 510type containsNested struct { 511 NotNested int 512 Nested struct { 513 Inside int 514 } 515} 516 517type containsDoubleNested struct { 518 NotNested int 519 Nested struct { 520 InsideNested struct { 521 Inside int 522 } 523 } 524} 525 526type ptrNested struct { 527 Ptr *struct{ Inside int } 528} 529 530type dup struct { // more than one field of the same struct type 531 A, B allBoolean 532} 533 534func TestNestedInference(t *testing.T) { 535 testCases := []struct { 536 in interface{} 537 want Schema 538 }{ 539 { 540 in: containsNested{}, 541 want: Schema{ 542 reqField("NotNested", "INTEGER"), 543 &FieldSchema{ 544 Name: "Nested", 545 Required: true, 546 Type: "RECORD", 547 Schema: Schema{reqField("Inside", "INTEGER")}, 548 }, 549 }, 550 }, 551 { 552 in: containsDoubleNested{}, 553 want: Schema{ 554 reqField("NotNested", "INTEGER"), 555 &FieldSchema{ 556 Name: "Nested", 557 Required: true, 558 Type: "RECORD", 559 Schema: Schema{ 560 { 561 Name: "InsideNested", 562 Required: true, 563 Type: "RECORD", 564 Schema: Schema{reqField("Inside", "INTEGER")}, 565 }, 566 }, 567 }, 568 }, 569 }, 570 { 571 in: ptrNested{}, 572 want: Schema{ 573 &FieldSchema{ 574 Name: "Ptr", 575 Required: true, 576 Type: "RECORD", 577 Schema: Schema{reqField("Inside", "INTEGER")}, 578 }, 579 }, 580 }, 581 { 582 in: dup{}, 583 want: Schema{ 584 &FieldSchema{ 585 Name: "A", 586 Required: true, 587 Type: "RECORD", 588 Schema: Schema{reqField("Bool", "BOOLEAN")}, 589 }, 590 &FieldSchema{ 591 Name: "B", 592 Required: true, 593 Type: "RECORD", 594 Schema: Schema{reqField("Bool", "BOOLEAN")}, 595 }, 596 }, 597 }, 598 } 599 600 for _, tc := range testCases { 601 got, err := InferSchema(tc.in) 602 if err != nil { 603 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err) 604 } 605 if !testutil.Equal(got, tc.want) { 606 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in, 607 pretty.Value(got), pretty.Value(tc.want)) 608 } 609 } 610} 611 612type repeated struct { 613 NotRepeated []byte 614 RepeatedByteSlice [][]byte 615 Slice []int 616 Array [5]bool 617} 618 619type nestedRepeated struct { 620 NotRepeated int 621 Repeated []struct { 622 Inside int 623 } 624 RepeatedPtr []*struct{ Inside int } 625} 626 627func repField(name, typ string) *FieldSchema { 628 return &FieldSchema{ 629 Name: name, 630 Type: FieldType(typ), 631 Repeated: true, 632 } 633} 634 635func TestRepeatedInference(t *testing.T) { 636 testCases := []struct { 637 in interface{} 638 want Schema 639 }{ 640 { 641 in: repeated{}, 642 want: Schema{ 643 reqField("NotRepeated", "BYTES"), 644 repField("RepeatedByteSlice", "BYTES"), 645 repField("Slice", "INTEGER"), 646 repField("Array", "BOOLEAN"), 647 }, 648 }, 649 { 650 in: nestedRepeated{}, 651 want: Schema{ 652 reqField("NotRepeated", "INTEGER"), 653 { 654 Name: "Repeated", 655 Repeated: true, 656 Type: "RECORD", 657 Schema: Schema{reqField("Inside", "INTEGER")}, 658 }, 659 { 660 Name: "RepeatedPtr", 661 Repeated: true, 662 Type: "RECORD", 663 Schema: Schema{reqField("Inside", "INTEGER")}, 664 }, 665 }, 666 }, 667 } 668 669 for i, tc := range testCases { 670 got, err := InferSchema(tc.in) 671 if err != nil { 672 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 673 } 674 if !testutil.Equal(got, tc.want) { 675 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 676 pretty.Value(got), pretty.Value(tc.want)) 677 } 678 } 679} 680 681type allNulls struct { 682 A NullInt64 683 B NullFloat64 684 C NullBool 685 D NullString 686 E NullTimestamp 687 F NullTime 688 G NullDate 689 H NullDateTime 690 I NullGeography 691} 692 693func TestNullInference(t *testing.T) { 694 got, err := InferSchema(allNulls{}) 695 if err != nil { 696 t.Fatal(err) 697 } 698 want := Schema{ 699 optField("A", "INTEGER"), 700 optField("B", "FLOAT"), 701 optField("C", "BOOLEAN"), 702 optField("D", "STRING"), 703 optField("E", "TIMESTAMP"), 704 optField("F", "TIME"), 705 optField("G", "DATE"), 706 optField("H", "DATETIME"), 707 optField("I", "GEOGRAPHY"), 708 } 709 if diff := testutil.Diff(got, want); diff != "" { 710 t.Error(diff) 711 } 712} 713 714type Embedded struct { 715 Embedded int 716} 717 718type embedded struct { 719 Embedded2 int 720} 721 722type nestedEmbedded struct { 723 Embedded 724 embedded 725} 726 727func TestEmbeddedInference(t *testing.T) { 728 got, err := InferSchema(nestedEmbedded{}) 729 if err != nil { 730 t.Fatal(err) 731 } 732 want := Schema{ 733 reqField("Embedded", "INTEGER"), 734 reqField("Embedded2", "INTEGER"), 735 } 736 if !testutil.Equal(got, want) { 737 t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want)) 738 } 739} 740 741func TestRecursiveInference(t *testing.T) { 742 type List struct { 743 Val int 744 Next *List 745 } 746 747 _, err := InferSchema(List{}) 748 if err == nil { 749 t.Fatal("got nil, want error") 750 } 751} 752 753type withTags struct { 754 NoTag int 755 ExcludeTag int `bigquery:"-"` 756 SimpleTag int `bigquery:"simple_tag"` 757 UnderscoreTag int `bigquery:"_id"` 758 MixedCase int `bigquery:"MIXEDcase"` 759 Nullable []byte `bigquery:",nullable"` 760 NullNumeric *big.Rat `bigquery:",nullable"` 761} 762 763type withTagsNested struct { 764 Nested withTags `bigquery:"nested"` 765 NestedAnonymous struct { 766 ExcludeTag int `bigquery:"-"` 767 Inside int `bigquery:"inside"` 768 } `bigquery:"anon"` 769 PNested *struct{ X int } // not nullable, for backwards compatibility 770 PNestedNullable *struct{ X int } `bigquery:",nullable"` 771} 772 773type withTagsRepeated struct { 774 Repeated []withTags `bigquery:"repeated"` 775 RepeatedAnonymous []struct { 776 ExcludeTag int `bigquery:"-"` 777 Inside int `bigquery:"inside"` 778 } `bigquery:"anon"` 779} 780 781type withTagsEmbedded struct { 782 withTags 783} 784 785var withTagsSchema = Schema{ 786 reqField("NoTag", "INTEGER"), 787 reqField("simple_tag", "INTEGER"), 788 reqField("_id", "INTEGER"), 789 reqField("MIXEDcase", "INTEGER"), 790 optField("Nullable", "BYTES"), 791 optField("NullNumeric", "NUMERIC"), 792} 793 794func TestTagInference(t *testing.T) { 795 testCases := []struct { 796 in interface{} 797 want Schema 798 }{ 799 { 800 in: withTags{}, 801 want: withTagsSchema, 802 }, 803 { 804 in: withTagsNested{}, 805 want: Schema{ 806 &FieldSchema{ 807 Name: "nested", 808 Required: true, 809 Type: "RECORD", 810 Schema: withTagsSchema, 811 }, 812 &FieldSchema{ 813 Name: "anon", 814 Required: true, 815 Type: "RECORD", 816 Schema: Schema{reqField("inside", "INTEGER")}, 817 }, 818 &FieldSchema{ 819 Name: "PNested", 820 Required: true, 821 Type: "RECORD", 822 Schema: Schema{reqField("X", "INTEGER")}, 823 }, 824 &FieldSchema{ 825 Name: "PNestedNullable", 826 Required: false, 827 Type: "RECORD", 828 Schema: Schema{reqField("X", "INTEGER")}, 829 }, 830 }, 831 }, 832 { 833 in: withTagsRepeated{}, 834 want: Schema{ 835 &FieldSchema{ 836 Name: "repeated", 837 Repeated: true, 838 Type: "RECORD", 839 Schema: withTagsSchema, 840 }, 841 &FieldSchema{ 842 Name: "anon", 843 Repeated: true, 844 Type: "RECORD", 845 Schema: Schema{reqField("inside", "INTEGER")}, 846 }, 847 }, 848 }, 849 { 850 in: withTagsEmbedded{}, 851 want: withTagsSchema, 852 }, 853 } 854 for i, tc := range testCases { 855 got, err := InferSchema(tc.in) 856 if err != nil { 857 t.Fatalf("%d: error inferring TableSchema: %v", i, err) 858 } 859 if !testutil.Equal(got, tc.want) { 860 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, 861 pretty.Value(got), pretty.Value(tc.want)) 862 } 863 } 864} 865 866func TestTagInferenceErrors(t *testing.T) { 867 testCases := []interface{}{ 868 struct { 869 LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"` 870 }{}, 871 struct { 872 UnsupporedStartChar int `bigquery:"øab"` 873 }{}, 874 struct { 875 UnsupportedEndChar int `bigquery:"abø"` 876 }{}, 877 struct { 878 UnsupportedMiddleChar int `bigquery:"aøb"` 879 }{}, 880 struct { 881 StartInt int `bigquery:"1abc"` 882 }{}, 883 struct { 884 Hyphens int `bigquery:"a-b"` 885 }{}, 886 } 887 for i, tc := range testCases { 888 889 _, got := InferSchema(tc) 890 if _, ok := got.(invalidFieldNameError); !ok { 891 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant invalidFieldNameError", i, got) 892 } 893 } 894 895 _, err := InferSchema(struct { 896 X int `bigquery:",optional"` 897 }{}) 898 if err == nil { 899 t.Error("got nil, want error") 900 } 901} 902 903func TestSchemaErrors(t *testing.T) { 904 testCases := []struct { 905 in interface{} 906 want interface{} 907 }{ 908 { 909 in: []byte{}, 910 want: noStructError{}, 911 }, 912 { 913 in: new(int), 914 want: noStructError{}, 915 }, 916 { 917 in: struct{ Uint uint }{}, 918 want: unsupportedFieldTypeError{}, 919 }, 920 { 921 in: struct{ Uint64 uint64 }{}, 922 want: unsupportedFieldTypeError{}, 923 }, 924 { 925 in: struct{ Uintptr uintptr }{}, 926 want: unsupportedFieldTypeError{}, 927 }, 928 { 929 in: struct{ Complex complex64 }{}, 930 want: unsupportedFieldTypeError{}, 931 }, 932 { 933 in: struct{ Map map[string]int }{}, 934 want: unsupportedFieldTypeError{}, 935 }, 936 { 937 in: struct{ Chan chan bool }{}, 938 want: unsupportedFieldTypeError{}, 939 }, 940 { 941 in: struct{ Ptr *int }{}, 942 want: unsupportedFieldTypeError{}, 943 }, 944 { 945 in: struct{ Interface interface{} }{}, 946 want: unsupportedFieldTypeError{}, 947 }, 948 { 949 in: struct{ MultiDimensional [][]int }{}, 950 want: unsupportedFieldTypeError{}, 951 }, 952 { 953 in: struct{ MultiDimensional [][][]byte }{}, 954 want: unsupportedFieldTypeError{}, 955 }, 956 { 957 in: struct{ SliceOfPointer []*int }{}, 958 want: unsupportedFieldTypeError{}, 959 }, 960 { 961 in: struct{ SliceOfNull []NullInt64 }{}, 962 want: unsupportedFieldTypeError{}, 963 }, 964 { 965 in: struct{ ChanSlice []chan bool }{}, 966 want: unsupportedFieldTypeError{}, 967 }, 968 { 969 in: struct{ NestedChan struct{ Chan []chan bool } }{}, 970 want: unsupportedFieldTypeError{}, 971 }, 972 { 973 in: struct { 974 X int `bigquery:",nullable"` 975 }{}, 976 want: badNullableError{}, 977 }, 978 { 979 in: struct { 980 X bool `bigquery:",nullable"` 981 }{}, 982 want: badNullableError{}, 983 }, 984 { 985 in: struct { 986 X struct{ N int } `bigquery:",nullable"` 987 }{}, 988 want: badNullableError{}, 989 }, 990 { 991 in: struct { 992 X []int `bigquery:",nullable"` 993 }{}, 994 want: badNullableError{}, 995 }, 996 { 997 in: struct{ X *[]byte }{}, 998 want: unsupportedFieldTypeError{}, 999 }, 1000 { 1001 in: struct{ X *[]int }{}, 1002 want: unsupportedFieldTypeError{}, 1003 }, 1004 { 1005 in: struct{ X *int }{}, 1006 want: unsupportedFieldTypeError{}, 1007 }, 1008 } 1009 for _, tc := range testCases { 1010 _, got := InferSchema(tc.in) 1011 if reflect.TypeOf(got) != reflect.TypeOf(tc.want) { 1012 t.Errorf("%#v: got:\n%#v\nwant type %T", tc.in, got, tc.want) 1013 } 1014 } 1015} 1016 1017func TestHasRecursiveType(t *testing.T) { 1018 type ( 1019 nonStruct int 1020 nonRec struct{ A string } 1021 dup struct{ A, B nonRec } 1022 rec struct { 1023 A int 1024 B *rec 1025 } 1026 recUnexported struct { 1027 A int 1028 } 1029 hasRec struct { 1030 A int 1031 R *rec 1032 } 1033 recSlicePointer struct { 1034 A []*recSlicePointer 1035 } 1036 ) 1037 for _, test := range []struct { 1038 in interface{} 1039 want bool 1040 }{ 1041 {nonStruct(0), false}, 1042 {nonRec{}, false}, 1043 {dup{}, false}, 1044 {rec{}, true}, 1045 {recUnexported{}, false}, 1046 {hasRec{}, true}, 1047 {&recSlicePointer{}, true}, 1048 } { 1049 got, err := hasRecursiveType(reflect.TypeOf(test.in), nil) 1050 if err != nil { 1051 t.Fatal(err) 1052 } 1053 if got != test.want { 1054 t.Errorf("%T: got %t, want %t", test.in, got, test.want) 1055 } 1056 } 1057} 1058 1059func TestSchemaFromJSON(t *testing.T) { 1060 testCasesExpectingSuccess := []struct { 1061 bqSchemaJSON []byte 1062 description string 1063 expectedSchema Schema 1064 }{ 1065 { 1066 description: "Flat table with a mixture of NULLABLE and REQUIRED fields", 1067 bqSchemaJSON: []byte(` 1068[ 1069 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1070 {"name":"flat_bytes","type":"BYTES","mode":"REQUIRED","description":"Flat required BYTES"}, 1071 {"name":"flat_integer","type":"INTEGER","mode":"NULLABLE","description":"Flat nullable INTEGER"}, 1072 {"name":"flat_float","type":"FLOAT","mode":"REQUIRED","description":"Flat required FLOAT"}, 1073 {"name":"flat_boolean","type":"BOOLEAN","mode":"NULLABLE","description":"Flat nullable BOOLEAN"}, 1074 {"name":"flat_timestamp","type":"TIMESTAMP","mode":"REQUIRED","description":"Flat required TIMESTAMP"}, 1075 {"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"}, 1076 {"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"}, 1077 {"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"}, 1078 {"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat required NUMERIC"}, 1079 {"name":"flat_bignumeric","type":"BIGNUMERIC","mode":"NULLABLE","description":"Flat nullable BIGNUMERIC"}, 1080 {"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"}, 1081 {"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"}, 1082 {"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"}, 1083 {"name":"aliased_float","type":"FLOAT64","mode":"REQUIRED","description":"Aliased required float"}, 1084 {"name":"aliased_record","type":"STRUCT","mode":"NULLABLE","description":"Aliased nullable record"}, 1085 {"name":"aliased_bignumeric","type":"BIGDECIMAL","mode":"NULLABLE","description":"Aliased nullable bignumeric"} 1086]`), 1087 expectedSchema: Schema{ 1088 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1089 fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true, nil), 1090 fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false, nil), 1091 fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true, nil), 1092 fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false, nil), 1093 fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true, nil), 1094 fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil), 1095 fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil), 1096 fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil), 1097 fieldSchema("Flat required NUMERIC", "flat_numeric", "NUMERIC", false, true, nil), 1098 fieldSchema("Flat nullable BIGNUMERIC", "flat_bignumeric", "BIGNUMERIC", false, false, nil), 1099 fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil), 1100 fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil), 1101 fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil), 1102 fieldSchema("Aliased required float", "aliased_float", "FLOAT", false, true, nil), 1103 fieldSchema("Aliased nullable record", "aliased_record", "RECORD", false, false, nil), 1104 fieldSchema("Aliased nullable bignumeric", "aliased_bignumeric", "BIGNUMERIC", false, false, nil), 1105 }, 1106 }, 1107 { 1108 description: "Table with a nested RECORD", 1109 bqSchemaJSON: []byte(` 1110[ 1111 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1112 {"name":"nested_record","type":"RECORD","mode":"NULLABLE","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 1113]`), 1114 expectedSchema: Schema{ 1115 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1116 &FieldSchema{ 1117 Description: "Nested nullable RECORD", 1118 Name: "nested_record", 1119 Required: false, 1120 Type: "RECORD", 1121 Schema: Schema{ 1122 { 1123 Description: "First nested record field", 1124 Name: "record_field_1", 1125 Required: false, 1126 Type: "STRING", 1127 }, 1128 { 1129 Description: "Second nested record field", 1130 Name: "record_field_2", 1131 Required: true, 1132 Type: "INTEGER", 1133 }, 1134 }, 1135 }, 1136 }, 1137 }, 1138 { 1139 description: "Table with a repeated RECORD", 1140 bqSchemaJSON: []byte(` 1141[ 1142 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"}, 1143 {"name":"nested_record","type":"RECORD","mode":"REPEATED","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]} 1144]`), 1145 expectedSchema: Schema{ 1146 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil), 1147 &FieldSchema{ 1148 Description: "Nested nullable RECORD", 1149 Name: "nested_record", 1150 Repeated: true, 1151 Required: false, 1152 Type: "RECORD", 1153 Schema: Schema{ 1154 { 1155 Description: "First nested record field", 1156 Name: "record_field_1", 1157 Required: false, 1158 Type: "STRING", 1159 }, 1160 { 1161 Description: "Second nested record field", 1162 Name: "record_field_2", 1163 Required: true, 1164 Type: "INTEGER", 1165 }, 1166 }, 1167 }, 1168 }, 1169 }, 1170 } 1171 for _, tc := range testCasesExpectingSuccess { 1172 convertedSchema, err := SchemaFromJSON(tc.bqSchemaJSON) 1173 if err != nil { 1174 t.Errorf("encountered an error when converting JSON table schema (%s): %v", tc.description, err) 1175 continue 1176 } 1177 if !testutil.Equal(convertedSchema, tc.expectedSchema) { 1178 t.Errorf("generated JSON table schema (%s) differs from the expected schema", tc.description) 1179 } 1180 } 1181 1182 testCasesExpectingFailure := []struct { 1183 bqSchemaJSON []byte 1184 description string 1185 }{ 1186 { 1187 description: "Schema with invalid JSON", 1188 bqSchemaJSON: []byte(`This is not JSON`), 1189 }, 1190 { 1191 description: "Schema with unknown field type", 1192 bqSchemaJSON: []byte(`[{"name":"strange_type","type":"STRANGE","description":"This type should not exist"}]`), 1193 }, 1194 { 1195 description: "Schema with zero length", 1196 bqSchemaJSON: []byte(``), 1197 }, 1198 } 1199 for _, tc := range testCasesExpectingFailure { 1200 _, err := SchemaFromJSON(tc.bqSchemaJSON) 1201 if err == nil { 1202 t.Errorf("converting this schema should have returned an error (%s): %v", tc.description, err) 1203 continue 1204 } 1205 } 1206} 1207