1// Copyright 2017 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package bigquery
16
17import (
18	"testing"
19	"time"
20
21	"cloud.google.com/go/internal/testutil"
22	bq "google.golang.org/api/bigquery/v2"
23)
24
25func TestBQToTableMetadata(t *testing.T) {
26	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
27	aTimeMillis := aTime.UnixNano() / 1e6
28	aDurationMillis := int64(1800000)
29	aDuration := time.Duration(aDurationMillis) * time.Millisecond
30	for _, test := range []struct {
31		in   *bq.Table
32		want *TableMetadata
33	}{
34		{&bq.Table{}, &TableMetadata{}}, // test minimal case
35		{
36			&bq.Table{
37				CreationTime:     aTimeMillis,
38				Description:      "desc",
39				Etag:             "etag",
40				ExpirationTime:   aTimeMillis,
41				FriendlyName:     "fname",
42				Id:               "id",
43				LastModifiedTime: uint64(aTimeMillis),
44				Location:         "loc",
45				NumBytes:         123,
46				NumLongTermBytes: 23,
47				NumRows:          7,
48				StreamingBuffer: &bq.Streamingbuffer{
49					EstimatedBytes:  11,
50					EstimatedRows:   3,
51					OldestEntryTime: uint64(aTimeMillis),
52				},
53				MaterializedView: &bq.MaterializedViewDefinition{
54					EnableRefresh:     true,
55					Query:             "mat view query",
56					LastRefreshTime:   aTimeMillis,
57					RefreshIntervalMs: aDurationMillis,
58				},
59				TimePartitioning: &bq.TimePartitioning{
60					ExpirationMs: 7890,
61					Type:         "DAY",
62					Field:        "pfield",
63				},
64				Clustering: &bq.Clustering{
65					Fields: []string{"cfield1", "cfield2"},
66				},
67				RequirePartitionFilter:  true,
68				EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
69				Type:                    "EXTERNAL",
70				View:                    &bq.ViewDefinition{Query: "view-query"},
71				Labels:                  map[string]string{"a": "b"},
72				ExternalDataConfiguration: &bq.ExternalDataConfiguration{
73					SourceFormat: "GOOGLE_SHEETS",
74				},
75			},
76			&TableMetadata{
77				Description:        "desc",
78				Name:               "fname",
79				Location:           "loc",
80				ViewQuery:          "view-query",
81				FullID:             "id",
82				Type:               ExternalTable,
83				Labels:             map[string]string{"a": "b"},
84				ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets},
85				ExpirationTime:     aTime.Truncate(time.Millisecond),
86				CreationTime:       aTime.Truncate(time.Millisecond),
87				LastModifiedTime:   aTime.Truncate(time.Millisecond),
88				NumBytes:           123,
89				NumLongTermBytes:   23,
90				NumRows:            7,
91				MaterializedView: &MaterializedViewDefinition{
92					EnableRefresh:   true,
93					Query:           "mat view query",
94					LastRefreshTime: aTime,
95					RefreshInterval: aDuration,
96				},
97				TimePartitioning: &TimePartitioning{
98					Type:       DayPartitioningType,
99					Expiration: 7890 * time.Millisecond,
100					Field:      "pfield",
101				},
102				Clustering: &Clustering{
103					Fields: []string{"cfield1", "cfield2"},
104				},
105				RequirePartitionFilter: true,
106				StreamingBuffer: &StreamingBuffer{
107					EstimatedBytes:  11,
108					EstimatedRows:   3,
109					OldestEntryTime: aTime,
110				},
111				EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
112				ETag:             "etag",
113			},
114		},
115	} {
116		got, err := bqToTableMetadata(test.in)
117		if err != nil {
118			t.Fatal(err)
119		}
120		if diff := testutil.Diff(got, test.want); diff != "" {
121			t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
122		}
123	}
124}
125
126func TestTableMetadataToBQ(t *testing.T) {
127	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
128	aTimeMillis := aTime.UnixNano() / 1e6
129	sc := Schema{fieldSchema("desc", "name", "STRING", false, true, nil)}
130
131	for _, test := range []struct {
132		in   *TableMetadata
133		want *bq.Table
134	}{
135		{nil, &bq.Table{}},
136		{&TableMetadata{}, &bq.Table{}},
137		{
138			&TableMetadata{
139				Name:               "n",
140				Description:        "d",
141				Schema:             sc,
142				ExpirationTime:     aTime,
143				Labels:             map[string]string{"a": "b"},
144				ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
145				EncryptionConfig:   &EncryptionConfig{KMSKeyName: "keyName"},
146			},
147			&bq.Table{
148				FriendlyName: "n",
149				Description:  "d",
150				Schema: &bq.TableSchema{
151					Fields: []*bq.TableFieldSchema{
152						bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
153					},
154				},
155				ExpirationTime:            aTimeMillis,
156				Labels:                    map[string]string{"a": "b"},
157				ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
158				EncryptionConfiguration:   &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
159			},
160		},
161		{
162			&TableMetadata{ViewQuery: "q"},
163			&bq.Table{
164				View: &bq.ViewDefinition{
165					Query:           "q",
166					UseLegacySql:    false,
167					ForceSendFields: []string{"UseLegacySql"},
168				},
169			},
170		},
171		{
172			&TableMetadata{
173				ViewQuery:              "q",
174				UseLegacySQL:           true,
175				TimePartitioning:       &TimePartitioning{},
176				RequirePartitionFilter: true,
177			},
178			&bq.Table{
179				View: &bq.ViewDefinition{
180					Query:        "q",
181					UseLegacySql: true,
182				},
183				TimePartitioning: &bq.TimePartitioning{
184					Type:         "DAY",
185					ExpirationMs: 0,
186				},
187				RequirePartitionFilter: true,
188			},
189		},
190		{
191			&TableMetadata{
192				ViewQuery:      "q",
193				UseStandardSQL: true,
194				TimePartitioning: &TimePartitioning{
195					Type:       HourPartitioningType,
196					Expiration: time.Second,
197					Field:      "ofDreams",
198				},
199				Clustering: &Clustering{
200					Fields: []string{"cfield1"},
201				},
202			},
203			&bq.Table{
204				View: &bq.ViewDefinition{
205					Query:           "q",
206					UseLegacySql:    false,
207					ForceSendFields: []string{"UseLegacySql"},
208				},
209				TimePartitioning: &bq.TimePartitioning{
210					Type:         "HOUR",
211					ExpirationMs: 1000,
212					Field:        "ofDreams",
213				},
214				Clustering: &bq.Clustering{
215					Fields: []string{"cfield1"},
216				},
217			},
218		},
219		{
220			&TableMetadata{
221				RangePartitioning: &RangePartitioning{
222					Field: "ofNumbers",
223					Range: &RangePartitioningRange{
224						Start:    1,
225						End:      100,
226						Interval: 5,
227					},
228				},
229				Clustering: &Clustering{
230					Fields: []string{"cfield1"},
231				},
232			},
233			&bq.Table{
234
235				RangePartitioning: &bq.RangePartitioning{
236					Field: "ofNumbers",
237					Range: &bq.RangePartitioningRange{
238						Start:           1,
239						End:             100,
240						Interval:        5,
241						ForceSendFields: []string{"Start", "End", "Interval"},
242					},
243				},
244				Clustering: &bq.Clustering{
245					Fields: []string{"cfield1"},
246				},
247			},
248		},
249		{
250			&TableMetadata{ExpirationTime: NeverExpire},
251			&bq.Table{ExpirationTime: 0},
252		},
253	} {
254		got, err := test.in.toBQ()
255		if err != nil {
256			t.Fatalf("%+v: %v", test.in, err)
257		}
258		if diff := testutil.Diff(got, test.want); diff != "" {
259			t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
260		}
261	}
262
263	// Errors
264	for _, in := range []*TableMetadata{
265		{Schema: sc, ViewQuery: "q"}, // can't have both schema and query
266		{UseLegacySQL: true},         // UseLegacySQL without query
267		{UseStandardSQL: true},       // UseStandardSQL without query
268		// read-only fields
269		{FullID: "x"},
270		{Type: "x"},
271		{CreationTime: aTime},
272		{LastModifiedTime: aTime},
273		{NumBytes: 1},
274		{NumLongTermBytes: 1},
275		{NumRows: 1},
276		{StreamingBuffer: &StreamingBuffer{}},
277		{ETag: "x"},
278		// expiration time outside allowable range is invalid
279		// See https://godoc.org/time#Time.UnixNano
280		{ExpirationTime: time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC).Add(-1)},
281		{ExpirationTime: time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC).Add(1)},
282	} {
283		_, err := in.toBQ()
284		if err == nil {
285			t.Errorf("%+v: got nil, want error", in)
286		}
287	}
288}
289
290func TestTableMetadataToUpdateToBQ(t *testing.T) {
291	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
292	for _, test := range []struct {
293		tm   TableMetadataToUpdate
294		want *bq.Table
295	}{
296		{
297			tm:   TableMetadataToUpdate{},
298			want: &bq.Table{},
299		},
300		{
301			tm: TableMetadataToUpdate{
302				Description: "d",
303				Name:        "n",
304			},
305			want: &bq.Table{
306				Description:     "d",
307				FriendlyName:    "n",
308				ForceSendFields: []string{"Description", "FriendlyName"},
309			},
310		},
311		{
312			tm: TableMetadataToUpdate{
313				Schema:         Schema{fieldSchema("desc", "name", "STRING", false, true, nil)},
314				ExpirationTime: aTime,
315			},
316			want: &bq.Table{
317				Schema: &bq.TableSchema{
318					Fields: []*bq.TableFieldSchema{
319						bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
320					},
321				},
322				ExpirationTime:  aTime.UnixNano() / 1e6,
323				ForceSendFields: []string{"Schema", "ExpirationTime"},
324			},
325		},
326		{
327			tm: TableMetadataToUpdate{ViewQuery: "q"},
328			want: &bq.Table{
329				View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}},
330			},
331		},
332		{
333			tm: TableMetadataToUpdate{UseLegacySQL: false},
334			want: &bq.Table{
335				View: &bq.ViewDefinition{
336					UseLegacySql:    false,
337					ForceSendFields: []string{"UseLegacySql"},
338				},
339			},
340		},
341		{
342			tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true},
343			want: &bq.Table{
344				View: &bq.ViewDefinition{
345					Query:           "q",
346					UseLegacySql:    true,
347					ForceSendFields: []string{"Query", "UseLegacySql"},
348				},
349			},
350		},
351		{
352			tm: func() (tm TableMetadataToUpdate) {
353				tm.SetLabel("L", "V")
354				tm.DeleteLabel("D")
355				return tm
356			}(),
357			want: &bq.Table{
358				Labels:     map[string]string{"L": "V"},
359				NullFields: []string{"Labels.D"},
360			},
361		},
362		{
363			tm: TableMetadataToUpdate{ExpirationTime: NeverExpire},
364			want: &bq.Table{
365				NullFields: []string{"ExpirationTime"},
366			},
367		},
368		{
369			tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: 0}},
370			want: &bq.Table{
371				TimePartitioning: &bq.TimePartitioning{
372					Type:            "DAY",
373					ForceSendFields: []string{"RequirePartitionFilter"},
374					NullFields:      []string{"ExpirationMs"},
375				},
376			},
377		},
378		{
379			tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: time.Duration(time.Hour)}},
380			want: &bq.Table{
381				TimePartitioning: &bq.TimePartitioning{
382					ExpirationMs:    3600000,
383					Type:            "DAY",
384					ForceSendFields: []string{"RequirePartitionFilter"},
385				},
386			},
387		},
388		{
389			tm: TableMetadataToUpdate{RequirePartitionFilter: false},
390			want: &bq.Table{
391				RequirePartitionFilter: false,
392				ForceSendFields:        []string{"RequirePartitionFilter"},
393			},
394		},
395		{
396			tm: TableMetadataToUpdate{RequirePartitionFilter: true},
397			want: &bq.Table{
398				RequirePartitionFilter: true,
399				ForceSendFields:        []string{"RequirePartitionFilter"},
400			},
401		},
402	} {
403		got, _ := test.tm.toBQ()
404		if !testutil.Equal(got, test.want) {
405			t.Errorf("%+v:\ngot  %+v\nwant %+v", test.tm, got, test.want)
406		}
407	}
408}
409
410func TestTableMetadataToUpdateToBQErrors(t *testing.T) {
411	// See https://godoc.org/time#Time.UnixNano
412	start := time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC)
413	end := time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC)
414
415	for _, test := range []struct {
416		desc    string
417		aTime   time.Time
418		wantErr bool
419	}{
420		{desc: "ignored zero value", aTime: time.Time{}, wantErr: false},
421		{desc: "earliest valid time", aTime: start, wantErr: false},
422		{desc: "latested valid time", aTime: end, wantErr: false},
423		{desc: "invalid times before 1678", aTime: start.Add(-1), wantErr: true},
424		{desc: "invalid times after 2262", aTime: end.Add(1), wantErr: true},
425		{desc: "valid times after 1678", aTime: start.Add(1), wantErr: false},
426		{desc: "valid times before 2262", aTime: end.Add(-1), wantErr: false},
427	} {
428		tm := &TableMetadataToUpdate{ExpirationTime: test.aTime}
429		_, err := tm.toBQ()
430		if test.wantErr && err == nil {
431			t.Errorf("[%s] got no error, want error", test.desc)
432		}
433		if !test.wantErr && err != nil {
434			t.Errorf("[%s] got error, want no error", test.desc)
435		}
436	}
437}
438