1// Copyright 2015 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "io" 19 20 bq "google.golang.org/api/bigquery/v2" 21) 22 23// GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute 24// an input or output to a BigQuery operation. 25type GCSReference struct { 26 // URIs refer to Google Cloud Storage objects. 27 URIs []string 28 29 FileConfig 30 31 // DestinationFormat is the format to use when writing exported files. 32 // Allowed values are: CSV, Avro, JSON. The default is CSV. 33 // CSV is not supported for tables with nested or repeated fields. 34 DestinationFormat DataFormat 35 36 // Compression specifies the type of compression to apply when writing data 37 // to Google Cloud Storage, or using this GCSReference as an ExternalData 38 // source with CSV or JSON SourceFormat. Default is None. 39 Compression Compression 40} 41 42// NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination. 43// In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object. 44// Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided. 45// Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name. 46// For more information about the treatment of wildcards and multiple URIs, 47// see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple 48func NewGCSReference(uri ...string) *GCSReference { 49 return &GCSReference{URIs: uri} 50} 51 52// Compression is the type of compression to apply when writing data to Google Cloud Storage. 53type Compression string 54 55const ( 56 // None specifies no compression. 57 None Compression = "NONE" 58 // Gzip specifies gzip compression. 59 Gzip Compression = "GZIP" 60) 61 62func (gcs *GCSReference) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader { 63 lc.SourceUris = gcs.URIs 64 gcs.FileConfig.populateLoadConfig(lc) 65 return nil 66} 67 68func (gcs *GCSReference) toBQ() bq.ExternalDataConfiguration { 69 conf := bq.ExternalDataConfiguration{ 70 Compression: string(gcs.Compression), 71 SourceUris: append([]string{}, gcs.URIs...), 72 } 73 gcs.FileConfig.populateExternalDataConfig(&conf) 74 return conf 75} 76