1// Copyright 2015 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bigquery 16 17import ( 18 "io" 19 20 bq "google.golang.org/api/bigquery/v2" 21) 22 23// GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute 24// an input or output to a BigQuery operation. 25type GCSReference struct { 26 // URIs refer to Google Cloud Storage objects. 27 URIs []string 28 29 FileConfig 30 31 // DestinationFormat is the format to use when writing exported files. 32 // Allowed values are: CSV, Avro, JSON. The default is CSV. 33 // CSV is not supported for tables with nested or repeated fields. 34 DestinationFormat DataFormat 35 36 // Compression specifies the type of compression to apply when writing data 37 // to Google Cloud Storage, or using this GCSReference as an ExternalData 38 // source with CSV or JSON SourceFormat. Default is None. 39 // 40 // Avro files allow additional compression types: DEFLATE and SNAPPY. 41 Compression Compression 42} 43 44// NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination. 45// In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object. 46// Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided. 47// Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name. 48// For more information about the treatment of wildcards and multiple URIs, 49// see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple 50func NewGCSReference(uri ...string) *GCSReference { 51 return &GCSReference{URIs: uri} 52} 53 54// Compression is the type of compression to apply when writing data to Google Cloud Storage. 55type Compression string 56 57const ( 58 // None specifies no compression. 59 None Compression = "NONE" 60 // Gzip specifies gzip compression. 61 Gzip Compression = "GZIP" 62 // Deflate specifies DEFLATE compression for Avro files. 63 Deflate Compression = "DEFLATE" 64 // Snappy specifies SNAPPY compression for Avro files. 65 Snappy Compression = "SNAPPY" 66) 67 68func (gcs *GCSReference) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader { 69 lc.SourceUris = gcs.URIs 70 gcs.FileConfig.populateLoadConfig(lc) 71 return nil 72} 73 74func (gcs *GCSReference) toBQ() bq.ExternalDataConfiguration { 75 conf := bq.ExternalDataConfiguration{ 76 Compression: string(gcs.Compression), 77 SourceUris: append([]string{}, gcs.URIs...), 78 } 79 gcs.FileConfig.populateExternalDataConfig(&conf) 80 return conf 81} 82