1// Copyright 2021 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// https://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Code generated by protoc-gen-go_gapic. DO NOT EDIT. 16 17package storage 18 19import ( 20 "context" 21 "fmt" 22 "math" 23 "net/url" 24 "time" 25 26 gax "github.com/googleapis/gax-go/v2" 27 "google.golang.org/api/option" 28 "google.golang.org/api/option/internaloption" 29 gtransport "google.golang.org/api/transport/grpc" 30 storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1" 31 "google.golang.org/grpc" 32 "google.golang.org/grpc/codes" 33 "google.golang.org/grpc/metadata" 34) 35 36var newBigQueryWriteClientHook clientHook 37 38// BigQueryWriteCallOptions contains the retry settings for each method of BigQueryWriteClient. 39type BigQueryWriteCallOptions struct { 40 CreateWriteStream []gax.CallOption 41 AppendRows []gax.CallOption 42 GetWriteStream []gax.CallOption 43 FinalizeWriteStream []gax.CallOption 44 BatchCommitWriteStreams []gax.CallOption 45 FlushRows []gax.CallOption 46} 47 48func defaultBigQueryWriteGRPCClientOptions() []option.ClientOption { 49 return []option.ClientOption{ 50 internaloption.WithDefaultEndpoint("bigquerystorage.googleapis.com:443"), 51 internaloption.WithDefaultMTLSEndpoint("bigquerystorage.mtls.googleapis.com:443"), 52 internaloption.WithDefaultAudience("https://bigquerystorage.googleapis.com/"), 53 internaloption.WithDefaultScopes(DefaultAuthScopes()...), 54 internaloption.EnableJwtWithScope(), 55 option.WithGRPCDialOption(grpc.WithDisableServiceConfig()), 56 option.WithGRPCDialOption(grpc.WithDefaultCallOptions( 57 grpc.MaxCallRecvMsgSize(math.MaxInt32))), 58 } 59} 60 61func defaultBigQueryWriteCallOptions() *BigQueryWriteCallOptions { 62 return &BigQueryWriteCallOptions{ 63 CreateWriteStream: []gax.CallOption{ 64 gax.WithRetry(func() gax.Retryer { 65 return gax.OnCodes([]codes.Code{ 66 codes.DeadlineExceeded, 67 codes.Unavailable, 68 }, gax.Backoff{ 69 Initial: 100 * time.Millisecond, 70 Max: 60000 * time.Millisecond, 71 Multiplier: 1.30, 72 }) 73 }), 74 }, 75 AppendRows: []gax.CallOption{ 76 gax.WithRetry(func() gax.Retryer { 77 return gax.OnCodes([]codes.Code{ 78 codes.Unavailable, 79 }, gax.Backoff{ 80 Initial: 100 * time.Millisecond, 81 Max: 60000 * time.Millisecond, 82 Multiplier: 1.30, 83 }) 84 }), 85 }, 86 GetWriteStream: []gax.CallOption{ 87 gax.WithRetry(func() gax.Retryer { 88 return gax.OnCodes([]codes.Code{ 89 codes.DeadlineExceeded, 90 codes.Unavailable, 91 }, gax.Backoff{ 92 Initial: 100 * time.Millisecond, 93 Max: 60000 * time.Millisecond, 94 Multiplier: 1.30, 95 }) 96 }), 97 }, 98 FinalizeWriteStream: []gax.CallOption{ 99 gax.WithRetry(func() gax.Retryer { 100 return gax.OnCodes([]codes.Code{ 101 codes.DeadlineExceeded, 102 codes.Unavailable, 103 }, gax.Backoff{ 104 Initial: 100 * time.Millisecond, 105 Max: 60000 * time.Millisecond, 106 Multiplier: 1.30, 107 }) 108 }), 109 }, 110 BatchCommitWriteStreams: []gax.CallOption{ 111 gax.WithRetry(func() gax.Retryer { 112 return gax.OnCodes([]codes.Code{ 113 codes.DeadlineExceeded, 114 codes.Unavailable, 115 }, gax.Backoff{ 116 Initial: 100 * time.Millisecond, 117 Max: 60000 * time.Millisecond, 118 Multiplier: 1.30, 119 }) 120 }), 121 }, 122 FlushRows: []gax.CallOption{ 123 gax.WithRetry(func() gax.Retryer { 124 return gax.OnCodes([]codes.Code{ 125 codes.DeadlineExceeded, 126 codes.Unavailable, 127 }, gax.Backoff{ 128 Initial: 100 * time.Millisecond, 129 Max: 60000 * time.Millisecond, 130 Multiplier: 1.30, 131 }) 132 }), 133 }, 134 } 135} 136 137// internalBigQueryWriteClient is an interface that defines the methods availaible from BigQuery Storage API. 138type internalBigQueryWriteClient interface { 139 Close() error 140 setGoogleClientInfo(...string) 141 Connection() *grpc.ClientConn 142 CreateWriteStream(context.Context, *storagepb.CreateWriteStreamRequest, ...gax.CallOption) (*storagepb.WriteStream, error) 143 AppendRows(context.Context, ...gax.CallOption) (storagepb.BigQueryWrite_AppendRowsClient, error) 144 GetWriteStream(context.Context, *storagepb.GetWriteStreamRequest, ...gax.CallOption) (*storagepb.WriteStream, error) 145 FinalizeWriteStream(context.Context, *storagepb.FinalizeWriteStreamRequest, ...gax.CallOption) (*storagepb.FinalizeWriteStreamResponse, error) 146 BatchCommitWriteStreams(context.Context, *storagepb.BatchCommitWriteStreamsRequest, ...gax.CallOption) (*storagepb.BatchCommitWriteStreamsResponse, error) 147 FlushRows(context.Context, *storagepb.FlushRowsRequest, ...gax.CallOption) (*storagepb.FlushRowsResponse, error) 148} 149 150// BigQueryWriteClient is a client for interacting with BigQuery Storage API. 151// Methods, except Close, may be called concurrently. However, fields must not be modified concurrently with method calls. 152// 153// BigQuery Write API. 154// 155// The Write API can be used to write data to BigQuery. 156// 157// For supplementary information about the Write API, see: 158// https://cloud.google.com/bigquery/docs/write-api (at https://cloud.google.com/bigquery/docs/write-api) 159type BigQueryWriteClient struct { 160 // The internal transport-dependent client. 161 internalClient internalBigQueryWriteClient 162 163 // The call options for this service. 164 CallOptions *BigQueryWriteCallOptions 165} 166 167// Wrapper methods routed to the internal client. 168 169// Close closes the connection to the API service. The user should invoke this when 170// the client is no longer required. 171func (c *BigQueryWriteClient) Close() error { 172 return c.internalClient.Close() 173} 174 175// setGoogleClientInfo sets the name and version of the application in 176// the `x-goog-api-client` header passed on each request. Intended for 177// use by Google-written clients. 178func (c *BigQueryWriteClient) setGoogleClientInfo(keyval ...string) { 179 c.internalClient.setGoogleClientInfo(keyval...) 180} 181 182// Connection returns a connection to the API service. 183// 184// Deprecated. 185func (c *BigQueryWriteClient) Connection() *grpc.ClientConn { 186 return c.internalClient.Connection() 187} 188 189// CreateWriteStream creates a write stream to the given table. 190// Additionally, every table has a special stream named ‘_default’ 191// to which data can be written. This stream doesn’t need to be created using 192// CreateWriteStream. It is a stream that can be used simultaneously by any 193// number of clients. Data written to this stream is considered committed as 194// soon as an acknowledgement is received. 195func (c *BigQueryWriteClient) CreateWriteStream(ctx context.Context, req *storagepb.CreateWriteStreamRequest, opts ...gax.CallOption) (*storagepb.WriteStream, error) { 196 return c.internalClient.CreateWriteStream(ctx, req, opts...) 197} 198 199// AppendRows appends data to the given stream. 200// 201// If offset is specified, the offset is checked against the end of 202// stream. The server returns OUT_OF_RANGE in AppendRowsResponse if an 203// attempt is made to append to an offset beyond the current end of the stream 204// or ALREADY_EXISTS if user provides an offset that has already been 205// written to. User can retry with adjusted offset within the same RPC 206// connection. If offset is not specified, append happens at the end of the 207// stream. 208// 209// The response contains an optional offset at which the append 210// happened. No offset information will be returned for appends to a 211// default stream. 212// 213// Responses are received in the same order in which requests are sent. 214// There will be one response for each successful inserted request. Responses 215// may optionally embed error information if the originating AppendRequest was 216// not successfully processed. 217// 218// The specifics of when successfully appended data is made visible to the 219// table are governed by the type of stream: 220// 221// For COMMITTED streams (which includes the default stream), data is 222// visible immediately upon successful append. 223// 224// For BUFFERED streams, data is made visible via a subsequent FlushRows 225// rpc which advances a cursor to a newer offset in the stream. 226// 227// For PENDING streams, data is not made visible until the stream itself is 228// finalized (via the FinalizeWriteStream rpc), and the stream is explicitly 229// committed via the BatchCommitWriteStreams rpc. 230func (c *BigQueryWriteClient) AppendRows(ctx context.Context, opts ...gax.CallOption) (storagepb.BigQueryWrite_AppendRowsClient, error) { 231 return c.internalClient.AppendRows(ctx, opts...) 232} 233 234// GetWriteStream gets information about a write stream. 235func (c *BigQueryWriteClient) GetWriteStream(ctx context.Context, req *storagepb.GetWriteStreamRequest, opts ...gax.CallOption) (*storagepb.WriteStream, error) { 236 return c.internalClient.GetWriteStream(ctx, req, opts...) 237} 238 239// FinalizeWriteStream finalize a write stream so that no new data can be appended to the 240// stream. Finalize is not supported on the ‘_default’ stream. 241func (c *BigQueryWriteClient) FinalizeWriteStream(ctx context.Context, req *storagepb.FinalizeWriteStreamRequest, opts ...gax.CallOption) (*storagepb.FinalizeWriteStreamResponse, error) { 242 return c.internalClient.FinalizeWriteStream(ctx, req, opts...) 243} 244 245// BatchCommitWriteStreams atomically commits a group of PENDING streams that belong to the same 246// parent table. 247// 248// Streams must be finalized before commit and cannot be committed multiple 249// times. Once a stream is committed, data in the stream becomes available 250// for read operations. 251func (c *BigQueryWriteClient) BatchCommitWriteStreams(ctx context.Context, req *storagepb.BatchCommitWriteStreamsRequest, opts ...gax.CallOption) (*storagepb.BatchCommitWriteStreamsResponse, error) { 252 return c.internalClient.BatchCommitWriteStreams(ctx, req, opts...) 253} 254 255// FlushRows flushes rows to a BUFFERED stream. 256// 257// If users are appending rows to BUFFERED stream, flush operation is 258// required in order for the rows to become available for reading. A 259// Flush operation flushes up to any previously flushed offset in a BUFFERED 260// stream, to the offset specified in the request. 261// 262// Flush is not supported on the _default stream, since it is not BUFFERED. 263func (c *BigQueryWriteClient) FlushRows(ctx context.Context, req *storagepb.FlushRowsRequest, opts ...gax.CallOption) (*storagepb.FlushRowsResponse, error) { 264 return c.internalClient.FlushRows(ctx, req, opts...) 265} 266 267// bigQueryWriteGRPCClient is a client for interacting with BigQuery Storage API over gRPC transport. 268// 269// Methods, except Close, may be called concurrently. However, fields must not be modified concurrently with method calls. 270type bigQueryWriteGRPCClient struct { 271 // Connection pool of gRPC connections to the service. 272 connPool gtransport.ConnPool 273 274 // flag to opt out of default deadlines via GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE 275 disableDeadlines bool 276 277 // Points back to the CallOptions field of the containing BigQueryWriteClient 278 CallOptions **BigQueryWriteCallOptions 279 280 // The gRPC API client. 281 bigQueryWriteClient storagepb.BigQueryWriteClient 282 283 // The x-goog-* metadata to be sent with each request. 284 xGoogMetadata metadata.MD 285} 286 287// NewBigQueryWriteClient creates a new big query write client based on gRPC. 288// The returned client must be Closed when it is done being used to clean up its underlying connections. 289// 290// BigQuery Write API. 291// 292// The Write API can be used to write data to BigQuery. 293// 294// For supplementary information about the Write API, see: 295// https://cloud.google.com/bigquery/docs/write-api (at https://cloud.google.com/bigquery/docs/write-api) 296func NewBigQueryWriteClient(ctx context.Context, opts ...option.ClientOption) (*BigQueryWriteClient, error) { 297 clientOpts := defaultBigQueryWriteGRPCClientOptions() 298 if newBigQueryWriteClientHook != nil { 299 hookOpts, err := newBigQueryWriteClientHook(ctx, clientHookParams{}) 300 if err != nil { 301 return nil, err 302 } 303 clientOpts = append(clientOpts, hookOpts...) 304 } 305 306 disableDeadlines, err := checkDisableDeadlines() 307 if err != nil { 308 return nil, err 309 } 310 311 connPool, err := gtransport.DialPool(ctx, append(clientOpts, opts...)...) 312 if err != nil { 313 return nil, err 314 } 315 client := BigQueryWriteClient{CallOptions: defaultBigQueryWriteCallOptions()} 316 317 c := &bigQueryWriteGRPCClient{ 318 connPool: connPool, 319 disableDeadlines: disableDeadlines, 320 bigQueryWriteClient: storagepb.NewBigQueryWriteClient(connPool), 321 CallOptions: &client.CallOptions, 322 } 323 c.setGoogleClientInfo() 324 325 client.internalClient = c 326 327 return &client, nil 328} 329 330// Connection returns a connection to the API service. 331// 332// Deprecated. 333func (c *bigQueryWriteGRPCClient) Connection() *grpc.ClientConn { 334 return c.connPool.Conn() 335} 336 337// setGoogleClientInfo sets the name and version of the application in 338// the `x-goog-api-client` header passed on each request. Intended for 339// use by Google-written clients. 340func (c *bigQueryWriteGRPCClient) setGoogleClientInfo(keyval ...string) { 341 kv := append([]string{"gl-go", versionGo()}, keyval...) 342 kv = append(kv, "gapic", versionClient, "gax", gax.Version, "grpc", grpc.Version) 343 c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...)) 344} 345 346// Close closes the connection to the API service. The user should invoke this when 347// the client is no longer required. 348func (c *bigQueryWriteGRPCClient) Close() error { 349 return c.connPool.Close() 350} 351 352func (c *bigQueryWriteGRPCClient) CreateWriteStream(ctx context.Context, req *storagepb.CreateWriteStreamRequest, opts ...gax.CallOption) (*storagepb.WriteStream, error) { 353 if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { 354 cctx, cancel := context.WithTimeout(ctx, 600000*time.Millisecond) 355 defer cancel() 356 ctx = cctx 357 } 358 md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "parent", url.QueryEscape(req.GetParent()))) 359 ctx = insertMetadata(ctx, c.xGoogMetadata, md) 360 opts = append((*c.CallOptions).CreateWriteStream[0:len((*c.CallOptions).CreateWriteStream):len((*c.CallOptions).CreateWriteStream)], opts...) 361 var resp *storagepb.WriteStream 362 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 363 var err error 364 resp, err = c.bigQueryWriteClient.CreateWriteStream(ctx, req, settings.GRPC...) 365 return err 366 }, opts...) 367 if err != nil { 368 return nil, err 369 } 370 return resp, nil 371} 372 373func (c *bigQueryWriteGRPCClient) AppendRows(ctx context.Context, opts ...gax.CallOption) (storagepb.BigQueryWrite_AppendRowsClient, error) { 374 ctx = insertMetadata(ctx, c.xGoogMetadata) 375 var resp storagepb.BigQueryWrite_AppendRowsClient 376 opts = append((*c.CallOptions).AppendRows[0:len((*c.CallOptions).AppendRows):len((*c.CallOptions).AppendRows)], opts...) 377 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 378 var err error 379 resp, err = c.bigQueryWriteClient.AppendRows(ctx, settings.GRPC...) 380 return err 381 }, opts...) 382 if err != nil { 383 return nil, err 384 } 385 return resp, nil 386} 387 388func (c *bigQueryWriteGRPCClient) GetWriteStream(ctx context.Context, req *storagepb.GetWriteStreamRequest, opts ...gax.CallOption) (*storagepb.WriteStream, error) { 389 if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { 390 cctx, cancel := context.WithTimeout(ctx, 600000*time.Millisecond) 391 defer cancel() 392 ctx = cctx 393 } 394 md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "name", url.QueryEscape(req.GetName()))) 395 ctx = insertMetadata(ctx, c.xGoogMetadata, md) 396 opts = append((*c.CallOptions).GetWriteStream[0:len((*c.CallOptions).GetWriteStream):len((*c.CallOptions).GetWriteStream)], opts...) 397 var resp *storagepb.WriteStream 398 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 399 var err error 400 resp, err = c.bigQueryWriteClient.GetWriteStream(ctx, req, settings.GRPC...) 401 return err 402 }, opts...) 403 if err != nil { 404 return nil, err 405 } 406 return resp, nil 407} 408 409func (c *bigQueryWriteGRPCClient) FinalizeWriteStream(ctx context.Context, req *storagepb.FinalizeWriteStreamRequest, opts ...gax.CallOption) (*storagepb.FinalizeWriteStreamResponse, error) { 410 if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { 411 cctx, cancel := context.WithTimeout(ctx, 600000*time.Millisecond) 412 defer cancel() 413 ctx = cctx 414 } 415 md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "name", url.QueryEscape(req.GetName()))) 416 ctx = insertMetadata(ctx, c.xGoogMetadata, md) 417 opts = append((*c.CallOptions).FinalizeWriteStream[0:len((*c.CallOptions).FinalizeWriteStream):len((*c.CallOptions).FinalizeWriteStream)], opts...) 418 var resp *storagepb.FinalizeWriteStreamResponse 419 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 420 var err error 421 resp, err = c.bigQueryWriteClient.FinalizeWriteStream(ctx, req, settings.GRPC...) 422 return err 423 }, opts...) 424 if err != nil { 425 return nil, err 426 } 427 return resp, nil 428} 429 430func (c *bigQueryWriteGRPCClient) BatchCommitWriteStreams(ctx context.Context, req *storagepb.BatchCommitWriteStreamsRequest, opts ...gax.CallOption) (*storagepb.BatchCommitWriteStreamsResponse, error) { 431 if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { 432 cctx, cancel := context.WithTimeout(ctx, 600000*time.Millisecond) 433 defer cancel() 434 ctx = cctx 435 } 436 md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "parent", url.QueryEscape(req.GetParent()))) 437 ctx = insertMetadata(ctx, c.xGoogMetadata, md) 438 opts = append((*c.CallOptions).BatchCommitWriteStreams[0:len((*c.CallOptions).BatchCommitWriteStreams):len((*c.CallOptions).BatchCommitWriteStreams)], opts...) 439 var resp *storagepb.BatchCommitWriteStreamsResponse 440 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 441 var err error 442 resp, err = c.bigQueryWriteClient.BatchCommitWriteStreams(ctx, req, settings.GRPC...) 443 return err 444 }, opts...) 445 if err != nil { 446 return nil, err 447 } 448 return resp, nil 449} 450 451func (c *bigQueryWriteGRPCClient) FlushRows(ctx context.Context, req *storagepb.FlushRowsRequest, opts ...gax.CallOption) (*storagepb.FlushRowsResponse, error) { 452 if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { 453 cctx, cancel := context.WithTimeout(ctx, 600000*time.Millisecond) 454 defer cancel() 455 ctx = cctx 456 } 457 md := metadata.Pairs("x-goog-request-params", fmt.Sprintf("%s=%v", "write_stream", url.QueryEscape(req.GetWriteStream()))) 458 ctx = insertMetadata(ctx, c.xGoogMetadata, md) 459 opts = append((*c.CallOptions).FlushRows[0:len((*c.CallOptions).FlushRows):len((*c.CallOptions).FlushRows)], opts...) 460 var resp *storagepb.FlushRowsResponse 461 err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { 462 var err error 463 resp, err = c.bigQueryWriteClient.FlushRows(ctx, req, settings.GRPC...) 464 return err 465 }, opts...) 466 if err != nil { 467 return nil, err 468 } 469 return resp, nil 470} 471