1// Copyright 2016, Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above 11// copyright notice, this list of conditions and the following disclaimer 12// in the documentation and/or other materials provided with the 13// distribution. 14// * Neither the name of Google Inc. nor the names of its 15// contributors may be used to endorse or promote products derived from 16// this software without specific prior written permission. 17// 18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30package gax 31 32import ( 33 "math/rand" 34 "time" 35 36 "google.golang.org/grpc" 37 "google.golang.org/grpc/codes" 38 "google.golang.org/grpc/status" 39) 40 41// CallOption is an option used by Invoke to control behaviors of RPC calls. 42// CallOption works by modifying relevant fields of CallSettings. 43type CallOption interface { 44 // Resolve applies the option by modifying cs. 45 Resolve(cs *CallSettings) 46} 47 48// Retryer is used by Invoke to determine retry behavior. 49type Retryer interface { 50 // Retry reports whether a request should be retriedand how long to pause before retrying 51 // if the previous attempt returned with err. Invoke never calls Retry with nil error. 52 Retry(err error) (pause time.Duration, shouldRetry bool) 53} 54 55type retryerOption func() Retryer 56 57func (o retryerOption) Resolve(s *CallSettings) { 58 s.Retry = o 59} 60 61// WithRetry sets CallSettings.Retry to fn. 62func WithRetry(fn func() Retryer) CallOption { 63 return retryerOption(fn) 64} 65 66// OnCodes returns a Retryer that retries if and only if 67// the previous attempt returns a GRPC error whose error code is stored in cc. 68// Pause times between retries are specified by bo. 69// 70// bo is only used for its parameters; each Retryer has its own copy. 71func OnCodes(cc []codes.Code, bo Backoff) Retryer { 72 return &boRetryer{ 73 backoff: bo, 74 codes: append([]codes.Code(nil), cc...), 75 } 76} 77 78type boRetryer struct { 79 backoff Backoff 80 codes []codes.Code 81} 82 83func (r *boRetryer) Retry(err error) (time.Duration, bool) { 84 st, ok := status.FromError(err) 85 if !ok { 86 return 0, false 87 } 88 c := st.Code() 89 for _, rc := range r.codes { 90 if c == rc { 91 return r.backoff.Pause(), true 92 } 93 } 94 return 0, false 95} 96 97// Backoff implements exponential backoff. 98// The wait time between retries is a random value between 0 and the "retry envelope". 99// The envelope starts at Initial and increases by the factor of Multiplier every retry, 100// but is capped at Max. 101type Backoff struct { 102 // Initial is the initial value of the retry envelope, defaults to 1 second. 103 Initial time.Duration 104 105 // Max is the maximum value of the retry envelope, defaults to 30 seconds. 106 Max time.Duration 107 108 // Multiplier is the factor by which the retry envelope increases. 109 // It should be greater than 1 and defaults to 2. 110 Multiplier float64 111 112 // cur is the current retry envelope 113 cur time.Duration 114} 115 116// Pause returns the next time.Duration that the caller should use to backoff. 117func (bo *Backoff) Pause() time.Duration { 118 if bo.Initial == 0 { 119 bo.Initial = time.Second 120 } 121 if bo.cur == 0 { 122 bo.cur = bo.Initial 123 } 124 if bo.Max == 0 { 125 bo.Max = 30 * time.Second 126 } 127 if bo.Multiplier < 1 { 128 bo.Multiplier = 2 129 } 130 // Select a duration between 1ns and the current max. It might seem 131 // counterintuitive to have so much jitter, but 132 // https://www.awsarchitectureblog.com/2015/03/backoff.html argues that 133 // that is the best strategy. 134 d := time.Duration(1 + rand.Int63n(int64(bo.cur))) 135 bo.cur = time.Duration(float64(bo.cur) * bo.Multiplier) 136 if bo.cur > bo.Max { 137 bo.cur = bo.Max 138 } 139 return d 140} 141 142type grpcOpt []grpc.CallOption 143 144func (o grpcOpt) Resolve(s *CallSettings) { 145 s.GRPC = o 146} 147 148// WithGRPCOptions allows passing gRPC call options during client creation. 149func WithGRPCOptions(opt ...grpc.CallOption) CallOption { 150 return grpcOpt(append([]grpc.CallOption(nil), opt...)) 151} 152 153// CallSettings allow fine-grained control over how calls are made. 154type CallSettings struct { 155 // Retry returns a Retryer to be used to control retry logic of a method call. 156 // If Retry is nil or the returned Retryer is nil, the call will not be retried. 157 Retry func() Retryer 158 159 // CallOptions to be forwarded to GRPC. 160 GRPC []grpc.CallOption 161} 162