1package azblob_test 2 3import ( 4 "context" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "strings" 10 "time" 11 12 chk "gopkg.in/check.v1" 13 14 "github.com/Azure/azure-pipeline-go/pipeline" 15 "github.com/Azure/azure-storage-blob-go/azblob" 16) 17 18// For testing docs, see: https://labix.org/gocheck 19// To test a specific test: go test -check.f MyTestSuite 20 21type retryTestScenario int32 22 23const ( 24 // Retry until success. Max reties hit. Operation time out prevents additional retries 25 retryTestScenarioRetryUntilSuccess retryTestScenario = 1 26 retryTestScenarioRetryUntilOperationCancel retryTestScenario = 2 27 retryTestScenarioRetryUntilMaxRetries retryTestScenario = 3 28) 29 30func (s *aztestsSuite) TestRetryTestScenarioUntilSuccess(c *chk.C) { 31 testRetryTestScenario(c, retryTestScenarioRetryUntilSuccess) 32} 33 34func (s *aztestsSuite) TestRetryTestScenarioUntilOperationCancel(c *chk.C) { 35 testRetryTestScenario(c, retryTestScenarioRetryUntilOperationCancel) 36} 37func (s *aztestsSuite) TestRetryTestScenarioUntilMaxRetries(c *chk.C) { 38 testRetryTestScenario(c, retryTestScenarioRetryUntilMaxRetries) 39} 40func newRetryTestPolicyFactory(c *chk.C, scenario retryTestScenario, maxRetries int32, cancel context.CancelFunc) *retryTestPolicyFactory { 41 return &retryTestPolicyFactory{c: c, scenario: scenario, maxRetries: maxRetries, cancel: cancel} 42} 43 44type retryTestPolicyFactory struct { 45 c *chk.C 46 scenario retryTestScenario 47 maxRetries int32 48 cancel context.CancelFunc 49 try int32 50} 51 52func (f *retryTestPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 53 f.try = 0 // Reset this for each test 54 return &retryTestPolicy{factory: f, next: next} 55} 56 57type retryTestPolicy struct { 58 next pipeline.Policy 59 factory *retryTestPolicyFactory 60} 61 62type retryError struct { 63 temporary, timeout bool 64} 65 66func (e *retryError) Temporary() bool { return e.temporary } 67func (e *retryError) Timeout() bool { return e.timeout } 68func (e *retryError) Error() string { 69 return fmt.Sprintf("Temporary=%t, Timeout=%t", e.Temporary(), e.Timeout()) 70} 71 72type httpResponse struct { 73 response *http.Response 74} 75 76func (r *httpResponse) Response() *http.Response { return r.response } 77 78func (p *retryTestPolicy) Do(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) { 79 c := p.factory.c 80 p.factory.try++ // Increment the try 81 c.Assert(p.factory.try <= p.factory.maxRetries, chk.Equals, true) // Ensure # of tries < MaxRetries 82 req := request.Request 83 84 // Validate the expected pre-conditions for each try 85 expectedHost := "PrimaryDC" 86 if p.factory.try%2 == 0 { 87 if p.factory.scenario != retryTestScenarioRetryUntilSuccess || p.factory.try <= 4 { 88 expectedHost = "SecondaryDC" 89 } 90 } 91 c.Assert(req.URL.Host, chk.Equals, expectedHost) // Ensure we got the expected primary/secondary DC 92 93 // Ensure that any headers & query parameters this method adds (later) are removed/reset for each try 94 c.Assert(req.Header.Get("TestHeader"), chk.Equals, "") // Ensure our "TestHeader" is not in the HTTP request 95 values := req.URL.Query() 96 c.Assert(len(values["TestQueryParam"]), chk.Equals, 0) // TestQueryParam shouldn't be in the HTTP request 97 98 if seeker, ok := req.Body.(io.ReadSeeker); !ok { 99 c.Fail() // Body must be an io.ReadSeeker 100 } else { 101 pos, err := seeker.Seek(0, io.SeekCurrent) 102 c.Assert(err, chk.IsNil) // Ensure that body was seekable 103 c.Assert(pos, chk.Equals, int64(0)) // Ensure body seeked back to position 0 104 } 105 106 // Add a query param & header; these not be here on the next try 107 values["TestQueryParam"] = []string{"TestQueryParamValue"} 108 req.Header.Set("TestHeader", "TestValue") // Add a header this not exist with each try 109 b := []byte{0} 110 n, err := req.Body.Read(b) 111 c.Assert(n, chk.Equals, 1) // Read failed 112 113 switch p.factory.scenario { 114 case retryTestScenarioRetryUntilSuccess: 115 switch p.factory.try { 116 case 1: 117 if deadline, ok := ctx.Deadline(); ok { 118 time.Sleep(time.Until(deadline) + time.Second) // Let the context timeout expire 119 } 120 err = ctx.Err() 121 case 2: 122 err = &retryError{temporary: true} 123 case 3: 124 err = &retryError{timeout: true} 125 case 4: 126 response = &httpResponse{response: &http.Response{StatusCode: http.StatusNotFound}} 127 case 5: 128 err = &retryError{temporary: true} // These attempts all fail but we're making sure we never see the secondary DC again 129 case 6: 130 response = &httpResponse{response: &http.Response{StatusCode: http.StatusOK}} // Stop retries with valid response 131 default: 132 c.Fail() // Retries should have stopped so we shouldn't get here 133 } 134 case retryTestScenarioRetryUntilOperationCancel: 135 switch p.factory.try { 136 case 1: 137 p.factory.cancel() 138 err = context.Canceled 139 default: 140 c.Fail() // Retries should have stopped so we shouldn't get here 141 } 142 case retryTestScenarioRetryUntilMaxRetries: 143 err = &retryError{temporary: true} // Keep retrying until maxRetries is hit 144 } 145 return response, err // Return the response & err 146} 147 148func testRetryTestScenario(c *chk.C, scenario retryTestScenario) { 149 u, _ := url.Parse("http://PrimaryDC") 150 retryOptions := azblob.RetryOptions{ 151 Policy: azblob.RetryPolicyExponential, 152 MaxTries: 6, 153 TryTimeout: 2 * time.Second, 154 RetryDelay: 1 * time.Second, 155 MaxRetryDelay: 4 * time.Second, 156 RetryReadsFromSecondaryHost: "SecondaryDC", 157 } 158 minExpectedTimeToMaxRetries := (retryOptions.MaxRetryDelay * time.Duration(retryOptions.MaxTries-3)) / 2 // a very rough approximation, of a lower bound, given assumption that we hit the cap early in the retry count, and pessimistically assuming that all get halved by random jitter calcs 159 ctx := context.Background() 160 ctx, cancel := context.WithTimeout(ctx, 64 /*2^MaxTries(6)*/ *retryOptions.TryTimeout) 161 retrytestPolicyFactory := newRetryTestPolicyFactory(c, scenario, retryOptions.MaxTries, cancel) 162 factories := [...]pipeline.Factory{ 163 azblob.NewRetryPolicyFactory(retryOptions), 164 retrytestPolicyFactory, 165 } 166 p := pipeline.NewPipeline(factories[:], pipeline.Options{}) 167 request, err := pipeline.NewRequest(http.MethodGet, *u, strings.NewReader("TestData")) 168 start := time.Now() 169 response, err := p.Do(ctx, nil, request) 170 switch scenario { 171 case retryTestScenarioRetryUntilSuccess: 172 if err != nil || response == nil || response.Response() == nil || response.Response().StatusCode != http.StatusOK { 173 c.Fail() // Operation didn't run to success 174 } 175 case retryTestScenarioRetryUntilMaxRetries: 176 c.Assert(err, chk.NotNil) // Ensure we ended with an error 177 c.Assert(response, chk.IsNil) // Ensure we ended without a valid response 178 c.Assert(retrytestPolicyFactory.try, chk.Equals, retryOptions.MaxTries) // Ensure the operation ends with the exact right number of tries 179 c.Assert(time.Since(start) > minExpectedTimeToMaxRetries, chk.Equals, true) // Ensure it took about as long to get here as we expect (bearing in mind randomness in the jitter), as a basic sanity check of our delay duration calculations 180 case retryTestScenarioRetryUntilOperationCancel: 181 c.Assert(err, chk.Equals, context.Canceled) // Ensure we ended due to cancellation 182 c.Assert(response, chk.IsNil) // Ensure we ended without a valid response 183 c.Assert(retrytestPolicyFactory.try <= retryOptions.MaxTries, chk.Equals, true) // Ensure we didn't end due to reaching max tries 184 } 185 cancel() 186} 187 188/* 189 Fail primary; retry should be on secondary URL - maybe do this twice 190 Fail secondary; and never see primary again 191 192 Make sure any mutations are lost on each retry 193 Make sure body is reset on each retry 194 195 Timeout a try; should retry (unless no more) 196 timeout an operation; should not retry 197 check timeout query param; should be try timeout 198 199 Return Temporary() = true; should retry (unless max) 200 Return Timeout() true; should retry (unless max) 201 202 Secondary try returns 404; no more tries against secondary 203 204 error where Temporary() and Timeout() return false; no retry 205 error where Temporary() & Timeout don't exist; no retry 206 no error; no retry; return success, nil 207*/ 208