1package azblob_test
2
3import (
4	"context"
5	"fmt"
6	"io"
7	"net/http"
8	"net/url"
9	"strings"
10	"time"
11
12	chk "gopkg.in/check.v1"
13
14	"github.com/Azure/azure-pipeline-go/pipeline"
15	"github.com/Azure/azure-storage-blob-go/azblob"
16)
17
18// For testing docs, see: https://labix.org/gocheck
19// To test a specific test: go test -check.f MyTestSuite
20
21type retryTestScenario int32
22
23const (
24	// Retry until success. Max reties hit. Operation time out prevents additional retries
25	retryTestScenarioRetryUntilSuccess         retryTestScenario = 1
26	retryTestScenarioRetryUntilOperationCancel retryTestScenario = 2
27	retryTestScenarioRetryUntilMaxRetries      retryTestScenario = 3
28)
29
30func (s *aztestsSuite) TestRetryTestScenarioUntilSuccess(c *chk.C) {
31	testRetryTestScenario(c, retryTestScenarioRetryUntilSuccess)
32}
33
34func (s *aztestsSuite) TestRetryTestScenarioUntilOperationCancel(c *chk.C) {
35	testRetryTestScenario(c, retryTestScenarioRetryUntilOperationCancel)
36}
37func (s *aztestsSuite) TestRetryTestScenarioUntilMaxRetries(c *chk.C) {
38	testRetryTestScenario(c, retryTestScenarioRetryUntilMaxRetries)
39}
40func newRetryTestPolicyFactory(c *chk.C, scenario retryTestScenario, maxRetries int32, cancel context.CancelFunc) *retryTestPolicyFactory {
41	return &retryTestPolicyFactory{c: c, scenario: scenario, maxRetries: maxRetries, cancel: cancel}
42}
43
44type retryTestPolicyFactory struct {
45	c          *chk.C
46	scenario   retryTestScenario
47	maxRetries int32
48	cancel     context.CancelFunc
49	try        int32
50}
51
52func (f *retryTestPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
53	f.try = 0 // Reset this for each test
54	return &retryTestPolicy{factory: f, next: next}
55}
56
57type retryTestPolicy struct {
58	next    pipeline.Policy
59	factory *retryTestPolicyFactory
60}
61
62type retryError struct {
63	temporary, timeout bool
64}
65
66func (e *retryError) Temporary() bool { return e.temporary }
67func (e *retryError) Timeout() bool   { return e.timeout }
68func (e *retryError) Error() string {
69	return fmt.Sprintf("Temporary=%t, Timeout=%t", e.Temporary(), e.Timeout())
70}
71
72type httpResponse struct {
73	response *http.Response
74}
75
76func (r *httpResponse) Response() *http.Response { return r.response }
77
78func (p *retryTestPolicy) Do(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) {
79	c := p.factory.c
80	p.factory.try++                                                   // Increment the try
81	c.Assert(p.factory.try <= p.factory.maxRetries, chk.Equals, true) // Ensure # of tries < MaxRetries
82	req := request.Request
83
84	// Validate the expected pre-conditions for each try
85	expectedHost := "PrimaryDC"
86	if p.factory.try%2 == 0 {
87		if p.factory.scenario != retryTestScenarioRetryUntilSuccess || p.factory.try <= 4 {
88			expectedHost = "SecondaryDC"
89		}
90	}
91	c.Assert(req.URL.Host, chk.Equals, expectedHost) // Ensure we got the expected primary/secondary DC
92
93	// Ensure that any headers & query parameters this method adds (later) are removed/reset for each try
94	c.Assert(req.Header.Get("TestHeader"), chk.Equals, "") // Ensure our "TestHeader" is not in the HTTP request
95	values := req.URL.Query()
96	c.Assert(len(values["TestQueryParam"]), chk.Equals, 0) // TestQueryParam shouldn't be in the HTTP request
97
98	if seeker, ok := req.Body.(io.ReadSeeker); !ok {
99		c.Fail() // Body must be an io.ReadSeeker
100	} else {
101		pos, err := seeker.Seek(0, io.SeekCurrent)
102		c.Assert(err, chk.IsNil)            // Ensure that body was seekable
103		c.Assert(pos, chk.Equals, int64(0)) // Ensure body seeked back to position 0
104	}
105
106	// Add a query param & header; these not be here on the next try
107	values["TestQueryParam"] = []string{"TestQueryParamValue"}
108	req.Header.Set("TestHeader", "TestValue") // Add a header this not exist with each try
109	b := []byte{0}
110	n, err := req.Body.Read(b)
111	c.Assert(n, chk.Equals, 1) // Read failed
112
113	switch p.factory.scenario {
114	case retryTestScenarioRetryUntilSuccess:
115		switch p.factory.try {
116		case 1:
117			if deadline, ok := ctx.Deadline(); ok {
118				time.Sleep(time.Until(deadline) + time.Second) // Let the context timeout expire
119			}
120			err = ctx.Err()
121		case 2:
122			err = &retryError{temporary: true}
123		case 3:
124			err = &retryError{timeout: true}
125		case 4:
126			response = &httpResponse{response: &http.Response{StatusCode: http.StatusNotFound}}
127		case 5:
128			err = &retryError{temporary: true} // These attempts all fail but we're making sure we never see the secondary DC again
129		case 6:
130			response = &httpResponse{response: &http.Response{StatusCode: http.StatusOK}} // Stop retries with valid response
131		default:
132			c.Fail() // Retries should have stopped so we shouldn't get here
133		}
134	case retryTestScenarioRetryUntilOperationCancel:
135		switch p.factory.try {
136		case 1:
137			p.factory.cancel()
138			err = context.Canceled
139		default:
140			c.Fail() // Retries should have stopped so we shouldn't get here
141		}
142	case retryTestScenarioRetryUntilMaxRetries:
143		err = &retryError{temporary: true} // Keep retrying until maxRetries is hit
144	}
145	return response, err // Return the response & err
146}
147
148func testRetryTestScenario(c *chk.C, scenario retryTestScenario) {
149	u, _ := url.Parse("http://PrimaryDC")
150	retryOptions := azblob.RetryOptions{
151		Policy:                      azblob.RetryPolicyExponential,
152		MaxTries:                    6,
153		TryTimeout:                  2 * time.Second,
154		RetryDelay:                  1 * time.Second,
155		MaxRetryDelay:               4 * time.Second,
156		RetryReadsFromSecondaryHost: "SecondaryDC",
157	}
158	minExpectedTimeToMaxRetries := (retryOptions.MaxRetryDelay * time.Duration(retryOptions.MaxTries-3)) / 2 // a very rough approximation, of a lower bound, given assumption that we hit the cap early in the retry count, and pessimistically assuming that all get halved by random jitter calcs
159	ctx := context.Background()
160	ctx, cancel := context.WithTimeout(ctx, 64 /*2^MaxTries(6)*/ *retryOptions.TryTimeout)
161	retrytestPolicyFactory := newRetryTestPolicyFactory(c, scenario, retryOptions.MaxTries, cancel)
162	factories := [...]pipeline.Factory{
163		azblob.NewRetryPolicyFactory(retryOptions),
164		retrytestPolicyFactory,
165	}
166	p := pipeline.NewPipeline(factories[:], pipeline.Options{})
167	request, err := pipeline.NewRequest(http.MethodGet, *u, strings.NewReader("TestData"))
168	start := time.Now()
169	response, err := p.Do(ctx, nil, request)
170	switch scenario {
171	case retryTestScenarioRetryUntilSuccess:
172		if err != nil || response == nil || response.Response() == nil || response.Response().StatusCode != http.StatusOK {
173			c.Fail() // Operation didn't run to success
174		}
175	case retryTestScenarioRetryUntilMaxRetries:
176		c.Assert(err, chk.NotNil)                                                   // Ensure we ended with an error
177		c.Assert(response, chk.IsNil)                                               // Ensure we ended without a valid response
178		c.Assert(retrytestPolicyFactory.try, chk.Equals, retryOptions.MaxTries)     // Ensure the operation ends with the exact right number of tries
179		c.Assert(time.Since(start) > minExpectedTimeToMaxRetries, chk.Equals, true) // Ensure it took about as long to get here as we expect (bearing in mind randomness in the jitter), as a basic sanity check of our delay duration calculations
180	case retryTestScenarioRetryUntilOperationCancel:
181		c.Assert(err, chk.Equals, context.Canceled)                                     // Ensure we ended due to cancellation
182		c.Assert(response, chk.IsNil)                                                   // Ensure we ended without a valid response
183		c.Assert(retrytestPolicyFactory.try <= retryOptions.MaxTries, chk.Equals, true) // Ensure we didn't end due to reaching max tries
184	}
185	cancel()
186}
187
188/*
189   	Fail primary; retry should be on secondary URL - maybe do this twice
190   	Fail secondary; and never see primary again
191
192   	Make sure any mutations are lost on each retry
193   	Make sure body is reset on each retry
194
195   	Timeout a try; should retry (unless no more)
196   	timeout an operation; should not retry
197   	check timeout query param; should be try timeout
198
199   	Return Temporary() = true; should retry (unless max)
200   	Return Timeout() true; should retry (unless max)
201
202   	Secondary try returns 404; no more tries against secondary
203
204   	error where Temporary() and Timeout() return false; no retry
205   	error where Temporary() & Timeout don't exist; no retry
206    no error; no retry; return success, nil
207*/
208