1package azblob_test
2
3import (
4	"context"
5	"io/ioutil"
6	"os"
7
8	"github.com/Azure/azure-storage-blob-go/azblob"
9	chk "gopkg.in/check.v1"
10)
11
12// create a test file
13func generateFile(fileName string, fileSize int) []byte {
14	// generate random data
15	_, bigBuff := getRandomDataAndReader(fileSize)
16
17	// write to file and return the data
18	ioutil.WriteFile(fileName, bigBuff, 0666)
19	return bigBuff
20}
21
22func performUploadStreamToBlockBlobTest(c *chk.C, blobSize, bufferSize, maxBuffers int) {
23	// Set up test container
24	bsu := getBSU()
25	containerURL, _ := createNewContainer(c, bsu)
26	defer deleteContainer(c, containerURL)
27
28	// Set up test blob
29	blobURL, _ := getBlockBlobURL(c, containerURL)
30
31	// Create some data to test the upload stream
32	blobContentReader, blobData := getRandomDataAndReader(blobSize)
33
34	// Perform UploadStreamToBlockBlob
35	uploadResp, err := azblob.UploadStreamToBlockBlob(ctx, blobContentReader, blobURL,
36		azblob.UploadStreamToBlockBlobOptions{BufferSize: bufferSize, MaxBuffers: maxBuffers})
37
38	// Assert that upload was successful
39	c.Assert(err, chk.Equals, nil)
40	c.Assert(uploadResp.Response().StatusCode, chk.Equals, 201)
41
42	// Download the blob to verify
43	downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false)
44	c.Assert(err, chk.IsNil)
45
46	// Assert that the content is correct
47	actualBlobData, err := ioutil.ReadAll(downloadResponse.Response().Body)
48	c.Assert(err, chk.IsNil)
49	c.Assert(len(actualBlobData), chk.Equals, blobSize)
50	c.Assert(actualBlobData, chk.DeepEquals, blobData)
51}
52
53func (s *aztestsSuite) TestUploadStreamToBlockBlobInChunks(c *chk.C) {
54	blobSize := 8 * 1024
55	bufferSize := 1024
56	maxBuffers := 3
57	performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers)
58}
59
60func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleBuffer(c *chk.C) {
61	blobSize := 8 * 1024
62	bufferSize := 1024
63	maxBuffers := 1
64	performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers)
65}
66
67func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleIO(c *chk.C) {
68	blobSize := 1024
69	bufferSize := 8 * 1024
70	maxBuffers := 3
71	performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers)
72}
73
74func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleIOEdgeCase(c *chk.C) {
75	blobSize := 8 * 1024
76	bufferSize := 8 * 1024
77	maxBuffers := 3
78	performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers)
79}
80
81func (s *aztestsSuite) TestUploadStreamToBlockBlobEmpty(c *chk.C) {
82	blobSize := 0
83	bufferSize := 8 * 1024
84	maxBuffers := 3
85	performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers)
86}
87
88func performUploadAndDownloadFileTest(c *chk.C, fileSize, blockSize, parallelism, downloadOffset, downloadCount int) {
89	// Set up file to upload
90	fileName := "BigFile.bin"
91	fileData := generateFile(fileName, fileSize)
92
93	// Open the file to upload
94	file, err := os.Open(fileName)
95	c.Assert(err, chk.Equals, nil)
96	defer file.Close()
97	defer os.Remove(fileName)
98
99	// Set up test container
100	bsu := getBSU()
101	containerURL, _ := createNewContainer(c, bsu)
102	defer deleteContainer(c, containerURL)
103
104	// Set up test blob
105	blockBlobURL, _ := getBlockBlobURL(c, containerURL)
106
107	// Upload the file to a block blob
108	response, err := azblob.UploadFileToBlockBlob(context.Background(), file, blockBlobURL,
109		azblob.UploadToBlockBlobOptions{
110			BlockSize:   int64(blockSize),
111			Parallelism: uint16(parallelism),
112			// If Progress is non-nil, this function is called periodically as bytes are uploaded.
113			Progress: func(bytesTransferred int64) {
114				c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(fileSize), chk.Equals, true)
115			},
116		})
117	c.Assert(err, chk.Equals, nil)
118	c.Assert(response.Response().StatusCode, chk.Equals, 201)
119
120	// Set up file to download the blob to
121	destFileName := "BigFile-downloaded.bin"
122	destFile, err := os.Create(destFileName)
123	c.Assert(err, chk.Equals, nil)
124	defer destFile.Close()
125	defer os.Remove(destFileName)
126
127	// Perform download
128	err = azblob.DownloadBlobToFile(context.Background(), blockBlobURL.BlobURL, int64(downloadOffset), int64(downloadCount),
129		destFile,
130		azblob.DownloadFromBlobOptions{
131			BlockSize:   int64(blockSize),
132			Parallelism: uint16(parallelism),
133			// If Progress is non-nil, this function is called periodically as bytes are uploaded.
134			Progress: func(bytesTransferred int64) {
135				c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(fileSize), chk.Equals, true)
136			}})
137
138	// Assert download was successful
139	c.Assert(err, chk.Equals, nil)
140
141	// Assert downloaded data is consistent
142	var destBuffer []byte
143	if downloadCount == azblob.CountToEnd {
144		destBuffer = make([]byte, fileSize-downloadOffset)
145	} else {
146		destBuffer = make([]byte, downloadCount)
147	}
148
149	n, err := destFile.Read(destBuffer)
150	c.Assert(err, chk.Equals, nil)
151
152	if downloadOffset == 0 && downloadCount == 0 {
153		c.Assert(destBuffer, chk.DeepEquals, fileData)
154	} else {
155		if downloadCount == 0 {
156			c.Assert(n, chk.Equals, fileSize-downloadOffset)
157			c.Assert(destBuffer, chk.DeepEquals, fileData[downloadOffset:])
158		} else {
159			c.Assert(n, chk.Equals, downloadCount)
160			c.Assert(destBuffer, chk.DeepEquals, fileData[downloadOffset:downloadOffset+downloadCount])
161		}
162	}
163}
164
165func (s *aztestsSuite) TestUploadAndDownloadFileInChunks(c *chk.C) {
166	fileSize := 8 * 1024
167	blockSize := 1024
168	parallelism := 3
169	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0)
170}
171
172func (s *aztestsSuite) TestUploadAndDownloadFileSingleIO(c *chk.C) {
173	fileSize := 1024
174	blockSize := 2048
175	parallelism := 3
176	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0)
177}
178
179func (s *aztestsSuite) TestUploadAndDownloadFileSingleRoutine(c *chk.C) {
180	fileSize := 8 * 1024
181	blockSize := 1024
182	parallelism := 1
183	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0)
184}
185
186func (s *aztestsSuite) TestUploadAndDownloadFileEmpty(c *chk.C) {
187	fileSize := 0
188	blockSize := 1024
189	parallelism := 3
190	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0)
191}
192
193func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroOffset(c *chk.C) {
194	fileSize := 8 * 1024
195	blockSize := 1024
196	parallelism := 3
197	downloadOffset := 1000
198	downloadCount := 0
199	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount)
200}
201
202func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroCount(c *chk.C) {
203	fileSize := 8 * 1024
204	blockSize := 1024
205	parallelism := 3
206	downloadOffset := 0
207	downloadCount := 6000
208	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount)
209}
210
211func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroOffsetAndCount(c *chk.C) {
212	fileSize := 8 * 1024
213	blockSize := 1024
214	parallelism := 3
215	downloadOffset := 1000
216	downloadCount := 6000
217	performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount)
218}
219
220func performUploadAndDownloadBufferTest(c *chk.C, blobSize, blockSize, parallelism, downloadOffset, downloadCount int) {
221	// Set up buffer to upload
222	_, bytesToUpload := getRandomDataAndReader(blobSize)
223
224	// Set up test container
225	bsu := getBSU()
226	containerURL, _ := createNewContainer(c, bsu)
227	defer deleteContainer(c, containerURL)
228
229	// Set up test blob
230	blockBlobURL, _ := getBlockBlobURL(c, containerURL)
231
232	// Pass the Context, stream, stream size, block blob URL, and options to StreamToBlockBlob
233	response, err := azblob.UploadBufferToBlockBlob(context.Background(), bytesToUpload, blockBlobURL,
234		azblob.UploadToBlockBlobOptions{
235			BlockSize:   int64(blockSize),
236			Parallelism: uint16(parallelism),
237			// If Progress is non-nil, this function is called periodically as bytes are uploaded.
238			Progress: func(bytesTransferred int64) {
239				c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(blobSize), chk.Equals, true)
240			},
241		})
242	c.Assert(err, chk.Equals, nil)
243	c.Assert(response.Response().StatusCode, chk.Equals, 201)
244
245	// Set up buffer to download the blob to
246	var destBuffer []byte
247	if downloadCount == azblob.CountToEnd {
248		destBuffer = make([]byte, blobSize-downloadOffset)
249	} else {
250		destBuffer = make([]byte, downloadCount)
251	}
252
253	// Download the blob to a buffer
254	err = azblob.DownloadBlobToBuffer(context.Background(), blockBlobURL.BlobURL, int64(downloadOffset), int64(downloadCount),
255		destBuffer, azblob.DownloadFromBlobOptions{
256			AccessConditions: azblob.BlobAccessConditions{},
257			BlockSize:        int64(blockSize),
258			Parallelism:      uint16(parallelism),
259			// If Progress is non-nil, this function is called periodically as bytes are uploaded.
260			Progress: func(bytesTransferred int64) {
261				c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(blobSize), chk.Equals, true)
262			},
263		})
264
265	c.Assert(err, chk.Equals, nil)
266
267	if downloadOffset == 0 && downloadCount == 0 {
268		c.Assert(destBuffer, chk.DeepEquals, bytesToUpload)
269	} else {
270		if downloadCount == 0 {
271			c.Assert(destBuffer, chk.DeepEquals, bytesToUpload[downloadOffset:])
272		} else {
273			c.Assert(destBuffer, chk.DeepEquals, bytesToUpload[downloadOffset:downloadOffset+downloadCount])
274		}
275	}
276}
277
278func (s *aztestsSuite) TestUploadAndDownloadBufferInChunks(c *chk.C) {
279	blobSize := 8 * 1024
280	blockSize := 1024
281	parallelism := 3
282	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0)
283}
284
285func (s *aztestsSuite) TestUploadAndDownloadBufferSingleIO(c *chk.C) {
286	blobSize := 1024
287	blockSize := 8 * 1024
288	parallelism := 3
289	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0)
290}
291
292func (s *aztestsSuite) TestUploadAndDownloadBufferSingleRoutine(c *chk.C) {
293	blobSize := 8 * 1024
294	blockSize := 1024
295	parallelism := 1
296	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0)
297}
298
299func (s *aztestsSuite) TestUploadAndDownloadBufferEmpty(c *chk.C) {
300	blobSize := 0
301	blockSize := 1024
302	parallelism := 3
303	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0)
304}
305
306func (s *aztestsSuite) TestDownloadBufferWithNonZeroOffset(c *chk.C) {
307	blobSize := 8 * 1024
308	blockSize := 1024
309	parallelism := 3
310	downloadOffset := 1000
311	downloadCount := 0
312	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount)
313}
314
315func (s *aztestsSuite) TestDownloadBufferWithNonZeroCount(c *chk.C) {
316	blobSize := 8 * 1024
317	blockSize := 1024
318	parallelism := 3
319	downloadOffset := 0
320	downloadCount := 6000
321	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount)
322}
323
324func (s *aztestsSuite) TestDownloadBufferWithNonZeroOffsetAndCount(c *chk.C) {
325	blobSize := 8 * 1024
326	blockSize := 1024
327	parallelism := 3
328	downloadOffset := 2000
329	downloadCount := 6 * 1024
330	performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount)
331}
332