1package azblob_test 2 3import ( 4 "context" 5 "io/ioutil" 6 "os" 7 8 "github.com/Azure/azure-storage-blob-go/azblob" 9 chk "gopkg.in/check.v1" 10) 11 12// create a test file 13func generateFile(fileName string, fileSize int) []byte { 14 // generate random data 15 _, bigBuff := getRandomDataAndReader(fileSize) 16 17 // write to file and return the data 18 ioutil.WriteFile(fileName, bigBuff, 0666) 19 return bigBuff 20} 21 22func performUploadStreamToBlockBlobTest(c *chk.C, blobSize, bufferSize, maxBuffers int) { 23 // Set up test container 24 bsu := getBSU() 25 containerURL, _ := createNewContainer(c, bsu) 26 defer deleteContainer(c, containerURL) 27 28 // Set up test blob 29 blobURL, _ := getBlockBlobURL(c, containerURL) 30 31 // Create some data to test the upload stream 32 blobContentReader, blobData := getRandomDataAndReader(blobSize) 33 34 // Perform UploadStreamToBlockBlob 35 uploadResp, err := azblob.UploadStreamToBlockBlob(ctx, blobContentReader, blobURL, 36 azblob.UploadStreamToBlockBlobOptions{BufferSize: bufferSize, MaxBuffers: maxBuffers}) 37 38 // Assert that upload was successful 39 c.Assert(err, chk.Equals, nil) 40 c.Assert(uploadResp.Response().StatusCode, chk.Equals, 201) 41 42 // Download the blob to verify 43 downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false) 44 c.Assert(err, chk.IsNil) 45 46 // Assert that the content is correct 47 actualBlobData, err := ioutil.ReadAll(downloadResponse.Response().Body) 48 c.Assert(err, chk.IsNil) 49 c.Assert(len(actualBlobData), chk.Equals, blobSize) 50 c.Assert(actualBlobData, chk.DeepEquals, blobData) 51} 52 53func (s *aztestsSuite) TestUploadStreamToBlockBlobInChunks(c *chk.C) { 54 blobSize := 8 * 1024 55 bufferSize := 1024 56 maxBuffers := 3 57 performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers) 58} 59 60func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleBuffer(c *chk.C) { 61 blobSize := 8 * 1024 62 bufferSize := 1024 63 maxBuffers := 1 64 performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers) 65} 66 67func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleIO(c *chk.C) { 68 blobSize := 1024 69 bufferSize := 8 * 1024 70 maxBuffers := 3 71 performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers) 72} 73 74func (s *aztestsSuite) TestUploadStreamToBlockBlobSingleIOEdgeCase(c *chk.C) { 75 blobSize := 8 * 1024 76 bufferSize := 8 * 1024 77 maxBuffers := 3 78 performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers) 79} 80 81func (s *aztestsSuite) TestUploadStreamToBlockBlobEmpty(c *chk.C) { 82 blobSize := 0 83 bufferSize := 8 * 1024 84 maxBuffers := 3 85 performUploadStreamToBlockBlobTest(c, blobSize, bufferSize, maxBuffers) 86} 87 88func performUploadAndDownloadFileTest(c *chk.C, fileSize, blockSize, parallelism, downloadOffset, downloadCount int) { 89 // Set up file to upload 90 fileName := "BigFile.bin" 91 fileData := generateFile(fileName, fileSize) 92 93 // Open the file to upload 94 file, err := os.Open(fileName) 95 c.Assert(err, chk.Equals, nil) 96 defer file.Close() 97 defer os.Remove(fileName) 98 99 // Set up test container 100 bsu := getBSU() 101 containerURL, _ := createNewContainer(c, bsu) 102 defer deleteContainer(c, containerURL) 103 104 // Set up test blob 105 blockBlobURL, _ := getBlockBlobURL(c, containerURL) 106 107 // Upload the file to a block blob 108 response, err := azblob.UploadFileToBlockBlob(context.Background(), file, blockBlobURL, 109 azblob.UploadToBlockBlobOptions{ 110 BlockSize: int64(blockSize), 111 Parallelism: uint16(parallelism), 112 // If Progress is non-nil, this function is called periodically as bytes are uploaded. 113 Progress: func(bytesTransferred int64) { 114 c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(fileSize), chk.Equals, true) 115 }, 116 }) 117 c.Assert(err, chk.Equals, nil) 118 c.Assert(response.Response().StatusCode, chk.Equals, 201) 119 120 // Set up file to download the blob to 121 destFileName := "BigFile-downloaded.bin" 122 destFile, err := os.Create(destFileName) 123 c.Assert(err, chk.Equals, nil) 124 defer destFile.Close() 125 defer os.Remove(destFileName) 126 127 // Perform download 128 err = azblob.DownloadBlobToFile(context.Background(), blockBlobURL.BlobURL, int64(downloadOffset), int64(downloadCount), 129 destFile, 130 azblob.DownloadFromBlobOptions{ 131 BlockSize: int64(blockSize), 132 Parallelism: uint16(parallelism), 133 // If Progress is non-nil, this function is called periodically as bytes are uploaded. 134 Progress: func(bytesTransferred int64) { 135 c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(fileSize), chk.Equals, true) 136 }}) 137 138 // Assert download was successful 139 c.Assert(err, chk.Equals, nil) 140 141 // Assert downloaded data is consistent 142 var destBuffer []byte 143 if downloadCount == azblob.CountToEnd { 144 destBuffer = make([]byte, fileSize-downloadOffset) 145 } else { 146 destBuffer = make([]byte, downloadCount) 147 } 148 149 n, err := destFile.Read(destBuffer) 150 c.Assert(err, chk.Equals, nil) 151 152 if downloadOffset == 0 && downloadCount == 0 { 153 c.Assert(destBuffer, chk.DeepEquals, fileData) 154 } else { 155 if downloadCount == 0 { 156 c.Assert(n, chk.Equals, fileSize-downloadOffset) 157 c.Assert(destBuffer, chk.DeepEquals, fileData[downloadOffset:]) 158 } else { 159 c.Assert(n, chk.Equals, downloadCount) 160 c.Assert(destBuffer, chk.DeepEquals, fileData[downloadOffset:downloadOffset+downloadCount]) 161 } 162 } 163} 164 165func (s *aztestsSuite) TestUploadAndDownloadFileInChunks(c *chk.C) { 166 fileSize := 8 * 1024 167 blockSize := 1024 168 parallelism := 3 169 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0) 170} 171 172func (s *aztestsSuite) TestUploadAndDownloadFileSingleIO(c *chk.C) { 173 fileSize := 1024 174 blockSize := 2048 175 parallelism := 3 176 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0) 177} 178 179func (s *aztestsSuite) TestUploadAndDownloadFileSingleRoutine(c *chk.C) { 180 fileSize := 8 * 1024 181 blockSize := 1024 182 parallelism := 1 183 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0) 184} 185 186func (s *aztestsSuite) TestUploadAndDownloadFileEmpty(c *chk.C) { 187 fileSize := 0 188 blockSize := 1024 189 parallelism := 3 190 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, 0, 0) 191} 192 193func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroOffset(c *chk.C) { 194 fileSize := 8 * 1024 195 blockSize := 1024 196 parallelism := 3 197 downloadOffset := 1000 198 downloadCount := 0 199 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount) 200} 201 202func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroCount(c *chk.C) { 203 fileSize := 8 * 1024 204 blockSize := 1024 205 parallelism := 3 206 downloadOffset := 0 207 downloadCount := 6000 208 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount) 209} 210 211func (s *aztestsSuite) TestUploadAndDownloadFileNonZeroOffsetAndCount(c *chk.C) { 212 fileSize := 8 * 1024 213 blockSize := 1024 214 parallelism := 3 215 downloadOffset := 1000 216 downloadCount := 6000 217 performUploadAndDownloadFileTest(c, fileSize, blockSize, parallelism, downloadOffset, downloadCount) 218} 219 220func performUploadAndDownloadBufferTest(c *chk.C, blobSize, blockSize, parallelism, downloadOffset, downloadCount int) { 221 // Set up buffer to upload 222 _, bytesToUpload := getRandomDataAndReader(blobSize) 223 224 // Set up test container 225 bsu := getBSU() 226 containerURL, _ := createNewContainer(c, bsu) 227 defer deleteContainer(c, containerURL) 228 229 // Set up test blob 230 blockBlobURL, _ := getBlockBlobURL(c, containerURL) 231 232 // Pass the Context, stream, stream size, block blob URL, and options to StreamToBlockBlob 233 response, err := azblob.UploadBufferToBlockBlob(context.Background(), bytesToUpload, blockBlobURL, 234 azblob.UploadToBlockBlobOptions{ 235 BlockSize: int64(blockSize), 236 Parallelism: uint16(parallelism), 237 // If Progress is non-nil, this function is called periodically as bytes are uploaded. 238 Progress: func(bytesTransferred int64) { 239 c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(blobSize), chk.Equals, true) 240 }, 241 }) 242 c.Assert(err, chk.Equals, nil) 243 c.Assert(response.Response().StatusCode, chk.Equals, 201) 244 245 // Set up buffer to download the blob to 246 var destBuffer []byte 247 if downloadCount == azblob.CountToEnd { 248 destBuffer = make([]byte, blobSize-downloadOffset) 249 } else { 250 destBuffer = make([]byte, downloadCount) 251 } 252 253 // Download the blob to a buffer 254 err = azblob.DownloadBlobToBuffer(context.Background(), blockBlobURL.BlobURL, int64(downloadOffset), int64(downloadCount), 255 destBuffer, azblob.DownloadFromBlobOptions{ 256 AccessConditions: azblob.BlobAccessConditions{}, 257 BlockSize: int64(blockSize), 258 Parallelism: uint16(parallelism), 259 // If Progress is non-nil, this function is called periodically as bytes are uploaded. 260 Progress: func(bytesTransferred int64) { 261 c.Assert(bytesTransferred > 0 && bytesTransferred <= int64(blobSize), chk.Equals, true) 262 }, 263 }) 264 265 c.Assert(err, chk.Equals, nil) 266 267 if downloadOffset == 0 && downloadCount == 0 { 268 c.Assert(destBuffer, chk.DeepEquals, bytesToUpload) 269 } else { 270 if downloadCount == 0 { 271 c.Assert(destBuffer, chk.DeepEquals, bytesToUpload[downloadOffset:]) 272 } else { 273 c.Assert(destBuffer, chk.DeepEquals, bytesToUpload[downloadOffset:downloadOffset+downloadCount]) 274 } 275 } 276} 277 278func (s *aztestsSuite) TestUploadAndDownloadBufferInChunks(c *chk.C) { 279 blobSize := 8 * 1024 280 blockSize := 1024 281 parallelism := 3 282 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0) 283} 284 285func (s *aztestsSuite) TestUploadAndDownloadBufferSingleIO(c *chk.C) { 286 blobSize := 1024 287 blockSize := 8 * 1024 288 parallelism := 3 289 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0) 290} 291 292func (s *aztestsSuite) TestUploadAndDownloadBufferSingleRoutine(c *chk.C) { 293 blobSize := 8 * 1024 294 blockSize := 1024 295 parallelism := 1 296 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0) 297} 298 299func (s *aztestsSuite) TestUploadAndDownloadBufferEmpty(c *chk.C) { 300 blobSize := 0 301 blockSize := 1024 302 parallelism := 3 303 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, 0, 0) 304} 305 306func (s *aztestsSuite) TestDownloadBufferWithNonZeroOffset(c *chk.C) { 307 blobSize := 8 * 1024 308 blockSize := 1024 309 parallelism := 3 310 downloadOffset := 1000 311 downloadCount := 0 312 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount) 313} 314 315func (s *aztestsSuite) TestDownloadBufferWithNonZeroCount(c *chk.C) { 316 blobSize := 8 * 1024 317 blockSize := 1024 318 parallelism := 3 319 downloadOffset := 0 320 downloadCount := 6000 321 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount) 322} 323 324func (s *aztestsSuite) TestDownloadBufferWithNonZeroOffsetAndCount(c *chk.C) { 325 blobSize := 8 * 1024 326 blockSize := 1024 327 parallelism := 3 328 downloadOffset := 2000 329 downloadCount := 6 * 1024 330 performUploadAndDownloadBufferTest(c, blobSize, blockSize, parallelism, downloadOffset, downloadCount) 331} 332