1package s3store 2 3import ( 4 "bytes" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "testing" 9 10 "github.com/golang/mock/gomock" 11 "github.com/stretchr/testify/assert" 12 13 "github.com/aws/aws-sdk-go/aws" 14 "github.com/aws/aws-sdk-go/aws/awserr" 15 "github.com/aws/aws-sdk-go/service/s3" 16 "github.com/tus/tusd" 17) 18 19//go:generate mockgen -destination=./s3store_mock_test.go -package=s3store github.com/tus/tusd/s3store S3API 20 21// Test interface implementations 22var _ tusd.DataStore = S3Store{} 23var _ tusd.GetReaderDataStore = S3Store{} 24var _ tusd.TerminaterDataStore = S3Store{} 25var _ tusd.FinisherDataStore = S3Store{} 26var _ tusd.ConcaterDataStore = S3Store{} 27 28func TestNewUpload(t *testing.T) { 29 mockCtrl := gomock.NewController(t) 30 defer mockCtrl.Finish() 31 assert := assert.New(t) 32 33 s3obj := NewMockS3API(mockCtrl) 34 store := New("bucket", s3obj) 35 36 assert.Equal("bucket", store.Bucket) 37 assert.Equal(s3obj, store.Service) 38 39 s1 := "hello" 40 s2 := "men?" 41 42 gomock.InOrder( 43 s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{ 44 Bucket: aws.String("bucket"), 45 Key: aws.String("uploadId"), 46 Metadata: map[string]*string{ 47 "foo": &s1, 48 "bar": &s2, 49 }, 50 }).Return(&s3.CreateMultipartUploadOutput{ 51 UploadId: aws.String("multipartId"), 52 }, nil), 53 s3obj.EXPECT().PutObject(&s3.PutObjectInput{ 54 Bucket: aws.String("bucket"), 55 Key: aws.String("uploadId.info"), 56 Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), 57 ContentLength: aws.Int64(int64(171)), 58 }), 59 ) 60 61 info := tusd.FileInfo{ 62 ID: "uploadId", 63 Size: 500, 64 MetaData: map[string]string{ 65 "foo": "hello", 66 "bar": "menü", 67 }, 68 } 69 70 id, err := store.NewUpload(info) 71 assert.Nil(err) 72 assert.Equal("uploadId+multipartId", id) 73} 74 75func TestNewUploadWithObjectPrefix(t *testing.T) { 76 mockCtrl := gomock.NewController(t) 77 defer mockCtrl.Finish() 78 assert := assert.New(t) 79 80 s3obj := NewMockS3API(mockCtrl) 81 store := New("bucket", s3obj) 82 store.ObjectPrefix = "my/uploaded/files" 83 84 assert.Equal("bucket", store.Bucket) 85 assert.Equal(s3obj, store.Service) 86 87 s1 := "hello" 88 s2 := "men?" 89 90 gomock.InOrder( 91 s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{ 92 Bucket: aws.String("bucket"), 93 Key: aws.String("my/uploaded/files/uploadId"), 94 Metadata: map[string]*string{ 95 "foo": &s1, 96 "bar": &s2, 97 }, 98 }).Return(&s3.CreateMultipartUploadOutput{ 99 UploadId: aws.String("multipartId"), 100 }, nil), 101 s3obj.EXPECT().PutObject(&s3.PutObjectInput{ 102 Bucket: aws.String("bucket"), 103 Key: aws.String("my/uploaded/files/uploadId.info"), 104 Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), 105 ContentLength: aws.Int64(int64(171)), 106 }), 107 ) 108 109 info := tusd.FileInfo{ 110 ID: "uploadId", 111 Size: 500, 112 MetaData: map[string]string{ 113 "foo": "hello", 114 "bar": "menü", 115 }, 116 } 117 118 id, err := store.NewUpload(info) 119 assert.Nil(err) 120 assert.Equal("uploadId+multipartId", id) 121} 122 123func TestNewUploadLargerMaxObjectSize(t *testing.T) { 124 mockCtrl := gomock.NewController(t) 125 defer mockCtrl.Finish() 126 assert := assert.New(t) 127 128 s3obj := NewMockS3API(mockCtrl) 129 store := New("bucket", s3obj) 130 131 assert.Equal("bucket", store.Bucket) 132 assert.Equal(s3obj, store.Service) 133 134 info := tusd.FileInfo{ 135 ID: "uploadId", 136 Size: store.MaxObjectSize + 1, 137 } 138 139 id, err := store.NewUpload(info) 140 assert.NotNil(err) 141 assert.EqualError(err, fmt.Sprintf("s3store: upload size of %v bytes exceeds MaxObjectSize of %v bytes", info.Size, store.MaxObjectSize)) 142 assert.Equal("", id) 143} 144 145func TestGetInfoNotFound(t *testing.T) { 146 mockCtrl := gomock.NewController(t) 147 defer mockCtrl.Finish() 148 assert := assert.New(t) 149 150 s3obj := NewMockS3API(mockCtrl) 151 store := New("bucket", s3obj) 152 153 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 154 Bucket: aws.String("bucket"), 155 Key: aws.String("uploadId.info"), 156 }).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil)) 157 158 _, err := store.GetInfo("uploadId+multipartId") 159 assert.Equal(tusd.ErrNotFound, err) 160} 161 162func TestGetInfo(t *testing.T) { 163 mockCtrl := gomock.NewController(t) 164 defer mockCtrl.Finish() 165 assert := assert.New(t) 166 167 s3obj := NewMockS3API(mockCtrl) 168 store := New("bucket", s3obj) 169 170 gomock.InOrder( 171 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 172 Bucket: aws.String("bucket"), 173 Key: aws.String("uploadId.info"), 174 }).Return(&s3.GetObjectOutput{ 175 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 176 }, nil), 177 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 178 Bucket: aws.String("bucket"), 179 Key: aws.String("uploadId"), 180 UploadId: aws.String("multipartId"), 181 PartNumberMarker: aws.Int64(0), 182 }).Return(&s3.ListPartsOutput{ 183 Parts: []*s3.Part{ 184 { 185 Size: aws.Int64(100), 186 }, 187 { 188 Size: aws.Int64(200), 189 }, 190 }, 191 NextPartNumberMarker: aws.Int64(2), 192 IsTruncated: aws.Bool(true), 193 }, nil), 194 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 195 Bucket: aws.String("bucket"), 196 Key: aws.String("uploadId"), 197 UploadId: aws.String("multipartId"), 198 PartNumberMarker: aws.Int64(2), 199 }).Return(&s3.ListPartsOutput{ 200 Parts: []*s3.Part{ 201 { 202 Size: aws.Int64(100), 203 }, 204 }, 205 }, nil), 206 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 207 Bucket: aws.String("bucket"), 208 Key: aws.String("uploadId.part"), 209 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)), 210 ) 211 212 info, err := store.GetInfo("uploadId+multipartId") 213 assert.Nil(err) 214 assert.Equal(int64(500), info.Size) 215 assert.Equal(int64(400), info.Offset) 216 assert.Equal("uploadId+multipartId", info.ID) 217 assert.Equal("hello", info.MetaData["foo"]) 218 assert.Equal("menü", info.MetaData["bar"]) 219} 220 221func TestGetInfoWithIncompletePart(t *testing.T) { 222 mockCtrl := gomock.NewController(t) 223 defer mockCtrl.Finish() 224 assert := assert.New(t) 225 226 s3obj := NewMockS3API(mockCtrl) 227 store := New("bucket", s3obj) 228 229 gomock.InOrder( 230 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 231 Bucket: aws.String("bucket"), 232 Key: aws.String("uploadId.info"), 233 }).Return(&s3.GetObjectOutput{ 234 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 235 }, nil), 236 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 237 Bucket: aws.String("bucket"), 238 Key: aws.String("uploadId"), 239 UploadId: aws.String("multipartId"), 240 PartNumberMarker: aws.Int64(0), 241 }).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil), 242 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 243 Bucket: aws.String("bucket"), 244 Key: aws.String("uploadId.part"), 245 }).Return(&s3.GetObjectOutput{ 246 ContentLength: aws.Int64(10), 247 Body: ioutil.NopCloser(bytes.NewReader([]byte("0123456789"))), 248 }, nil), 249 ) 250 251 info, err := store.GetInfo("uploadId+multipartId") 252 assert.Nil(err) 253 assert.Equal(int64(10), info.Offset) 254 assert.Equal("uploadId+multipartId", info.ID) 255} 256 257func TestGetInfoFinished(t *testing.T) { 258 mockCtrl := gomock.NewController(t) 259 defer mockCtrl.Finish() 260 assert := assert.New(t) 261 262 s3obj := NewMockS3API(mockCtrl) 263 store := New("bucket", s3obj) 264 265 gomock.InOrder( 266 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 267 Bucket: aws.String("bucket"), 268 Key: aws.String("uploadId.info"), 269 }).Return(&s3.GetObjectOutput{ 270 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 271 }, nil), 272 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 273 Bucket: aws.String("bucket"), 274 Key: aws.String("uploadId"), 275 UploadId: aws.String("multipartId"), 276 PartNumberMarker: aws.Int64(0), 277 }).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)), 278 ) 279 280 info, err := store.GetInfo("uploadId+multipartId") 281 assert.Nil(err) 282 assert.Equal(int64(500), info.Size) 283 assert.Equal(int64(500), info.Offset) 284} 285 286func TestGetReader(t *testing.T) { 287 mockCtrl := gomock.NewController(t) 288 defer mockCtrl.Finish() 289 assert := assert.New(t) 290 291 s3obj := NewMockS3API(mockCtrl) 292 store := New("bucket", s3obj) 293 294 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 295 Bucket: aws.String("bucket"), 296 Key: aws.String("uploadId"), 297 }).Return(&s3.GetObjectOutput{ 298 Body: ioutil.NopCloser(bytes.NewReader([]byte(`hello world`))), 299 }, nil) 300 301 content, err := store.GetReader("uploadId+multipartId") 302 assert.Nil(err) 303 assert.Equal(ioutil.NopCloser(bytes.NewReader([]byte(`hello world`))), content) 304} 305 306func TestGetReaderNotFound(t *testing.T) { 307 mockCtrl := gomock.NewController(t) 308 defer mockCtrl.Finish() 309 assert := assert.New(t) 310 311 s3obj := NewMockS3API(mockCtrl) 312 store := New("bucket", s3obj) 313 314 gomock.InOrder( 315 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 316 Bucket: aws.String("bucket"), 317 Key: aws.String("uploadId"), 318 }).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 319 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 320 Bucket: aws.String("bucket"), 321 Key: aws.String("uploadId"), 322 UploadId: aws.String("multipartId"), 323 MaxParts: aws.Int64(0), 324 }).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)), 325 ) 326 327 content, err := store.GetReader("uploadId+multipartId") 328 assert.Nil(content) 329 assert.Equal(tusd.ErrNotFound, err) 330} 331 332func TestGetReaderNotFinished(t *testing.T) { 333 mockCtrl := gomock.NewController(t) 334 defer mockCtrl.Finish() 335 assert := assert.New(t) 336 337 s3obj := NewMockS3API(mockCtrl) 338 store := New("bucket", s3obj) 339 340 gomock.InOrder( 341 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 342 Bucket: aws.String("bucket"), 343 Key: aws.String("uploadId"), 344 }).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 345 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 346 Bucket: aws.String("bucket"), 347 Key: aws.String("uploadId"), 348 UploadId: aws.String("multipartId"), 349 MaxParts: aws.Int64(0), 350 }).Return(&s3.ListPartsOutput{ 351 Parts: []*s3.Part{}, 352 }, nil), 353 ) 354 355 content, err := store.GetReader("uploadId+multipartId") 356 assert.Nil(content) 357 assert.Equal("cannot stream non-finished upload", err.Error()) 358} 359 360func TestDeclareLength(t *testing.T) { 361 mockCtrl := gomock.NewController(t) 362 defer mockCtrl.Finish() 363 assert := assert.New(t) 364 365 s3obj := NewMockS3API(mockCtrl) 366 store := New("bucket", s3obj) 367 368 gomock.InOrder( 369 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 370 Bucket: aws.String("bucket"), 371 Key: aws.String("uploadId.info"), 372 }).Return(&s3.GetObjectOutput{ 373 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":0,"SizeIsDeferred":true,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 374 }, nil), 375 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 376 Bucket: aws.String("bucket"), 377 Key: aws.String("uploadId"), 378 UploadId: aws.String("multipartId"), 379 PartNumberMarker: aws.Int64(0), 380 }).Return(&s3.ListPartsOutput{ 381 Parts: []*s3.Part{}, 382 }, nil), 383 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 384 Bucket: aws.String("bucket"), 385 Key: aws.String("uploadId.part"), 386 }).Return(nil, awserr.New("NotFound", "Not Found", nil)), 387 s3obj.EXPECT().PutObject(&s3.PutObjectInput{ 388 Bucket: aws.String("bucket"), 389 Key: aws.String("uploadId.info"), 390 Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), 391 ContentLength: aws.Int64(int64(144)), 392 }), 393 ) 394 395 err := store.DeclareLength("uploadId+multipartId", 500) 396 assert.Nil(err) 397} 398 399func TestFinishUpload(t *testing.T) { 400 mockCtrl := gomock.NewController(t) 401 defer mockCtrl.Finish() 402 assert := assert.New(t) 403 404 s3obj := NewMockS3API(mockCtrl) 405 store := New("bucket", s3obj) 406 407 gomock.InOrder( 408 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 409 Bucket: aws.String("bucket"), 410 Key: aws.String("uploadId"), 411 UploadId: aws.String("multipartId"), 412 PartNumberMarker: aws.Int64(0), 413 }).Return(&s3.ListPartsOutput{ 414 Parts: []*s3.Part{ 415 { 416 Size: aws.Int64(100), 417 ETag: aws.String("foo"), 418 PartNumber: aws.Int64(1), 419 }, 420 { 421 Size: aws.Int64(200), 422 ETag: aws.String("bar"), 423 PartNumber: aws.Int64(2), 424 }, 425 }, 426 NextPartNumberMarker: aws.Int64(2), 427 IsTruncated: aws.Bool(true), 428 }, nil), 429 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 430 Bucket: aws.String("bucket"), 431 Key: aws.String("uploadId"), 432 UploadId: aws.String("multipartId"), 433 PartNumberMarker: aws.Int64(2), 434 }).Return(&s3.ListPartsOutput{ 435 Parts: []*s3.Part{ 436 { 437 Size: aws.Int64(100), 438 ETag: aws.String("foobar"), 439 PartNumber: aws.Int64(3), 440 }, 441 }, 442 }, nil), 443 s3obj.EXPECT().CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ 444 Bucket: aws.String("bucket"), 445 Key: aws.String("uploadId"), 446 UploadId: aws.String("multipartId"), 447 MultipartUpload: &s3.CompletedMultipartUpload{ 448 Parts: []*s3.CompletedPart{ 449 { 450 ETag: aws.String("foo"), 451 PartNumber: aws.Int64(1), 452 }, 453 { 454 ETag: aws.String("bar"), 455 PartNumber: aws.Int64(2), 456 }, 457 { 458 ETag: aws.String("foobar"), 459 PartNumber: aws.Int64(3), 460 }, 461 }, 462 }, 463 }).Return(nil, nil), 464 ) 465 466 err := store.FinishUpload("uploadId+multipartId") 467 assert.Nil(err) 468} 469 470func TestWriteChunk(t *testing.T) { 471 mockCtrl := gomock.NewController(t) 472 defer mockCtrl.Finish() 473 assert := assert.New(t) 474 475 s3obj := NewMockS3API(mockCtrl) 476 store := New("bucket", s3obj) 477 store.MaxPartSize = 8 478 store.MinPartSize = 4 479 store.MaxMultipartParts = 10000 480 store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024 481 482 gomock.InOrder( 483 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 484 Bucket: aws.String("bucket"), 485 Key: aws.String("uploadId.info"), 486 }).Return(&s3.GetObjectOutput{ 487 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 488 }, nil), 489 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 490 Bucket: aws.String("bucket"), 491 Key: aws.String("uploadId"), 492 UploadId: aws.String("multipartId"), 493 PartNumberMarker: aws.Int64(0), 494 }).Return(&s3.ListPartsOutput{ 495 Parts: []*s3.Part{ 496 { 497 Size: aws.Int64(100), 498 }, 499 { 500 Size: aws.Int64(200), 501 }, 502 }, 503 }, nil), 504 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 505 Bucket: aws.String("bucket"), 506 Key: aws.String("uploadId.part"), 507 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)), 508 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 509 Bucket: aws.String("bucket"), 510 Key: aws.String("uploadId"), 511 UploadId: aws.String("multipartId"), 512 PartNumberMarker: aws.Int64(0), 513 }).Return(&s3.ListPartsOutput{ 514 Parts: []*s3.Part{ 515 { 516 Size: aws.Int64(100), 517 }, 518 { 519 Size: aws.Int64(200), 520 }, 521 }, 522 }, nil), 523 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 524 Bucket: aws.String("bucket"), 525 Key: aws.String("uploadId.part"), 526 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 527 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 528 Bucket: aws.String("bucket"), 529 Key: aws.String("uploadId"), 530 UploadId: aws.String("multipartId"), 531 PartNumber: aws.Int64(3), 532 Body: bytes.NewReader([]byte("1234")), 533 })).Return(nil, nil), 534 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 535 Bucket: aws.String("bucket"), 536 Key: aws.String("uploadId"), 537 UploadId: aws.String("multipartId"), 538 PartNumber: aws.Int64(4), 539 Body: bytes.NewReader([]byte("5678")), 540 })).Return(nil, nil), 541 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 542 Bucket: aws.String("bucket"), 543 Key: aws.String("uploadId"), 544 UploadId: aws.String("multipartId"), 545 PartNumber: aws.Int64(5), 546 Body: bytes.NewReader([]byte("90AB")), 547 })).Return(nil, nil), 548 s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{ 549 Bucket: aws.String("bucket"), 550 Key: aws.String("uploadId.part"), 551 Body: bytes.NewReader([]byte("CD")), 552 })).Return(nil, nil), 553 ) 554 555 bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, bytes.NewReader([]byte("1234567890ABCD"))) 556 assert.Nil(err) 557 assert.Equal(int64(14), bytesRead) 558} 559 560// TestWriteChunkWithUnexpectedEOF ensures that WriteChunk does not error out 561// if the io.Reader returns an io.ErrUnexpectedEOF. This happens when a HTTP 562// PATCH request gets interrupted. 563func TestWriteChunkWithUnexpectedEOF(t *testing.T) { 564 mockCtrl := gomock.NewController(t) 565 defer mockCtrl.Finish() 566 assert := assert.New(t) 567 568 s3obj := NewMockS3API(mockCtrl) 569 store := New("bucket", s3obj) 570 store.MaxPartSize = 500 571 store.MinPartSize = 100 572 store.MaxMultipartParts = 10000 573 store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024 574 575 gomock.InOrder( 576 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 577 Bucket: aws.String("bucket"), 578 Key: aws.String("uploadId.info"), 579 }).Return(&s3.GetObjectOutput{ 580 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 581 }, nil), 582 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 583 Bucket: aws.String("bucket"), 584 Key: aws.String("uploadId"), 585 UploadId: aws.String("multipartId"), 586 PartNumberMarker: aws.Int64(0), 587 }).Return(&s3.ListPartsOutput{ 588 Parts: []*s3.Part{ 589 { 590 Size: aws.Int64(100), 591 }, 592 { 593 Size: aws.Int64(200), 594 }, 595 }, 596 }, nil), 597 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 598 Bucket: aws.String("bucket"), 599 Key: aws.String("uploadId.part"), 600 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)), 601 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 602 Bucket: aws.String("bucket"), 603 Key: aws.String("uploadId"), 604 UploadId: aws.String("multipartId"), 605 PartNumberMarker: aws.Int64(0), 606 }).Return(&s3.ListPartsOutput{ 607 Parts: []*s3.Part{ 608 { 609 Size: aws.Int64(100), 610 }, 611 { 612 Size: aws.Int64(200), 613 }, 614 }, 615 }, nil), 616 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 617 Bucket: aws.String("bucket"), 618 Key: aws.String("uploadId.part"), 619 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 620 s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{ 621 Bucket: aws.String("bucket"), 622 Key: aws.String("uploadId.part"), 623 Body: bytes.NewReader([]byte("1234567890ABCD")), 624 })).Return(nil, nil), 625 ) 626 627 reader, writer := io.Pipe() 628 629 go func() { 630 writer.Write([]byte("1234567890ABCD")) 631 writer.CloseWithError(io.ErrUnexpectedEOF) 632 }() 633 634 bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, reader) 635 assert.Nil(err) 636 assert.Equal(int64(14), bytesRead) 637} 638 639func TestWriteChunkWriteIncompletePartBecauseTooSmall(t *testing.T) { 640 mockCtrl := gomock.NewController(t) 641 defer mockCtrl.Finish() 642 assert := assert.New(t) 643 644 s3obj := NewMockS3API(mockCtrl) 645 store := New("bucket", s3obj) 646 647 gomock.InOrder( 648 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 649 Bucket: aws.String("bucket"), 650 Key: aws.String("uploadId.info"), 651 }).Return(&s3.GetObjectOutput{ 652 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 653 }, nil), 654 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 655 Bucket: aws.String("bucket"), 656 Key: aws.String("uploadId"), 657 UploadId: aws.String("multipartId"), 658 PartNumberMarker: aws.Int64(0), 659 }).Return(&s3.ListPartsOutput{ 660 Parts: []*s3.Part{ 661 { 662 Size: aws.Int64(100), 663 }, 664 { 665 Size: aws.Int64(200), 666 }, 667 }, 668 }, nil), 669 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 670 Bucket: aws.String("bucket"), 671 Key: aws.String("uploadId.part"), 672 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist", nil)), 673 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 674 Bucket: aws.String("bucket"), 675 Key: aws.String("uploadId"), 676 UploadId: aws.String("multipartId"), 677 PartNumberMarker: aws.Int64(0), 678 }).Return(&s3.ListPartsOutput{ 679 Parts: []*s3.Part{ 680 { 681 Size: aws.Int64(100), 682 }, 683 { 684 Size: aws.Int64(200), 685 }, 686 }, 687 }, nil), 688 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 689 Bucket: aws.String("bucket"), 690 Key: aws.String("uploadId.part"), 691 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 692 s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{ 693 Bucket: aws.String("bucket"), 694 Key: aws.String("uploadId.part"), 695 Body: bytes.NewReader([]byte("1234567890")), 696 })).Return(nil, nil), 697 ) 698 699 bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, bytes.NewReader([]byte("1234567890"))) 700 assert.Nil(err) 701 assert.Equal(int64(10), bytesRead) 702} 703 704func TestWriteChunkPrependsIncompletePart(t *testing.T) { 705 mockCtrl := gomock.NewController(t) 706 defer mockCtrl.Finish() 707 assert := assert.New(t) 708 709 s3obj := NewMockS3API(mockCtrl) 710 store := New("bucket", s3obj) 711 store.MaxPartSize = 8 712 store.MinPartSize = 4 713 store.MaxMultipartParts = 10000 714 store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024 715 716 gomock.InOrder( 717 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 718 Bucket: aws.String("bucket"), 719 Key: aws.String("uploadId.info"), 720 }).Return(&s3.GetObjectOutput{ 721 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":5,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 722 }, nil), 723 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 724 Bucket: aws.String("bucket"), 725 Key: aws.String("uploadId"), 726 UploadId: aws.String("multipartId"), 727 PartNumberMarker: aws.Int64(0), 728 }).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil), 729 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 730 Bucket: aws.String("bucket"), 731 Key: aws.String("uploadId.part"), 732 }).Return(&s3.GetObjectOutput{ 733 ContentLength: aws.Int64(3), 734 Body: ioutil.NopCloser(bytes.NewReader([]byte("123"))), 735 }, nil), 736 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 737 Bucket: aws.String("bucket"), 738 Key: aws.String("uploadId"), 739 UploadId: aws.String("multipartId"), 740 PartNumberMarker: aws.Int64(0), 741 }).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil), 742 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 743 Bucket: aws.String("bucket"), 744 Key: aws.String("uploadId.part"), 745 }).Return(&s3.GetObjectOutput{ 746 ContentLength: aws.Int64(3), 747 Body: ioutil.NopCloser(bytes.NewReader([]byte("123"))), 748 }, nil), 749 s3obj.EXPECT().DeleteObject(&s3.DeleteObjectInput{ 750 Bucket: aws.String(store.Bucket), 751 Key: aws.String("uploadId.part"), 752 }).Return(&s3.DeleteObjectOutput{}, nil), 753 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 754 Bucket: aws.String("bucket"), 755 Key: aws.String("uploadId"), 756 UploadId: aws.String("multipartId"), 757 PartNumber: aws.Int64(1), 758 Body: bytes.NewReader([]byte("1234")), 759 })).Return(nil, nil), 760 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 761 Bucket: aws.String("bucket"), 762 Key: aws.String("uploadId"), 763 UploadId: aws.String("multipartId"), 764 PartNumber: aws.Int64(2), 765 Body: bytes.NewReader([]byte("5")), 766 })).Return(nil, nil), 767 ) 768 769 bytesRead, err := store.WriteChunk("uploadId+multipartId", 3, bytes.NewReader([]byte("45"))) 770 assert.Nil(err) 771 assert.Equal(int64(2), bytesRead) 772} 773 774func TestWriteChunkPrependsIncompletePartAndWritesANewIncompletePart(t *testing.T) { 775 mockCtrl := gomock.NewController(t) 776 defer mockCtrl.Finish() 777 assert := assert.New(t) 778 779 s3obj := NewMockS3API(mockCtrl) 780 store := New("bucket", s3obj) 781 store.MaxPartSize = 8 782 store.MinPartSize = 4 783 store.MaxMultipartParts = 10000 784 store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024 785 786 gomock.InOrder( 787 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 788 Bucket: aws.String("bucket"), 789 Key: aws.String("uploadId.info"), 790 }).Return(&s3.GetObjectOutput{ 791 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":10,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 792 }, nil), 793 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 794 Bucket: aws.String("bucket"), 795 Key: aws.String("uploadId"), 796 UploadId: aws.String("multipartId"), 797 PartNumberMarker: aws.Int64(0), 798 }).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil), 799 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 800 Bucket: aws.String("bucket"), 801 Key: aws.String("uploadId.part"), 802 }).Return(&s3.GetObjectOutput{ 803 ContentLength: aws.Int64(3), 804 Body: ioutil.NopCloser(bytes.NewReader([]byte("123"))), 805 }, nil), 806 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 807 Bucket: aws.String("bucket"), 808 Key: aws.String("uploadId"), 809 UploadId: aws.String("multipartId"), 810 PartNumberMarker: aws.Int64(0), 811 }).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil), 812 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 813 Bucket: aws.String("bucket"), 814 Key: aws.String("uploadId.part"), 815 }).Return(&s3.GetObjectOutput{ 816 Body: ioutil.NopCloser(bytes.NewReader([]byte("123"))), 817 ContentLength: aws.Int64(3), 818 }, nil), 819 s3obj.EXPECT().DeleteObject(&s3.DeleteObjectInput{ 820 Bucket: aws.String(store.Bucket), 821 Key: aws.String("uploadId.part"), 822 }).Return(&s3.DeleteObjectOutput{}, nil), 823 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 824 Bucket: aws.String("bucket"), 825 Key: aws.String("uploadId"), 826 UploadId: aws.String("multipartId"), 827 PartNumber: aws.Int64(1), 828 Body: bytes.NewReader([]byte("1234")), 829 })).Return(nil, nil), 830 s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{ 831 Bucket: aws.String("bucket"), 832 Key: aws.String("uploadId.part"), 833 Body: bytes.NewReader([]byte("5")), 834 })).Return(nil, nil), 835 ) 836 837 bytesRead, err := store.WriteChunk("uploadId+multipartId", 3, bytes.NewReader([]byte("45"))) 838 assert.Nil(err) 839 assert.Equal(int64(2), bytesRead) 840} 841 842func TestWriteChunkAllowTooSmallLast(t *testing.T) { 843 mockCtrl := gomock.NewController(t) 844 defer mockCtrl.Finish() 845 assert := assert.New(t) 846 847 s3obj := NewMockS3API(mockCtrl) 848 store := New("bucket", s3obj) 849 store.MinPartSize = 20 850 851 gomock.InOrder( 852 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 853 Bucket: aws.String("bucket"), 854 Key: aws.String("uploadId.info"), 855 }).Return(&s3.GetObjectOutput{ 856 Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))), 857 }, nil), 858 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 859 Bucket: aws.String("bucket"), 860 Key: aws.String("uploadId"), 861 UploadId: aws.String("multipartId"), 862 PartNumberMarker: aws.Int64(0), 863 }).Return(&s3.ListPartsOutput{ 864 Parts: []*s3.Part{ 865 { 866 Size: aws.Int64(400), 867 }, 868 { 869 Size: aws.Int64(90), 870 }, 871 }, 872 }, nil), 873 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 874 Bucket: aws.String("bucket"), 875 Key: aws.String("uploadId.part"), 876 }).Return(&s3.GetObjectOutput{}, awserr.New("AccessDenied", "Access Denied.", nil)), 877 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 878 Bucket: aws.String("bucket"), 879 Key: aws.String("uploadId"), 880 UploadId: aws.String("multipartId"), 881 PartNumberMarker: aws.Int64(0), 882 }).Return(&s3.ListPartsOutput{ 883 Parts: []*s3.Part{ 884 { 885 Size: aws.Int64(400), 886 }, 887 { 888 Size: aws.Int64(90), 889 }, 890 }, 891 }, nil), 892 s3obj.EXPECT().GetObject(&s3.GetObjectInput{ 893 Bucket: aws.String("bucket"), 894 Key: aws.String("uploadId.part"), 895 }).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)), 896 s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{ 897 Bucket: aws.String("bucket"), 898 Key: aws.String("uploadId"), 899 UploadId: aws.String("multipartId"), 900 PartNumber: aws.Int64(3), 901 Body: bytes.NewReader([]byte("1234567890")), 902 })).Return(nil, nil), 903 ) 904 905 // 10 bytes are missing for the upload to be finished (offset at 490 for 500 906 // bytes file) but the minimum chunk size is higher (20). The chunk is 907 // still uploaded since the last part may be smaller than the minimum. 908 bytesRead, err := store.WriteChunk("uploadId+multipartId", 490, bytes.NewReader([]byte("1234567890"))) 909 assert.Nil(err) 910 assert.Equal(int64(10), bytesRead) 911} 912 913func TestTerminate(t *testing.T) { 914 mockCtrl := gomock.NewController(t) 915 defer mockCtrl.Finish() 916 assert := assert.New(t) 917 918 s3obj := NewMockS3API(mockCtrl) 919 store := New("bucket", s3obj) 920 921 // Order is not important in this situation. 922 s3obj.EXPECT().AbortMultipartUpload(&s3.AbortMultipartUploadInput{ 923 Bucket: aws.String("bucket"), 924 Key: aws.String("uploadId"), 925 UploadId: aws.String("multipartId"), 926 }).Return(nil, nil) 927 928 s3obj.EXPECT().DeleteObjects(&s3.DeleteObjectsInput{ 929 Bucket: aws.String("bucket"), 930 Delete: &s3.Delete{ 931 Objects: []*s3.ObjectIdentifier{ 932 { 933 Key: aws.String("uploadId"), 934 }, 935 { 936 Key: aws.String("uploadId.part"), 937 }, 938 { 939 Key: aws.String("uploadId.info"), 940 }, 941 }, 942 Quiet: aws.Bool(true), 943 }, 944 }).Return(&s3.DeleteObjectsOutput{}, nil) 945 946 err := store.Terminate("uploadId+multipartId") 947 assert.Nil(err) 948} 949 950func TestTerminateWithErrors(t *testing.T) { 951 mockCtrl := gomock.NewController(t) 952 defer mockCtrl.Finish() 953 assert := assert.New(t) 954 955 s3obj := NewMockS3API(mockCtrl) 956 store := New("bucket", s3obj) 957 958 // Order is not important in this situation. 959 // NoSuchUpload errors should be ignored 960 s3obj.EXPECT().AbortMultipartUpload(&s3.AbortMultipartUploadInput{ 961 Bucket: aws.String("bucket"), 962 Key: aws.String("uploadId"), 963 UploadId: aws.String("multipartId"), 964 }).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)) 965 966 s3obj.EXPECT().DeleteObjects(&s3.DeleteObjectsInput{ 967 Bucket: aws.String("bucket"), 968 Delete: &s3.Delete{ 969 Objects: []*s3.ObjectIdentifier{ 970 { 971 Key: aws.String("uploadId"), 972 }, 973 { 974 Key: aws.String("uploadId.part"), 975 }, 976 { 977 Key: aws.String("uploadId.info"), 978 }, 979 }, 980 Quiet: aws.Bool(true), 981 }, 982 }).Return(&s3.DeleteObjectsOutput{ 983 Errors: []*s3.Error{ 984 { 985 Code: aws.String("hello"), 986 Key: aws.String("uploadId"), 987 Message: aws.String("it's me."), 988 }, 989 }, 990 }, nil) 991 992 err := store.Terminate("uploadId+multipartId") 993 assert.Equal("Multiple errors occurred:\n\tAWS S3 Error (hello) for object uploadId: it's me.\n", err.Error()) 994} 995 996func TestConcatUploads(t *testing.T) { 997 mockCtrl := gomock.NewController(t) 998 defer mockCtrl.Finish() 999 assert := assert.New(t) 1000 1001 s3obj := NewMockS3API(mockCtrl) 1002 store := New("bucket", s3obj) 1003 1004 s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{ 1005 Bucket: aws.String("bucket"), 1006 Key: aws.String("uploadId"), 1007 UploadId: aws.String("multipartId"), 1008 CopySource: aws.String("bucket/aaa"), 1009 PartNumber: aws.Int64(1), 1010 }).Return(nil, nil) 1011 1012 s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{ 1013 Bucket: aws.String("bucket"), 1014 Key: aws.String("uploadId"), 1015 UploadId: aws.String("multipartId"), 1016 CopySource: aws.String("bucket/bbb"), 1017 PartNumber: aws.Int64(2), 1018 }).Return(nil, nil) 1019 1020 s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{ 1021 Bucket: aws.String("bucket"), 1022 Key: aws.String("uploadId"), 1023 UploadId: aws.String("multipartId"), 1024 CopySource: aws.String("bucket/ccc"), 1025 PartNumber: aws.Int64(3), 1026 }).Return(nil, nil) 1027 1028 // Output from s3Store.FinishUpload 1029 gomock.InOrder( 1030 s3obj.EXPECT().ListParts(&s3.ListPartsInput{ 1031 Bucket: aws.String("bucket"), 1032 Key: aws.String("uploadId"), 1033 UploadId: aws.String("multipartId"), 1034 PartNumberMarker: aws.Int64(0), 1035 }).Return(&s3.ListPartsOutput{ 1036 Parts: []*s3.Part{ 1037 { 1038 ETag: aws.String("foo"), 1039 PartNumber: aws.Int64(1), 1040 }, 1041 { 1042 ETag: aws.String("bar"), 1043 PartNumber: aws.Int64(2), 1044 }, 1045 { 1046 ETag: aws.String("baz"), 1047 PartNumber: aws.Int64(3), 1048 }, 1049 }, 1050 }, nil), 1051 s3obj.EXPECT().CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ 1052 Bucket: aws.String("bucket"), 1053 Key: aws.String("uploadId"), 1054 UploadId: aws.String("multipartId"), 1055 MultipartUpload: &s3.CompletedMultipartUpload{ 1056 Parts: []*s3.CompletedPart{ 1057 { 1058 ETag: aws.String("foo"), 1059 PartNumber: aws.Int64(1), 1060 }, 1061 { 1062 ETag: aws.String("bar"), 1063 PartNumber: aws.Int64(2), 1064 }, 1065 { 1066 ETag: aws.String("baz"), 1067 PartNumber: aws.Int64(3), 1068 }, 1069 }, 1070 }, 1071 }).Return(nil, nil), 1072 ) 1073 1074 err := store.ConcatUploads("uploadId+multipartId", []string{ 1075 "aaa+AAA", 1076 "bbb+BBB", 1077 "ccc+CCC", 1078 }) 1079 assert.Nil(err) 1080} 1081