1package s3store
2
3import (
4	"bytes"
5	"fmt"
6	"io"
7	"io/ioutil"
8	"testing"
9
10	"github.com/golang/mock/gomock"
11	"github.com/stretchr/testify/assert"
12
13	"github.com/aws/aws-sdk-go/aws"
14	"github.com/aws/aws-sdk-go/aws/awserr"
15	"github.com/aws/aws-sdk-go/service/s3"
16	"github.com/tus/tusd"
17)
18
19//go:generate mockgen -destination=./s3store_mock_test.go -package=s3store github.com/tus/tusd/s3store S3API
20
21// Test interface implementations
22var _ tusd.DataStore = S3Store{}
23var _ tusd.GetReaderDataStore = S3Store{}
24var _ tusd.TerminaterDataStore = S3Store{}
25var _ tusd.FinisherDataStore = S3Store{}
26var _ tusd.ConcaterDataStore = S3Store{}
27
28func TestNewUpload(t *testing.T) {
29	mockCtrl := gomock.NewController(t)
30	defer mockCtrl.Finish()
31	assert := assert.New(t)
32
33	s3obj := NewMockS3API(mockCtrl)
34	store := New("bucket", s3obj)
35
36	assert.Equal("bucket", store.Bucket)
37	assert.Equal(s3obj, store.Service)
38
39	s1 := "hello"
40	s2 := "men?"
41
42	gomock.InOrder(
43		s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{
44			Bucket: aws.String("bucket"),
45			Key:    aws.String("uploadId"),
46			Metadata: map[string]*string{
47				"foo": &s1,
48				"bar": &s2,
49			},
50		}).Return(&s3.CreateMultipartUploadOutput{
51			UploadId: aws.String("multipartId"),
52		}, nil),
53		s3obj.EXPECT().PutObject(&s3.PutObjectInput{
54			Bucket:        aws.String("bucket"),
55			Key:           aws.String("uploadId.info"),
56			Body:          bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
57			ContentLength: aws.Int64(int64(171)),
58		}),
59	)
60
61	info := tusd.FileInfo{
62		ID:   "uploadId",
63		Size: 500,
64		MetaData: map[string]string{
65			"foo": "hello",
66			"bar": "menü",
67		},
68	}
69
70	id, err := store.NewUpload(info)
71	assert.Nil(err)
72	assert.Equal("uploadId+multipartId", id)
73}
74
75func TestNewUploadWithObjectPrefix(t *testing.T) {
76	mockCtrl := gomock.NewController(t)
77	defer mockCtrl.Finish()
78	assert := assert.New(t)
79
80	s3obj := NewMockS3API(mockCtrl)
81	store := New("bucket", s3obj)
82	store.ObjectPrefix = "my/uploaded/files"
83
84	assert.Equal("bucket", store.Bucket)
85	assert.Equal(s3obj, store.Service)
86
87	s1 := "hello"
88	s2 := "men?"
89
90	gomock.InOrder(
91		s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{
92			Bucket: aws.String("bucket"),
93			Key:    aws.String("my/uploaded/files/uploadId"),
94			Metadata: map[string]*string{
95				"foo": &s1,
96				"bar": &s2,
97			},
98		}).Return(&s3.CreateMultipartUploadOutput{
99			UploadId: aws.String("multipartId"),
100		}, nil),
101		s3obj.EXPECT().PutObject(&s3.PutObjectInput{
102			Bucket:        aws.String("bucket"),
103			Key:           aws.String("my/uploaded/files/uploadId.info"),
104			Body:          bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
105			ContentLength: aws.Int64(int64(171)),
106		}),
107	)
108
109	info := tusd.FileInfo{
110		ID:   "uploadId",
111		Size: 500,
112		MetaData: map[string]string{
113			"foo": "hello",
114			"bar": "menü",
115		},
116	}
117
118	id, err := store.NewUpload(info)
119	assert.Nil(err)
120	assert.Equal("uploadId+multipartId", id)
121}
122
123func TestNewUploadLargerMaxObjectSize(t *testing.T) {
124	mockCtrl := gomock.NewController(t)
125	defer mockCtrl.Finish()
126	assert := assert.New(t)
127
128	s3obj := NewMockS3API(mockCtrl)
129	store := New("bucket", s3obj)
130
131	assert.Equal("bucket", store.Bucket)
132	assert.Equal(s3obj, store.Service)
133
134	info := tusd.FileInfo{
135		ID:   "uploadId",
136		Size: store.MaxObjectSize + 1,
137	}
138
139	id, err := store.NewUpload(info)
140	assert.NotNil(err)
141	assert.EqualError(err, fmt.Sprintf("s3store: upload size of %v bytes exceeds MaxObjectSize of %v bytes", info.Size, store.MaxObjectSize))
142	assert.Equal("", id)
143}
144
145func TestGetInfoNotFound(t *testing.T) {
146	mockCtrl := gomock.NewController(t)
147	defer mockCtrl.Finish()
148	assert := assert.New(t)
149
150	s3obj := NewMockS3API(mockCtrl)
151	store := New("bucket", s3obj)
152
153	s3obj.EXPECT().GetObject(&s3.GetObjectInput{
154		Bucket: aws.String("bucket"),
155		Key:    aws.String("uploadId.info"),
156	}).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil))
157
158	_, err := store.GetInfo("uploadId+multipartId")
159	assert.Equal(tusd.ErrNotFound, err)
160}
161
162func TestGetInfo(t *testing.T) {
163	mockCtrl := gomock.NewController(t)
164	defer mockCtrl.Finish()
165	assert := assert.New(t)
166
167	s3obj := NewMockS3API(mockCtrl)
168	store := New("bucket", s3obj)
169
170	gomock.InOrder(
171		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
172			Bucket: aws.String("bucket"),
173			Key:    aws.String("uploadId.info"),
174		}).Return(&s3.GetObjectOutput{
175			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
176		}, nil),
177		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
178			Bucket:           aws.String("bucket"),
179			Key:              aws.String("uploadId"),
180			UploadId:         aws.String("multipartId"),
181			PartNumberMarker: aws.Int64(0),
182		}).Return(&s3.ListPartsOutput{
183			Parts: []*s3.Part{
184				{
185					Size: aws.Int64(100),
186				},
187				{
188					Size: aws.Int64(200),
189				},
190			},
191			NextPartNumberMarker: aws.Int64(2),
192			IsTruncated:          aws.Bool(true),
193		}, nil),
194		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
195			Bucket:           aws.String("bucket"),
196			Key:              aws.String("uploadId"),
197			UploadId:         aws.String("multipartId"),
198			PartNumberMarker: aws.Int64(2),
199		}).Return(&s3.ListPartsOutput{
200			Parts: []*s3.Part{
201				{
202					Size: aws.Int64(100),
203				},
204			},
205		}, nil),
206		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
207			Bucket: aws.String("bucket"),
208			Key:    aws.String("uploadId.part"),
209		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)),
210	)
211
212	info, err := store.GetInfo("uploadId+multipartId")
213	assert.Nil(err)
214	assert.Equal(int64(500), info.Size)
215	assert.Equal(int64(400), info.Offset)
216	assert.Equal("uploadId+multipartId", info.ID)
217	assert.Equal("hello", info.MetaData["foo"])
218	assert.Equal("menü", info.MetaData["bar"])
219}
220
221func TestGetInfoWithIncompletePart(t *testing.T) {
222	mockCtrl := gomock.NewController(t)
223	defer mockCtrl.Finish()
224	assert := assert.New(t)
225
226	s3obj := NewMockS3API(mockCtrl)
227	store := New("bucket", s3obj)
228
229	gomock.InOrder(
230		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
231			Bucket: aws.String("bucket"),
232			Key:    aws.String("uploadId.info"),
233		}).Return(&s3.GetObjectOutput{
234			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
235		}, nil),
236		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
237			Bucket:           aws.String("bucket"),
238			Key:              aws.String("uploadId"),
239			UploadId:         aws.String("multipartId"),
240			PartNumberMarker: aws.Int64(0),
241		}).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil),
242		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
243			Bucket: aws.String("bucket"),
244			Key:    aws.String("uploadId.part"),
245		}).Return(&s3.GetObjectOutput{
246			ContentLength: 	aws.Int64(10),
247			Body: 			ioutil.NopCloser(bytes.NewReader([]byte("0123456789"))),
248		}, nil),
249	)
250
251	info, err := store.GetInfo("uploadId+multipartId")
252	assert.Nil(err)
253	assert.Equal(int64(10), info.Offset)
254	assert.Equal("uploadId+multipartId", info.ID)
255}
256
257func TestGetInfoFinished(t *testing.T) {
258	mockCtrl := gomock.NewController(t)
259	defer mockCtrl.Finish()
260	assert := assert.New(t)
261
262	s3obj := NewMockS3API(mockCtrl)
263	store := New("bucket", s3obj)
264
265	gomock.InOrder(
266		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
267			Bucket: aws.String("bucket"),
268			Key:    aws.String("uploadId.info"),
269		}).Return(&s3.GetObjectOutput{
270			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
271		}, nil),
272		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
273			Bucket:           aws.String("bucket"),
274			Key:              aws.String("uploadId"),
275			UploadId:         aws.String("multipartId"),
276			PartNumberMarker: aws.Int64(0),
277		}).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)),
278	)
279
280	info, err := store.GetInfo("uploadId+multipartId")
281	assert.Nil(err)
282	assert.Equal(int64(500), info.Size)
283	assert.Equal(int64(500), info.Offset)
284}
285
286func TestGetReader(t *testing.T) {
287	mockCtrl := gomock.NewController(t)
288	defer mockCtrl.Finish()
289	assert := assert.New(t)
290
291	s3obj := NewMockS3API(mockCtrl)
292	store := New("bucket", s3obj)
293
294	s3obj.EXPECT().GetObject(&s3.GetObjectInput{
295		Bucket: aws.String("bucket"),
296		Key:    aws.String("uploadId"),
297	}).Return(&s3.GetObjectOutput{
298		Body: ioutil.NopCloser(bytes.NewReader([]byte(`hello world`))),
299	}, nil)
300
301	content, err := store.GetReader("uploadId+multipartId")
302	assert.Nil(err)
303	assert.Equal(ioutil.NopCloser(bytes.NewReader([]byte(`hello world`))), content)
304}
305
306func TestGetReaderNotFound(t *testing.T) {
307	mockCtrl := gomock.NewController(t)
308	defer mockCtrl.Finish()
309	assert := assert.New(t)
310
311	s3obj := NewMockS3API(mockCtrl)
312	store := New("bucket", s3obj)
313
314	gomock.InOrder(
315		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
316			Bucket: aws.String("bucket"),
317			Key:    aws.String("uploadId"),
318		}).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
319		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
320			Bucket:   aws.String("bucket"),
321			Key:      aws.String("uploadId"),
322			UploadId: aws.String("multipartId"),
323			MaxParts: aws.Int64(0),
324		}).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil)),
325	)
326
327	content, err := store.GetReader("uploadId+multipartId")
328	assert.Nil(content)
329	assert.Equal(tusd.ErrNotFound, err)
330}
331
332func TestGetReaderNotFinished(t *testing.T) {
333	mockCtrl := gomock.NewController(t)
334	defer mockCtrl.Finish()
335	assert := assert.New(t)
336
337	s3obj := NewMockS3API(mockCtrl)
338	store := New("bucket", s3obj)
339
340	gomock.InOrder(
341		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
342			Bucket: aws.String("bucket"),
343			Key:    aws.String("uploadId"),
344		}).Return(nil, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
345		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
346			Bucket:   aws.String("bucket"),
347			Key:      aws.String("uploadId"),
348			UploadId: aws.String("multipartId"),
349			MaxParts: aws.Int64(0),
350		}).Return(&s3.ListPartsOutput{
351			Parts: []*s3.Part{},
352		}, nil),
353	)
354
355	content, err := store.GetReader("uploadId+multipartId")
356	assert.Nil(content)
357	assert.Equal("cannot stream non-finished upload", err.Error())
358}
359
360func TestDeclareLength(t *testing.T) {
361	mockCtrl := gomock.NewController(t)
362	defer mockCtrl.Finish()
363	assert := assert.New(t)
364
365	s3obj := NewMockS3API(mockCtrl)
366	store := New("bucket", s3obj)
367
368	gomock.InOrder(
369		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
370			Bucket: aws.String("bucket"),
371			Key:    aws.String("uploadId.info"),
372		}).Return(&s3.GetObjectOutput{
373			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":0,"SizeIsDeferred":true,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
374		}, nil),
375		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
376			Bucket:           aws.String("bucket"),
377			Key:              aws.String("uploadId"),
378			UploadId:         aws.String("multipartId"),
379			PartNumberMarker: aws.Int64(0),
380		}).Return(&s3.ListPartsOutput{
381			Parts: []*s3.Part{},
382		}, nil),
383		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
384			Bucket: aws.String("bucket"),
385			Key:    aws.String("uploadId.part"),
386		}).Return(nil, awserr.New("NotFound", "Not Found", nil)),
387		s3obj.EXPECT().PutObject(&s3.PutObjectInput{
388			Bucket:        aws.String("bucket"),
389			Key:           aws.String("uploadId.info"),
390			Body:          bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
391			ContentLength: aws.Int64(int64(144)),
392		}),
393	)
394
395	err := store.DeclareLength("uploadId+multipartId", 500)
396	assert.Nil(err)
397}
398
399func TestFinishUpload(t *testing.T) {
400	mockCtrl := gomock.NewController(t)
401	defer mockCtrl.Finish()
402	assert := assert.New(t)
403
404	s3obj := NewMockS3API(mockCtrl)
405	store := New("bucket", s3obj)
406
407	gomock.InOrder(
408		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
409			Bucket:           aws.String("bucket"),
410			Key:              aws.String("uploadId"),
411			UploadId:         aws.String("multipartId"),
412			PartNumberMarker: aws.Int64(0),
413		}).Return(&s3.ListPartsOutput{
414			Parts: []*s3.Part{
415				{
416					Size:       aws.Int64(100),
417					ETag:       aws.String("foo"),
418					PartNumber: aws.Int64(1),
419				},
420				{
421					Size:       aws.Int64(200),
422					ETag:       aws.String("bar"),
423					PartNumber: aws.Int64(2),
424				},
425			},
426			NextPartNumberMarker: aws.Int64(2),
427			IsTruncated:          aws.Bool(true),
428		}, nil),
429		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
430			Bucket:           aws.String("bucket"),
431			Key:              aws.String("uploadId"),
432			UploadId:         aws.String("multipartId"),
433			PartNumberMarker: aws.Int64(2),
434		}).Return(&s3.ListPartsOutput{
435			Parts: []*s3.Part{
436				{
437					Size:       aws.Int64(100),
438					ETag:       aws.String("foobar"),
439					PartNumber: aws.Int64(3),
440				},
441			},
442		}, nil),
443		s3obj.EXPECT().CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
444			Bucket:   aws.String("bucket"),
445			Key:      aws.String("uploadId"),
446			UploadId: aws.String("multipartId"),
447			MultipartUpload: &s3.CompletedMultipartUpload{
448				Parts: []*s3.CompletedPart{
449					{
450						ETag:       aws.String("foo"),
451						PartNumber: aws.Int64(1),
452					},
453					{
454						ETag:       aws.String("bar"),
455						PartNumber: aws.Int64(2),
456					},
457					{
458						ETag:       aws.String("foobar"),
459						PartNumber: aws.Int64(3),
460					},
461				},
462			},
463		}).Return(nil, nil),
464	)
465
466	err := store.FinishUpload("uploadId+multipartId")
467	assert.Nil(err)
468}
469
470func TestWriteChunk(t *testing.T) {
471	mockCtrl := gomock.NewController(t)
472	defer mockCtrl.Finish()
473	assert := assert.New(t)
474
475	s3obj := NewMockS3API(mockCtrl)
476	store := New("bucket", s3obj)
477	store.MaxPartSize = 8
478	store.MinPartSize = 4
479	store.MaxMultipartParts = 10000
480	store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024
481
482	gomock.InOrder(
483		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
484			Bucket: aws.String("bucket"),
485			Key:    aws.String("uploadId.info"),
486		}).Return(&s3.GetObjectOutput{
487			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
488		}, nil),
489		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
490			Bucket:           aws.String("bucket"),
491			Key:              aws.String("uploadId"),
492			UploadId:         aws.String("multipartId"),
493			PartNumberMarker: aws.Int64(0),
494		}).Return(&s3.ListPartsOutput{
495			Parts: []*s3.Part{
496				{
497					Size: aws.Int64(100),
498				},
499				{
500					Size: aws.Int64(200),
501				},
502			},
503		}, nil),
504		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
505			Bucket: aws.String("bucket"),
506			Key:    aws.String("uploadId.part"),
507		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)),
508		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
509			Bucket:           aws.String("bucket"),
510			Key:              aws.String("uploadId"),
511			UploadId:         aws.String("multipartId"),
512			PartNumberMarker: aws.Int64(0),
513		}).Return(&s3.ListPartsOutput{
514			Parts: []*s3.Part{
515				{
516					Size: aws.Int64(100),
517				},
518				{
519					Size: aws.Int64(200),
520				},
521			},
522		}, nil),
523		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
524			Bucket: aws.String("bucket"),
525			Key:    aws.String("uploadId.part"),
526		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
527		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
528			Bucket:     aws.String("bucket"),
529			Key:        aws.String("uploadId"),
530			UploadId:   aws.String("multipartId"),
531			PartNumber: aws.Int64(3),
532			Body:       bytes.NewReader([]byte("1234")),
533		})).Return(nil, nil),
534		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
535			Bucket:     aws.String("bucket"),
536			Key:        aws.String("uploadId"),
537			UploadId:   aws.String("multipartId"),
538			PartNumber: aws.Int64(4),
539			Body:       bytes.NewReader([]byte("5678")),
540		})).Return(nil, nil),
541		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
542			Bucket:     aws.String("bucket"),
543			Key:        aws.String("uploadId"),
544			UploadId:   aws.String("multipartId"),
545			PartNumber: aws.Int64(5),
546			Body:       bytes.NewReader([]byte("90AB")),
547		})).Return(nil, nil),
548		s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{
549			Bucket: aws.String("bucket"),
550			Key:    aws.String("uploadId.part"),
551			Body:   bytes.NewReader([]byte("CD")),
552		})).Return(nil, nil),
553	)
554
555	bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, bytes.NewReader([]byte("1234567890ABCD")))
556	assert.Nil(err)
557	assert.Equal(int64(14), bytesRead)
558}
559
560// TestWriteChunkWithUnexpectedEOF ensures that WriteChunk does not error out
561// if the io.Reader returns an io.ErrUnexpectedEOF. This happens when a HTTP
562// PATCH request gets interrupted.
563func TestWriteChunkWithUnexpectedEOF(t *testing.T) {
564	mockCtrl := gomock.NewController(t)
565	defer mockCtrl.Finish()
566	assert := assert.New(t)
567
568	s3obj := NewMockS3API(mockCtrl)
569	store := New("bucket", s3obj)
570	store.MaxPartSize = 500
571	store.MinPartSize = 100
572	store.MaxMultipartParts = 10000
573	store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024
574
575	gomock.InOrder(
576		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
577			Bucket: aws.String("bucket"),
578			Key:    aws.String("uploadId.info"),
579		}).Return(&s3.GetObjectOutput{
580			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
581		}, nil),
582		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
583			Bucket:           aws.String("bucket"),
584			Key:              aws.String("uploadId"),
585			UploadId:         aws.String("multipartId"),
586			PartNumberMarker: aws.Int64(0),
587		}).Return(&s3.ListPartsOutput{
588			Parts: []*s3.Part{
589				{
590					Size: aws.Int64(100),
591				},
592				{
593					Size: aws.Int64(200),
594				},
595			},
596		}, nil),
597		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
598			Bucket: aws.String("bucket"),
599			Key:    aws.String("uploadId.part"),
600		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "Not found", nil)),
601		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
602			Bucket:           aws.String("bucket"),
603			Key:              aws.String("uploadId"),
604			UploadId:         aws.String("multipartId"),
605			PartNumberMarker: aws.Int64(0),
606		}).Return(&s3.ListPartsOutput{
607			Parts: []*s3.Part{
608				{
609					Size: aws.Int64(100),
610				},
611				{
612					Size: aws.Int64(200),
613				},
614			},
615		}, nil),
616		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
617			Bucket: aws.String("bucket"),
618			Key:    aws.String("uploadId.part"),
619		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
620		s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{
621			Bucket: aws.String("bucket"),
622			Key:    aws.String("uploadId.part"),
623			Body:   bytes.NewReader([]byte("1234567890ABCD")),
624		})).Return(nil, nil),
625	)
626
627	reader, writer := io.Pipe()
628
629	go func() {
630		writer.Write([]byte("1234567890ABCD"))
631		writer.CloseWithError(io.ErrUnexpectedEOF)
632	}()
633
634	bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, reader)
635	assert.Nil(err)
636	assert.Equal(int64(14), bytesRead)
637}
638
639func TestWriteChunkWriteIncompletePartBecauseTooSmall(t *testing.T) {
640	mockCtrl := gomock.NewController(t)
641	defer mockCtrl.Finish()
642	assert := assert.New(t)
643
644	s3obj := NewMockS3API(mockCtrl)
645	store := New("bucket", s3obj)
646
647	gomock.InOrder(
648		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
649			Bucket: aws.String("bucket"),
650			Key:    aws.String("uploadId.info"),
651		}).Return(&s3.GetObjectOutput{
652			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
653		}, nil),
654		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
655			Bucket:           aws.String("bucket"),
656			Key:              aws.String("uploadId"),
657			UploadId:         aws.String("multipartId"),
658			PartNumberMarker: aws.Int64(0),
659		}).Return(&s3.ListPartsOutput{
660			Parts: []*s3.Part{
661				{
662					Size: aws.Int64(100),
663				},
664				{
665					Size: aws.Int64(200),
666				},
667			},
668		}, nil),
669		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
670			Bucket: aws.String("bucket"),
671			Key:    aws.String("uploadId.part"),
672		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist", nil)),
673		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
674			Bucket:           aws.String("bucket"),
675			Key:              aws.String("uploadId"),
676			UploadId:         aws.String("multipartId"),
677			PartNumberMarker: aws.Int64(0),
678		}).Return(&s3.ListPartsOutput{
679			Parts: []*s3.Part{
680				{
681					Size: aws.Int64(100),
682				},
683				{
684					Size: aws.Int64(200),
685				},
686			},
687		}, nil),
688		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
689			Bucket: aws.String("bucket"),
690			Key:    aws.String("uploadId.part"),
691		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
692		s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{
693			Bucket: aws.String("bucket"),
694			Key:    aws.String("uploadId.part"),
695			Body:   bytes.NewReader([]byte("1234567890")),
696		})).Return(nil, nil),
697	)
698
699	bytesRead, err := store.WriteChunk("uploadId+multipartId", 300, bytes.NewReader([]byte("1234567890")))
700	assert.Nil(err)
701	assert.Equal(int64(10), bytesRead)
702}
703
704func TestWriteChunkPrependsIncompletePart(t *testing.T) {
705	mockCtrl := gomock.NewController(t)
706	defer mockCtrl.Finish()
707	assert := assert.New(t)
708
709	s3obj := NewMockS3API(mockCtrl)
710	store := New("bucket", s3obj)
711	store.MaxPartSize = 8
712	store.MinPartSize = 4
713	store.MaxMultipartParts = 10000
714	store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024
715
716	gomock.InOrder(
717		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
718			Bucket: aws.String("bucket"),
719			Key:    aws.String("uploadId.info"),
720		}).Return(&s3.GetObjectOutput{
721			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":5,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
722		}, nil),
723		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
724			Bucket:           aws.String("bucket"),
725			Key:              aws.String("uploadId"),
726			UploadId:         aws.String("multipartId"),
727			PartNumberMarker: aws.Int64(0),
728		}).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil),
729		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
730			Bucket: aws.String("bucket"),
731			Key:    aws.String("uploadId.part"),
732		}).Return(&s3.GetObjectOutput{
733			ContentLength: 	aws.Int64(3),
734			Body:			ioutil.NopCloser(bytes.NewReader([]byte("123"))),
735		}, nil),
736		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
737			Bucket:           aws.String("bucket"),
738			Key:              aws.String("uploadId"),
739			UploadId:         aws.String("multipartId"),
740			PartNumberMarker: aws.Int64(0),
741		}).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil),
742		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
743			Bucket: aws.String("bucket"),
744			Key:    aws.String("uploadId.part"),
745		}).Return(&s3.GetObjectOutput{
746			ContentLength: aws.Int64(3),
747			Body:          ioutil.NopCloser(bytes.NewReader([]byte("123"))),
748		}, nil),
749		s3obj.EXPECT().DeleteObject(&s3.DeleteObjectInput{
750			Bucket: aws.String(store.Bucket),
751			Key:    aws.String("uploadId.part"),
752		}).Return(&s3.DeleteObjectOutput{}, nil),
753		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
754			Bucket:     aws.String("bucket"),
755			Key:        aws.String("uploadId"),
756			UploadId:   aws.String("multipartId"),
757			PartNumber: aws.Int64(1),
758			Body:       bytes.NewReader([]byte("1234")),
759		})).Return(nil, nil),
760		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
761			Bucket:     aws.String("bucket"),
762			Key:        aws.String("uploadId"),
763			UploadId:   aws.String("multipartId"),
764			PartNumber: aws.Int64(2),
765			Body:       bytes.NewReader([]byte("5")),
766		})).Return(nil, nil),
767	)
768
769	bytesRead, err := store.WriteChunk("uploadId+multipartId", 3, bytes.NewReader([]byte("45")))
770	assert.Nil(err)
771	assert.Equal(int64(2), bytesRead)
772}
773
774func TestWriteChunkPrependsIncompletePartAndWritesANewIncompletePart(t *testing.T) {
775	mockCtrl := gomock.NewController(t)
776	defer mockCtrl.Finish()
777	assert := assert.New(t)
778
779	s3obj := NewMockS3API(mockCtrl)
780	store := New("bucket", s3obj)
781	store.MaxPartSize = 8
782	store.MinPartSize = 4
783	store.MaxMultipartParts = 10000
784	store.MaxObjectSize = 5 * 1024 * 1024 * 1024 * 1024
785
786	gomock.InOrder(
787		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
788			Bucket: aws.String("bucket"),
789			Key:    aws.String("uploadId.info"),
790		}).Return(&s3.GetObjectOutput{
791			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":10,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
792		}, nil),
793		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
794			Bucket:           aws.String("bucket"),
795			Key:              aws.String("uploadId"),
796			UploadId:         aws.String("multipartId"),
797			PartNumberMarker: aws.Int64(0),
798		}).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil),
799		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
800			Bucket: aws.String("bucket"),
801			Key:    aws.String("uploadId.part"),
802		}).Return(&s3.GetObjectOutput{
803			ContentLength: aws.Int64(3),
804			Body:          ioutil.NopCloser(bytes.NewReader([]byte("123"))),
805		}, nil),
806		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
807			Bucket:           aws.String("bucket"),
808			Key:              aws.String("uploadId"),
809			UploadId:         aws.String("multipartId"),
810			PartNumberMarker: aws.Int64(0),
811		}).Return(&s3.ListPartsOutput{Parts: []*s3.Part{}}, nil),
812		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
813			Bucket: aws.String("bucket"),
814			Key:    aws.String("uploadId.part"),
815		}).Return(&s3.GetObjectOutput{
816			Body:          ioutil.NopCloser(bytes.NewReader([]byte("123"))),
817			ContentLength: aws.Int64(3),
818		}, nil),
819		s3obj.EXPECT().DeleteObject(&s3.DeleteObjectInput{
820			Bucket: aws.String(store.Bucket),
821			Key:    aws.String("uploadId.part"),
822		}).Return(&s3.DeleteObjectOutput{}, nil),
823		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
824			Bucket:     aws.String("bucket"),
825			Key:        aws.String("uploadId"),
826			UploadId:   aws.String("multipartId"),
827			PartNumber: aws.Int64(1),
828			Body:       bytes.NewReader([]byte("1234")),
829		})).Return(nil, nil),
830		s3obj.EXPECT().PutObject(NewPutObjectInputMatcher(&s3.PutObjectInput{
831			Bucket: aws.String("bucket"),
832			Key:    aws.String("uploadId.part"),
833			Body:   bytes.NewReader([]byte("5")),
834		})).Return(nil, nil),
835	)
836
837	bytesRead, err := store.WriteChunk("uploadId+multipartId", 3, bytes.NewReader([]byte("45")))
838	assert.Nil(err)
839	assert.Equal(int64(2), bytesRead)
840}
841
842func TestWriteChunkAllowTooSmallLast(t *testing.T) {
843	mockCtrl := gomock.NewController(t)
844	defer mockCtrl.Finish()
845	assert := assert.New(t)
846
847	s3obj := NewMockS3API(mockCtrl)
848	store := New("bucket", s3obj)
849	store.MinPartSize = 20
850
851	gomock.InOrder(
852		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
853			Bucket: aws.String("bucket"),
854			Key:    aws.String("uploadId.info"),
855		}).Return(&s3.GetObjectOutput{
856			Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"ID":"uploadId","Size":500,"Offset":0,"MetaData":null,"IsPartial":false,"IsFinal":false,"PartialUploads":null}`))),
857		}, nil),
858		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
859			Bucket:           aws.String("bucket"),
860			Key:              aws.String("uploadId"),
861			UploadId:         aws.String("multipartId"),
862			PartNumberMarker: aws.Int64(0),
863		}).Return(&s3.ListPartsOutput{
864			Parts: []*s3.Part{
865				{
866					Size: aws.Int64(400),
867				},
868				{
869					Size: aws.Int64(90),
870				},
871			},
872		}, nil),
873		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
874			Bucket: aws.String("bucket"),
875			Key:    aws.String("uploadId.part"),
876		}).Return(&s3.GetObjectOutput{}, awserr.New("AccessDenied", "Access Denied.", nil)),
877		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
878			Bucket:           aws.String("bucket"),
879			Key:              aws.String("uploadId"),
880			UploadId:         aws.String("multipartId"),
881			PartNumberMarker: aws.Int64(0),
882		}).Return(&s3.ListPartsOutput{
883			Parts: []*s3.Part{
884				{
885					Size: aws.Int64(400),
886				},
887				{
888					Size: aws.Int64(90),
889				},
890			},
891		}, nil),
892		s3obj.EXPECT().GetObject(&s3.GetObjectInput{
893			Bucket: aws.String("bucket"),
894			Key:    aws.String("uploadId.part"),
895		}).Return(&s3.GetObjectOutput{}, awserr.New("NoSuchKey", "The specified key does not exist.", nil)),
896		s3obj.EXPECT().UploadPart(NewUploadPartInputMatcher(&s3.UploadPartInput{
897			Bucket:     aws.String("bucket"),
898			Key:        aws.String("uploadId"),
899			UploadId:   aws.String("multipartId"),
900			PartNumber: aws.Int64(3),
901			Body:       bytes.NewReader([]byte("1234567890")),
902		})).Return(nil, nil),
903	)
904
905	// 10 bytes are missing for the upload to be finished (offset at 490 for 500
906	// bytes file) but the minimum chunk size is higher (20). The chunk is
907	// still uploaded since the last part may be smaller than the minimum.
908	bytesRead, err := store.WriteChunk("uploadId+multipartId", 490, bytes.NewReader([]byte("1234567890")))
909	assert.Nil(err)
910	assert.Equal(int64(10), bytesRead)
911}
912
913func TestTerminate(t *testing.T) {
914	mockCtrl := gomock.NewController(t)
915	defer mockCtrl.Finish()
916	assert := assert.New(t)
917
918	s3obj := NewMockS3API(mockCtrl)
919	store := New("bucket", s3obj)
920
921	// Order is not important in this situation.
922	s3obj.EXPECT().AbortMultipartUpload(&s3.AbortMultipartUploadInput{
923		Bucket:   aws.String("bucket"),
924		Key:      aws.String("uploadId"),
925		UploadId: aws.String("multipartId"),
926	}).Return(nil, nil)
927
928	s3obj.EXPECT().DeleteObjects(&s3.DeleteObjectsInput{
929		Bucket: aws.String("bucket"),
930		Delete: &s3.Delete{
931			Objects: []*s3.ObjectIdentifier{
932				{
933					Key: aws.String("uploadId"),
934				},
935				{
936					Key: aws.String("uploadId.part"),
937				},
938				{
939					Key: aws.String("uploadId.info"),
940				},
941			},
942			Quiet: aws.Bool(true),
943		},
944	}).Return(&s3.DeleteObjectsOutput{}, nil)
945
946	err := store.Terminate("uploadId+multipartId")
947	assert.Nil(err)
948}
949
950func TestTerminateWithErrors(t *testing.T) {
951	mockCtrl := gomock.NewController(t)
952	defer mockCtrl.Finish()
953	assert := assert.New(t)
954
955	s3obj := NewMockS3API(mockCtrl)
956	store := New("bucket", s3obj)
957
958	// Order is not important in this situation.
959	// NoSuchUpload errors should be ignored
960	s3obj.EXPECT().AbortMultipartUpload(&s3.AbortMultipartUploadInput{
961		Bucket:   aws.String("bucket"),
962		Key:      aws.String("uploadId"),
963		UploadId: aws.String("multipartId"),
964	}).Return(nil, awserr.New("NoSuchUpload", "The specified upload does not exist.", nil))
965
966	s3obj.EXPECT().DeleteObjects(&s3.DeleteObjectsInput{
967		Bucket: aws.String("bucket"),
968		Delete: &s3.Delete{
969			Objects: []*s3.ObjectIdentifier{
970				{
971					Key: aws.String("uploadId"),
972				},
973				{
974					Key: aws.String("uploadId.part"),
975				},
976				{
977					Key: aws.String("uploadId.info"),
978				},
979			},
980			Quiet: aws.Bool(true),
981		},
982	}).Return(&s3.DeleteObjectsOutput{
983		Errors: []*s3.Error{
984			{
985				Code:    aws.String("hello"),
986				Key:     aws.String("uploadId"),
987				Message: aws.String("it's me."),
988			},
989		},
990	}, nil)
991
992	err := store.Terminate("uploadId+multipartId")
993	assert.Equal("Multiple errors occurred:\n\tAWS S3 Error (hello) for object uploadId: it's me.\n", err.Error())
994}
995
996func TestConcatUploads(t *testing.T) {
997	mockCtrl := gomock.NewController(t)
998	defer mockCtrl.Finish()
999	assert := assert.New(t)
1000
1001	s3obj := NewMockS3API(mockCtrl)
1002	store := New("bucket", s3obj)
1003
1004	s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{
1005		Bucket:     aws.String("bucket"),
1006		Key:        aws.String("uploadId"),
1007		UploadId:   aws.String("multipartId"),
1008		CopySource: aws.String("bucket/aaa"),
1009		PartNumber: aws.Int64(1),
1010	}).Return(nil, nil)
1011
1012	s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{
1013		Bucket:     aws.String("bucket"),
1014		Key:        aws.String("uploadId"),
1015		UploadId:   aws.String("multipartId"),
1016		CopySource: aws.String("bucket/bbb"),
1017		PartNumber: aws.Int64(2),
1018	}).Return(nil, nil)
1019
1020	s3obj.EXPECT().UploadPartCopy(&s3.UploadPartCopyInput{
1021		Bucket:     aws.String("bucket"),
1022		Key:        aws.String("uploadId"),
1023		UploadId:   aws.String("multipartId"),
1024		CopySource: aws.String("bucket/ccc"),
1025		PartNumber: aws.Int64(3),
1026	}).Return(nil, nil)
1027
1028	// Output from s3Store.FinishUpload
1029	gomock.InOrder(
1030		s3obj.EXPECT().ListParts(&s3.ListPartsInput{
1031			Bucket:           aws.String("bucket"),
1032			Key:              aws.String("uploadId"),
1033			UploadId:         aws.String("multipartId"),
1034			PartNumberMarker: aws.Int64(0),
1035		}).Return(&s3.ListPartsOutput{
1036			Parts: []*s3.Part{
1037				{
1038					ETag:       aws.String("foo"),
1039					PartNumber: aws.Int64(1),
1040				},
1041				{
1042					ETag:       aws.String("bar"),
1043					PartNumber: aws.Int64(2),
1044				},
1045				{
1046					ETag:       aws.String("baz"),
1047					PartNumber: aws.Int64(3),
1048				},
1049			},
1050		}, nil),
1051		s3obj.EXPECT().CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
1052			Bucket:   aws.String("bucket"),
1053			Key:      aws.String("uploadId"),
1054			UploadId: aws.String("multipartId"),
1055			MultipartUpload: &s3.CompletedMultipartUpload{
1056				Parts: []*s3.CompletedPart{
1057					{
1058						ETag:       aws.String("foo"),
1059						PartNumber: aws.Int64(1),
1060					},
1061					{
1062						ETag:       aws.String("bar"),
1063						PartNumber: aws.Int64(2),
1064					},
1065					{
1066						ETag:       aws.String("baz"),
1067						PartNumber: aws.Int64(3),
1068					},
1069				},
1070			},
1071		}).Return(nil, nil),
1072	)
1073
1074	err := store.ConcatUploads("uploadId+multipartId", []string{
1075		"aaa+AAA",
1076		"bbb+BBB",
1077		"ccc+CCC",
1078	})
1079	assert.Nil(err)
1080}
1081