1/*
2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
3 * Copyright 2015-2020 MinIO, Inc.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 *     http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package minio
19
20import (
21	"bytes"
22	"context"
23	"encoding/xml"
24	"io"
25	"net/http"
26	"net/url"
27	"time"
28
29	"github.com/minio/minio-go/v7/pkg/s3utils"
30)
31
32// RemoveBucket deletes the bucket name.
33//
34//  All objects (including all object versions and delete markers).
35//  in the bucket must be deleted before successfully attempting this request.
36func (c Client) RemoveBucket(ctx context.Context, bucketName string) error {
37	// Input validation.
38	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
39		return err
40	}
41	// Execute DELETE on bucket.
42	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
43		bucketName:       bucketName,
44		contentSHA256Hex: emptySHA256Hex,
45	})
46	defer closeResponse(resp)
47	if err != nil {
48		return err
49	}
50	if resp != nil {
51		if resp.StatusCode != http.StatusNoContent {
52			return httpRespToErrorResponse(resp, bucketName, "")
53		}
54	}
55
56	// Remove the location from cache on a successful delete.
57	c.bucketLocCache.Delete(bucketName)
58
59	return nil
60}
61
62// AdvancedRemoveOptions intended for internal use by replication
63type AdvancedRemoveOptions struct {
64	ReplicationDeleteMarker bool
65	ReplicationStatus       ReplicationStatus
66	ReplicationMTime        time.Time
67	ReplicationRequest      bool
68}
69
70// RemoveObjectOptions represents options specified by user for RemoveObject call
71type RemoveObjectOptions struct {
72	GovernanceBypass bool
73	VersionID        string
74	Internal         AdvancedRemoveOptions
75}
76
77// RemoveObject removes an object from a bucket.
78func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error {
79	// Input validation.
80	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
81		return err
82	}
83	if err := s3utils.CheckValidObjectName(objectName); err != nil {
84		return err
85	}
86
87	return c.removeObject(ctx, bucketName, objectName, opts)
88}
89
90func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error {
91
92	// Get resources properly escaped and lined up before
93	// using them in http request.
94	urlValues := make(url.Values)
95
96	if opts.VersionID != "" {
97		urlValues.Set("versionId", opts.VersionID)
98	}
99
100	// Build headers.
101	headers := make(http.Header)
102
103	if opts.GovernanceBypass {
104		// Set the bypass goverenance retention header
105		headers.Set(amzBypassGovernance, "true")
106	}
107	if opts.Internal.ReplicationDeleteMarker {
108		headers.Set(minIOBucketReplicationDeleteMarker, "true")
109	}
110	if !opts.Internal.ReplicationMTime.IsZero() {
111		headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339Nano))
112	}
113	if !opts.Internal.ReplicationStatus.Empty() {
114		headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
115	}
116	if opts.Internal.ReplicationRequest {
117		headers.Set(minIOBucketReplicationRequest, "")
118	}
119	// Execute DELETE on objectName.
120	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
121		bucketName:       bucketName,
122		objectName:       objectName,
123		contentSHA256Hex: emptySHA256Hex,
124		queryValues:      urlValues,
125		customHeader:     headers,
126	})
127	defer closeResponse(resp)
128	if err != nil {
129		return err
130	}
131	if resp != nil {
132		// if some unexpected error happened and max retry is reached, we want to let client know
133		if resp.StatusCode != http.StatusNoContent {
134			return httpRespToErrorResponse(resp, bucketName, objectName)
135		}
136	}
137
138	// DeleteObject always responds with http '204' even for
139	// objects which do not exist. So no need to handle them
140	// specifically.
141	return nil
142}
143
144// RemoveObjectError - container of Multi Delete S3 API error
145type RemoveObjectError struct {
146	ObjectName string
147	VersionID  string
148	Err        error
149}
150
151// generateRemoveMultiObjects - generate the XML request for remove multi objects request
152func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
153	delObjects := []deleteObject{}
154	for _, obj := range objects {
155		delObjects = append(delObjects, deleteObject{
156			Key:       obj.Key,
157			VersionID: obj.VersionID,
158		})
159	}
160	xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true})
161	return xmlBytes
162}
163
164// processRemoveMultiObjectsResponse - parse the remove multi objects web service
165// and return the success/failure result status for each object
166func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) {
167	// Parse multi delete XML response
168	rmResult := &deleteMultiObjectsResult{}
169	err := xmlDecoder(body, rmResult)
170	if err != nil {
171		errorCh <- RemoveObjectError{ObjectName: "", Err: err}
172		return
173	}
174
175	// Fill deletion that returned an error.
176	for _, obj := range rmResult.UnDeletedObjects {
177		// Version does not exist is not an error ignore and continue.
178		switch obj.Code {
179		case "InvalidArgument", "NoSuchVersion":
180			continue
181		}
182		errorCh <- RemoveObjectError{
183			ObjectName: obj.Key,
184			VersionID:  obj.VersionID,
185			Err: ErrorResponse{
186				Code:    obj.Code,
187				Message: obj.Message,
188			},
189		}
190	}
191}
192
193// RemoveObjectsOptions represents options specified by user for RemoveObjects call
194type RemoveObjectsOptions struct {
195	GovernanceBypass bool
196}
197
198// RemoveObjects removes multiple objects from a bucket while
199// it is possible to specify objects versions which are received from
200// objectsCh. Remove failures are sent back via error channel.
201func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError {
202	errorCh := make(chan RemoveObjectError, 1)
203
204	// Validate if bucket name is valid.
205	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
206		defer close(errorCh)
207		errorCh <- RemoveObjectError{
208			Err: err,
209		}
210		return errorCh
211	}
212	// Validate objects channel to be properly allocated.
213	if objectsCh == nil {
214		defer close(errorCh)
215		errorCh <- RemoveObjectError{
216			Err: errInvalidArgument("Objects channel cannot be nil"),
217		}
218		return errorCh
219	}
220
221	go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts)
222	return errorCh
223}
224
225// Return true if the character is within the allowed characters in an XML 1.0 document
226// The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets
227func validXMLChar(r rune) (ok bool) {
228	return r == 0x09 ||
229		r == 0x0A ||
230		r == 0x0D ||
231		r >= 0x20 && r <= 0xD7FF ||
232		r >= 0xE000 && r <= 0xFFFD ||
233		r >= 0x10000 && r <= 0x10FFFF
234}
235
236func hasInvalidXMLChar(str string) bool {
237	for _, s := range str {
238		if !validXMLChar(s) {
239			return true
240		}
241	}
242	return false
243}
244
245// Generate and call MultiDelete S3 requests based on entries received from objectsCh
246func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) {
247	maxEntries := 1000
248	finish := false
249	urlValues := make(url.Values)
250	urlValues.Set("delete", "")
251
252	// Close error channel when Multi delete finishes.
253	defer close(errorCh)
254
255	// Loop over entries by 1000 and call MultiDelete requests
256	for {
257		if finish {
258			break
259		}
260		count := 0
261		var batch []ObjectInfo
262
263		// Try to gather 1000 entries
264		for object := range objectsCh {
265			if hasInvalidXMLChar(object.Key) {
266				// Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document.
267				err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
268					VersionID:        object.VersionID,
269					GovernanceBypass: opts.GovernanceBypass,
270				})
271				if err != nil {
272					// Version does not exist is not an error ignore and continue.
273					switch ToErrorResponse(err).Code {
274					case "InvalidArgument", "NoSuchVersion":
275						continue
276					}
277					errorCh <- RemoveObjectError{
278						ObjectName: object.Key,
279						VersionID:  object.VersionID,
280						Err:        err,
281					}
282				}
283				continue
284			}
285
286			batch = append(batch, object)
287			if count++; count >= maxEntries {
288				break
289			}
290		}
291		if count == 0 {
292			// Multi Objects Delete API doesn't accept empty object list, quit immediately
293			break
294		}
295		if count < maxEntries {
296			// We didn't have 1000 entries, so this is the last batch
297			finish = true
298		}
299
300		// Build headers.
301		headers := make(http.Header)
302		if opts.GovernanceBypass {
303			// Set the bypass goverenance retention header
304			headers.Set(amzBypassGovernance, "true")
305		}
306
307		// Generate remove multi objects XML request
308		removeBytes := generateRemoveMultiObjectsRequest(batch)
309		// Execute GET on bucket to list objects.
310		resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{
311			bucketName:       bucketName,
312			queryValues:      urlValues,
313			contentBody:      bytes.NewReader(removeBytes),
314			contentLength:    int64(len(removeBytes)),
315			contentMD5Base64: sumMD5Base64(removeBytes),
316			contentSHA256Hex: sum256Hex(removeBytes),
317			customHeader:     headers,
318		})
319		if resp != nil {
320			if resp.StatusCode != http.StatusOK {
321				e := httpRespToErrorResponse(resp, bucketName, "")
322				errorCh <- RemoveObjectError{ObjectName: "", Err: e}
323			}
324		}
325		if err != nil {
326			for _, b := range batch {
327				errorCh <- RemoveObjectError{
328					ObjectName: b.Key,
329					VersionID:  b.VersionID,
330					Err:        err,
331				}
332			}
333			continue
334		}
335
336		// Process multiobjects remove xml response
337		processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)
338
339		closeResponse(resp)
340	}
341}
342
343// RemoveIncompleteUpload aborts an partially uploaded object.
344func (c Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error {
345	// Input validation.
346	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
347		return err
348	}
349	if err := s3utils.CheckValidObjectName(objectName); err != nil {
350		return err
351	}
352	// Find multipart upload ids of the object to be aborted.
353	uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName)
354	if err != nil {
355		return err
356	}
357
358	for _, uploadID := range uploadIDs {
359		// abort incomplete multipart upload, based on the upload id passed.
360		err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID)
361		if err != nil {
362			return err
363		}
364	}
365
366	return nil
367}
368
369// abortMultipartUpload aborts a multipart upload for the given
370// uploadID, all previously uploaded parts are deleted.
371func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error {
372	// Input validation.
373	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
374		return err
375	}
376	if err := s3utils.CheckValidObjectName(objectName); err != nil {
377		return err
378	}
379
380	// Initialize url queries.
381	urlValues := make(url.Values)
382	urlValues.Set("uploadId", uploadID)
383
384	// Execute DELETE on multipart upload.
385	resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
386		bucketName:       bucketName,
387		objectName:       objectName,
388		queryValues:      urlValues,
389		contentSHA256Hex: emptySHA256Hex,
390	})
391	defer closeResponse(resp)
392	if err != nil {
393		return err
394	}
395	if resp != nil {
396		if resp.StatusCode != http.StatusNoContent {
397			// Abort has no response body, handle it for any errors.
398			var errorResponse ErrorResponse
399			switch resp.StatusCode {
400			case http.StatusNotFound:
401				// This is needed specifically for abort and it cannot
402				// be converged into default case.
403				errorResponse = ErrorResponse{
404					Code:       "NoSuchUpload",
405					Message:    "The specified multipart upload does not exist.",
406					BucketName: bucketName,
407					Key:        objectName,
408					RequestID:  resp.Header.Get("x-amz-request-id"),
409					HostID:     resp.Header.Get("x-amz-id-2"),
410					Region:     resp.Header.Get("x-amz-bucket-region"),
411				}
412			default:
413				return httpRespToErrorResponse(resp, bucketName, objectName)
414			}
415			return errorResponse
416		}
417	}
418	return nil
419}
420