1/* 2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage 3 * Copyright 2015-2020 MinIO, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package minio 19 20import ( 21 "bytes" 22 "context" 23 "encoding/xml" 24 "io" 25 "net/http" 26 "net/url" 27 "time" 28 29 "github.com/minio/minio-go/v7/pkg/s3utils" 30) 31 32// RemoveBucket deletes the bucket name. 33// 34// All objects (including all object versions and delete markers). 35// in the bucket must be deleted before successfully attempting this request. 36func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { 37 // Input validation. 38 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 39 return err 40 } 41 // Execute DELETE on bucket. 42 resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ 43 bucketName: bucketName, 44 contentSHA256Hex: emptySHA256Hex, 45 }) 46 defer closeResponse(resp) 47 if err != nil { 48 return err 49 } 50 if resp != nil { 51 if resp.StatusCode != http.StatusNoContent { 52 return httpRespToErrorResponse(resp, bucketName, "") 53 } 54 } 55 56 // Remove the location from cache on a successful delete. 57 c.bucketLocCache.Delete(bucketName) 58 59 return nil 60} 61 62// AdvancedRemoveOptions intended for internal use by replication 63type AdvancedRemoveOptions struct { 64 ReplicationDeleteMarker bool 65 ReplicationStatus ReplicationStatus 66 ReplicationMTime time.Time 67 ReplicationRequest bool 68} 69 70// RemoveObjectOptions represents options specified by user for RemoveObject call 71type RemoveObjectOptions struct { 72 GovernanceBypass bool 73 VersionID string 74 Internal AdvancedRemoveOptions 75} 76 77// RemoveObject removes an object from a bucket. 78func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { 79 // Input validation. 80 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 81 return err 82 } 83 if err := s3utils.CheckValidObjectName(objectName); err != nil { 84 return err 85 } 86 87 return c.removeObject(ctx, bucketName, objectName, opts) 88} 89 90func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { 91 92 // Get resources properly escaped and lined up before 93 // using them in http request. 94 urlValues := make(url.Values) 95 96 if opts.VersionID != "" { 97 urlValues.Set("versionId", opts.VersionID) 98 } 99 100 // Build headers. 101 headers := make(http.Header) 102 103 if opts.GovernanceBypass { 104 // Set the bypass goverenance retention header 105 headers.Set(amzBypassGovernance, "true") 106 } 107 if opts.Internal.ReplicationDeleteMarker { 108 headers.Set(minIOBucketReplicationDeleteMarker, "true") 109 } 110 if !opts.Internal.ReplicationMTime.IsZero() { 111 headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339Nano)) 112 } 113 if !opts.Internal.ReplicationStatus.Empty() { 114 headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus)) 115 } 116 if opts.Internal.ReplicationRequest { 117 headers.Set(minIOBucketReplicationRequest, "") 118 } 119 // Execute DELETE on objectName. 120 resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ 121 bucketName: bucketName, 122 objectName: objectName, 123 contentSHA256Hex: emptySHA256Hex, 124 queryValues: urlValues, 125 customHeader: headers, 126 }) 127 defer closeResponse(resp) 128 if err != nil { 129 return err 130 } 131 if resp != nil { 132 // if some unexpected error happened and max retry is reached, we want to let client know 133 if resp.StatusCode != http.StatusNoContent { 134 return httpRespToErrorResponse(resp, bucketName, objectName) 135 } 136 } 137 138 // DeleteObject always responds with http '204' even for 139 // objects which do not exist. So no need to handle them 140 // specifically. 141 return nil 142} 143 144// RemoveObjectError - container of Multi Delete S3 API error 145type RemoveObjectError struct { 146 ObjectName string 147 VersionID string 148 Err error 149} 150 151// generateRemoveMultiObjects - generate the XML request for remove multi objects request 152func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { 153 delObjects := []deleteObject{} 154 for _, obj := range objects { 155 delObjects = append(delObjects, deleteObject{ 156 Key: obj.Key, 157 VersionID: obj.VersionID, 158 }) 159 } 160 xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true}) 161 return xmlBytes 162} 163 164// processRemoveMultiObjectsResponse - parse the remove multi objects web service 165// and return the success/failure result status for each object 166func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) { 167 // Parse multi delete XML response 168 rmResult := &deleteMultiObjectsResult{} 169 err := xmlDecoder(body, rmResult) 170 if err != nil { 171 errorCh <- RemoveObjectError{ObjectName: "", Err: err} 172 return 173 } 174 175 // Fill deletion that returned an error. 176 for _, obj := range rmResult.UnDeletedObjects { 177 // Version does not exist is not an error ignore and continue. 178 switch obj.Code { 179 case "InvalidArgument", "NoSuchVersion": 180 continue 181 } 182 errorCh <- RemoveObjectError{ 183 ObjectName: obj.Key, 184 VersionID: obj.VersionID, 185 Err: ErrorResponse{ 186 Code: obj.Code, 187 Message: obj.Message, 188 }, 189 } 190 } 191} 192 193// RemoveObjectsOptions represents options specified by user for RemoveObjects call 194type RemoveObjectsOptions struct { 195 GovernanceBypass bool 196} 197 198// RemoveObjects removes multiple objects from a bucket while 199// it is possible to specify objects versions which are received from 200// objectsCh. Remove failures are sent back via error channel. 201func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError { 202 errorCh := make(chan RemoveObjectError, 1) 203 204 // Validate if bucket name is valid. 205 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 206 defer close(errorCh) 207 errorCh <- RemoveObjectError{ 208 Err: err, 209 } 210 return errorCh 211 } 212 // Validate objects channel to be properly allocated. 213 if objectsCh == nil { 214 defer close(errorCh) 215 errorCh <- RemoveObjectError{ 216 Err: errInvalidArgument("Objects channel cannot be nil"), 217 } 218 return errorCh 219 } 220 221 go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) 222 return errorCh 223} 224 225// Return true if the character is within the allowed characters in an XML 1.0 document 226// The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets 227func validXMLChar(r rune) (ok bool) { 228 return r == 0x09 || 229 r == 0x0A || 230 r == 0x0D || 231 r >= 0x20 && r <= 0xD7FF || 232 r >= 0xE000 && r <= 0xFFFD || 233 r >= 0x10000 && r <= 0x10FFFF 234} 235 236func hasInvalidXMLChar(str string) bool { 237 for _, s := range str { 238 if !validXMLChar(s) { 239 return true 240 } 241 } 242 return false 243} 244 245// Generate and call MultiDelete S3 requests based on entries received from objectsCh 246func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { 247 maxEntries := 1000 248 finish := false 249 urlValues := make(url.Values) 250 urlValues.Set("delete", "") 251 252 // Close error channel when Multi delete finishes. 253 defer close(errorCh) 254 255 // Loop over entries by 1000 and call MultiDelete requests 256 for { 257 if finish { 258 break 259 } 260 count := 0 261 var batch []ObjectInfo 262 263 // Try to gather 1000 entries 264 for object := range objectsCh { 265 if hasInvalidXMLChar(object.Key) { 266 // Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document. 267 err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ 268 VersionID: object.VersionID, 269 GovernanceBypass: opts.GovernanceBypass, 270 }) 271 if err != nil { 272 // Version does not exist is not an error ignore and continue. 273 switch ToErrorResponse(err).Code { 274 case "InvalidArgument", "NoSuchVersion": 275 continue 276 } 277 errorCh <- RemoveObjectError{ 278 ObjectName: object.Key, 279 VersionID: object.VersionID, 280 Err: err, 281 } 282 } 283 continue 284 } 285 286 batch = append(batch, object) 287 if count++; count >= maxEntries { 288 break 289 } 290 } 291 if count == 0 { 292 // Multi Objects Delete API doesn't accept empty object list, quit immediately 293 break 294 } 295 if count < maxEntries { 296 // We didn't have 1000 entries, so this is the last batch 297 finish = true 298 } 299 300 // Build headers. 301 headers := make(http.Header) 302 if opts.GovernanceBypass { 303 // Set the bypass goverenance retention header 304 headers.Set(amzBypassGovernance, "true") 305 } 306 307 // Generate remove multi objects XML request 308 removeBytes := generateRemoveMultiObjectsRequest(batch) 309 // Execute GET on bucket to list objects. 310 resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ 311 bucketName: bucketName, 312 queryValues: urlValues, 313 contentBody: bytes.NewReader(removeBytes), 314 contentLength: int64(len(removeBytes)), 315 contentMD5Base64: sumMD5Base64(removeBytes), 316 contentSHA256Hex: sum256Hex(removeBytes), 317 customHeader: headers, 318 }) 319 if resp != nil { 320 if resp.StatusCode != http.StatusOK { 321 e := httpRespToErrorResponse(resp, bucketName, "") 322 errorCh <- RemoveObjectError{ObjectName: "", Err: e} 323 } 324 } 325 if err != nil { 326 for _, b := range batch { 327 errorCh <- RemoveObjectError{ 328 ObjectName: b.Key, 329 VersionID: b.VersionID, 330 Err: err, 331 } 332 } 333 continue 334 } 335 336 // Process multiobjects remove xml response 337 processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) 338 339 closeResponse(resp) 340 } 341} 342 343// RemoveIncompleteUpload aborts an partially uploaded object. 344func (c Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error { 345 // Input validation. 346 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 347 return err 348 } 349 if err := s3utils.CheckValidObjectName(objectName); err != nil { 350 return err 351 } 352 // Find multipart upload ids of the object to be aborted. 353 uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName) 354 if err != nil { 355 return err 356 } 357 358 for _, uploadID := range uploadIDs { 359 // abort incomplete multipart upload, based on the upload id passed. 360 err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) 361 if err != nil { 362 return err 363 } 364 } 365 366 return nil 367} 368 369// abortMultipartUpload aborts a multipart upload for the given 370// uploadID, all previously uploaded parts are deleted. 371func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error { 372 // Input validation. 373 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 374 return err 375 } 376 if err := s3utils.CheckValidObjectName(objectName); err != nil { 377 return err 378 } 379 380 // Initialize url queries. 381 urlValues := make(url.Values) 382 urlValues.Set("uploadId", uploadID) 383 384 // Execute DELETE on multipart upload. 385 resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ 386 bucketName: bucketName, 387 objectName: objectName, 388 queryValues: urlValues, 389 contentSHA256Hex: emptySHA256Hex, 390 }) 391 defer closeResponse(resp) 392 if err != nil { 393 return err 394 } 395 if resp != nil { 396 if resp.StatusCode != http.StatusNoContent { 397 // Abort has no response body, handle it for any errors. 398 var errorResponse ErrorResponse 399 switch resp.StatusCode { 400 case http.StatusNotFound: 401 // This is needed specifically for abort and it cannot 402 // be converged into default case. 403 errorResponse = ErrorResponse{ 404 Code: "NoSuchUpload", 405 Message: "The specified multipart upload does not exist.", 406 BucketName: bucketName, 407 Key: objectName, 408 RequestID: resp.Header.Get("x-amz-request-id"), 409 HostID: resp.Header.Get("x-amz-id-2"), 410 Region: resp.Header.Get("x-amz-bucket-region"), 411 } 412 default: 413 return httpRespToErrorResponse(resp, bucketName, objectName) 414 } 415 return errorResponse 416 } 417 } 418 return nil 419} 420