1package swift 2 3import ( 4 "bufio" 5 "bytes" 6 "crypto/hmac" 7 "crypto/md5" 8 "crypto/sha1" 9 "encoding/hex" 10 "encoding/json" 11 "fmt" 12 "hash" 13 "io" 14 "io/ioutil" 15 "mime" 16 "net/http" 17 "net/url" 18 "os" 19 "path" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24) 25 26const ( 27 DefaultUserAgent = "goswift/1.0" // Default user agent 28 DefaultRetries = 3 // Default number of retries on token expiry 29 TimeFormat = "2006-01-02T15:04:05" // Python date format for json replies parsed as UTC 30 UploadTar = "tar" // Data format specifier for Connection.BulkUpload(). 31 UploadTarGzip = "tar.gz" // Data format specifier for Connection.BulkUpload(). 32 UploadTarBzip2 = "tar.bz2" // Data format specifier for Connection.BulkUpload(). 33 allContainersLimit = 10000 // Number of containers to fetch at once 34 allObjectsLimit = 10000 // Number objects to fetch at once 35 allObjectsChanLimit = 1000 // ...when fetching to a channel 36) 37 38// ObjectType is the type of the swift object, regular, static large, 39// or dynamic large. 40type ObjectType int 41 42// Values that ObjectType can take 43const ( 44 RegularObjectType ObjectType = iota 45 StaticLargeObjectType 46 DynamicLargeObjectType 47) 48 49// Connection holds the details of the connection to the swift server. 50// 51// You need to provide UserName, ApiKey and AuthUrl when you create a 52// connection then call Authenticate on it. 53// 54// The auth version in use will be detected from the AuthURL - you can 55// override this with the AuthVersion parameter. 56// 57// If using v2 auth you can also set Region in the Connection 58// structure. If you don't set Region you will get the default region 59// which may not be what you want. 60// 61// For reference some common AuthUrls looks like this: 62// 63// Rackspace US https://auth.api.rackspacecloud.com/v1.0 64// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0 65// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 66// Memset Memstore UK https://auth.storage.memset.com/v1.0 67// Memstore v2 https://auth.storage.memset.com/v2.0 68// 69// When using Google Appengine you must provide the Connection with an 70// appengine-specific Transport: 71// 72// import ( 73// "appengine/urlfetch" 74// "fmt" 75// "github.com/ncw/swift" 76// ) 77// 78// func handler(w http.ResponseWriter, r *http.Request) { 79// ctx := appengine.NewContext(r) 80// tr := urlfetch.Transport{Context: ctx} 81// c := swift.Connection{ 82// UserName: "user", 83// ApiKey: "key", 84// AuthUrl: "auth_url", 85// Transport: tr, 86// } 87// _ := c.Authenticate() 88// containers, _ := c.ContainerNames(nil) 89// fmt.Fprintf(w, "containers: %q", containers) 90// } 91// 92// If you don't supply a Transport, one is made which relies on 93// http.ProxyFromEnvironment (http://golang.org/pkg/net/http/#ProxyFromEnvironment). 94// This means that the connection will respect the HTTP proxy specified by the 95// environment variables $HTTP_PROXY and $NO_PROXY. 96type Connection struct { 97 // Parameters - fill these in before calling Authenticate 98 // They are all optional except UserName, ApiKey and AuthUrl 99 Domain string // User's domain name 100 DomainId string // User's domain Id 101 UserName string // UserName for api 102 UserId string // User Id 103 ApiKey string // Key for api access 104 AuthUrl string // Auth URL 105 Retries int // Retries on error (default is 3) 106 UserAgent string // Http User agent (default goswift/1.0) 107 ConnectTimeout time.Duration // Connect channel timeout (default 10s) 108 Timeout time.Duration // Data channel timeout (default 60s) 109 Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only) 110 AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect 111 Internal bool // Set this to true to use the the internal / service network 112 Tenant string // Name of the tenant (v2,v3 auth only) 113 TenantId string // Id of the tenant (v2,v3 auth only) 114 EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set) 115 TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain 116 TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain 117 TrustId string // Id of the trust (v3 auth only) 118 Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) 119 // These are filled in after Authenticate is called as are the defaults for above 120 StorageUrl string 121 AuthToken string 122 client *http.Client 123 Auth Authenticator `json:"-" xml:"-"` // the current authenticator 124 authLock sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth 125 // swiftInfo is filled after QueryInfo is called 126 swiftInfo SwiftInfo 127} 128 129// setFromEnv reads the value that param points to (it must be a 130// pointer), if it isn't the zero value then it reads the environment 131// variable name passed in, parses it according to the type and writes 132// it to the pointer. 133func setFromEnv(param interface{}, name string) (err error) { 134 val := os.Getenv(name) 135 if val == "" { 136 return 137 } 138 switch result := param.(type) { 139 case *string: 140 if *result == "" { 141 *result = val 142 } 143 case *int: 144 if *result == 0 { 145 *result, err = strconv.Atoi(val) 146 } 147 case *bool: 148 if *result == false { 149 *result, err = strconv.ParseBool(val) 150 } 151 case *time.Duration: 152 if *result == 0 { 153 *result, err = time.ParseDuration(val) 154 } 155 case *EndpointType: 156 if *result == EndpointType("") { 157 *result = EndpointType(val) 158 } 159 default: 160 return newErrorf(0, "can't set var of type %T", param) 161 } 162 return err 163} 164 165// ApplyEnvironment reads environment variables and applies them to 166// the Connection structure. It won't overwrite any parameters which 167// are already set in the Connection struct. 168// 169// To make a new Connection object entirely from the environment you 170// would do: 171// 172// c := new(Connection) 173// err := c.ApplyEnvironment() 174// if err != nil { log.Fatal(err) } 175// 176// The naming of these variables follows the official Openstack naming 177// scheme so it should be compatible with OpenStack rc files. 178// 179// For v1 authentication (obsolete) 180// ST_AUTH - Auth URL 181// ST_USER - UserName for api 182// ST_KEY - Key for api access 183// 184// For v2 authentication 185// OS_AUTH_URL - Auth URL 186// OS_USERNAME - UserName for api 187// OS_PASSWORD - Key for api access 188// OS_TENANT_NAME - Name of the tenant 189// OS_TENANT_ID - Id of the tenant 190// OS_REGION_NAME - Region to use - default is use first region 191// 192// For v3 authentication 193// OS_AUTH_URL - Auth URL 194// OS_USERNAME - UserName for api 195// OS_USER_ID - User Id 196// OS_PASSWORD - Key for api access 197// OS_USER_DOMAIN_NAME - User's domain name 198// OS_USER_DOMAIN_ID - User's domain Id 199// OS_PROJECT_NAME - Name of the project 200// OS_PROJECT_DOMAIN_NAME - Name of the tenant's domain, only needed if it differs from the user domain 201// OS_PROJECT_DOMAIN_ID - Id of the tenant's domain, only needed if it differs the from user domain 202// OS_TRUST_ID - If of the trust 203// OS_REGION_NAME - Region to use - default is use first region 204// 205// Other 206// OS_ENDPOINT_TYPE - Endpoint type public, internal or admin 207// ST_AUTH_VERSION - Choose auth version - 1, 2 or 3 or leave at 0 for autodetect 208// 209// For manual authentication 210// OS_STORAGE_URL - storage URL from alternate authentication 211// OS_AUTH_TOKEN - Auth Token from alternate authentication 212// 213// Library specific 214// GOSWIFT_RETRIES - Retries on error (default is 3) 215// GOSWIFT_USER_AGENT - HTTP User agent (default goswift/1.0) 216// GOSWIFT_CONNECT_TIMEOUT - Connect channel timeout with unit, eg "10s", "100ms" (default "10s") 217// GOSWIFT_TIMEOUT - Data channel timeout with unit, eg "10s", "100ms" (default "60s") 218// GOSWIFT_INTERNAL - Set this to "true" to use the the internal network (obsolete - use OS_ENDPOINT_TYPE) 219func (c *Connection) ApplyEnvironment() (err error) { 220 for _, item := range []struct { 221 result interface{} 222 name string 223 }{ 224 // Environment variables - keep in same order as Connection 225 {&c.Domain, "OS_USER_DOMAIN_NAME"}, 226 {&c.DomainId, "OS_USER_DOMAIN_ID"}, 227 {&c.UserName, "OS_USERNAME"}, 228 {&c.UserId, "OS_USER_ID"}, 229 {&c.ApiKey, "OS_PASSWORD"}, 230 {&c.AuthUrl, "OS_AUTH_URL"}, 231 {&c.Retries, "GOSWIFT_RETRIES"}, 232 {&c.UserAgent, "GOSWIFT_USER_AGENT"}, 233 {&c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT"}, 234 {&c.Timeout, "GOSWIFT_TIMEOUT"}, 235 {&c.Region, "OS_REGION_NAME"}, 236 {&c.AuthVersion, "ST_AUTH_VERSION"}, 237 {&c.Internal, "GOSWIFT_INTERNAL"}, 238 {&c.Tenant, "OS_TENANT_NAME"}, //v2 239 {&c.Tenant, "OS_PROJECT_NAME"}, // v3 240 {&c.TenantId, "OS_TENANT_ID"}, 241 {&c.EndpointType, "OS_ENDPOINT_TYPE"}, 242 {&c.TenantDomain, "OS_PROJECT_DOMAIN_NAME"}, 243 {&c.TenantDomainId, "OS_PROJECT_DOMAIN_ID"}, 244 {&c.TrustId, "OS_TRUST_ID"}, 245 {&c.StorageUrl, "OS_STORAGE_URL"}, 246 {&c.AuthToken, "OS_AUTH_TOKEN"}, 247 // v1 auth alternatives 248 {&c.ApiKey, "ST_KEY"}, 249 {&c.UserName, "ST_USER"}, 250 {&c.AuthUrl, "ST_AUTH"}, 251 } { 252 err = setFromEnv(item.result, item.name) 253 if err != nil { 254 return newErrorf(0, "failed to read env var %q: %v", item.name, err) 255 } 256 } 257 return nil 258} 259 260// Error - all errors generated by this package are of this type. Other error 261// may be passed on from library functions though. 262type Error struct { 263 StatusCode int // HTTP status code if relevant or 0 if not 264 Text string 265} 266 267// Error satisfy the error interface. 268func (e *Error) Error() string { 269 return e.Text 270} 271 272// newError make a new error from a string. 273func newError(StatusCode int, Text string) *Error { 274 return &Error{ 275 StatusCode: StatusCode, 276 Text: Text, 277 } 278} 279 280// newErrorf makes a new error from sprintf parameters. 281func newErrorf(StatusCode int, Text string, Parameters ...interface{}) *Error { 282 return newError(StatusCode, fmt.Sprintf(Text, Parameters...)) 283} 284 285// errorMap defines http error codes to error mappings. 286type errorMap map[int]error 287 288var ( 289 // Specific Errors you might want to check for equality 290 NotModified = newError(304, "Not Modified") 291 BadRequest = newError(400, "Bad Request") 292 AuthorizationFailed = newError(401, "Authorization Failed") 293 ContainerNotFound = newError(404, "Container Not Found") 294 ContainerNotEmpty = newError(409, "Container Not Empty") 295 ObjectNotFound = newError(404, "Object Not Found") 296 ObjectCorrupted = newError(422, "Object Corrupted") 297 TimeoutError = newError(408, "Timeout when reading or writing data") 298 Forbidden = newError(403, "Operation forbidden") 299 TooLargeObject = newError(413, "Too Large Object") 300 RateLimit = newError(498, "Rate Limit") 301 302 // Mappings for authentication errors 303 authErrorMap = errorMap{ 304 400: BadRequest, 305 401: AuthorizationFailed, 306 403: Forbidden, 307 } 308 309 // Mappings for container errors 310 ContainerErrorMap = errorMap{ 311 400: BadRequest, 312 403: Forbidden, 313 404: ContainerNotFound, 314 409: ContainerNotEmpty, 315 498: RateLimit, 316 } 317 318 // Mappings for object errors 319 objectErrorMap = errorMap{ 320 304: NotModified, 321 400: BadRequest, 322 403: Forbidden, 323 404: ObjectNotFound, 324 413: TooLargeObject, 325 422: ObjectCorrupted, 326 498: RateLimit, 327 } 328) 329 330// checkClose is used to check the return from Close in a defer 331// statement. 332func checkClose(c io.Closer, err *error) { 333 cerr := c.Close() 334 if *err == nil { 335 *err = cerr 336 } 337} 338 339// drainAndClose discards all data from rd and closes it. 340// If an error occurs during Read, it is discarded. 341func drainAndClose(rd io.ReadCloser, err *error) { 342 if rd == nil { 343 return 344 } 345 346 _, _ = io.Copy(ioutil.Discard, rd) 347 cerr := rd.Close() 348 if err != nil && *err == nil { 349 *err = cerr 350 } 351} 352 353// parseHeaders checks a response for errors and translates into 354// standard errors if necessary. If an error is returned, resp.Body 355// has been drained and closed. 356func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error { 357 if errorMap != nil { 358 if err, ok := errorMap[resp.StatusCode]; ok { 359 drainAndClose(resp.Body, nil) 360 return err 361 } 362 } 363 if resp.StatusCode < 200 || resp.StatusCode > 299 { 364 drainAndClose(resp.Body, nil) 365 return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status) 366 } 367 return nil 368} 369 370// readHeaders returns a Headers object from the http.Response. 371// 372// If it receives multiple values for a key (which should never 373// happen) it will use the first one 374func readHeaders(resp *http.Response) Headers { 375 headers := Headers{} 376 for key, values := range resp.Header { 377 headers[key] = values[0] 378 } 379 return headers 380} 381 382// Headers stores HTTP headers (can only have one of each header like Swift). 383type Headers map[string]string 384 385// Does an http request using the running timer passed in 386func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*http.Response, error) { 387 // Do the request in the background so we can check the timeout 388 type result struct { 389 resp *http.Response 390 err error 391 } 392 done := make(chan result, 1) 393 go func() { 394 resp, err := c.client.Do(req) 395 done <- result{resp, err} 396 }() 397 // Wait for the read or the timeout 398 select { 399 case r := <-done: 400 return r.resp, r.err 401 case <-timer.C: 402 // Kill the connection on timeout so we don't leak sockets or goroutines 403 cancelRequest(c.Transport, req) 404 return nil, TimeoutError 405 } 406 panic("unreachable") // For Go 1.0 407} 408 409// Set defaults for any unset values 410// 411// Call with authLock held 412func (c *Connection) setDefaults() { 413 if c.UserAgent == "" { 414 c.UserAgent = DefaultUserAgent 415 } 416 if c.Retries == 0 { 417 c.Retries = DefaultRetries 418 } 419 if c.ConnectTimeout == 0 { 420 c.ConnectTimeout = 10 * time.Second 421 } 422 if c.Timeout == 0 { 423 c.Timeout = 60 * time.Second 424 } 425 if c.Transport == nil { 426 c.Transport = &http.Transport{ 427 // TLSClientConfig: &tls.Config{RootCAs: pool}, 428 // DisableCompression: true, 429 Proxy: http.ProxyFromEnvironment, 430 MaxIdleConnsPerHost: 2048, 431 } 432 } 433 if c.client == nil { 434 c.client = &http.Client{ 435 // CheckRedirect: redirectPolicyFunc, 436 Transport: c.Transport, 437 } 438 } 439} 440 441// Authenticate connects to the Swift server. 442// 443// If you don't call it before calling one of the connection methods 444// then it will be called for you on the first access. 445func (c *Connection) Authenticate() (err error) { 446 c.authLock.Lock() 447 defer c.authLock.Unlock() 448 return c.authenticate() 449} 450 451// Internal implementation of Authenticate 452// 453// Call with authLock held 454func (c *Connection) authenticate() (err error) { 455 c.setDefaults() 456 457 // Flush the keepalives connection - if we are 458 // re-authenticating then stuff has gone wrong 459 flushKeepaliveConnections(c.Transport) 460 461 if c.Auth == nil { 462 c.Auth, err = newAuth(c) 463 if err != nil { 464 return 465 } 466 } 467 468 retries := 1 469again: 470 var req *http.Request 471 req, err = c.Auth.Request(c) 472 if err != nil { 473 return 474 } 475 if req != nil { 476 timer := time.NewTimer(c.ConnectTimeout) 477 defer timer.Stop() 478 var resp *http.Response 479 resp, err = c.doTimeoutRequest(timer, req) 480 if err != nil { 481 return 482 } 483 defer func() { 484 drainAndClose(resp.Body, &err) 485 // Flush the auth connection - we don't want to keep 486 // it open if keepalives were enabled 487 flushKeepaliveConnections(c.Transport) 488 }() 489 if err = c.parseHeaders(resp, authErrorMap); err != nil { 490 // Try again for a limited number of times on 491 // AuthorizationFailed or BadRequest. This allows us 492 // to try some alternate forms of the request 493 if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { 494 retries-- 495 goto again 496 } 497 return 498 } 499 err = c.Auth.Response(resp) 500 if err != nil { 501 return 502 } 503 } 504 if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" { 505 c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType) 506 } else { 507 c.StorageUrl = c.Auth.StorageUrl(c.Internal) 508 } 509 c.AuthToken = c.Auth.Token() 510 if !c.authenticated() { 511 err = newError(0, "Response didn't have storage url and auth token") 512 return 513 } 514 return 515} 516 517// Get an authToken and url 518// 519// The Url may be updated if it needed to authenticate using the OnReAuth function 520func (c *Connection) getUrlAndAuthToken(targetUrlIn string, OnReAuth func() (string, error)) (targetUrlOut, authToken string, err error) { 521 c.authLock.Lock() 522 defer c.authLock.Unlock() 523 targetUrlOut = targetUrlIn 524 if !c.authenticated() { 525 err = c.authenticate() 526 if err != nil { 527 return 528 } 529 if OnReAuth != nil { 530 targetUrlOut, err = OnReAuth() 531 if err != nil { 532 return 533 } 534 } 535 } 536 authToken = c.AuthToken 537 return 538} 539 540// flushKeepaliveConnections is called to flush pending requests after an error. 541func flushKeepaliveConnections(transport http.RoundTripper) { 542 if tr, ok := transport.(interface { 543 CloseIdleConnections() 544 }); ok { 545 tr.CloseIdleConnections() 546 } 547} 548 549// UnAuthenticate removes the authentication from the Connection. 550func (c *Connection) UnAuthenticate() { 551 c.authLock.Lock() 552 c.StorageUrl = "" 553 c.AuthToken = "" 554 c.authLock.Unlock() 555} 556 557// Authenticated returns a boolean to show if the current connection 558// is authenticated. 559// 560// Doesn't actually check the credentials against the server. 561func (c *Connection) Authenticated() bool { 562 c.authLock.Lock() 563 defer c.authLock.Unlock() 564 return c.authenticated() 565} 566 567// Internal version of Authenticated() 568// 569// Call with authLock held 570func (c *Connection) authenticated() bool { 571 return c.StorageUrl != "" && c.AuthToken != "" 572} 573 574// SwiftInfo contains the JSON object returned by Swift when the /info 575// route is queried. The object contains, among others, the Swift version, 576// the enabled middlewares and their configuration 577type SwiftInfo map[string]interface{} 578 579func (i SwiftInfo) SupportsBulkDelete() bool { 580 _, val := i["bulk_delete"] 581 return val 582} 583 584func (i SwiftInfo) SupportsSLO() bool { 585 _, val := i["slo"] 586 return val 587} 588 589func (i SwiftInfo) SLOMinSegmentSize() int64 { 590 if slo, ok := i["slo"].(map[string]interface{}); ok { 591 val, _ := slo["min_segment_size"].(float64) 592 return int64(val) 593 } 594 return 1 595} 596 597// Discover Swift configuration by doing a request against /info 598func (c *Connection) QueryInfo() (infos SwiftInfo, err error) { 599 infoUrl, err := url.Parse(c.StorageUrl) 600 if err != nil { 601 return nil, err 602 } 603 infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info") 604 resp, err := c.client.Get(infoUrl.String()) 605 if err == nil { 606 if resp.StatusCode != http.StatusOK { 607 drainAndClose(resp.Body, nil) 608 return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode) 609 } 610 err = readJson(resp, &infos) 611 if err == nil { 612 c.authLock.Lock() 613 c.swiftInfo = infos 614 c.authLock.Unlock() 615 } 616 return infos, err 617 } 618 return nil, err 619} 620 621func (c *Connection) cachedQueryInfo() (infos SwiftInfo, err error) { 622 c.authLock.Lock() 623 infos = c.swiftInfo 624 c.authLock.Unlock() 625 if infos == nil { 626 infos, err = c.QueryInfo() 627 if err != nil { 628 return 629 } 630 } 631 return infos, nil 632} 633 634// RequestOpts contains parameters for Connection.storage. 635type RequestOpts struct { 636 Container string 637 ObjectName string 638 Operation string 639 Parameters url.Values 640 Headers Headers 641 ErrorMap errorMap 642 NoResponse bool 643 Body io.Reader 644 Retries int 645 // if set this is called on re-authentication to refresh the targetUrl 646 OnReAuth func() (string, error) 647} 648 649// Call runs a remote command on the targetUrl, returns a 650// response, headers and possible error. 651// 652// operation is GET, HEAD etc 653// container is the name of a container 654// Any other parameters (if not None) are added to the targetUrl 655// 656// Returns a response or an error. If response is returned then 657// the resp.Body must be read completely and 658// resp.Body.Close() must be called on it, unless noResponse is set in 659// which case the body will be closed in this function 660// 661// If "Content-Length" is set in p.Headers it will be used - this can 662// be used to override the default chunked transfer encoding for 663// uploads. 664// 665// This will Authenticate if necessary, and re-authenticate if it 666// receives a 401 error which means the token has expired 667// 668// This method is exported so extensions can call it. 669func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, headers Headers, err error) { 670 c.authLock.Lock() 671 c.setDefaults() 672 c.authLock.Unlock() 673 retries := p.Retries 674 if retries == 0 { 675 retries = c.Retries 676 } 677 var req *http.Request 678 for { 679 var authToken string 680 if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil { 681 return //authentication failure 682 } 683 var URL *url.URL 684 URL, err = url.Parse(targetUrl) 685 if err != nil { 686 return 687 } 688 if p.Container != "" { 689 URL.Path += "/" + p.Container 690 if p.ObjectName != "" { 691 URL.Path += "/" + p.ObjectName 692 } 693 } 694 if p.Parameters != nil { 695 URL.RawQuery = p.Parameters.Encode() 696 } 697 timer := time.NewTimer(c.ConnectTimeout) 698 defer timer.Stop() 699 reader := p.Body 700 if reader != nil { 701 reader = newWatchdogReader(reader, c.Timeout, timer) 702 } 703 req, err = http.NewRequest(p.Operation, URL.String(), reader) 704 if err != nil { 705 return 706 } 707 if p.Headers != nil { 708 for k, v := range p.Headers { 709 // Set ContentLength in req if the user passed it in in the headers 710 if k == "Content-Length" { 711 contentLength, err := strconv.ParseInt(v, 10, 64) 712 if err != nil { 713 return nil, nil, fmt.Errorf("Invalid %q header %q: %v", k, v, err) 714 } 715 req.ContentLength = contentLength 716 } else { 717 req.Header.Add(k, v) 718 } 719 } 720 } 721 req.Header.Add("User-Agent", c.UserAgent) 722 req.Header.Add("X-Auth-Token", authToken) 723 resp, err = c.doTimeoutRequest(timer, req) 724 if err != nil { 725 if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 { 726 retries-- 727 continue 728 } 729 return nil, nil, err 730 } 731 // Check to see if token has expired 732 if resp.StatusCode == 401 && retries > 0 { 733 drainAndClose(resp.Body, nil) 734 c.UnAuthenticate() 735 retries-- 736 } else { 737 break 738 } 739 } 740 741 if err = c.parseHeaders(resp, p.ErrorMap); err != nil { 742 return nil, nil, err 743 } 744 headers = readHeaders(resp) 745 if p.NoResponse { 746 var err error 747 drainAndClose(resp.Body, &err) 748 if err != nil { 749 return nil, nil, err 750 } 751 } else { 752 // Cancel the request on timeout 753 cancel := func() { 754 cancelRequest(c.Transport, req) 755 } 756 // Wrap resp.Body to make it obey an idle timeout 757 resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel) 758 } 759 return 760} 761 762// storage runs a remote command on a the storage url, returns a 763// response, headers and possible error. 764// 765// operation is GET, HEAD etc 766// container is the name of a container 767// Any other parameters (if not None) are added to the storage url 768// 769// Returns a response or an error. If response is returned then 770// resp.Body.Close() must be called on it, unless noResponse is set in 771// which case the body will be closed in this function 772// 773// This will Authenticate if necessary, and re-authenticate if it 774// receives a 401 error which means the token has expired 775func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Headers, err error) { 776 p.OnReAuth = func() (string, error) { 777 return c.StorageUrl, nil 778 } 779 c.authLock.Lock() 780 url := c.StorageUrl 781 c.authLock.Unlock() 782 return c.Call(url, p) 783} 784 785// readLines reads the response into an array of strings. 786// 787// Closes the response when done 788func readLines(resp *http.Response) (lines []string, err error) { 789 defer drainAndClose(resp.Body, &err) 790 reader := bufio.NewReader(resp.Body) 791 buffer := bytes.NewBuffer(make([]byte, 0, 128)) 792 var part []byte 793 var prefix bool 794 for { 795 if part, prefix, err = reader.ReadLine(); err != nil { 796 break 797 } 798 buffer.Write(part) 799 if !prefix { 800 lines = append(lines, buffer.String()) 801 buffer.Reset() 802 } 803 } 804 if err == io.EOF { 805 err = nil 806 } 807 return 808} 809 810// readJson reads the response into the json type passed in 811// 812// Closes the response when done 813func readJson(resp *http.Response, result interface{}) (err error) { 814 defer drainAndClose(resp.Body, &err) 815 decoder := json.NewDecoder(resp.Body) 816 return decoder.Decode(result) 817} 818 819/* ------------------------------------------------------------ */ 820 821// ContainersOpts is options for Containers() and ContainerNames() 822type ContainersOpts struct { 823 Limit int // For an integer value n, limits the number of results to at most n values. 824 Prefix string // Given a string value x, return container names matching the specified prefix. 825 Marker string // Given a string value x, return container names greater in value than the specified marker. 826 EndMarker string // Given a string value x, return container names less in value than the specified marker. 827 Headers Headers // Any additional HTTP headers - can be nil 828} 829 830// parse the ContainerOpts 831func (opts *ContainersOpts) parse() (url.Values, Headers) { 832 v := url.Values{} 833 var h Headers 834 if opts != nil { 835 if opts.Limit > 0 { 836 v.Set("limit", strconv.Itoa(opts.Limit)) 837 } 838 if opts.Prefix != "" { 839 v.Set("prefix", opts.Prefix) 840 } 841 if opts.Marker != "" { 842 v.Set("marker", opts.Marker) 843 } 844 if opts.EndMarker != "" { 845 v.Set("end_marker", opts.EndMarker) 846 } 847 h = opts.Headers 848 } 849 return v, h 850} 851 852// ContainerNames returns a slice of names of containers in this account. 853func (c *Connection) ContainerNames(opts *ContainersOpts) ([]string, error) { 854 v, h := opts.parse() 855 resp, _, err := c.storage(RequestOpts{ 856 Operation: "GET", 857 Parameters: v, 858 ErrorMap: ContainerErrorMap, 859 Headers: h, 860 }) 861 if err != nil { 862 return nil, err 863 } 864 lines, err := readLines(resp) 865 return lines, err 866} 867 868// Container contains information about a container 869type Container struct { 870 Name string // Name of the container 871 Count int64 // Number of objects in the container 872 Bytes int64 // Total number of bytes used in the container 873} 874 875// Containers returns a slice of structures with full information as 876// described in Container. 877func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) { 878 v, h := opts.parse() 879 v.Set("format", "json") 880 resp, _, err := c.storage(RequestOpts{ 881 Operation: "GET", 882 Parameters: v, 883 ErrorMap: ContainerErrorMap, 884 Headers: h, 885 }) 886 if err != nil { 887 return nil, err 888 } 889 var containers []Container 890 err = readJson(resp, &containers) 891 return containers, err 892} 893 894// containersAllOpts makes a copy of opts if set or makes a new one and 895// overrides Limit and Marker 896func containersAllOpts(opts *ContainersOpts) *ContainersOpts { 897 var newOpts ContainersOpts 898 if opts != nil { 899 newOpts = *opts 900 } 901 if newOpts.Limit == 0 { 902 newOpts.Limit = allContainersLimit 903 } 904 newOpts.Marker = "" 905 return &newOpts 906} 907 908// ContainersAll is like Containers but it returns all the Containers 909// 910// It calls Containers multiple times using the Marker parameter 911// 912// It has a default Limit parameter but you may pass in your own 913func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) { 914 opts = containersAllOpts(opts) 915 containers := make([]Container, 0) 916 for { 917 newContainers, err := c.Containers(opts) 918 if err != nil { 919 return nil, err 920 } 921 containers = append(containers, newContainers...) 922 if len(newContainers) < opts.Limit { 923 break 924 } 925 opts.Marker = newContainers[len(newContainers)-1].Name 926 } 927 return containers, nil 928} 929 930// ContainerNamesAll is like ContainerNamess but it returns all the Containers 931// 932// It calls ContainerNames multiple times using the Marker parameter 933// 934// It has a default Limit parameter but you may pass in your own 935func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) { 936 opts = containersAllOpts(opts) 937 containers := make([]string, 0) 938 for { 939 newContainers, err := c.ContainerNames(opts) 940 if err != nil { 941 return nil, err 942 } 943 containers = append(containers, newContainers...) 944 if len(newContainers) < opts.Limit { 945 break 946 } 947 opts.Marker = newContainers[len(newContainers)-1] 948 } 949 return containers, nil 950} 951 952/* ------------------------------------------------------------ */ 953 954// ObjectOpts is options for Objects() and ObjectNames() 955type ObjectsOpts struct { 956 Limit int // For an integer value n, limits the number of results to at most n values. 957 Marker string // Given a string value x, return object names greater in value than the specified marker. 958 EndMarker string // Given a string value x, return object names less in value than the specified marker 959 Prefix string // For a string value x, causes the results to be limited to object names beginning with the substring x. 960 Path string // For a string value x, return the object names nested in the pseudo path 961 Delimiter rune // For a character c, return all the object names nested in the container 962 Headers Headers // Any additional HTTP headers - can be nil 963} 964 965// parse reads values out of ObjectsOpts 966func (opts *ObjectsOpts) parse() (url.Values, Headers) { 967 v := url.Values{} 968 var h Headers 969 if opts != nil { 970 if opts.Limit > 0 { 971 v.Set("limit", strconv.Itoa(opts.Limit)) 972 } 973 if opts.Marker != "" { 974 v.Set("marker", opts.Marker) 975 } 976 if opts.EndMarker != "" { 977 v.Set("end_marker", opts.EndMarker) 978 } 979 if opts.Prefix != "" { 980 v.Set("prefix", opts.Prefix) 981 } 982 if opts.Path != "" { 983 v.Set("path", opts.Path) 984 } 985 if opts.Delimiter != 0 { 986 v.Set("delimiter", string(opts.Delimiter)) 987 } 988 h = opts.Headers 989 } 990 return v, h 991} 992 993// ObjectNames returns a slice of names of objects in a given container. 994func (c *Connection) ObjectNames(container string, opts *ObjectsOpts) ([]string, error) { 995 v, h := opts.parse() 996 resp, _, err := c.storage(RequestOpts{ 997 Container: container, 998 Operation: "GET", 999 Parameters: v, 1000 ErrorMap: ContainerErrorMap, 1001 Headers: h, 1002 }) 1003 if err != nil { 1004 return nil, err 1005 } 1006 return readLines(resp) 1007} 1008 1009// Object contains information about an object 1010type Object struct { 1011 Name string `json:"name"` // object name 1012 ContentType string `json:"content_type"` // eg application/directory 1013 Bytes int64 `json:"bytes"` // size in bytes 1014 ServerLastModified string `json:"last_modified"` // Last modified time, eg '2011-06-30T08:20:47.736680' as a string supplied by the server 1015 LastModified time.Time // Last modified time converted to a time.Time 1016 Hash string `json:"hash"` // MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" 1017 PseudoDirectory bool // Set when using delimiter to show that this directory object does not really exist 1018 SubDir string `json:"subdir"` // returned only when using delimiter to mark "pseudo directories" 1019 ObjectType ObjectType // type of this object 1020} 1021 1022// Objects returns a slice of Object with information about each 1023// object in the container. 1024// 1025// If Delimiter is set in the opts then PseudoDirectory may be set, 1026// with ContentType 'application/directory'. These are not real 1027// objects but represent directories of objects which haven't had an 1028// object created for them. 1029func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, error) { 1030 v, h := opts.parse() 1031 v.Set("format", "json") 1032 resp, _, err := c.storage(RequestOpts{ 1033 Container: container, 1034 Operation: "GET", 1035 Parameters: v, 1036 ErrorMap: ContainerErrorMap, 1037 Headers: h, 1038 }) 1039 if err != nil { 1040 return nil, err 1041 } 1042 var objects []Object 1043 err = readJson(resp, &objects) 1044 // Convert Pseudo directories and dates 1045 for i := range objects { 1046 object := &objects[i] 1047 if object.SubDir != "" { 1048 object.Name = object.SubDir 1049 object.PseudoDirectory = true 1050 object.ContentType = "application/directory" 1051 } 1052 if object.ServerLastModified != "" { 1053 // 2012-11-11T14:49:47.887250 1054 // 1055 // Remove fractional seconds if present. This 1056 // then keeps it consistent with Object 1057 // which can only return timestamps accurate 1058 // to 1 second 1059 // 1060 // The TimeFormat will parse fractional 1061 // seconds if desired though 1062 datetime := strings.SplitN(object.ServerLastModified, ".", 2)[0] 1063 object.LastModified, err = time.Parse(TimeFormat, datetime) 1064 if err != nil { 1065 return nil, err 1066 } 1067 } 1068 } 1069 return objects, err 1070} 1071 1072// objectsAllOpts makes a copy of opts if set or makes a new one and 1073// overrides Limit and Marker 1074func objectsAllOpts(opts *ObjectsOpts, Limit int) *ObjectsOpts { 1075 var newOpts ObjectsOpts 1076 if opts != nil { 1077 newOpts = *opts 1078 } 1079 if newOpts.Limit == 0 { 1080 newOpts.Limit = Limit 1081 } 1082 newOpts.Marker = "" 1083 return &newOpts 1084} 1085 1086// A closure defined by the caller to iterate through all objects 1087// 1088// Call Objects or ObjectNames from here with the *ObjectOpts passed in 1089// 1090// Do whatever is required with the results then return them 1091type ObjectsWalkFn func(*ObjectsOpts) (interface{}, error) 1092 1093// ObjectsWalk is uses to iterate through all the objects in chunks as 1094// returned by Objects or ObjectNames using the Marker and Limit 1095// parameters in the ObjectsOpts. 1096// 1097// Pass in a closure `walkFn` which calls Objects or ObjectNames with 1098// the *ObjectsOpts passed to it and does something with the results. 1099// 1100// Errors will be returned from this function 1101// 1102// It has a default Limit parameter but you may pass in your own 1103func (c *Connection) ObjectsWalk(container string, opts *ObjectsOpts, walkFn ObjectsWalkFn) error { 1104 opts = objectsAllOpts(opts, allObjectsChanLimit) 1105 for { 1106 objects, err := walkFn(opts) 1107 if err != nil { 1108 return err 1109 } 1110 var n int 1111 var last string 1112 switch objects := objects.(type) { 1113 case []string: 1114 n = len(objects) 1115 if n > 0 { 1116 last = objects[len(objects)-1] 1117 } 1118 case []Object: 1119 n = len(objects) 1120 if n > 0 { 1121 last = objects[len(objects)-1].Name 1122 } 1123 default: 1124 panic("Unknown type returned to ObjectsWalk") 1125 } 1126 if n < opts.Limit { 1127 break 1128 } 1129 opts.Marker = last 1130 } 1131 return nil 1132} 1133 1134// ObjectsAll is like Objects but it returns an unlimited number of Objects in a slice 1135// 1136// It calls Objects multiple times using the Marker parameter 1137func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) { 1138 objects := make([]Object, 0) 1139 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 1140 newObjects, err := c.Objects(container, opts) 1141 if err == nil { 1142 objects = append(objects, newObjects...) 1143 } 1144 return newObjects, err 1145 }) 1146 return objects, err 1147} 1148 1149// ObjectNamesAll is like ObjectNames but it returns all the Objects 1150// 1151// It calls ObjectNames multiple times using the Marker parameter 1152// 1153// It has a default Limit parameter but you may pass in your own 1154func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) { 1155 objects := make([]string, 0) 1156 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 1157 newObjects, err := c.ObjectNames(container, opts) 1158 if err == nil { 1159 objects = append(objects, newObjects...) 1160 } 1161 return newObjects, err 1162 }) 1163 return objects, err 1164} 1165 1166// Account contains information about this account. 1167type Account struct { 1168 BytesUsed int64 // total number of bytes used 1169 Containers int64 // total number of containers 1170 Objects int64 // total number of objects 1171} 1172 1173// getInt64FromHeader is a helper function to decode int64 from header. 1174func getInt64FromHeader(resp *http.Response, header string) (result int64, err error) { 1175 value := resp.Header.Get(header) 1176 result, err = strconv.ParseInt(value, 10, 64) 1177 if err != nil { 1178 err = newErrorf(0, "Bad Header '%s': '%s': %s", header, value, err) 1179 } 1180 return 1181} 1182 1183// Account returns info about the account in an Account struct. 1184func (c *Connection) Account() (info Account, headers Headers, err error) { 1185 var resp *http.Response 1186 resp, headers, err = c.storage(RequestOpts{ 1187 Operation: "HEAD", 1188 ErrorMap: ContainerErrorMap, 1189 NoResponse: true, 1190 }) 1191 if err != nil { 1192 return 1193 } 1194 // Parse the headers into a dict 1195 // 1196 // {'Accept-Ranges': 'bytes', 1197 // 'Content-Length': '0', 1198 // 'Date': 'Tue, 05 Jul 2011 16:37:06 GMT', 1199 // 'X-Account-Bytes-Used': '316598182', 1200 // 'X-Account-Container-Count': '4', 1201 // 'X-Account-Object-Count': '1433'} 1202 if info.BytesUsed, err = getInt64FromHeader(resp, "X-Account-Bytes-Used"); err != nil { 1203 return 1204 } 1205 if info.Containers, err = getInt64FromHeader(resp, "X-Account-Container-Count"); err != nil { 1206 return 1207 } 1208 if info.Objects, err = getInt64FromHeader(resp, "X-Account-Object-Count"); err != nil { 1209 return 1210 } 1211 return 1212} 1213 1214// AccountUpdate adds, replaces or remove account metadata. 1215// 1216// Add or update keys by mentioning them in the Headers. 1217// 1218// Remove keys by setting them to an empty string. 1219func (c *Connection) AccountUpdate(h Headers) error { 1220 _, _, err := c.storage(RequestOpts{ 1221 Operation: "POST", 1222 ErrorMap: ContainerErrorMap, 1223 NoResponse: true, 1224 Headers: h, 1225 }) 1226 return err 1227} 1228 1229// ContainerCreate creates a container. 1230// 1231// If you don't want to add Headers just pass in nil 1232// 1233// No error is returned if it already exists but the metadata if any will be updated. 1234func (c *Connection) ContainerCreate(container string, h Headers) error { 1235 _, _, err := c.storage(RequestOpts{ 1236 Container: container, 1237 Operation: "PUT", 1238 ErrorMap: ContainerErrorMap, 1239 NoResponse: true, 1240 Headers: h, 1241 }) 1242 return err 1243} 1244 1245// ContainerDelete deletes a container. 1246// 1247// May return ContainerDoesNotExist or ContainerNotEmpty 1248func (c *Connection) ContainerDelete(container string) error { 1249 _, _, err := c.storage(RequestOpts{ 1250 Container: container, 1251 Operation: "DELETE", 1252 ErrorMap: ContainerErrorMap, 1253 NoResponse: true, 1254 }) 1255 return err 1256} 1257 1258// Container returns info about a single container including any 1259// metadata in the headers. 1260func (c *Connection) Container(container string) (info Container, headers Headers, err error) { 1261 var resp *http.Response 1262 resp, headers, err = c.storage(RequestOpts{ 1263 Container: container, 1264 Operation: "HEAD", 1265 ErrorMap: ContainerErrorMap, 1266 NoResponse: true, 1267 }) 1268 if err != nil { 1269 return 1270 } 1271 // Parse the headers into the struct 1272 info.Name = container 1273 if info.Bytes, err = getInt64FromHeader(resp, "X-Container-Bytes-Used"); err != nil { 1274 return 1275 } 1276 if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil { 1277 return 1278 } 1279 return 1280} 1281 1282// ContainerUpdate adds, replaces or removes container metadata. 1283// 1284// Add or update keys by mentioning them in the Metadata. 1285// 1286// Remove keys by setting them to an empty string. 1287// 1288// Container metadata can only be read with Container() not with Containers(). 1289func (c *Connection) ContainerUpdate(container string, h Headers) error { 1290 _, _, err := c.storage(RequestOpts{ 1291 Container: container, 1292 Operation: "POST", 1293 ErrorMap: ContainerErrorMap, 1294 NoResponse: true, 1295 Headers: h, 1296 }) 1297 return err 1298} 1299 1300// ------------------------------------------------------------ 1301 1302// ObjectCreateFile represents a swift object open for writing 1303type ObjectCreateFile struct { 1304 checkHash bool // whether we are checking the hash 1305 pipeReader *io.PipeReader // pipe for the caller to use 1306 pipeWriter *io.PipeWriter 1307 hash hash.Hash // hash being build up as we go along 1308 done chan struct{} // signals when the upload has finished 1309 resp *http.Response // valid when done has signalled 1310 err error // ditto 1311 headers Headers // ditto 1312} 1313 1314// Write bytes to the object - see io.Writer 1315func (file *ObjectCreateFile) Write(p []byte) (n int, err error) { 1316 n, err = file.pipeWriter.Write(p) 1317 if err == io.ErrClosedPipe { 1318 if file.err != nil { 1319 return 0, file.err 1320 } 1321 return 0, newError(500, "Write on closed file") 1322 } 1323 if err == nil && file.checkHash { 1324 _, _ = file.hash.Write(p) 1325 } 1326 return 1327} 1328 1329// Close the object and checks the md5sum if it was required. 1330// 1331// Also returns any other errors from the server (eg container not 1332// found) so it is very important to check the errors on this method. 1333func (file *ObjectCreateFile) Close() error { 1334 // Close the body 1335 err := file.pipeWriter.Close() 1336 if err != nil { 1337 return err 1338 } 1339 1340 // Wait for the HTTP operation to complete 1341 <-file.done 1342 1343 // Check errors 1344 if file.err != nil { 1345 return file.err 1346 } 1347 if file.checkHash { 1348 receivedMd5 := strings.ToLower(file.headers["Etag"]) 1349 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1350 if receivedMd5 != calculatedMd5 { 1351 return ObjectCorrupted 1352 } 1353 } 1354 return nil 1355} 1356 1357// Headers returns the response headers from the created object if the upload 1358// has been completed. The Close() method must be called on an ObjectCreateFile 1359// before this method. 1360func (file *ObjectCreateFile) Headers() (Headers, error) { 1361 // error out if upload is not complete. 1362 select { 1363 case <-file.done: 1364 default: 1365 return nil, fmt.Errorf("Cannot get metadata, object upload failed or has not yet completed.") 1366 } 1367 return file.headers, nil 1368} 1369 1370// Check it satisfies the interface 1371var _ io.WriteCloser = &ObjectCreateFile{} 1372 1373// objectPutHeaders create a set of headers for a PUT 1374// 1375// It guesses the contentType from the objectName if it isn't set 1376// 1377// checkHash may be changed 1378func objectPutHeaders(objectName string, checkHash *bool, Hash string, contentType string, h Headers) Headers { 1379 if contentType == "" { 1380 contentType = mime.TypeByExtension(path.Ext(objectName)) 1381 if contentType == "" { 1382 contentType = "application/octet-stream" 1383 } 1384 } 1385 // Meta stuff 1386 extraHeaders := map[string]string{ 1387 "Content-Type": contentType, 1388 } 1389 for key, value := range h { 1390 extraHeaders[key] = value 1391 } 1392 if Hash != "" { 1393 extraHeaders["Etag"] = Hash 1394 *checkHash = false // the server will do it 1395 } 1396 return extraHeaders 1397} 1398 1399// ObjectCreate creates or updates the object in the container. It 1400// returns an io.WriteCloser you should write the contents to. You 1401// MUST call Close() on it and you MUST check the error return from 1402// Close(). 1403// 1404// If checkHash is True then it will calculate the MD5 Hash of the 1405// file as it is being uploaded and check it against that returned 1406// from the server. If it is wrong then it will return 1407// ObjectCorrupted on Close() 1408// 1409// If you know the MD5 hash of the object ahead of time then set the 1410// Hash parameter and it will be sent to the server (as an Etag 1411// header) and the server will check the MD5 itself after the upload, 1412// and this will return ObjectCorrupted on Close() if it is incorrect. 1413// 1414// If you don't want any error protection (not recommended) then set 1415// checkHash to false and Hash to "". 1416// 1417// If contentType is set it will be used, otherwise one will be 1418// guessed from objectName using mime.TypeByExtension 1419func (c *Connection) ObjectCreate(container string, objectName string, checkHash bool, Hash string, contentType string, h Headers) (file *ObjectCreateFile, err error) { 1420 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1421 pipeReader, pipeWriter := io.Pipe() 1422 file = &ObjectCreateFile{ 1423 hash: md5.New(), 1424 checkHash: checkHash, 1425 pipeReader: pipeReader, 1426 pipeWriter: pipeWriter, 1427 done: make(chan struct{}), 1428 } 1429 // Run the PUT in the background piping it data 1430 go func() { 1431 opts := RequestOpts{ 1432 Container: container, 1433 ObjectName: objectName, 1434 Operation: "PUT", 1435 Headers: extraHeaders, 1436 Body: pipeReader, 1437 NoResponse: true, 1438 ErrorMap: objectErrorMap, 1439 } 1440 file.resp, file.headers, file.err = c.storage(opts) 1441 // Signal finished 1442 pipeReader.Close() 1443 close(file.done) 1444 }() 1445 return 1446} 1447 1448func (c *Connection) objectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers, parameters url.Values) (headers Headers, err error) { 1449 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1450 hash := md5.New() 1451 var body io.Reader = contents 1452 if checkHash { 1453 body = io.TeeReader(contents, hash) 1454 } 1455 _, headers, err = c.storage(RequestOpts{ 1456 Container: container, 1457 ObjectName: objectName, 1458 Operation: "PUT", 1459 Headers: extraHeaders, 1460 Body: body, 1461 NoResponse: true, 1462 ErrorMap: objectErrorMap, 1463 Parameters: parameters, 1464 }) 1465 if err != nil { 1466 return 1467 } 1468 if checkHash { 1469 receivedMd5 := strings.ToLower(headers["Etag"]) 1470 calculatedMd5 := fmt.Sprintf("%x", hash.Sum(nil)) 1471 if receivedMd5 != calculatedMd5 { 1472 err = ObjectCorrupted 1473 return 1474 } 1475 } 1476 return 1477} 1478 1479// ObjectPut creates or updates the path in the container from 1480// contents. contents should be an open io.Reader which will have all 1481// its contents read. 1482// 1483// This is a low level interface. 1484// 1485// If checkHash is True then it will calculate the MD5 Hash of the 1486// file as it is being uploaded and check it against that returned 1487// from the server. If it is wrong then it will return 1488// ObjectCorrupted. 1489// 1490// If you know the MD5 hash of the object ahead of time then set the 1491// Hash parameter and it will be sent to the server (as an Etag 1492// header) and the server will check the MD5 itself after the upload, 1493// and this will return ObjectCorrupted if it is incorrect. 1494// 1495// If you don't want any error protection (not recommended) then set 1496// checkHash to false and Hash to "". 1497// 1498// If contentType is set it will be used, otherwise one will be 1499// guessed from objectName using mime.TypeByExtension 1500func (c *Connection) ObjectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers) (headers Headers, err error) { 1501 return c.objectPut(container, objectName, contents, checkHash, Hash, contentType, h, nil) 1502} 1503 1504// ObjectPutBytes creates an object from a []byte in a container. 1505// 1506// This is a simplified interface which checks the MD5. 1507func (c *Connection) ObjectPutBytes(container string, objectName string, contents []byte, contentType string) (err error) { 1508 buf := bytes.NewBuffer(contents) 1509 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1510 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1511 return 1512} 1513 1514// ObjectPutString creates an object from a string in a container. 1515// 1516// This is a simplified interface which checks the MD5 1517func (c *Connection) ObjectPutString(container string, objectName string, contents string, contentType string) (err error) { 1518 buf := strings.NewReader(contents) 1519 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1520 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1521 return 1522} 1523 1524// ObjectOpenFile represents a swift object open for reading 1525type ObjectOpenFile struct { 1526 connection *Connection // stored copy of Connection used in Open 1527 container string // stored copy of container used in Open 1528 objectName string // stored copy of objectName used in Open 1529 headers Headers // stored copy of headers used in Open 1530 resp *http.Response // http connection 1531 body io.Reader // read data from this 1532 checkHash bool // true if checking MD5 1533 hash hash.Hash // currently accumulating MD5 1534 bytes int64 // number of bytes read on this connection 1535 eof bool // whether we have read end of file 1536 pos int64 // current position when reading 1537 lengthOk bool // whether length is valid 1538 length int64 // length of the object if read 1539 seeked bool // whether we have seeked this file or not 1540 overSeeked bool // set if we have seeked to the end or beyond 1541} 1542 1543// Read bytes from the object - see io.Reader 1544func (file *ObjectOpenFile) Read(p []byte) (n int, err error) { 1545 if file.overSeeked { 1546 return 0, io.EOF 1547 } 1548 n, err = file.body.Read(p) 1549 file.bytes += int64(n) 1550 file.pos += int64(n) 1551 if err == io.EOF { 1552 file.eof = true 1553 } 1554 return 1555} 1556 1557// Seek sets the offset for the next Read to offset, interpreted 1558// according to whence: 0 means relative to the origin of the file, 1 1559// means relative to the current offset, and 2 means relative to the 1560// end. Seek returns the new offset and an Error, if any. 1561// 1562// Seek uses HTTP Range headers which, if the file pointer is moved, 1563// will involve reopening the HTTP connection. 1564// 1565// Note that you can't seek to the end of a file or beyond; HTTP Range 1566// requests don't support the file pointer being outside the data, 1567// unlike os.File 1568// 1569// Seek(0, 1) will return the current file pointer. 1570func (file *ObjectOpenFile) Seek(offset int64, whence int) (newPos int64, err error) { 1571 file.overSeeked = false 1572 switch whence { 1573 case 0: // relative to start 1574 newPos = offset 1575 case 1: // relative to current 1576 newPos = file.pos + offset 1577 case 2: // relative to end 1578 if !file.lengthOk { 1579 return file.pos, newError(0, "Length of file unknown so can't seek from end") 1580 } 1581 newPos = file.length + offset 1582 if offset >= 0 { 1583 file.overSeeked = true 1584 return 1585 } 1586 default: 1587 panic("Unknown whence in ObjectOpenFile.Seek") 1588 } 1589 // If at correct position (quite likely), do nothing 1590 if newPos == file.pos { 1591 return 1592 } 1593 // Close the file... 1594 file.seeked = true 1595 err = file.Close() 1596 if err != nil { 1597 return 1598 } 1599 // ...and re-open with a Range header 1600 if file.headers == nil { 1601 file.headers = Headers{} 1602 } 1603 if newPos > 0 { 1604 file.headers["Range"] = fmt.Sprintf("bytes=%d-", newPos) 1605 } else { 1606 delete(file.headers, "Range") 1607 } 1608 newFile, _, err := file.connection.ObjectOpen(file.container, file.objectName, false, file.headers) 1609 if err != nil { 1610 return 1611 } 1612 // Update the file 1613 file.resp = newFile.resp 1614 file.body = newFile.body 1615 file.checkHash = false 1616 file.pos = newPos 1617 return 1618} 1619 1620// Length gets the objects content length either from a cached copy or 1621// from the server. 1622func (file *ObjectOpenFile) Length() (int64, error) { 1623 if !file.lengthOk { 1624 info, _, err := file.connection.Object(file.container, file.objectName) 1625 file.length = info.Bytes 1626 file.lengthOk = (err == nil) 1627 return file.length, err 1628 } 1629 return file.length, nil 1630} 1631 1632// Close the object and checks the length and md5sum if it was 1633// required and all the object was read 1634func (file *ObjectOpenFile) Close() (err error) { 1635 // Close the body at the end 1636 defer checkClose(file.resp.Body, &err) 1637 1638 // If not end of file or seeked then can't check anything 1639 if !file.eof || file.seeked { 1640 return 1641 } 1642 1643 // Check the MD5 sum if requested 1644 if file.checkHash { 1645 receivedMd5 := strings.ToLower(file.resp.Header.Get("Etag")) 1646 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1647 if receivedMd5 != calculatedMd5 { 1648 err = ObjectCorrupted 1649 return 1650 } 1651 } 1652 1653 // Check to see we read the correct number of bytes 1654 if file.lengthOk && file.length != file.bytes { 1655 err = ObjectCorrupted 1656 return 1657 } 1658 return 1659} 1660 1661// Check it satisfies the interfaces 1662var _ io.ReadCloser = &ObjectOpenFile{} 1663var _ io.Seeker = &ObjectOpenFile{} 1664 1665func (c *Connection) objectOpenBase(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1666 var resp *http.Response 1667 opts := RequestOpts{ 1668 Container: container, 1669 ObjectName: objectName, 1670 Operation: "GET", 1671 ErrorMap: objectErrorMap, 1672 Headers: h, 1673 Parameters: parameters, 1674 } 1675 resp, headers, err = c.storage(opts) 1676 if err != nil { 1677 return 1678 } 1679 // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set 1680 if checkHash && headers.IsLargeObject() { 1681 // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) 1682 checkHash = false 1683 } 1684 file = &ObjectOpenFile{ 1685 connection: c, 1686 container: container, 1687 objectName: objectName, 1688 headers: h, 1689 resp: resp, 1690 checkHash: checkHash, 1691 body: resp.Body, 1692 } 1693 if checkHash { 1694 file.hash = md5.New() 1695 file.body = io.TeeReader(resp.Body, file.hash) 1696 } 1697 // Read Content-Length 1698 if resp.Header.Get("Content-Length") != "" { 1699 file.length, err = getInt64FromHeader(resp, "Content-Length") 1700 file.lengthOk = (err == nil) 1701 } 1702 return 1703} 1704 1705func (c *Connection) objectOpen(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1706 err = withLORetry(0, func() (Headers, int64, error) { 1707 file, headers, err = c.objectOpenBase(container, objectName, checkHash, h, parameters) 1708 if err != nil { 1709 return headers, 0, err 1710 } 1711 return headers, file.length, nil 1712 }) 1713 return 1714} 1715 1716// ObjectOpen returns an ObjectOpenFile for reading the contents of 1717// the object. This satisfies the io.ReadCloser and the io.Seeker 1718// interfaces. 1719// 1720// You must call Close() on contents when finished 1721// 1722// Returns the headers of the response. 1723// 1724// If checkHash is true then it will calculate the md5sum of the file 1725// as it is being received and check it against that returned from the 1726// server. If it is wrong then it will return ObjectCorrupted. It 1727// will also check the length returned. No checking will be done if 1728// you don't read all the contents. 1729// 1730// Note that objects with X-Object-Manifest or X-Static-Large-Object 1731// set won't ever have their md5sum's checked as the md5sum reported 1732// on the object is actually the md5sum of the md5sums of the 1733// parts. This isn't very helpful to detect a corrupted download as 1734// the size of the parts aren't known without doing more operations. 1735// If you want to ensure integrity of an object with a manifest then 1736// you will need to download everything in the manifest separately. 1737// 1738// headers["Content-Type"] will give the content type if desired. 1739func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { 1740 return c.objectOpen(container, objectName, checkHash, h, nil) 1741} 1742 1743// ObjectGet gets the object into the io.Writer contents. 1744// 1745// Returns the headers of the response. 1746// 1747// If checkHash is true then it will calculate the md5sum of the file 1748// as it is being received and check it against that returned from the 1749// server. If it is wrong then it will return ObjectCorrupted. 1750// 1751// headers["Content-Type"] will give the content type if desired. 1752func (c *Connection) ObjectGet(container string, objectName string, contents io.Writer, checkHash bool, h Headers) (headers Headers, err error) { 1753 file, headers, err := c.ObjectOpen(container, objectName, checkHash, h) 1754 if err != nil { 1755 return 1756 } 1757 defer checkClose(file, &err) 1758 _, err = io.Copy(contents, file) 1759 return 1760} 1761 1762// ObjectGetBytes returns an object as a []byte. 1763// 1764// This is a simplified interface which checks the MD5 1765func (c *Connection) ObjectGetBytes(container string, objectName string) (contents []byte, err error) { 1766 var buf bytes.Buffer 1767 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1768 contents = buf.Bytes() 1769 return 1770} 1771 1772// ObjectGetString returns an object as a string. 1773// 1774// This is a simplified interface which checks the MD5 1775func (c *Connection) ObjectGetString(container string, objectName string) (contents string, err error) { 1776 var buf bytes.Buffer 1777 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1778 contents = buf.String() 1779 return 1780} 1781 1782// ObjectDelete deletes the object. 1783// 1784// May return ObjectNotFound if the object isn't found 1785func (c *Connection) ObjectDelete(container string, objectName string) error { 1786 _, _, err := c.storage(RequestOpts{ 1787 Container: container, 1788 ObjectName: objectName, 1789 Operation: "DELETE", 1790 ErrorMap: objectErrorMap, 1791 }) 1792 return err 1793} 1794 1795// ObjectTempUrl returns a temporary URL for an object 1796func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string { 1797 mac := hmac.New(sha1.New, []byte(secretKey)) 1798 prefix, _ := url.Parse(c.StorageUrl) 1799 body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName) 1800 mac.Write([]byte(body)) 1801 sig := hex.EncodeToString(mac.Sum(nil)) 1802 return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix()) 1803} 1804 1805// parseResponseStatus parses string like "200 OK" and returns Error. 1806// 1807// For status codes beween 200 and 299, this returns nil. 1808func parseResponseStatus(resp string, errorMap errorMap) error { 1809 code := 0 1810 reason := resp 1811 t := strings.SplitN(resp, " ", 2) 1812 if len(t) == 2 { 1813 ncode, err := strconv.Atoi(t[0]) 1814 if err == nil { 1815 code = ncode 1816 reason = t[1] 1817 } 1818 } 1819 if errorMap != nil { 1820 if err, ok := errorMap[code]; ok { 1821 return err 1822 } 1823 } 1824 if 200 <= code && code <= 299 { 1825 return nil 1826 } 1827 return newError(code, reason) 1828} 1829 1830// BulkDeleteResult stores results of BulkDelete(). 1831// 1832// Individual errors may (or may not) be returned by Errors. 1833// Errors is a map whose keys are a full path of where the object was 1834// to be deleted, and whose values are Error objects. A full path of 1835// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 1836type BulkDeleteResult struct { 1837 NumberNotFound int64 // # of objects not found. 1838 NumberDeleted int64 // # of deleted objects. 1839 Errors map[string]error // Mapping between object name and an error. 1840 Headers Headers // Response HTTP headers. 1841} 1842 1843func (c *Connection) doBulkDelete(objects []string) (result BulkDeleteResult, err error) { 1844 var buffer bytes.Buffer 1845 for _, s := range objects { 1846 u := url.URL{Path: s} 1847 buffer.WriteString(u.String() + "\n") 1848 } 1849 resp, headers, err := c.storage(RequestOpts{ 1850 Operation: "DELETE", 1851 Parameters: url.Values{"bulk-delete": []string{"1"}}, 1852 Headers: Headers{ 1853 "Accept": "application/json", 1854 "Content-Type": "text/plain", 1855 "Content-Length": strconv.Itoa(buffer.Len()), 1856 }, 1857 ErrorMap: ContainerErrorMap, 1858 Body: &buffer, 1859 }) 1860 if err != nil { 1861 return 1862 } 1863 var jsonResult struct { 1864 NotFound int64 `json:"Number Not Found"` 1865 Status string `json:"Response Status"` 1866 Errors [][]string 1867 Deleted int64 `json:"Number Deleted"` 1868 } 1869 err = readJson(resp, &jsonResult) 1870 if err != nil { 1871 return 1872 } 1873 1874 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 1875 result.NumberNotFound = jsonResult.NotFound 1876 result.NumberDeleted = jsonResult.Deleted 1877 result.Headers = headers 1878 el := make(map[string]error, len(jsonResult.Errors)) 1879 for _, t := range jsonResult.Errors { 1880 if len(t) != 2 { 1881 continue 1882 } 1883 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 1884 } 1885 result.Errors = el 1886 return 1887} 1888 1889// BulkDelete deletes multiple objectNames from container in one operation. 1890// 1891// Some servers may not accept bulk-delete requests since bulk-delete is 1892// an optional feature of swift - these will return the Forbidden error. 1893// 1894// See also: 1895// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html 1896// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html 1897func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) { 1898 fullPaths := make([]string, len(objectNames)) 1899 for i, name := range objectNames { 1900 fullPaths[i] = fmt.Sprintf("/%s/%s", container, name) 1901 } 1902 return c.doBulkDelete(fullPaths) 1903} 1904 1905// BulkUploadResult stores results of BulkUpload(). 1906// 1907// Individual errors may (or may not) be returned by Errors. 1908// Errors is a map whose keys are a full path of where an object was 1909// to be created, and whose values are Error objects. A full path of 1910// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 1911type BulkUploadResult struct { 1912 NumberCreated int64 // # of created objects. 1913 Errors map[string]error // Mapping between object name and an error. 1914 Headers Headers // Response HTTP headers. 1915} 1916 1917// BulkUpload uploads multiple files in one operation. 1918// 1919// uploadPath can be empty, a container name, or a pseudo-directory 1920// within a container. If uploadPath is empty, new containers may be 1921// automatically created. 1922// 1923// Files are read from dataStream. The format of the stream is specified 1924// by the format parameter. Available formats are: 1925// * UploadTar - Plain tar stream. 1926// * UploadTarGzip - Gzip compressed tar stream. 1927// * UploadTarBzip2 - Bzip2 compressed tar stream. 1928// 1929// Some servers may not accept bulk-upload requests since bulk-upload is 1930// an optional feature of swift - these will return the Forbidden error. 1931// 1932// See also: 1933// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-extract-archive.html 1934// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Extract_Archive-d1e2338.html 1935func (c *Connection) BulkUpload(uploadPath string, dataStream io.Reader, format string, h Headers) (result BulkUploadResult, err error) { 1936 extraHeaders := Headers{"Accept": "application/json"} 1937 for key, value := range h { 1938 extraHeaders[key] = value 1939 } 1940 // The following code abuses Container parameter intentionally. 1941 // The best fix might be to rename Container to UploadPath. 1942 resp, headers, err := c.storage(RequestOpts{ 1943 Container: uploadPath, 1944 Operation: "PUT", 1945 Parameters: url.Values{"extract-archive": []string{format}}, 1946 Headers: extraHeaders, 1947 ErrorMap: ContainerErrorMap, 1948 Body: dataStream, 1949 }) 1950 if err != nil { 1951 return 1952 } 1953 // Detect old servers which don't support this feature 1954 if headers["Content-Type"] != "application/json" { 1955 err = Forbidden 1956 return 1957 } 1958 var jsonResult struct { 1959 Created int64 `json:"Number Files Created"` 1960 Status string `json:"Response Status"` 1961 Errors [][]string 1962 } 1963 err = readJson(resp, &jsonResult) 1964 if err != nil { 1965 return 1966 } 1967 1968 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 1969 result.NumberCreated = jsonResult.Created 1970 result.Headers = headers 1971 el := make(map[string]error, len(jsonResult.Errors)) 1972 for _, t := range jsonResult.Errors { 1973 if len(t) != 2 { 1974 continue 1975 } 1976 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 1977 } 1978 result.Errors = el 1979 return 1980} 1981 1982// Object returns info about a single object including any metadata in the header. 1983// 1984// May return ObjectNotFound. 1985// 1986// Use headers.ObjectMetadata() to read the metadata in the Headers. 1987func (c *Connection) Object(container string, objectName string) (info Object, headers Headers, err error) { 1988 err = withLORetry(0, func() (Headers, int64, error) { 1989 info, headers, err = c.objectBase(container, objectName) 1990 if err != nil { 1991 return headers, 0, err 1992 } 1993 return headers, info.Bytes, nil 1994 }) 1995 return 1996} 1997 1998func (c *Connection) objectBase(container string, objectName string) (info Object, headers Headers, err error) { 1999 var resp *http.Response 2000 resp, headers, err = c.storage(RequestOpts{ 2001 Container: container, 2002 ObjectName: objectName, 2003 Operation: "HEAD", 2004 ErrorMap: objectErrorMap, 2005 NoResponse: true, 2006 }) 2007 if err != nil { 2008 return 2009 } 2010 // Parse the headers into the struct 2011 // HTTP/1.1 200 OK 2012 // Date: Thu, 07 Jun 2010 20:59:39 GMT 2013 // Server: Apache 2014 // Last-Modified: Fri, 12 Jun 2010 13:40:18 GMT 2015 // ETag: 8a964ee2a5e88be344f36c22562a6486 2016 // Content-Length: 512000 2017 // Content-Type: text/plain; charset=UTF-8 2018 // X-Object-Meta-Meat: Bacon 2019 // X-Object-Meta-Fruit: Bacon 2020 // X-Object-Meta-Veggie: Bacon 2021 // X-Object-Meta-Dairy: Bacon 2022 info.Name = objectName 2023 info.ContentType = resp.Header.Get("Content-Type") 2024 if resp.Header.Get("Content-Length") != "" { 2025 if info.Bytes, err = getInt64FromHeader(resp, "Content-Length"); err != nil { 2026 return 2027 } 2028 } 2029 // Currently ceph doesn't return a Last-Modified header for DLO manifests without any segments 2030 // See ceph http://tracker.ceph.com/issues/15812 2031 if resp.Header.Get("Last-Modified") != "" { 2032 info.ServerLastModified = resp.Header.Get("Last-Modified") 2033 if info.LastModified, err = time.Parse(http.TimeFormat, info.ServerLastModified); err != nil { 2034 return 2035 } 2036 } 2037 2038 info.Hash = resp.Header.Get("Etag") 2039 if resp.Header.Get("X-Object-Manifest") != "" { 2040 info.ObjectType = DynamicLargeObjectType 2041 } else if resp.Header.Get("X-Static-Large-Object") != "" { 2042 info.ObjectType = StaticLargeObjectType 2043 } 2044 2045 return 2046} 2047 2048// ObjectUpdate adds, replaces or removes object metadata. 2049// 2050// Add or Update keys by mentioning them in the Metadata. Use 2051// Metadata.ObjectHeaders and Headers.ObjectMetadata to convert your 2052// Metadata to and from normal HTTP headers. 2053// 2054// This removes all metadata previously added to the object and 2055// replaces it with that passed in so to delete keys, just don't 2056// mention them the headers you pass in. 2057// 2058// Object metadata can only be read with Object() not with Objects(). 2059// 2060// This can also be used to set headers not already assigned such as 2061// X-Delete-At or X-Delete-After for expiring objects. 2062// 2063// You cannot use this to change any of the object's other headers 2064// such as Content-Type, ETag, etc. 2065// 2066// Refer to copying an object when you need to update metadata or 2067// other headers such as Content-Type or CORS headers. 2068// 2069// May return ObjectNotFound. 2070func (c *Connection) ObjectUpdate(container string, objectName string, h Headers) error { 2071 _, _, err := c.storage(RequestOpts{ 2072 Container: container, 2073 ObjectName: objectName, 2074 Operation: "POST", 2075 ErrorMap: objectErrorMap, 2076 NoResponse: true, 2077 Headers: h, 2078 }) 2079 return err 2080} 2081 2082// urlPathEscape escapes URL path the in string using URL escaping rules 2083// 2084// This mimics url.PathEscape which only available from go 1.8 2085func urlPathEscape(in string) string { 2086 var u url.URL 2087 u.Path = in 2088 return u.String() 2089} 2090 2091// ObjectCopy does a server side copy of an object to a new position 2092// 2093// All metadata is preserved. If metadata is set in the headers then 2094// it overrides the old metadata on the copied object. 2095// 2096// The destination container must exist before the copy. 2097// 2098// You can use this to copy an object to itself - this is the only way 2099// to update the content type of an object. 2100func (c *Connection) ObjectCopy(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string, h Headers) (headers Headers, err error) { 2101 // Meta stuff 2102 extraHeaders := map[string]string{ 2103 "Destination": urlPathEscape(dstContainer + "/" + dstObjectName), 2104 } 2105 for key, value := range h { 2106 extraHeaders[key] = value 2107 } 2108 _, headers, err = c.storage(RequestOpts{ 2109 Container: srcContainer, 2110 ObjectName: srcObjectName, 2111 Operation: "COPY", 2112 ErrorMap: objectErrorMap, 2113 NoResponse: true, 2114 Headers: extraHeaders, 2115 }) 2116 return 2117} 2118 2119// ObjectMove does a server side move of an object to a new position 2120// 2121// This is a convenience method which calls ObjectCopy then ObjectDelete 2122// 2123// All metadata is preserved. 2124// 2125// The destination container must exist before the copy. 2126func (c *Connection) ObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) (err error) { 2127 _, err = c.ObjectCopy(srcContainer, srcObjectName, dstContainer, dstObjectName, nil) 2128 if err != nil { 2129 return 2130 } 2131 return c.ObjectDelete(srcContainer, srcObjectName) 2132} 2133 2134// ObjectUpdateContentType updates the content type of an object 2135// 2136// This is a convenience method which calls ObjectCopy 2137// 2138// All other metadata is preserved. 2139func (c *Connection) ObjectUpdateContentType(container string, objectName string, contentType string) (err error) { 2140 h := Headers{"Content-Type": contentType} 2141 _, err = c.ObjectCopy(container, objectName, container, objectName, h) 2142 return 2143} 2144 2145// ------------------------------------------------------------ 2146 2147// VersionContainerCreate is a helper method for creating and enabling version controlled containers. 2148// 2149// It builds the current object container, the non-current object version container, and enables versioning. 2150// 2151// If the server doesn't support versioning then it will return 2152// Forbidden however it will have created both the containers at that point. 2153func (c *Connection) VersionContainerCreate(current, version string) error { 2154 if err := c.ContainerCreate(version, nil); err != nil { 2155 return err 2156 } 2157 if err := c.ContainerCreate(current, nil); err != nil { 2158 return err 2159 } 2160 if err := c.VersionEnable(current, version); err != nil { 2161 return err 2162 } 2163 return nil 2164} 2165 2166// VersionEnable enables versioning on the current container with version as the tracking container. 2167// 2168// May return Forbidden if this isn't supported by the server 2169func (c *Connection) VersionEnable(current, version string) error { 2170 h := Headers{"X-Versions-Location": version} 2171 if err := c.ContainerUpdate(current, h); err != nil { 2172 return err 2173 } 2174 // Check to see if the header was set properly 2175 _, headers, err := c.Container(current) 2176 if err != nil { 2177 return err 2178 } 2179 // If failed to set versions header, return Forbidden as the server doesn't support this 2180 if headers["X-Versions-Location"] != version { 2181 return Forbidden 2182 } 2183 return nil 2184} 2185 2186// VersionDisable disables versioning on the current container. 2187func (c *Connection) VersionDisable(current string) error { 2188 h := Headers{"X-Versions-Location": ""} 2189 if err := c.ContainerUpdate(current, h); err != nil { 2190 return err 2191 } 2192 return nil 2193} 2194 2195// VersionObjectList returns a list of older versions of the object. 2196// 2197// Objects are returned in the format <length><object_name>/<timestamp> 2198func (c *Connection) VersionObjectList(version, object string) ([]string, error) { 2199 opts := &ObjectsOpts{ 2200 // <3-character zero-padded hexadecimal character length><object name>/ 2201 Prefix: fmt.Sprintf("%03x", len(object)) + object + "/", 2202 } 2203 return c.ObjectNames(version, opts) 2204} 2205