1package swift 2 3import ( 4 "bufio" 5 "bytes" 6 "crypto/hmac" 7 "crypto/md5" 8 "crypto/sha1" 9 "encoding/hex" 10 "encoding/json" 11 "fmt" 12 "hash" 13 "io" 14 "io/ioutil" 15 "mime" 16 "net/http" 17 "net/url" 18 "os" 19 "path" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24) 25 26const ( 27 DefaultUserAgent = "goswift/1.0" // Default user agent 28 DefaultRetries = 3 // Default number of retries on token expiry 29 TimeFormat = "2006-01-02T15:04:05" // Python date format for json replies parsed as UTC 30 UploadTar = "tar" // Data format specifier for Connection.BulkUpload(). 31 UploadTarGzip = "tar.gz" // Data format specifier for Connection.BulkUpload(). 32 UploadTarBzip2 = "tar.bz2" // Data format specifier for Connection.BulkUpload(). 33 allContainersLimit = 10000 // Number of containers to fetch at once 34 allObjectsLimit = 10000 // Number objects to fetch at once 35 allObjectsChanLimit = 1000 // ...when fetching to a channel 36) 37 38// ObjectType is the type of the swift object, regular, static large, 39// or dynamic large. 40type ObjectType int 41 42// Values that ObjectType can take 43const ( 44 RegularObjectType ObjectType = iota 45 StaticLargeObjectType 46 DynamicLargeObjectType 47) 48 49// Connection holds the details of the connection to the swift server. 50// 51// You need to provide UserName, ApiKey and AuthUrl when you create a 52// connection then call Authenticate on it. 53// 54// The auth version in use will be detected from the AuthURL - you can 55// override this with the AuthVersion parameter. 56// 57// If using v2 auth you can also set Region in the Connection 58// structure. If you don't set Region you will get the default region 59// which may not be what you want. 60// 61// For reference some common AuthUrls looks like this: 62// 63// Rackspace US https://auth.api.rackspacecloud.com/v1.0 64// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0 65// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 66// Memset Memstore UK https://auth.storage.memset.com/v1.0 67// Memstore v2 https://auth.storage.memset.com/v2.0 68// 69// When using Google Appengine you must provide the Connection with an 70// appengine-specific Transport: 71// 72// import ( 73// "appengine/urlfetch" 74// "fmt" 75// "github.com/ncw/swift" 76// ) 77// 78// func handler(w http.ResponseWriter, r *http.Request) { 79// ctx := appengine.NewContext(r) 80// tr := urlfetch.Transport{Context: ctx} 81// c := swift.Connection{ 82// UserName: "user", 83// ApiKey: "key", 84// AuthUrl: "auth_url", 85// Transport: tr, 86// } 87// _ := c.Authenticate() 88// containers, _ := c.ContainerNames(nil) 89// fmt.Fprintf(w, "containers: %q", containers) 90// } 91// 92// If you don't supply a Transport, one is made which relies on 93// http.ProxyFromEnvironment (http://golang.org/pkg/net/http/#ProxyFromEnvironment). 94// This means that the connection will respect the HTTP proxy specified by the 95// environment variables $HTTP_PROXY and $NO_PROXY. 96type Connection struct { 97 // Parameters - fill these in before calling Authenticate 98 // They are all optional except UserName, ApiKey and AuthUrl 99 Domain string // User's domain name 100 DomainId string // User's domain Id 101 UserName string // UserName for api 102 UserId string // User Id 103 ApiKey string // Key for api access 104 ApplicationCredentialId string // Application Credential ID 105 ApplicationCredentialName string // Application Credential Name 106 ApplicationCredentialSecret string // Application Credential Secret 107 AuthUrl string // Auth URL 108 Retries int // Retries on error (default is 3) 109 UserAgent string // Http User agent (default goswift/1.0) 110 ConnectTimeout time.Duration // Connect channel timeout (default 10s) 111 Timeout time.Duration // Data channel timeout (default 60s) 112 Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only) 113 AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect 114 Internal bool // Set this to true to use the the internal / service network 115 Tenant string // Name of the tenant (v2,v3 auth only) 116 TenantId string // Id of the tenant (v2,v3 auth only) 117 EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set) 118 TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain 119 TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain 120 TrustId string // Id of the trust (v3 auth only) 121 Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) 122 // These are filled in after Authenticate is called as are the defaults for above 123 StorageUrl string 124 AuthToken string 125 Expires time.Time // time the token expires, may be Zero if unknown 126 client *http.Client 127 Auth Authenticator `json:"-" xml:"-"` // the current authenticator 128 authLock *sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth 129 // swiftInfo is filled after QueryInfo is called 130 swiftInfo SwiftInfo 131} 132 133// setFromEnv reads the value that param points to (it must be a 134// pointer), if it isn't the zero value then it reads the environment 135// variable name passed in, parses it according to the type and writes 136// it to the pointer. 137func setFromEnv(param interface{}, name string) (err error) { 138 val := os.Getenv(name) 139 if val == "" { 140 return 141 } 142 switch result := param.(type) { 143 case *string: 144 if *result == "" { 145 *result = val 146 } 147 case *int: 148 if *result == 0 { 149 *result, err = strconv.Atoi(val) 150 } 151 case *bool: 152 if *result == false { 153 *result, err = strconv.ParseBool(val) 154 } 155 case *time.Duration: 156 if *result == 0 { 157 *result, err = time.ParseDuration(val) 158 } 159 case *EndpointType: 160 if *result == EndpointType("") { 161 *result = EndpointType(val) 162 } 163 default: 164 return newErrorf(0, "can't set var of type %T", param) 165 } 166 return err 167} 168 169// ApplyEnvironment reads environment variables and applies them to 170// the Connection structure. It won't overwrite any parameters which 171// are already set in the Connection struct. 172// 173// To make a new Connection object entirely from the environment you 174// would do: 175// 176// c := new(Connection) 177// err := c.ApplyEnvironment() 178// if err != nil { log.Fatal(err) } 179// 180// The naming of these variables follows the official Openstack naming 181// scheme so it should be compatible with OpenStack rc files. 182// 183// For v1 authentication (obsolete) 184// ST_AUTH - Auth URL 185// ST_USER - UserName for api 186// ST_KEY - Key for api access 187// 188// For v2 authentication 189// OS_AUTH_URL - Auth URL 190// OS_USERNAME - UserName for api 191// OS_PASSWORD - Key for api access 192// OS_TENANT_NAME - Name of the tenant 193// OS_TENANT_ID - Id of the tenant 194// OS_REGION_NAME - Region to use - default is use first region 195// 196// For v3 authentication 197// OS_AUTH_URL - Auth URL 198// OS_USERNAME - UserName for api 199// OS_USER_ID - User Id 200// OS_PASSWORD - Key for api access 201// OS_APPLICATION_CREDENTIAL_ID - Application Credential ID 202// OS_APPLICATION_CREDENTIAL_NAME - Application Credential Name 203// OS_APPLICATION_CREDENTIAL_SECRET - Application Credential Secret 204// OS_USER_DOMAIN_NAME - User's domain name 205// OS_USER_DOMAIN_ID - User's domain Id 206// OS_PROJECT_NAME - Name of the project 207// OS_PROJECT_DOMAIN_NAME - Name of the tenant's domain, only needed if it differs from the user domain 208// OS_PROJECT_DOMAIN_ID - Id of the tenant's domain, only needed if it differs the from user domain 209// OS_TRUST_ID - If of the trust 210// OS_REGION_NAME - Region to use - default is use first region 211// 212// Other 213// OS_ENDPOINT_TYPE - Endpoint type public, internal or admin 214// ST_AUTH_VERSION - Choose auth version - 1, 2 or 3 or leave at 0 for autodetect 215// 216// For manual authentication 217// OS_STORAGE_URL - storage URL from alternate authentication 218// OS_AUTH_TOKEN - Auth Token from alternate authentication 219// 220// Library specific 221// GOSWIFT_RETRIES - Retries on error (default is 3) 222// GOSWIFT_USER_AGENT - HTTP User agent (default goswift/1.0) 223// GOSWIFT_CONNECT_TIMEOUT - Connect channel timeout with unit, eg "10s", "100ms" (default "10s") 224// GOSWIFT_TIMEOUT - Data channel timeout with unit, eg "10s", "100ms" (default "60s") 225// GOSWIFT_INTERNAL - Set this to "true" to use the the internal network (obsolete - use OS_ENDPOINT_TYPE) 226func (c *Connection) ApplyEnvironment() (err error) { 227 for _, item := range []struct { 228 result interface{} 229 name string 230 }{ 231 // Environment variables - keep in same order as Connection 232 {&c.Domain, "OS_USER_DOMAIN_NAME"}, 233 {&c.DomainId, "OS_USER_DOMAIN_ID"}, 234 {&c.UserName, "OS_USERNAME"}, 235 {&c.UserId, "OS_USER_ID"}, 236 {&c.ApiKey, "OS_PASSWORD"}, 237 {&c.ApplicationCredentialId, "OS_APPLICATION_CREDENTIAL_ID"}, 238 {&c.ApplicationCredentialName, "OS_APPLICATION_CREDENTIAL_NAME"}, 239 {&c.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET"}, 240 {&c.AuthUrl, "OS_AUTH_URL"}, 241 {&c.Retries, "GOSWIFT_RETRIES"}, 242 {&c.UserAgent, "GOSWIFT_USER_AGENT"}, 243 {&c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT"}, 244 {&c.Timeout, "GOSWIFT_TIMEOUT"}, 245 {&c.Region, "OS_REGION_NAME"}, 246 {&c.AuthVersion, "ST_AUTH_VERSION"}, 247 {&c.Internal, "GOSWIFT_INTERNAL"}, 248 {&c.Tenant, "OS_TENANT_NAME"}, //v2 249 {&c.Tenant, "OS_PROJECT_NAME"}, // v3 250 {&c.TenantId, "OS_TENANT_ID"}, 251 {&c.EndpointType, "OS_ENDPOINT_TYPE"}, 252 {&c.TenantDomain, "OS_PROJECT_DOMAIN_NAME"}, 253 {&c.TenantDomainId, "OS_PROJECT_DOMAIN_ID"}, 254 {&c.TrustId, "OS_TRUST_ID"}, 255 {&c.StorageUrl, "OS_STORAGE_URL"}, 256 {&c.AuthToken, "OS_AUTH_TOKEN"}, 257 // v1 auth alternatives 258 {&c.ApiKey, "ST_KEY"}, 259 {&c.UserName, "ST_USER"}, 260 {&c.AuthUrl, "ST_AUTH"}, 261 } { 262 err = setFromEnv(item.result, item.name) 263 if err != nil { 264 return newErrorf(0, "failed to read env var %q: %v", item.name, err) 265 } 266 } 267 return nil 268} 269 270// Error - all errors generated by this package are of this type. Other error 271// may be passed on from library functions though. 272type Error struct { 273 StatusCode int // HTTP status code if relevant or 0 if not 274 Text string 275} 276 277// Error satisfy the error interface. 278func (e *Error) Error() string { 279 return e.Text 280} 281 282// newError make a new error from a string. 283func newError(StatusCode int, Text string) *Error { 284 return &Error{ 285 StatusCode: StatusCode, 286 Text: Text, 287 } 288} 289 290// newErrorf makes a new error from sprintf parameters. 291func newErrorf(StatusCode int, Text string, Parameters ...interface{}) *Error { 292 return newError(StatusCode, fmt.Sprintf(Text, Parameters...)) 293} 294 295// errorMap defines http error codes to error mappings. 296type errorMap map[int]error 297 298var ( 299 // Specific Errors you might want to check for equality 300 NotModified = newError(304, "Not Modified") 301 BadRequest = newError(400, "Bad Request") 302 AuthorizationFailed = newError(401, "Authorization Failed") 303 ContainerNotFound = newError(404, "Container Not Found") 304 ContainerNotEmpty = newError(409, "Container Not Empty") 305 ObjectNotFound = newError(404, "Object Not Found") 306 ObjectCorrupted = newError(422, "Object Corrupted") 307 TimeoutError = newError(408, "Timeout when reading or writing data") 308 Forbidden = newError(403, "Operation forbidden") 309 TooLargeObject = newError(413, "Too Large Object") 310 RateLimit = newError(498, "Rate Limit") 311 TooManyRequests = newError(429, "TooManyRequests") 312 313 // Mappings for authentication errors 314 authErrorMap = errorMap{ 315 400: BadRequest, 316 401: AuthorizationFailed, 317 403: Forbidden, 318 } 319 320 // Mappings for container errors 321 ContainerErrorMap = errorMap{ 322 400: BadRequest, 323 403: Forbidden, 324 404: ContainerNotFound, 325 409: ContainerNotEmpty, 326 498: RateLimit, 327 } 328 329 // Mappings for object errors 330 objectErrorMap = errorMap{ 331 304: NotModified, 332 400: BadRequest, 333 403: Forbidden, 334 404: ObjectNotFound, 335 413: TooLargeObject, 336 422: ObjectCorrupted, 337 429: TooManyRequests, 338 498: RateLimit, 339 } 340) 341 342// checkClose is used to check the return from Close in a defer 343// statement. 344func checkClose(c io.Closer, err *error) { 345 cerr := c.Close() 346 if *err == nil { 347 *err = cerr 348 } 349} 350 351// drainAndClose discards all data from rd and closes it. 352// If an error occurs during Read, it is discarded. 353func drainAndClose(rd io.ReadCloser, err *error) { 354 if rd == nil { 355 return 356 } 357 358 _, _ = io.Copy(ioutil.Discard, rd) 359 cerr := rd.Close() 360 if err != nil && *err == nil { 361 *err = cerr 362 } 363} 364 365// parseHeaders checks a response for errors and translates into 366// standard errors if necessary. If an error is returned, resp.Body 367// has been drained and closed. 368func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error { 369 if errorMap != nil { 370 if err, ok := errorMap[resp.StatusCode]; ok { 371 drainAndClose(resp.Body, nil) 372 return err 373 } 374 } 375 if resp.StatusCode < 200 || resp.StatusCode > 299 { 376 drainAndClose(resp.Body, nil) 377 return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status) 378 } 379 return nil 380} 381 382// readHeaders returns a Headers object from the http.Response. 383// 384// If it receives multiple values for a key (which should never 385// happen) it will use the first one 386func readHeaders(resp *http.Response) Headers { 387 headers := Headers{} 388 for key, values := range resp.Header { 389 headers[key] = values[0] 390 } 391 return headers 392} 393 394// Headers stores HTTP headers (can only have one of each header like Swift). 395type Headers map[string]string 396 397// Does an http request using the running timer passed in 398func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*http.Response, error) { 399 // Do the request in the background so we can check the timeout 400 type result struct { 401 resp *http.Response 402 err error 403 } 404 done := make(chan result, 1) 405 go func() { 406 resp, err := c.client.Do(req) 407 done <- result{resp, err} 408 }() 409 // Wait for the read or the timeout 410 select { 411 case r := <-done: 412 return r.resp, r.err 413 case <-timer.C: 414 // Kill the connection on timeout so we don't leak sockets or goroutines 415 cancelRequest(c.Transport, req) 416 return nil, TimeoutError 417 } 418 panic("unreachable") // For Go 1.0 419} 420 421// Set defaults for any unset values 422// 423// Call with authLock held 424func (c *Connection) setDefaults() { 425 if c.UserAgent == "" { 426 c.UserAgent = DefaultUserAgent 427 } 428 if c.Retries == 0 { 429 c.Retries = DefaultRetries 430 } 431 if c.ConnectTimeout == 0 { 432 c.ConnectTimeout = 10 * time.Second 433 } 434 if c.Timeout == 0 { 435 c.Timeout = 60 * time.Second 436 } 437 if c.Transport == nil { 438 t := &http.Transport{ 439 // TLSClientConfig: &tls.Config{RootCAs: pool}, 440 // DisableCompression: true, 441 Proxy: http.ProxyFromEnvironment, 442 // Half of linux's default open files limit (1024). 443 MaxIdleConnsPerHost: 512, 444 } 445 SetExpectContinueTimeout(t, 5*time.Second) 446 c.Transport = t 447 } 448 if c.client == nil { 449 c.client = &http.Client{ 450 // CheckRedirect: redirectPolicyFunc, 451 Transport: c.Transport, 452 } 453 } 454} 455 456// Authenticate connects to the Swift server. 457// 458// If you don't call it before calling one of the connection methods 459// then it will be called for you on the first access. 460func (c *Connection) Authenticate() (err error) { 461 if c.authLock == nil { 462 c.authLock = &sync.Mutex{} 463 } 464 c.authLock.Lock() 465 defer c.authLock.Unlock() 466 return c.authenticate() 467} 468 469// Internal implementation of Authenticate 470// 471// Call with authLock held 472func (c *Connection) authenticate() (err error) { 473 c.setDefaults() 474 475 // Flush the keepalives connection - if we are 476 // re-authenticating then stuff has gone wrong 477 flushKeepaliveConnections(c.Transport) 478 479 if c.Auth == nil { 480 c.Auth, err = newAuth(c) 481 if err != nil { 482 return 483 } 484 } 485 486 retries := 1 487again: 488 var req *http.Request 489 req, err = c.Auth.Request(c) 490 if err != nil { 491 return 492 } 493 if req != nil { 494 timer := time.NewTimer(c.ConnectTimeout) 495 defer timer.Stop() 496 var resp *http.Response 497 resp, err = c.doTimeoutRequest(timer, req) 498 if err != nil { 499 return 500 } 501 defer func() { 502 drainAndClose(resp.Body, &err) 503 // Flush the auth connection - we don't want to keep 504 // it open if keepalives were enabled 505 flushKeepaliveConnections(c.Transport) 506 }() 507 if err = c.parseHeaders(resp, authErrorMap); err != nil { 508 // Try again for a limited number of times on 509 // AuthorizationFailed or BadRequest. This allows us 510 // to try some alternate forms of the request 511 if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { 512 retries-- 513 goto again 514 } 515 return 516 } 517 err = c.Auth.Response(resp) 518 if err != nil { 519 return 520 } 521 } 522 if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" { 523 c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType) 524 } else { 525 c.StorageUrl = c.Auth.StorageUrl(c.Internal) 526 } 527 c.AuthToken = c.Auth.Token() 528 if do, ok := c.Auth.(Expireser); ok { 529 c.Expires = do.Expires() 530 } else { 531 c.Expires = time.Time{} 532 } 533 534 if !c.authenticated() { 535 err = newError(0, "Response didn't have storage url and auth token") 536 return 537 } 538 return 539} 540 541// Get an authToken and url 542// 543// The Url may be updated if it needed to authenticate using the OnReAuth function 544func (c *Connection) getUrlAndAuthToken(targetUrlIn string, OnReAuth func() (string, error)) (targetUrlOut, authToken string, err error) { 545 c.authLock.Lock() 546 defer c.authLock.Unlock() 547 targetUrlOut = targetUrlIn 548 if !c.authenticated() { 549 err = c.authenticate() 550 if err != nil { 551 return 552 } 553 if OnReAuth != nil { 554 targetUrlOut, err = OnReAuth() 555 if err != nil { 556 return 557 } 558 } 559 } 560 authToken = c.AuthToken 561 return 562} 563 564// flushKeepaliveConnections is called to flush pending requests after an error. 565func flushKeepaliveConnections(transport http.RoundTripper) { 566 if tr, ok := transport.(interface { 567 CloseIdleConnections() 568 }); ok { 569 tr.CloseIdleConnections() 570 } 571} 572 573// UnAuthenticate removes the authentication from the Connection. 574func (c *Connection) UnAuthenticate() { 575 c.authLock.Lock() 576 c.StorageUrl = "" 577 c.AuthToken = "" 578 c.authLock.Unlock() 579} 580 581// Authenticated returns a boolean to show if the current connection 582// is authenticated. 583// 584// Doesn't actually check the credentials against the server. 585func (c *Connection) Authenticated() bool { 586 if c.authLock == nil { 587 c.authLock = &sync.Mutex{} 588 } 589 c.authLock.Lock() 590 defer c.authLock.Unlock() 591 return c.authenticated() 592} 593 594// Internal version of Authenticated() 595// 596// Call with authLock held 597func (c *Connection) authenticated() bool { 598 if c.StorageUrl == "" || c.AuthToken == "" { 599 return false 600 } 601 if c.Expires.IsZero() { 602 return true 603 } 604 timeUntilExpiry := c.Expires.Sub(time.Now()) 605 return timeUntilExpiry >= 60*time.Second 606} 607 608// SwiftInfo contains the JSON object returned by Swift when the /info 609// route is queried. The object contains, among others, the Swift version, 610// the enabled middlewares and their configuration 611type SwiftInfo map[string]interface{} 612 613func (i SwiftInfo) SupportsBulkDelete() bool { 614 _, val := i["bulk_delete"] 615 return val 616} 617 618func (i SwiftInfo) SupportsSLO() bool { 619 _, val := i["slo"] 620 return val 621} 622 623func (i SwiftInfo) SLOMinSegmentSize() int64 { 624 if slo, ok := i["slo"].(map[string]interface{}); ok { 625 val, _ := slo["min_segment_size"].(float64) 626 return int64(val) 627 } 628 return 1 629} 630 631// Discover Swift configuration by doing a request against /info 632func (c *Connection) QueryInfo() (infos SwiftInfo, err error) { 633 infoUrl, err := url.Parse(c.StorageUrl) 634 if err != nil { 635 return nil, err 636 } 637 infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info") 638 resp, err := c.client.Get(infoUrl.String()) 639 if err == nil { 640 if resp.StatusCode != http.StatusOK { 641 drainAndClose(resp.Body, nil) 642 return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode) 643 } 644 err = readJson(resp, &infos) 645 if err == nil { 646 c.authLock.Lock() 647 c.swiftInfo = infos 648 c.authLock.Unlock() 649 } 650 return infos, err 651 } 652 return nil, err 653} 654 655func (c *Connection) cachedQueryInfo() (infos SwiftInfo, err error) { 656 c.authLock.Lock() 657 infos = c.swiftInfo 658 c.authLock.Unlock() 659 if infos == nil { 660 infos, err = c.QueryInfo() 661 if err != nil { 662 return 663 } 664 } 665 return infos, nil 666} 667 668// RequestOpts contains parameters for Connection.storage. 669type RequestOpts struct { 670 Container string 671 ObjectName string 672 Operation string 673 Parameters url.Values 674 Headers Headers 675 ErrorMap errorMap 676 NoResponse bool 677 Body io.Reader 678 Retries int 679 // if set this is called on re-authentication to refresh the targetUrl 680 OnReAuth func() (string, error) 681} 682 683// Call runs a remote command on the targetUrl, returns a 684// response, headers and possible error. 685// 686// operation is GET, HEAD etc 687// container is the name of a container 688// Any other parameters (if not None) are added to the targetUrl 689// 690// Returns a response or an error. If response is returned then 691// the resp.Body must be read completely and 692// resp.Body.Close() must be called on it, unless noResponse is set in 693// which case the body will be closed in this function 694// 695// If "Content-Length" is set in p.Headers it will be used - this can 696// be used to override the default chunked transfer encoding for 697// uploads. 698// 699// This will Authenticate if necessary, and re-authenticate if it 700// receives a 401 error which means the token has expired 701// 702// This method is exported so extensions can call it. 703func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, headers Headers, err error) { 704 c.authLock.Lock() 705 c.setDefaults() 706 c.authLock.Unlock() 707 retries := p.Retries 708 if retries == 0 { 709 retries = c.Retries 710 } 711 var req *http.Request 712 for { 713 var authToken string 714 if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil { 715 return //authentication failure 716 } 717 var URL *url.URL 718 URL, err = url.Parse(targetUrl) 719 if err != nil { 720 return 721 } 722 if p.Container != "" { 723 URL.Path += "/" + p.Container 724 if p.ObjectName != "" { 725 URL.Path += "/" + p.ObjectName 726 } 727 } 728 if p.Parameters != nil { 729 URL.RawQuery = p.Parameters.Encode() 730 } 731 timer := time.NewTimer(c.ConnectTimeout) 732 defer timer.Stop() 733 reader := p.Body 734 if reader != nil { 735 reader = newWatchdogReader(reader, c.Timeout, timer) 736 } 737 req, err = http.NewRequest(p.Operation, URL.String(), reader) 738 if err != nil { 739 return 740 } 741 if p.Headers != nil { 742 for k, v := range p.Headers { 743 // Set ContentLength in req if the user passed it in in the headers 744 if k == "Content-Length" { 745 req.ContentLength, err = strconv.ParseInt(v, 10, 64) 746 if err != nil { 747 err = fmt.Errorf("Invalid %q header %q: %v", k, v, err) 748 return 749 } 750 } else { 751 req.Header.Add(k, v) 752 } 753 } 754 } 755 req.Header.Add("User-Agent", c.UserAgent) 756 req.Header.Add("X-Auth-Token", authToken) 757 758 _, hasCL := p.Headers["Content-Length"] 759 AddExpectAndTransferEncoding(req, hasCL) 760 761 resp, err = c.doTimeoutRequest(timer, req) 762 if err != nil { 763 if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 { 764 retries-- 765 continue 766 } 767 return 768 } 769 // Check to see if token has expired 770 if resp.StatusCode == 401 && retries > 0 { 771 drainAndClose(resp.Body, nil) 772 c.UnAuthenticate() 773 retries-- 774 } else { 775 break 776 } 777 } 778 779 headers = readHeaders(resp) 780 if err = c.parseHeaders(resp, p.ErrorMap); err != nil { 781 return 782 } 783 if p.NoResponse { 784 drainAndClose(resp.Body, &err) 785 if err != nil { 786 return 787 } 788 } else { 789 // Cancel the request on timeout 790 cancel := func() { 791 cancelRequest(c.Transport, req) 792 } 793 // Wrap resp.Body to make it obey an idle timeout 794 resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel) 795 } 796 return 797} 798 799// storage runs a remote command on a the storage url, returns a 800// response, headers and possible error. 801// 802// operation is GET, HEAD etc 803// container is the name of a container 804// Any other parameters (if not None) are added to the storage url 805// 806// Returns a response or an error. If response is returned then 807// resp.Body.Close() must be called on it, unless noResponse is set in 808// which case the body will be closed in this function 809// 810// This will Authenticate if necessary, and re-authenticate if it 811// receives a 401 error which means the token has expired 812func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Headers, err error) { 813 p.OnReAuth = func() (string, error) { 814 return c.StorageUrl, nil 815 } 816 c.authLock.Lock() 817 url := c.StorageUrl 818 c.authLock.Unlock() 819 return c.Call(url, p) 820} 821 822// readLines reads the response into an array of strings. 823// 824// Closes the response when done 825func readLines(resp *http.Response) (lines []string, err error) { 826 defer drainAndClose(resp.Body, &err) 827 reader := bufio.NewReader(resp.Body) 828 buffer := bytes.NewBuffer(make([]byte, 0, 128)) 829 var part []byte 830 var prefix bool 831 for { 832 if part, prefix, err = reader.ReadLine(); err != nil { 833 break 834 } 835 buffer.Write(part) 836 if !prefix { 837 lines = append(lines, buffer.String()) 838 buffer.Reset() 839 } 840 } 841 if err == io.EOF { 842 err = nil 843 } 844 return 845} 846 847// readJson reads the response into the json type passed in 848// 849// Closes the response when done 850func readJson(resp *http.Response, result interface{}) (err error) { 851 defer drainAndClose(resp.Body, &err) 852 decoder := json.NewDecoder(resp.Body) 853 return decoder.Decode(result) 854} 855 856/* ------------------------------------------------------------ */ 857 858// ContainersOpts is options for Containers() and ContainerNames() 859type ContainersOpts struct { 860 Limit int // For an integer value n, limits the number of results to at most n values. 861 Prefix string // Given a string value x, return container names matching the specified prefix. 862 Marker string // Given a string value x, return container names greater in value than the specified marker. 863 EndMarker string // Given a string value x, return container names less in value than the specified marker. 864 Headers Headers // Any additional HTTP headers - can be nil 865} 866 867// parse the ContainerOpts 868func (opts *ContainersOpts) parse() (url.Values, Headers) { 869 v := url.Values{} 870 var h Headers 871 if opts != nil { 872 if opts.Limit > 0 { 873 v.Set("limit", strconv.Itoa(opts.Limit)) 874 } 875 if opts.Prefix != "" { 876 v.Set("prefix", opts.Prefix) 877 } 878 if opts.Marker != "" { 879 v.Set("marker", opts.Marker) 880 } 881 if opts.EndMarker != "" { 882 v.Set("end_marker", opts.EndMarker) 883 } 884 h = opts.Headers 885 } 886 return v, h 887} 888 889// ContainerNames returns a slice of names of containers in this account. 890func (c *Connection) ContainerNames(opts *ContainersOpts) ([]string, error) { 891 v, h := opts.parse() 892 resp, _, err := c.storage(RequestOpts{ 893 Operation: "GET", 894 Parameters: v, 895 ErrorMap: ContainerErrorMap, 896 Headers: h, 897 }) 898 if err != nil { 899 return nil, err 900 } 901 lines, err := readLines(resp) 902 return lines, err 903} 904 905// Container contains information about a container 906type Container struct { 907 Name string // Name of the container 908 Count int64 // Number of objects in the container 909 Bytes int64 // Total number of bytes used in the container 910} 911 912// Containers returns a slice of structures with full information as 913// described in Container. 914func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) { 915 v, h := opts.parse() 916 v.Set("format", "json") 917 resp, _, err := c.storage(RequestOpts{ 918 Operation: "GET", 919 Parameters: v, 920 ErrorMap: ContainerErrorMap, 921 Headers: h, 922 }) 923 if err != nil { 924 return nil, err 925 } 926 var containers []Container 927 err = readJson(resp, &containers) 928 return containers, err 929} 930 931// containersAllOpts makes a copy of opts if set or makes a new one and 932// overrides Limit and Marker 933func containersAllOpts(opts *ContainersOpts) *ContainersOpts { 934 var newOpts ContainersOpts 935 if opts != nil { 936 newOpts = *opts 937 } 938 if newOpts.Limit == 0 { 939 newOpts.Limit = allContainersLimit 940 } 941 newOpts.Marker = "" 942 return &newOpts 943} 944 945// ContainersAll is like Containers but it returns all the Containers 946// 947// It calls Containers multiple times using the Marker parameter 948// 949// It has a default Limit parameter but you may pass in your own 950func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) { 951 opts = containersAllOpts(opts) 952 containers := make([]Container, 0) 953 for { 954 newContainers, err := c.Containers(opts) 955 if err != nil { 956 return nil, err 957 } 958 containers = append(containers, newContainers...) 959 if len(newContainers) < opts.Limit { 960 break 961 } 962 opts.Marker = newContainers[len(newContainers)-1].Name 963 } 964 return containers, nil 965} 966 967// ContainerNamesAll is like ContainerNames but it returns all the Containers 968// 969// It calls ContainerNames multiple times using the Marker parameter 970// 971// It has a default Limit parameter but you may pass in your own 972func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) { 973 opts = containersAllOpts(opts) 974 containers := make([]string, 0) 975 for { 976 newContainers, err := c.ContainerNames(opts) 977 if err != nil { 978 return nil, err 979 } 980 containers = append(containers, newContainers...) 981 if len(newContainers) < opts.Limit { 982 break 983 } 984 opts.Marker = newContainers[len(newContainers)-1] 985 } 986 return containers, nil 987} 988 989/* ------------------------------------------------------------ */ 990 991// ObjectOpts is options for Objects() and ObjectNames() 992type ObjectsOpts struct { 993 Limit int // For an integer value n, limits the number of results to at most n values. 994 Marker string // Given a string value x, return object names greater in value than the specified marker. 995 EndMarker string // Given a string value x, return object names less in value than the specified marker 996 Prefix string // For a string value x, causes the results to be limited to object names beginning with the substring x. 997 Path string // For a string value x, return the object names nested in the pseudo path 998 Delimiter rune // For a character c, return all the object names nested in the container 999 Headers Headers // Any additional HTTP headers - can be nil 1000 KeepMarker bool // Do not reset Marker when using ObjectsAll or ObjectNamesAll 1001} 1002 1003// parse reads values out of ObjectsOpts 1004func (opts *ObjectsOpts) parse() (url.Values, Headers) { 1005 v := url.Values{} 1006 var h Headers 1007 if opts != nil { 1008 if opts.Limit > 0 { 1009 v.Set("limit", strconv.Itoa(opts.Limit)) 1010 } 1011 if opts.Marker != "" { 1012 v.Set("marker", opts.Marker) 1013 } 1014 if opts.EndMarker != "" { 1015 v.Set("end_marker", opts.EndMarker) 1016 } 1017 if opts.Prefix != "" { 1018 v.Set("prefix", opts.Prefix) 1019 } 1020 if opts.Path != "" { 1021 v.Set("path", opts.Path) 1022 } 1023 if opts.Delimiter != 0 { 1024 v.Set("delimiter", string(opts.Delimiter)) 1025 } 1026 h = opts.Headers 1027 } 1028 return v, h 1029} 1030 1031// ObjectNames returns a slice of names of objects in a given container. 1032func (c *Connection) ObjectNames(container string, opts *ObjectsOpts) ([]string, error) { 1033 v, h := opts.parse() 1034 resp, _, err := c.storage(RequestOpts{ 1035 Container: container, 1036 Operation: "GET", 1037 Parameters: v, 1038 ErrorMap: ContainerErrorMap, 1039 Headers: h, 1040 }) 1041 if err != nil { 1042 return nil, err 1043 } 1044 return readLines(resp) 1045} 1046 1047// Object contains information about an object 1048type Object struct { 1049 Name string `json:"name"` // object name 1050 ContentType string `json:"content_type"` // eg application/directory 1051 Bytes int64 `json:"bytes"` // size in bytes 1052 ServerLastModified string `json:"last_modified"` // Last modified time, eg '2011-06-30T08:20:47.736680' as a string supplied by the server 1053 LastModified time.Time // Last modified time converted to a time.Time 1054 Hash string `json:"hash"` // MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" 1055 SLOHash string `json:"slo_etag"` // MD5 hash of all segments' MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" 1056 PseudoDirectory bool // Set when using delimiter to show that this directory object does not really exist 1057 SubDir string `json:"subdir"` // returned only when using delimiter to mark "pseudo directories" 1058 ObjectType ObjectType // type of this object 1059} 1060 1061// Objects returns a slice of Object with information about each 1062// object in the container. 1063// 1064// If Delimiter is set in the opts then PseudoDirectory may be set, 1065// with ContentType 'application/directory'. These are not real 1066// objects but represent directories of objects which haven't had an 1067// object created for them. 1068func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, error) { 1069 v, h := opts.parse() 1070 v.Set("format", "json") 1071 resp, _, err := c.storage(RequestOpts{ 1072 Container: container, 1073 Operation: "GET", 1074 Parameters: v, 1075 ErrorMap: ContainerErrorMap, 1076 Headers: h, 1077 }) 1078 if err != nil { 1079 return nil, err 1080 } 1081 var objects []Object 1082 err = readJson(resp, &objects) 1083 // Convert Pseudo directories and dates 1084 for i := range objects { 1085 object := &objects[i] 1086 if object.SubDir != "" { 1087 object.Name = object.SubDir 1088 object.PseudoDirectory = true 1089 object.ContentType = "application/directory" 1090 } 1091 if object.ServerLastModified != "" { 1092 // 2012-11-11T14:49:47.887250 1093 // 1094 // Remove fractional seconds if present. This 1095 // then keeps it consistent with Object 1096 // which can only return timestamps accurate 1097 // to 1 second 1098 // 1099 // The TimeFormat will parse fractional 1100 // seconds if desired though 1101 datetime := strings.SplitN(object.ServerLastModified, ".", 2)[0] 1102 object.LastModified, err = time.Parse(TimeFormat, datetime) 1103 if err != nil { 1104 return nil, err 1105 } 1106 } 1107 if object.SLOHash != "" { 1108 object.ObjectType = StaticLargeObjectType 1109 } 1110 } 1111 return objects, err 1112} 1113 1114// objectsAllOpts makes a copy of opts if set or makes a new one and 1115// overrides Limit and Marker 1116// Marker is not overriden if KeepMarker is set 1117func objectsAllOpts(opts *ObjectsOpts, Limit int) *ObjectsOpts { 1118 var newOpts ObjectsOpts 1119 if opts != nil { 1120 newOpts = *opts 1121 } 1122 if newOpts.Limit == 0 { 1123 newOpts.Limit = Limit 1124 } 1125 if !newOpts.KeepMarker { 1126 newOpts.Marker = "" 1127 } 1128 return &newOpts 1129} 1130 1131// A closure defined by the caller to iterate through all objects 1132// 1133// Call Objects or ObjectNames from here with the *ObjectOpts passed in 1134// 1135// Do whatever is required with the results then return them 1136type ObjectsWalkFn func(*ObjectsOpts) (interface{}, error) 1137 1138// ObjectsWalk is uses to iterate through all the objects in chunks as 1139// returned by Objects or ObjectNames using the Marker and Limit 1140// parameters in the ObjectsOpts. 1141// 1142// Pass in a closure `walkFn` which calls Objects or ObjectNames with 1143// the *ObjectsOpts passed to it and does something with the results. 1144// 1145// Errors will be returned from this function 1146// 1147// It has a default Limit parameter but you may pass in your own 1148func (c *Connection) ObjectsWalk(container string, opts *ObjectsOpts, walkFn ObjectsWalkFn) error { 1149 opts = objectsAllOpts(opts, allObjectsChanLimit) 1150 for { 1151 objects, err := walkFn(opts) 1152 if err != nil { 1153 return err 1154 } 1155 var n int 1156 var last string 1157 switch objects := objects.(type) { 1158 case []string: 1159 n = len(objects) 1160 if n > 0 { 1161 last = objects[len(objects)-1] 1162 } 1163 case []Object: 1164 n = len(objects) 1165 if n > 0 { 1166 last = objects[len(objects)-1].Name 1167 } 1168 default: 1169 panic("Unknown type returned to ObjectsWalk") 1170 } 1171 if n < opts.Limit { 1172 break 1173 } 1174 opts.Marker = last 1175 } 1176 return nil 1177} 1178 1179// ObjectsAll is like Objects but it returns an unlimited number of Objects in a slice 1180// 1181// It calls Objects multiple times using the Marker parameter 1182func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) { 1183 objects := make([]Object, 0) 1184 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 1185 newObjects, err := c.Objects(container, opts) 1186 if err == nil { 1187 objects = append(objects, newObjects...) 1188 } 1189 return newObjects, err 1190 }) 1191 return objects, err 1192} 1193 1194// ObjectNamesAll is like ObjectNames but it returns all the Objects 1195// 1196// It calls ObjectNames multiple times using the Marker parameter. Marker is 1197// reset unless KeepMarker is set 1198// 1199// It has a default Limit parameter but you may pass in your own 1200func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) { 1201 objects := make([]string, 0) 1202 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 1203 newObjects, err := c.ObjectNames(container, opts) 1204 if err == nil { 1205 objects = append(objects, newObjects...) 1206 } 1207 return newObjects, err 1208 }) 1209 return objects, err 1210} 1211 1212// Account contains information about this account. 1213type Account struct { 1214 BytesUsed int64 // total number of bytes used 1215 Containers int64 // total number of containers 1216 Objects int64 // total number of objects 1217} 1218 1219// getInt64FromHeader is a helper function to decode int64 from header. 1220func getInt64FromHeader(resp *http.Response, header string) (result int64, err error) { 1221 value := resp.Header.Get(header) 1222 result, err = strconv.ParseInt(value, 10, 64) 1223 if err != nil { 1224 err = newErrorf(0, "Bad Header '%s': '%s': %s", header, value, err) 1225 } 1226 return 1227} 1228 1229// Account returns info about the account in an Account struct. 1230func (c *Connection) Account() (info Account, headers Headers, err error) { 1231 var resp *http.Response 1232 resp, headers, err = c.storage(RequestOpts{ 1233 Operation: "HEAD", 1234 ErrorMap: ContainerErrorMap, 1235 NoResponse: true, 1236 }) 1237 if err != nil { 1238 return 1239 } 1240 // Parse the headers into a dict 1241 // 1242 // {'Accept-Ranges': 'bytes', 1243 // 'Content-Length': '0', 1244 // 'Date': 'Tue, 05 Jul 2011 16:37:06 GMT', 1245 // 'X-Account-Bytes-Used': '316598182', 1246 // 'X-Account-Container-Count': '4', 1247 // 'X-Account-Object-Count': '1433'} 1248 if info.BytesUsed, err = getInt64FromHeader(resp, "X-Account-Bytes-Used"); err != nil { 1249 return 1250 } 1251 if info.Containers, err = getInt64FromHeader(resp, "X-Account-Container-Count"); err != nil { 1252 return 1253 } 1254 if info.Objects, err = getInt64FromHeader(resp, "X-Account-Object-Count"); err != nil { 1255 return 1256 } 1257 return 1258} 1259 1260// AccountUpdate adds, replaces or remove account metadata. 1261// 1262// Add or update keys by mentioning them in the Headers. 1263// 1264// Remove keys by setting them to an empty string. 1265func (c *Connection) AccountUpdate(h Headers) error { 1266 _, _, err := c.storage(RequestOpts{ 1267 Operation: "POST", 1268 ErrorMap: ContainerErrorMap, 1269 NoResponse: true, 1270 Headers: h, 1271 }) 1272 return err 1273} 1274 1275// ContainerCreate creates a container. 1276// 1277// If you don't want to add Headers just pass in nil 1278// 1279// No error is returned if it already exists but the metadata if any will be updated. 1280func (c *Connection) ContainerCreate(container string, h Headers) error { 1281 _, _, err := c.storage(RequestOpts{ 1282 Container: container, 1283 Operation: "PUT", 1284 ErrorMap: ContainerErrorMap, 1285 NoResponse: true, 1286 Headers: h, 1287 }) 1288 return err 1289} 1290 1291// ContainerDelete deletes a container. 1292// 1293// May return ContainerDoesNotExist or ContainerNotEmpty 1294func (c *Connection) ContainerDelete(container string) error { 1295 _, _, err := c.storage(RequestOpts{ 1296 Container: container, 1297 Operation: "DELETE", 1298 ErrorMap: ContainerErrorMap, 1299 NoResponse: true, 1300 }) 1301 return err 1302} 1303 1304// Container returns info about a single container including any 1305// metadata in the headers. 1306func (c *Connection) Container(container string) (info Container, headers Headers, err error) { 1307 var resp *http.Response 1308 resp, headers, err = c.storage(RequestOpts{ 1309 Container: container, 1310 Operation: "HEAD", 1311 ErrorMap: ContainerErrorMap, 1312 NoResponse: true, 1313 }) 1314 if err != nil { 1315 return 1316 } 1317 // Parse the headers into the struct 1318 info.Name = container 1319 if info.Bytes, err = getInt64FromHeader(resp, "X-Container-Bytes-Used"); err != nil { 1320 return 1321 } 1322 if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil { 1323 return 1324 } 1325 return 1326} 1327 1328// ContainerUpdate adds, replaces or removes container metadata. 1329// 1330// Add or update keys by mentioning them in the Metadata. 1331// 1332// Remove keys by setting them to an empty string. 1333// 1334// Container metadata can only be read with Container() not with Containers(). 1335func (c *Connection) ContainerUpdate(container string, h Headers) error { 1336 _, _, err := c.storage(RequestOpts{ 1337 Container: container, 1338 Operation: "POST", 1339 ErrorMap: ContainerErrorMap, 1340 NoResponse: true, 1341 Headers: h, 1342 }) 1343 return err 1344} 1345 1346// ------------------------------------------------------------ 1347 1348// ObjectCreateFile represents a swift object open for writing 1349type ObjectCreateFile struct { 1350 checkHash bool // whether we are checking the hash 1351 pipeReader *io.PipeReader // pipe for the caller to use 1352 pipeWriter *io.PipeWriter 1353 hash hash.Hash // hash being build up as we go along 1354 done chan struct{} // signals when the upload has finished 1355 resp *http.Response // valid when done has signalled 1356 err error // ditto 1357 headers Headers // ditto 1358} 1359 1360// Write bytes to the object - see io.Writer 1361func (file *ObjectCreateFile) Write(p []byte) (n int, err error) { 1362 n, err = file.pipeWriter.Write(p) 1363 if err == io.ErrClosedPipe { 1364 if file.err != nil { 1365 return 0, file.err 1366 } 1367 return 0, newError(500, "Write on closed file") 1368 } 1369 if err == nil && file.checkHash { 1370 _, _ = file.hash.Write(p) 1371 } 1372 return 1373} 1374 1375// CloseWithError closes the object, aborting the upload. 1376func (file *ObjectCreateFile) CloseWithError(err error) error { 1377 _ = file.pipeWriter.CloseWithError(err) 1378 <-file.done 1379 return nil 1380} 1381 1382// Close the object and checks the md5sum if it was required. 1383// 1384// Also returns any other errors from the server (eg container not 1385// found) so it is very important to check the errors on this method. 1386func (file *ObjectCreateFile) Close() error { 1387 // Close the body 1388 err := file.pipeWriter.Close() 1389 if err != nil { 1390 return err 1391 } 1392 1393 // Wait for the HTTP operation to complete 1394 <-file.done 1395 1396 // Check errors 1397 if file.err != nil { 1398 return file.err 1399 } 1400 if file.checkHash { 1401 receivedMd5 := strings.ToLower(file.headers["Etag"]) 1402 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1403 if receivedMd5 != calculatedMd5 { 1404 return ObjectCorrupted 1405 } 1406 } 1407 return nil 1408} 1409 1410// Headers returns the response headers from the created object if the upload 1411// has been completed. The Close() method must be called on an ObjectCreateFile 1412// before this method. 1413func (file *ObjectCreateFile) Headers() (Headers, error) { 1414 // error out if upload is not complete. 1415 select { 1416 case <-file.done: 1417 default: 1418 return nil, fmt.Errorf("Cannot get metadata, object upload failed or has not yet completed.") 1419 } 1420 return file.headers, nil 1421} 1422 1423// Check it satisfies the interface 1424var _ io.WriteCloser = &ObjectCreateFile{} 1425 1426// objectPutHeaders create a set of headers for a PUT 1427// 1428// It guesses the contentType from the objectName if it isn't set 1429// 1430// checkHash may be changed 1431func objectPutHeaders(objectName string, checkHash *bool, Hash string, contentType string, h Headers) Headers { 1432 if contentType == "" { 1433 contentType = mime.TypeByExtension(path.Ext(objectName)) 1434 if contentType == "" { 1435 contentType = "application/octet-stream" 1436 } 1437 } 1438 // Meta stuff 1439 extraHeaders := map[string]string{ 1440 "Content-Type": contentType, 1441 } 1442 for key, value := range h { 1443 extraHeaders[key] = value 1444 } 1445 if Hash != "" { 1446 extraHeaders["Etag"] = Hash 1447 *checkHash = false // the server will do it 1448 } 1449 return extraHeaders 1450} 1451 1452// ObjectCreate creates or updates the object in the container. It 1453// returns an io.WriteCloser you should write the contents to. You 1454// MUST call Close() on it and you MUST check the error return from 1455// Close(). 1456// 1457// If checkHash is True then it will calculate the MD5 Hash of the 1458// file as it is being uploaded and check it against that returned 1459// from the server. If it is wrong then it will return 1460// ObjectCorrupted on Close() 1461// 1462// If you know the MD5 hash of the object ahead of time then set the 1463// Hash parameter and it will be sent to the server (as an Etag 1464// header) and the server will check the MD5 itself after the upload, 1465// and this will return ObjectCorrupted on Close() if it is incorrect. 1466// 1467// If you don't want any error protection (not recommended) then set 1468// checkHash to false and Hash to "". 1469// 1470// If contentType is set it will be used, otherwise one will be 1471// guessed from objectName using mime.TypeByExtension 1472func (c *Connection) ObjectCreate(container string, objectName string, checkHash bool, Hash string, contentType string, h Headers) (file *ObjectCreateFile, err error) { 1473 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1474 pipeReader, pipeWriter := io.Pipe() 1475 file = &ObjectCreateFile{ 1476 hash: md5.New(), 1477 checkHash: checkHash, 1478 pipeReader: pipeReader, 1479 pipeWriter: pipeWriter, 1480 done: make(chan struct{}), 1481 } 1482 // Run the PUT in the background piping it data 1483 go func() { 1484 opts := RequestOpts{ 1485 Container: container, 1486 ObjectName: objectName, 1487 Operation: "PUT", 1488 Headers: extraHeaders, 1489 Body: pipeReader, 1490 NoResponse: true, 1491 ErrorMap: objectErrorMap, 1492 } 1493 file.resp, file.headers, file.err = c.storage(opts) 1494 // Signal finished 1495 pipeReader.Close() 1496 close(file.done) 1497 }() 1498 return 1499} 1500 1501func (c *Connection) ObjectSymlinkCreate(container string, symlink string, targetAccount string, targetContainer string, targetObject string, targetEtag string) (headers Headers, err error) { 1502 1503 EMPTY_MD5 := "d41d8cd98f00b204e9800998ecf8427e" 1504 symHeaders := Headers{} 1505 contents := bytes.NewBufferString("") 1506 if targetAccount != "" { 1507 symHeaders["X-Symlink-Target-Account"] = targetAccount 1508 } 1509 if targetEtag != "" { 1510 symHeaders["X-Symlink-Target-Etag"] = targetEtag 1511 } 1512 symHeaders["X-Symlink-Target"] = fmt.Sprintf("%s/%s", targetContainer, targetObject) 1513 _, err = c.ObjectPut(container, symlink, contents, true, EMPTY_MD5, "application/symlink", symHeaders) 1514 return 1515} 1516 1517func (c *Connection) objectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers, parameters url.Values) (headers Headers, err error) { 1518 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1519 hash := md5.New() 1520 var body io.Reader = contents 1521 if checkHash { 1522 body = io.TeeReader(contents, hash) 1523 } 1524 _, headers, err = c.storage(RequestOpts{ 1525 Container: container, 1526 ObjectName: objectName, 1527 Operation: "PUT", 1528 Headers: extraHeaders, 1529 Body: body, 1530 NoResponse: true, 1531 ErrorMap: objectErrorMap, 1532 Parameters: parameters, 1533 }) 1534 if err != nil { 1535 return 1536 } 1537 if checkHash { 1538 receivedMd5 := strings.ToLower(headers["Etag"]) 1539 calculatedMd5 := fmt.Sprintf("%x", hash.Sum(nil)) 1540 if receivedMd5 != calculatedMd5 { 1541 err = ObjectCorrupted 1542 return 1543 } 1544 } 1545 return 1546} 1547 1548// ObjectPut creates or updates the path in the container from 1549// contents. contents should be an open io.Reader which will have all 1550// its contents read. 1551// 1552// This is a low level interface. 1553// 1554// If checkHash is True then it will calculate the MD5 Hash of the 1555// file as it is being uploaded and check it against that returned 1556// from the server. If it is wrong then it will return 1557// ObjectCorrupted. 1558// 1559// If you know the MD5 hash of the object ahead of time then set the 1560// Hash parameter and it will be sent to the server (as an Etag 1561// header) and the server will check the MD5 itself after the upload, 1562// and this will return ObjectCorrupted if it is incorrect. 1563// 1564// If you don't want any error protection (not recommended) then set 1565// checkHash to false and Hash to "". 1566// 1567// If contentType is set it will be used, otherwise one will be 1568// guessed from objectName using mime.TypeByExtension 1569func (c *Connection) ObjectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers) (headers Headers, err error) { 1570 return c.objectPut(container, objectName, contents, checkHash, Hash, contentType, h, nil) 1571} 1572 1573// ObjectPutBytes creates an object from a []byte in a container. 1574// 1575// This is a simplified interface which checks the MD5. 1576func (c *Connection) ObjectPutBytes(container string, objectName string, contents []byte, contentType string) (err error) { 1577 buf := bytes.NewBuffer(contents) 1578 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1579 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1580 return 1581} 1582 1583// ObjectPutString creates an object from a string in a container. 1584// 1585// This is a simplified interface which checks the MD5 1586func (c *Connection) ObjectPutString(container string, objectName string, contents string, contentType string) (err error) { 1587 buf := strings.NewReader(contents) 1588 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1589 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1590 return 1591} 1592 1593// ObjectOpenFile represents a swift object open for reading 1594type ObjectOpenFile struct { 1595 connection *Connection // stored copy of Connection used in Open 1596 container string // stored copy of container used in Open 1597 objectName string // stored copy of objectName used in Open 1598 headers Headers // stored copy of headers used in Open 1599 resp *http.Response // http connection 1600 body io.Reader // read data from this 1601 checkHash bool // true if checking MD5 1602 hash hash.Hash // currently accumulating MD5 1603 bytes int64 // number of bytes read on this connection 1604 eof bool // whether we have read end of file 1605 pos int64 // current position when reading 1606 lengthOk bool // whether length is valid 1607 length int64 // length of the object if read 1608 seeked bool // whether we have seeked this file or not 1609 overSeeked bool // set if we have seeked to the end or beyond 1610} 1611 1612// Read bytes from the object - see io.Reader 1613func (file *ObjectOpenFile) Read(p []byte) (n int, err error) { 1614 if file.overSeeked { 1615 return 0, io.EOF 1616 } 1617 n, err = file.body.Read(p) 1618 file.bytes += int64(n) 1619 file.pos += int64(n) 1620 if err == io.EOF { 1621 file.eof = true 1622 } 1623 return 1624} 1625 1626// Seek sets the offset for the next Read to offset, interpreted 1627// according to whence: 0 means relative to the origin of the file, 1 1628// means relative to the current offset, and 2 means relative to the 1629// end. Seek returns the new offset and an Error, if any. 1630// 1631// Seek uses HTTP Range headers which, if the file pointer is moved, 1632// will involve reopening the HTTP connection. 1633// 1634// Note that you can't seek to the end of a file or beyond; HTTP Range 1635// requests don't support the file pointer being outside the data, 1636// unlike os.File 1637// 1638// Seek(0, 1) will return the current file pointer. 1639func (file *ObjectOpenFile) Seek(offset int64, whence int) (newPos int64, err error) { 1640 file.overSeeked = false 1641 switch whence { 1642 case 0: // relative to start 1643 newPos = offset 1644 case 1: // relative to current 1645 newPos = file.pos + offset 1646 case 2: // relative to end 1647 if !file.lengthOk { 1648 return file.pos, newError(0, "Length of file unknown so can't seek from end") 1649 } 1650 newPos = file.length + offset 1651 if offset >= 0 { 1652 file.overSeeked = true 1653 return 1654 } 1655 default: 1656 panic("Unknown whence in ObjectOpenFile.Seek") 1657 } 1658 // If at correct position (quite likely), do nothing 1659 if newPos == file.pos { 1660 return 1661 } 1662 // Close the file... 1663 file.seeked = true 1664 err = file.Close() 1665 if err != nil { 1666 return 1667 } 1668 // ...and re-open with a Range header 1669 if file.headers == nil { 1670 file.headers = Headers{} 1671 } 1672 if newPos > 0 { 1673 file.headers["Range"] = fmt.Sprintf("bytes=%d-", newPos) 1674 } else { 1675 delete(file.headers, "Range") 1676 } 1677 newFile, _, err := file.connection.ObjectOpen(file.container, file.objectName, false, file.headers) 1678 if err != nil { 1679 return 1680 } 1681 // Update the file 1682 file.resp = newFile.resp 1683 file.body = newFile.body 1684 file.checkHash = false 1685 file.pos = newPos 1686 return 1687} 1688 1689// Length gets the objects content length either from a cached copy or 1690// from the server. 1691func (file *ObjectOpenFile) Length() (int64, error) { 1692 if !file.lengthOk { 1693 info, _, err := file.connection.Object(file.container, file.objectName) 1694 file.length = info.Bytes 1695 file.lengthOk = (err == nil) 1696 return file.length, err 1697 } 1698 return file.length, nil 1699} 1700 1701// Close the object and checks the length and md5sum if it was 1702// required and all the object was read 1703func (file *ObjectOpenFile) Close() (err error) { 1704 // Close the body at the end 1705 defer checkClose(file.resp.Body, &err) 1706 1707 // If not end of file or seeked then can't check anything 1708 if !file.eof || file.seeked { 1709 return 1710 } 1711 1712 // Check the MD5 sum if requested 1713 if file.checkHash { 1714 receivedMd5 := strings.ToLower(file.resp.Header.Get("Etag")) 1715 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1716 if receivedMd5 != calculatedMd5 { 1717 err = ObjectCorrupted 1718 return 1719 } 1720 } 1721 1722 // Check to see we read the correct number of bytes 1723 if file.lengthOk && file.length != file.bytes { 1724 err = ObjectCorrupted 1725 return 1726 } 1727 return 1728} 1729 1730// Check it satisfies the interfaces 1731var _ io.ReadCloser = &ObjectOpenFile{} 1732var _ io.Seeker = &ObjectOpenFile{} 1733 1734func (c *Connection) objectOpenBase(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1735 var resp *http.Response 1736 opts := RequestOpts{ 1737 Container: container, 1738 ObjectName: objectName, 1739 Operation: "GET", 1740 ErrorMap: objectErrorMap, 1741 Headers: h, 1742 Parameters: parameters, 1743 } 1744 resp, headers, err = c.storage(opts) 1745 if err != nil { 1746 return 1747 } 1748 // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set 1749 if checkHash && headers.IsLargeObject() { 1750 // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) 1751 checkHash = false 1752 } 1753 file = &ObjectOpenFile{ 1754 connection: c, 1755 container: container, 1756 objectName: objectName, 1757 headers: h, 1758 resp: resp, 1759 checkHash: checkHash, 1760 body: resp.Body, 1761 } 1762 if checkHash { 1763 file.hash = md5.New() 1764 file.body = io.TeeReader(resp.Body, file.hash) 1765 } 1766 // Read Content-Length 1767 if resp.Header.Get("Content-Length") != "" { 1768 file.length, err = getInt64FromHeader(resp, "Content-Length") 1769 file.lengthOk = (err == nil) 1770 } 1771 return 1772} 1773 1774func (c *Connection) objectOpen(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1775 err = withLORetry(0, func() (Headers, int64, error) { 1776 file, headers, err = c.objectOpenBase(container, objectName, checkHash, h, parameters) 1777 if err != nil { 1778 return headers, 0, err 1779 } 1780 return headers, file.length, nil 1781 }) 1782 return 1783} 1784 1785// ObjectOpen returns an ObjectOpenFile for reading the contents of 1786// the object. This satisfies the io.ReadCloser and the io.Seeker 1787// interfaces. 1788// 1789// You must call Close() on contents when finished 1790// 1791// Returns the headers of the response. 1792// 1793// If checkHash is true then it will calculate the md5sum of the file 1794// as it is being received and check it against that returned from the 1795// server. If it is wrong then it will return ObjectCorrupted. It 1796// will also check the length returned. No checking will be done if 1797// you don't read all the contents. 1798// 1799// Note that objects with X-Object-Manifest or X-Static-Large-Object 1800// set won't ever have their md5sum's checked as the md5sum reported 1801// on the object is actually the md5sum of the md5sums of the 1802// parts. This isn't very helpful to detect a corrupted download as 1803// the size of the parts aren't known without doing more operations. 1804// If you want to ensure integrity of an object with a manifest then 1805// you will need to download everything in the manifest separately. 1806// 1807// headers["Content-Type"] will give the content type if desired. 1808func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { 1809 return c.objectOpen(container, objectName, checkHash, h, nil) 1810} 1811 1812// ObjectGet gets the object into the io.Writer contents. 1813// 1814// Returns the headers of the response. 1815// 1816// If checkHash is true then it will calculate the md5sum of the file 1817// as it is being received and check it against that returned from the 1818// server. If it is wrong then it will return ObjectCorrupted. 1819// 1820// headers["Content-Type"] will give the content type if desired. 1821func (c *Connection) ObjectGet(container string, objectName string, contents io.Writer, checkHash bool, h Headers) (headers Headers, err error) { 1822 file, headers, err := c.ObjectOpen(container, objectName, checkHash, h) 1823 if err != nil { 1824 return 1825 } 1826 defer checkClose(file, &err) 1827 _, err = io.Copy(contents, file) 1828 return 1829} 1830 1831// ObjectGetBytes returns an object as a []byte. 1832// 1833// This is a simplified interface which checks the MD5 1834func (c *Connection) ObjectGetBytes(container string, objectName string) (contents []byte, err error) { 1835 var buf bytes.Buffer 1836 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1837 contents = buf.Bytes() 1838 return 1839} 1840 1841// ObjectGetString returns an object as a string. 1842// 1843// This is a simplified interface which checks the MD5 1844func (c *Connection) ObjectGetString(container string, objectName string) (contents string, err error) { 1845 var buf bytes.Buffer 1846 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1847 contents = buf.String() 1848 return 1849} 1850 1851// ObjectDelete deletes the object. 1852// 1853// May return ObjectNotFound if the object isn't found 1854func (c *Connection) ObjectDelete(container string, objectName string) error { 1855 _, _, err := c.storage(RequestOpts{ 1856 Container: container, 1857 ObjectName: objectName, 1858 Operation: "DELETE", 1859 ErrorMap: objectErrorMap, 1860 }) 1861 return err 1862} 1863 1864// ObjectTempUrl returns a temporary URL for an object 1865func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string { 1866 mac := hmac.New(sha1.New, []byte(secretKey)) 1867 prefix, _ := url.Parse(c.StorageUrl) 1868 body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName) 1869 mac.Write([]byte(body)) 1870 sig := hex.EncodeToString(mac.Sum(nil)) 1871 return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix()) 1872} 1873 1874// parseResponseStatus parses string like "200 OK" and returns Error. 1875// 1876// For status codes beween 200 and 299, this returns nil. 1877func parseResponseStatus(resp string, errorMap errorMap) error { 1878 code := 0 1879 reason := resp 1880 t := strings.SplitN(resp, " ", 2) 1881 if len(t) == 2 { 1882 ncode, err := strconv.Atoi(t[0]) 1883 if err == nil { 1884 code = ncode 1885 reason = t[1] 1886 } 1887 } 1888 if errorMap != nil { 1889 if err, ok := errorMap[code]; ok { 1890 return err 1891 } 1892 } 1893 if 200 <= code && code <= 299 { 1894 return nil 1895 } 1896 return newError(code, reason) 1897} 1898 1899// BulkDeleteResult stores results of BulkDelete(). 1900// 1901// Individual errors may (or may not) be returned by Errors. 1902// Errors is a map whose keys are a full path of where the object was 1903// to be deleted, and whose values are Error objects. A full path of 1904// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 1905type BulkDeleteResult struct { 1906 NumberNotFound int64 // # of objects not found. 1907 NumberDeleted int64 // # of deleted objects. 1908 Errors map[string]error // Mapping between object name and an error. 1909 Headers Headers // Response HTTP headers. 1910} 1911 1912func (c *Connection) doBulkDelete(objects []string, h Headers) (result BulkDeleteResult, err error) { 1913 var buffer bytes.Buffer 1914 for _, s := range objects { 1915 u := url.URL{Path: s} 1916 buffer.WriteString(u.String() + "\n") 1917 } 1918 extraHeaders := Headers{ 1919 "Accept": "application/json", 1920 "Content-Type": "text/plain", 1921 "Content-Length": strconv.Itoa(buffer.Len()), 1922 } 1923 for key, value := range h { 1924 extraHeaders[key] = value 1925 } 1926 resp, headers, err := c.storage(RequestOpts{ 1927 Operation: "DELETE", 1928 Parameters: url.Values{"bulk-delete": []string{"1"}}, 1929 Headers: extraHeaders, 1930 ErrorMap: ContainerErrorMap, 1931 Body: &buffer, 1932 }) 1933 if err != nil { 1934 return 1935 } 1936 var jsonResult struct { 1937 NotFound int64 `json:"Number Not Found"` 1938 Status string `json:"Response Status"` 1939 Errors [][]string 1940 Deleted int64 `json:"Number Deleted"` 1941 } 1942 err = readJson(resp, &jsonResult) 1943 if err != nil { 1944 return 1945 } 1946 1947 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 1948 result.NumberNotFound = jsonResult.NotFound 1949 result.NumberDeleted = jsonResult.Deleted 1950 result.Headers = headers 1951 el := make(map[string]error, len(jsonResult.Errors)) 1952 for _, t := range jsonResult.Errors { 1953 if len(t) != 2 { 1954 continue 1955 } 1956 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 1957 } 1958 result.Errors = el 1959 return 1960} 1961 1962// BulkDelete deletes multiple objectNames from container in one operation. 1963// 1964// Some servers may not accept bulk-delete requests since bulk-delete is 1965// an optional feature of swift - these will return the Forbidden error. 1966// 1967// See also: 1968// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html 1969// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html 1970func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) { 1971 return c.BulkDeleteHeaders(container, objectNames, nil) 1972} 1973 1974// BulkDeleteHeaders deletes multiple objectNames from container in one operation. 1975// 1976// Some servers may not accept bulk-delete requests since bulk-delete is 1977// an optional feature of swift - these will return the Forbidden error. 1978// 1979// See also: 1980// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html 1981// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html 1982func (c *Connection) BulkDeleteHeaders(container string, objectNames []string, h Headers) (result BulkDeleteResult, err error) { 1983 if len(objectNames) == 0 { 1984 result.Errors = make(map[string]error) 1985 return 1986 } 1987 fullPaths := make([]string, len(objectNames)) 1988 for i, name := range objectNames { 1989 fullPaths[i] = fmt.Sprintf("/%s/%s", container, name) 1990 } 1991 return c.doBulkDelete(fullPaths, h) 1992} 1993 1994// BulkUploadResult stores results of BulkUpload(). 1995// 1996// Individual errors may (or may not) be returned by Errors. 1997// Errors is a map whose keys are a full path of where an object was 1998// to be created, and whose values are Error objects. A full path of 1999// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 2000type BulkUploadResult struct { 2001 NumberCreated int64 // # of created objects. 2002 Errors map[string]error // Mapping between object name and an error. 2003 Headers Headers // Response HTTP headers. 2004} 2005 2006// BulkUpload uploads multiple files in one operation. 2007// 2008// uploadPath can be empty, a container name, or a pseudo-directory 2009// within a container. If uploadPath is empty, new containers may be 2010// automatically created. 2011// 2012// Files are read from dataStream. The format of the stream is specified 2013// by the format parameter. Available formats are: 2014// * UploadTar - Plain tar stream. 2015// * UploadTarGzip - Gzip compressed tar stream. 2016// * UploadTarBzip2 - Bzip2 compressed tar stream. 2017// 2018// Some servers may not accept bulk-upload requests since bulk-upload is 2019// an optional feature of swift - these will return the Forbidden error. 2020// 2021// See also: 2022// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-extract-archive.html 2023// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Extract_Archive-d1e2338.html 2024func (c *Connection) BulkUpload(uploadPath string, dataStream io.Reader, format string, h Headers) (result BulkUploadResult, err error) { 2025 extraHeaders := Headers{"Accept": "application/json"} 2026 for key, value := range h { 2027 extraHeaders[key] = value 2028 } 2029 // The following code abuses Container parameter intentionally. 2030 // The best fix might be to rename Container to UploadPath. 2031 resp, headers, err := c.storage(RequestOpts{ 2032 Container: uploadPath, 2033 Operation: "PUT", 2034 Parameters: url.Values{"extract-archive": []string{format}}, 2035 Headers: extraHeaders, 2036 ErrorMap: ContainerErrorMap, 2037 Body: dataStream, 2038 }) 2039 if err != nil { 2040 return 2041 } 2042 // Detect old servers which don't support this feature 2043 if headers["Content-Type"] != "application/json" { 2044 err = Forbidden 2045 return 2046 } 2047 var jsonResult struct { 2048 Created int64 `json:"Number Files Created"` 2049 Status string `json:"Response Status"` 2050 Errors [][]string 2051 } 2052 err = readJson(resp, &jsonResult) 2053 if err != nil { 2054 return 2055 } 2056 2057 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 2058 result.NumberCreated = jsonResult.Created 2059 result.Headers = headers 2060 el := make(map[string]error, len(jsonResult.Errors)) 2061 for _, t := range jsonResult.Errors { 2062 if len(t) != 2 { 2063 continue 2064 } 2065 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 2066 } 2067 result.Errors = el 2068 return 2069} 2070 2071// Object returns info about a single object including any metadata in the header. 2072// 2073// May return ObjectNotFound. 2074// 2075// Use headers.ObjectMetadata() to read the metadata in the Headers. 2076func (c *Connection) Object(container string, objectName string) (info Object, headers Headers, err error) { 2077 err = withLORetry(0, func() (Headers, int64, error) { 2078 info, headers, err = c.objectBase(container, objectName) 2079 if err != nil { 2080 return headers, 0, err 2081 } 2082 return headers, info.Bytes, nil 2083 }) 2084 return 2085} 2086 2087func (c *Connection) objectBase(container string, objectName string) (info Object, headers Headers, err error) { 2088 var resp *http.Response 2089 resp, headers, err = c.storage(RequestOpts{ 2090 Container: container, 2091 ObjectName: objectName, 2092 Operation: "HEAD", 2093 ErrorMap: objectErrorMap, 2094 NoResponse: true, 2095 }) 2096 if err != nil { 2097 return 2098 } 2099 // Parse the headers into the struct 2100 // HTTP/1.1 200 OK 2101 // Date: Thu, 07 Jun 2010 20:59:39 GMT 2102 // Server: Apache 2103 // Last-Modified: Fri, 12 Jun 2010 13:40:18 GMT 2104 // ETag: 8a964ee2a5e88be344f36c22562a6486 2105 // Content-Length: 512000 2106 // Content-Type: text/plain; charset=UTF-8 2107 // X-Object-Meta-Meat: Bacon 2108 // X-Object-Meta-Fruit: Bacon 2109 // X-Object-Meta-Veggie: Bacon 2110 // X-Object-Meta-Dairy: Bacon 2111 info.Name = objectName 2112 info.ContentType = resp.Header.Get("Content-Type") 2113 if resp.Header.Get("Content-Length") != "" { 2114 if info.Bytes, err = getInt64FromHeader(resp, "Content-Length"); err != nil { 2115 return 2116 } 2117 } 2118 // Currently ceph doesn't return a Last-Modified header for DLO manifests without any segments 2119 // See ceph http://tracker.ceph.com/issues/15812 2120 if resp.Header.Get("Last-Modified") != "" { 2121 info.ServerLastModified = resp.Header.Get("Last-Modified") 2122 if info.LastModified, err = time.Parse(http.TimeFormat, info.ServerLastModified); err != nil { 2123 return 2124 } 2125 } 2126 2127 info.Hash = resp.Header.Get("Etag") 2128 if resp.Header.Get("X-Object-Manifest") != "" { 2129 info.ObjectType = DynamicLargeObjectType 2130 } else if resp.Header.Get("X-Static-Large-Object") != "" { 2131 info.ObjectType = StaticLargeObjectType 2132 } 2133 2134 return 2135} 2136 2137// ObjectUpdate adds, replaces or removes object metadata. 2138// 2139// Add or Update keys by mentioning them in the Metadata. Use 2140// Metadata.ObjectHeaders and Headers.ObjectMetadata to convert your 2141// Metadata to and from normal HTTP headers. 2142// 2143// This removes all metadata previously added to the object and 2144// replaces it with that passed in so to delete keys, just don't 2145// mention them the headers you pass in. 2146// 2147// Object metadata can only be read with Object() not with Objects(). 2148// 2149// This can also be used to set headers not already assigned such as 2150// X-Delete-At or X-Delete-After for expiring objects. 2151// 2152// You cannot use this to change any of the object's other headers 2153// such as Content-Type, ETag, etc. 2154// 2155// Refer to copying an object when you need to update metadata or 2156// other headers such as Content-Type or CORS headers. 2157// 2158// May return ObjectNotFound. 2159func (c *Connection) ObjectUpdate(container string, objectName string, h Headers) error { 2160 _, _, err := c.storage(RequestOpts{ 2161 Container: container, 2162 ObjectName: objectName, 2163 Operation: "POST", 2164 ErrorMap: objectErrorMap, 2165 NoResponse: true, 2166 Headers: h, 2167 }) 2168 return err 2169} 2170 2171// urlPathEscape escapes URL path the in string using URL escaping rules 2172// 2173// This mimics url.PathEscape which only available from go 1.8 2174func urlPathEscape(in string) string { 2175 var u url.URL 2176 u.Path = in 2177 return u.String() 2178} 2179 2180// ObjectCopy does a server side copy of an object to a new position 2181// 2182// All metadata is preserved. If metadata is set in the headers then 2183// it overrides the old metadata on the copied object. 2184// 2185// The destination container must exist before the copy. 2186// 2187// You can use this to copy an object to itself - this is the only way 2188// to update the content type of an object. 2189func (c *Connection) ObjectCopy(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string, h Headers) (headers Headers, err error) { 2190 // Meta stuff 2191 extraHeaders := map[string]string{ 2192 "Destination": urlPathEscape(dstContainer + "/" + dstObjectName), 2193 } 2194 for key, value := range h { 2195 extraHeaders[key] = value 2196 } 2197 _, headers, err = c.storage(RequestOpts{ 2198 Container: srcContainer, 2199 ObjectName: srcObjectName, 2200 Operation: "COPY", 2201 ErrorMap: objectErrorMap, 2202 NoResponse: true, 2203 Headers: extraHeaders, 2204 }) 2205 return 2206} 2207 2208// ObjectMove does a server side move of an object to a new position 2209// 2210// This is a convenience method which calls ObjectCopy then ObjectDelete 2211// 2212// All metadata is preserved. 2213// 2214// The destination container must exist before the copy. 2215func (c *Connection) ObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) (err error) { 2216 _, err = c.ObjectCopy(srcContainer, srcObjectName, dstContainer, dstObjectName, nil) 2217 if err != nil { 2218 return 2219 } 2220 return c.ObjectDelete(srcContainer, srcObjectName) 2221} 2222 2223// ObjectUpdateContentType updates the content type of an object 2224// 2225// This is a convenience method which calls ObjectCopy 2226// 2227// All other metadata is preserved. 2228func (c *Connection) ObjectUpdateContentType(container string, objectName string, contentType string) (err error) { 2229 h := Headers{"Content-Type": contentType} 2230 _, err = c.ObjectCopy(container, objectName, container, objectName, h) 2231 return 2232} 2233 2234// ------------------------------------------------------------ 2235 2236// VersionContainerCreate is a helper method for creating and enabling version controlled containers. 2237// 2238// It builds the current object container, the non-current object version container, and enables versioning. 2239// 2240// If the server doesn't support versioning then it will return 2241// Forbidden however it will have created both the containers at that point. 2242func (c *Connection) VersionContainerCreate(current, version string) error { 2243 if err := c.ContainerCreate(version, nil); err != nil { 2244 return err 2245 } 2246 if err := c.ContainerCreate(current, nil); err != nil { 2247 return err 2248 } 2249 if err := c.VersionEnable(current, version); err != nil { 2250 return err 2251 } 2252 return nil 2253} 2254 2255// VersionEnable enables versioning on the current container with version as the tracking container. 2256// 2257// May return Forbidden if this isn't supported by the server 2258func (c *Connection) VersionEnable(current, version string) error { 2259 h := Headers{"X-Versions-Location": version} 2260 if err := c.ContainerUpdate(current, h); err != nil { 2261 return err 2262 } 2263 // Check to see if the header was set properly 2264 _, headers, err := c.Container(current) 2265 if err != nil { 2266 return err 2267 } 2268 // If failed to set versions header, return Forbidden as the server doesn't support this 2269 if headers["X-Versions-Location"] != version { 2270 return Forbidden 2271 } 2272 return nil 2273} 2274 2275// VersionDisable disables versioning on the current container. 2276func (c *Connection) VersionDisable(current string) error { 2277 h := Headers{"X-Versions-Location": ""} 2278 if err := c.ContainerUpdate(current, h); err != nil { 2279 return err 2280 } 2281 return nil 2282} 2283 2284// VersionObjectList returns a list of older versions of the object. 2285// 2286// Objects are returned in the format <length><object_name>/<timestamp> 2287func (c *Connection) VersionObjectList(version, object string) ([]string, error) { 2288 opts := &ObjectsOpts{ 2289 // <3-character zero-padded hexadecimal character length><object name>/ 2290 Prefix: fmt.Sprintf("%03x", len(object)) + object + "/", 2291 } 2292 return c.ObjectNames(version, opts) 2293} 2294