1package swift 2 3import ( 4 "bufio" 5 "bytes" 6 "crypto/hmac" 7 "crypto/md5" 8 "crypto/sha1" 9 "encoding/hex" 10 "encoding/json" 11 "fmt" 12 "hash" 13 "io" 14 "mime" 15 "net/http" 16 "net/url" 17 "path" 18 "strconv" 19 "strings" 20 "sync" 21 "time" 22) 23 24const ( 25 DefaultUserAgent = "goswift/1.0" // Default user agent 26 DefaultRetries = 3 // Default number of retries on token expiry 27 TimeFormat = "2006-01-02T15:04:05" // Python date format for json replies parsed as UTC 28 UploadTar = "tar" // Data format specifier for Connection.BulkUpload(). 29 UploadTarGzip = "tar.gz" // Data format specifier for Connection.BulkUpload(). 30 UploadTarBzip2 = "tar.bz2" // Data format specifier for Connection.BulkUpload(). 31 allContainersLimit = 10000 // Number of containers to fetch at once 32 allObjectsLimit = 10000 // Number objects to fetch at once 33 allObjectsChanLimit = 1000 // ...when fetching to a channel 34) 35 36// ObjectType is the type of the swift object, regular, static large, 37// or dynamic large. 38type ObjectType int 39 40// Values that ObjectType can take 41const ( 42 RegularObjectType ObjectType = iota 43 StaticLargeObjectType 44 DynamicLargeObjectType 45) 46 47// Connection holds the details of the connection to the swift server. 48// 49// You need to provide UserName, ApiKey and AuthUrl when you create a 50// connection then call Authenticate on it. 51// 52// The auth version in use will be detected from the AuthURL - you can 53// override this with the AuthVersion parameter. 54// 55// If using v2 auth you can also set Region in the Connection 56// structure. If you don't set Region you will get the default region 57// which may not be what you want. 58// 59// For reference some common AuthUrls looks like this: 60// 61// Rackspace US https://auth.api.rackspacecloud.com/v1.0 62// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0 63// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 64// Memset Memstore UK https://auth.storage.memset.com/v1.0 65// Memstore v2 https://auth.storage.memset.com/v2.0 66// 67// When using Google Appengine you must provide the Connection with an 68// appengine-specific Transport: 69// 70// import ( 71// "appengine/urlfetch" 72// "fmt" 73// "github.com/ncw/swift" 74// ) 75// 76// func handler(w http.ResponseWriter, r *http.Request) { 77// ctx := appengine.NewContext(r) 78// tr := urlfetch.Transport{Context: ctx} 79// c := swift.Connection{ 80// UserName: "user", 81// ApiKey: "key", 82// AuthUrl: "auth_url", 83// Transport: tr, 84// } 85// _ := c.Authenticate() 86// containers, _ := c.ContainerNames(nil) 87// fmt.Fprintf(w, "containers: %q", containers) 88// } 89// 90// If you don't supply a Transport, one is made which relies on 91// http.ProxyFromEnvironment (http://golang.org/pkg/net/http/#ProxyFromEnvironment). 92// This means that the connection will respect the HTTP proxy specified by the 93// environment variables $HTTP_PROXY and $NO_PROXY. 94type Connection struct { 95 // Parameters - fill these in before calling Authenticate 96 // They are all optional except UserName, ApiKey and AuthUrl 97 Domain string // User's domain name 98 DomainId string // User's domain Id 99 UserName string // UserName for api 100 ApiKey string // Key for api access 101 AuthUrl string // Auth URL 102 Retries int // Retries on error (default is 3) 103 UserAgent string // Http User agent (default goswift/1.0) 104 ConnectTimeout time.Duration // Connect channel timeout (default 10s) 105 Timeout time.Duration // Data channel timeout (default 60s) 106 Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only) 107 AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect 108 Internal bool // Set this to true to use the the internal / service network 109 Tenant string // Name of the tenant (v2,v3 auth only) 110 TenantId string // Id of the tenant (v2,v3 auth only) 111 EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set) 112 TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain 113 TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain 114 TrustId string // Id of the trust (v3 auth only) 115 Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) 116 // These are filled in after Authenticate is called as are the defaults for above 117 StorageUrl string 118 AuthToken string 119 client *http.Client 120 Auth Authenticator `json:"-" xml:"-"` // the current authenticator 121 authLock sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth 122 // swiftInfo is filled after QueryInfo is called 123 swiftInfo SwiftInfo 124} 125 126// Error - all errors generated by this package are of this type. Other error 127// may be passed on from library functions though. 128type Error struct { 129 StatusCode int // HTTP status code if relevant or 0 if not 130 Text string 131} 132 133// Error satisfy the error interface. 134func (e *Error) Error() string { 135 return e.Text 136} 137 138// newError make a new error from a string. 139func newError(StatusCode int, Text string) *Error { 140 return &Error{ 141 StatusCode: StatusCode, 142 Text: Text, 143 } 144} 145 146// newErrorf makes a new error from sprintf parameters. 147func newErrorf(StatusCode int, Text string, Parameters ...interface{}) *Error { 148 return newError(StatusCode, fmt.Sprintf(Text, Parameters...)) 149} 150 151// errorMap defines http error codes to error mappings. 152type errorMap map[int]error 153 154var ( 155 // Specific Errors you might want to check for equality 156 BadRequest = newError(400, "Bad Request") 157 AuthorizationFailed = newError(401, "Authorization Failed") 158 ContainerNotFound = newError(404, "Container Not Found") 159 ContainerNotEmpty = newError(409, "Container Not Empty") 160 ObjectNotFound = newError(404, "Object Not Found") 161 ObjectCorrupted = newError(422, "Object Corrupted") 162 TimeoutError = newError(408, "Timeout when reading or writing data") 163 Forbidden = newError(403, "Operation forbidden") 164 TooLargeObject = newError(413, "Too Large Object") 165 166 // Mappings for authentication errors 167 authErrorMap = errorMap{ 168 400: BadRequest, 169 401: AuthorizationFailed, 170 403: Forbidden, 171 } 172 173 // Mappings for container errors 174 ContainerErrorMap = errorMap{ 175 400: BadRequest, 176 403: Forbidden, 177 404: ContainerNotFound, 178 409: ContainerNotEmpty, 179 } 180 181 // Mappings for object errors 182 objectErrorMap = errorMap{ 183 400: BadRequest, 184 403: Forbidden, 185 404: ObjectNotFound, 186 413: TooLargeObject, 187 422: ObjectCorrupted, 188 } 189) 190 191// checkClose is used to check the return from Close in a defer 192// statement. 193func checkClose(c io.Closer, err *error) { 194 cerr := c.Close() 195 if *err == nil { 196 *err = cerr 197 } 198} 199 200// parseHeaders checks a response for errors and translates into 201// standard errors if necessary. 202func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error { 203 if errorMap != nil { 204 if err, ok := errorMap[resp.StatusCode]; ok { 205 return err 206 } 207 } 208 if resp.StatusCode < 200 || resp.StatusCode > 299 { 209 return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status) 210 } 211 return nil 212} 213 214// readHeaders returns a Headers object from the http.Response. 215// 216// If it receives multiple values for a key (which should never 217// happen) it will use the first one 218func readHeaders(resp *http.Response) Headers { 219 headers := Headers{} 220 for key, values := range resp.Header { 221 headers[key] = values[0] 222 } 223 return headers 224} 225 226// Headers stores HTTP headers (can only have one of each header like Swift). 227type Headers map[string]string 228 229// Does an http request using the running timer passed in 230func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*http.Response, error) { 231 // Do the request in the background so we can check the timeout 232 type result struct { 233 resp *http.Response 234 err error 235 } 236 done := make(chan result, 1) 237 go func() { 238 resp, err := c.client.Do(req) 239 done <- result{resp, err} 240 }() 241 // Wait for the read or the timeout 242 select { 243 case r := <-done: 244 return r.resp, r.err 245 case <-timer.C: 246 // Kill the connection on timeout so we don't leak sockets or goroutines 247 cancelRequest(c.Transport, req) 248 return nil, TimeoutError 249 } 250 panic("unreachable") // For Go 1.0 251} 252 253// Set defaults for any unset values 254// 255// Call with authLock held 256func (c *Connection) setDefaults() { 257 if c.UserAgent == "" { 258 c.UserAgent = DefaultUserAgent 259 } 260 if c.Retries == 0 { 261 c.Retries = DefaultRetries 262 } 263 if c.ConnectTimeout == 0 { 264 c.ConnectTimeout = 10 * time.Second 265 } 266 if c.Timeout == 0 { 267 c.Timeout = 60 * time.Second 268 } 269 if c.Transport == nil { 270 c.Transport = &http.Transport{ 271 // TLSClientConfig: &tls.Config{RootCAs: pool}, 272 // DisableCompression: true, 273 Proxy: http.ProxyFromEnvironment, 274 MaxIdleConnsPerHost: 2048, 275 } 276 } 277 if c.client == nil { 278 c.client = &http.Client{ 279 // CheckRedirect: redirectPolicyFunc, 280 Transport: c.Transport, 281 } 282 } 283} 284 285// Authenticate connects to the Swift server. 286// 287// If you don't call it before calling one of the connection methods 288// then it will be called for you on the first access. 289func (c *Connection) Authenticate() (err error) { 290 c.authLock.Lock() 291 defer c.authLock.Unlock() 292 return c.authenticate() 293} 294 295// Internal implementation of Authenticate 296// 297// Call with authLock held 298func (c *Connection) authenticate() (err error) { 299 c.setDefaults() 300 301 // Flush the keepalives connection - if we are 302 // re-authenticating then stuff has gone wrong 303 flushKeepaliveConnections(c.Transport) 304 305 if c.Auth == nil { 306 c.Auth, err = newAuth(c) 307 if err != nil { 308 return 309 } 310 } 311 312 retries := 1 313again: 314 var req *http.Request 315 req, err = c.Auth.Request(c) 316 if err != nil { 317 return 318 } 319 if req != nil { 320 timer := time.NewTimer(c.ConnectTimeout) 321 var resp *http.Response 322 resp, err = c.doTimeoutRequest(timer, req) 323 if err != nil { 324 return 325 } 326 defer func() { 327 checkClose(resp.Body, &err) 328 // Flush the auth connection - we don't want to keep 329 // it open if keepalives were enabled 330 flushKeepaliveConnections(c.Transport) 331 }() 332 if err = c.parseHeaders(resp, authErrorMap); err != nil { 333 // Try again for a limited number of times on 334 // AuthorizationFailed or BadRequest. This allows us 335 // to try some alternate forms of the request 336 if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { 337 retries-- 338 goto again 339 } 340 return 341 } 342 err = c.Auth.Response(resp) 343 if err != nil { 344 return 345 } 346 } 347 if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" { 348 c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType) 349 } else { 350 c.StorageUrl = c.Auth.StorageUrl(c.Internal) 351 } 352 c.AuthToken = c.Auth.Token() 353 if !c.authenticated() { 354 err = newError(0, "Response didn't have storage url and auth token") 355 return 356 } 357 return 358} 359 360// Get an authToken and url 361// 362// The Url may be updated if it needed to authenticate using the OnReAuth function 363func (c *Connection) getUrlAndAuthToken(targetUrlIn string, OnReAuth func() (string, error)) (targetUrlOut, authToken string, err error) { 364 c.authLock.Lock() 365 defer c.authLock.Unlock() 366 targetUrlOut = targetUrlIn 367 if !c.authenticated() { 368 err = c.authenticate() 369 if err != nil { 370 return 371 } 372 if OnReAuth != nil { 373 targetUrlOut, err = OnReAuth() 374 if err != nil { 375 return 376 } 377 } 378 } 379 authToken = c.AuthToken 380 return 381} 382 383// flushKeepaliveConnections is called to flush pending requests after an error. 384func flushKeepaliveConnections(transport http.RoundTripper) { 385 if tr, ok := transport.(interface { 386 CloseIdleConnections() 387 }); ok { 388 tr.CloseIdleConnections() 389 } 390} 391 392// UnAuthenticate removes the authentication from the Connection. 393func (c *Connection) UnAuthenticate() { 394 c.authLock.Lock() 395 c.StorageUrl = "" 396 c.AuthToken = "" 397 c.authLock.Unlock() 398} 399 400// Authenticated returns a boolean to show if the current connection 401// is authenticated. 402// 403// Doesn't actually check the credentials against the server. 404func (c *Connection) Authenticated() bool { 405 c.authLock.Lock() 406 defer c.authLock.Unlock() 407 return c.authenticated() 408} 409 410// Internal version of Authenticated() 411// 412// Call with authLock held 413func (c *Connection) authenticated() bool { 414 return c.StorageUrl != "" && c.AuthToken != "" 415} 416 417// SwiftInfo contains the JSON object returned by Swift when the /info 418// route is queried. The object contains, among others, the Swift version, 419// the enabled middlewares and their configuration 420type SwiftInfo map[string]interface{} 421 422func (i SwiftInfo) SupportsBulkDelete() bool { 423 _, val := i["bulk_delete"] 424 return val 425} 426 427func (i SwiftInfo) SupportsSLO() bool { 428 _, val := i["slo"] 429 return val 430} 431 432func (i SwiftInfo) SLOMinSegmentSize() int64 { 433 if slo, ok := i["slo"].(map[string]interface{}); ok { 434 val, _ := slo["min_segment_size"].(float64) 435 return int64(val) 436 } 437 return 1 438} 439 440// Discover Swift configuration by doing a request against /info 441func (c *Connection) QueryInfo() (infos SwiftInfo, err error) { 442 infoUrl, err := url.Parse(c.StorageUrl) 443 if err != nil { 444 return nil, err 445 } 446 infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info") 447 resp, err := c.client.Get(infoUrl.String()) 448 if err == nil { 449 if resp.StatusCode != http.StatusOK { 450 resp.Body.Close() 451 return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode) 452 } 453 err = readJson(resp, &infos) 454 if err == nil { 455 c.authLock.Lock() 456 c.swiftInfo = infos 457 c.authLock.Unlock() 458 } 459 return infos, err 460 } 461 return nil, err 462} 463 464func (c *Connection) cachedQueryInfo() (infos SwiftInfo, err error) { 465 c.authLock.Lock() 466 infos = c.swiftInfo 467 c.authLock.Unlock() 468 if infos == nil { 469 infos, err = c.QueryInfo() 470 if err != nil { 471 return 472 } 473 } 474 return infos, nil 475} 476 477// RequestOpts contains parameters for Connection.storage. 478type RequestOpts struct { 479 Container string 480 ObjectName string 481 Operation string 482 Parameters url.Values 483 Headers Headers 484 ErrorMap errorMap 485 NoResponse bool 486 Body io.Reader 487 Retries int 488 // if set this is called on re-authentication to refresh the targetUrl 489 OnReAuth func() (string, error) 490} 491 492// Call runs a remote command on the targetUrl, returns a 493// response, headers and possible error. 494// 495// operation is GET, HEAD etc 496// container is the name of a container 497// Any other parameters (if not None) are added to the targetUrl 498// 499// Returns a response or an error. If response is returned then 500// resp.Body.Close() must be called on it, unless noResponse is set in 501// which case the body will be closed in this function 502// 503// If "Content-Length" is set in p.Headers it will be used - this can 504// be used to override the default chunked transfer encoding for 505// uploads. 506// 507// This will Authenticate if necessary, and re-authenticate if it 508// receives a 401 error which means the token has expired 509// 510// This method is exported so extensions can call it. 511func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, headers Headers, err error) { 512 c.authLock.Lock() 513 c.setDefaults() 514 c.authLock.Unlock() 515 retries := p.Retries 516 if retries == 0 { 517 retries = c.Retries 518 } 519 var req *http.Request 520 for { 521 var authToken string 522 if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil { 523 return //authentication failure 524 } 525 var URL *url.URL 526 URL, err = url.Parse(targetUrl) 527 if err != nil { 528 return 529 } 530 if p.Container != "" { 531 URL.Path += "/" + p.Container 532 if p.ObjectName != "" { 533 URL.Path += "/" + p.ObjectName 534 } 535 } 536 if p.Parameters != nil { 537 URL.RawQuery = p.Parameters.Encode() 538 } 539 timer := time.NewTimer(c.ConnectTimeout) 540 reader := p.Body 541 if reader != nil { 542 reader = newWatchdogReader(reader, c.Timeout, timer) 543 } 544 req, err = http.NewRequest(p.Operation, URL.String(), reader) 545 if err != nil { 546 return 547 } 548 if p.Headers != nil { 549 for k, v := range p.Headers { 550 // Set ContentLength in req if the user passed it in in the headers 551 if k == "Content-Length" { 552 contentLength, err := strconv.ParseInt(v, 10, 64) 553 if err != nil { 554 return nil, nil, fmt.Errorf("Invalid %q header %q: %v", k, v, err) 555 } 556 req.ContentLength = contentLength 557 } else { 558 req.Header.Add(k, v) 559 } 560 } 561 } 562 req.Header.Add("User-Agent", c.UserAgent) 563 req.Header.Add("X-Auth-Token", authToken) 564 resp, err = c.doTimeoutRequest(timer, req) 565 if err != nil { 566 if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 { 567 retries-- 568 continue 569 } 570 return nil, nil, err 571 } 572 // Check to see if token has expired 573 if resp.StatusCode == 401 && retries > 0 { 574 _ = resp.Body.Close() 575 c.UnAuthenticate() 576 retries-- 577 } else { 578 break 579 } 580 } 581 582 if err = c.parseHeaders(resp, p.ErrorMap); err != nil { 583 _ = resp.Body.Close() 584 return nil, nil, err 585 } 586 headers = readHeaders(resp) 587 if p.NoResponse { 588 err = resp.Body.Close() 589 if err != nil { 590 return nil, nil, err 591 } 592 } else { 593 // Cancel the request on timeout 594 cancel := func() { 595 cancelRequest(c.Transport, req) 596 } 597 // Wrap resp.Body to make it obey an idle timeout 598 resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel) 599 } 600 return 601} 602 603// storage runs a remote command on a the storage url, returns a 604// response, headers and possible error. 605// 606// operation is GET, HEAD etc 607// container is the name of a container 608// Any other parameters (if not None) are added to the storage url 609// 610// Returns a response or an error. If response is returned then 611// resp.Body.Close() must be called on it, unless noResponse is set in 612// which case the body will be closed in this function 613// 614// This will Authenticate if necessary, and re-authenticate if it 615// receives a 401 error which means the token has expired 616func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Headers, err error) { 617 p.OnReAuth = func() (string, error) { 618 return c.StorageUrl, nil 619 } 620 c.authLock.Lock() 621 url := c.StorageUrl 622 c.authLock.Unlock() 623 return c.Call(url, p) 624} 625 626// readLines reads the response into an array of strings. 627// 628// Closes the response when done 629func readLines(resp *http.Response) (lines []string, err error) { 630 defer checkClose(resp.Body, &err) 631 reader := bufio.NewReader(resp.Body) 632 buffer := bytes.NewBuffer(make([]byte, 0, 128)) 633 var part []byte 634 var prefix bool 635 for { 636 if part, prefix, err = reader.ReadLine(); err != nil { 637 break 638 } 639 buffer.Write(part) 640 if !prefix { 641 lines = append(lines, buffer.String()) 642 buffer.Reset() 643 } 644 } 645 if err == io.EOF { 646 err = nil 647 } 648 return 649} 650 651// readJson reads the response into the json type passed in 652// 653// Closes the response when done 654func readJson(resp *http.Response, result interface{}) (err error) { 655 defer checkClose(resp.Body, &err) 656 decoder := json.NewDecoder(resp.Body) 657 return decoder.Decode(result) 658} 659 660/* ------------------------------------------------------------ */ 661 662// ContainersOpts is options for Containers() and ContainerNames() 663type ContainersOpts struct { 664 Limit int // For an integer value n, limits the number of results to at most n values. 665 Prefix string // Given a string value x, return container names matching the specified prefix. 666 Marker string // Given a string value x, return container names greater in value than the specified marker. 667 EndMarker string // Given a string value x, return container names less in value than the specified marker. 668 Headers Headers // Any additional HTTP headers - can be nil 669} 670 671// parse the ContainerOpts 672func (opts *ContainersOpts) parse() (url.Values, Headers) { 673 v := url.Values{} 674 var h Headers 675 if opts != nil { 676 if opts.Limit > 0 { 677 v.Set("limit", strconv.Itoa(opts.Limit)) 678 } 679 if opts.Prefix != "" { 680 v.Set("prefix", opts.Prefix) 681 } 682 if opts.Marker != "" { 683 v.Set("marker", opts.Marker) 684 } 685 if opts.EndMarker != "" { 686 v.Set("end_marker", opts.EndMarker) 687 } 688 h = opts.Headers 689 } 690 return v, h 691} 692 693// ContainerNames returns a slice of names of containers in this account. 694func (c *Connection) ContainerNames(opts *ContainersOpts) ([]string, error) { 695 v, h := opts.parse() 696 resp, _, err := c.storage(RequestOpts{ 697 Operation: "GET", 698 Parameters: v, 699 ErrorMap: ContainerErrorMap, 700 Headers: h, 701 }) 702 if err != nil { 703 return nil, err 704 } 705 lines, err := readLines(resp) 706 return lines, err 707} 708 709// Container contains information about a container 710type Container struct { 711 Name string // Name of the container 712 Count int64 // Number of objects in the container 713 Bytes int64 // Total number of bytes used in the container 714} 715 716// Containers returns a slice of structures with full information as 717// described in Container. 718func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) { 719 v, h := opts.parse() 720 v.Set("format", "json") 721 resp, _, err := c.storage(RequestOpts{ 722 Operation: "GET", 723 Parameters: v, 724 ErrorMap: ContainerErrorMap, 725 Headers: h, 726 }) 727 if err != nil { 728 return nil, err 729 } 730 var containers []Container 731 err = readJson(resp, &containers) 732 return containers, err 733} 734 735// containersAllOpts makes a copy of opts if set or makes a new one and 736// overrides Limit and Marker 737func containersAllOpts(opts *ContainersOpts) *ContainersOpts { 738 var newOpts ContainersOpts 739 if opts != nil { 740 newOpts = *opts 741 } 742 if newOpts.Limit == 0 { 743 newOpts.Limit = allContainersLimit 744 } 745 newOpts.Marker = "" 746 return &newOpts 747} 748 749// ContainersAll is like Containers but it returns all the Containers 750// 751// It calls Containers multiple times using the Marker parameter 752// 753// It has a default Limit parameter but you may pass in your own 754func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) { 755 opts = containersAllOpts(opts) 756 containers := make([]Container, 0) 757 for { 758 newContainers, err := c.Containers(opts) 759 if err != nil { 760 return nil, err 761 } 762 containers = append(containers, newContainers...) 763 if len(newContainers) < opts.Limit { 764 break 765 } 766 opts.Marker = newContainers[len(newContainers)-1].Name 767 } 768 return containers, nil 769} 770 771// ContainerNamesAll is like ContainerNamess but it returns all the Containers 772// 773// It calls ContainerNames multiple times using the Marker parameter 774// 775// It has a default Limit parameter but you may pass in your own 776func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) { 777 opts = containersAllOpts(opts) 778 containers := make([]string, 0) 779 for { 780 newContainers, err := c.ContainerNames(opts) 781 if err != nil { 782 return nil, err 783 } 784 containers = append(containers, newContainers...) 785 if len(newContainers) < opts.Limit { 786 break 787 } 788 opts.Marker = newContainers[len(newContainers)-1] 789 } 790 return containers, nil 791} 792 793/* ------------------------------------------------------------ */ 794 795// ObjectOpts is options for Objects() and ObjectNames() 796type ObjectsOpts struct { 797 Limit int // For an integer value n, limits the number of results to at most n values. 798 Marker string // Given a string value x, return object names greater in value than the specified marker. 799 EndMarker string // Given a string value x, return object names less in value than the specified marker 800 Prefix string // For a string value x, causes the results to be limited to object names beginning with the substring x. 801 Path string // For a string value x, return the object names nested in the pseudo path 802 Delimiter rune // For a character c, return all the object names nested in the container 803 Headers Headers // Any additional HTTP headers - can be nil 804} 805 806// parse reads values out of ObjectsOpts 807func (opts *ObjectsOpts) parse() (url.Values, Headers) { 808 v := url.Values{} 809 var h Headers 810 if opts != nil { 811 if opts.Limit > 0 { 812 v.Set("limit", strconv.Itoa(opts.Limit)) 813 } 814 if opts.Marker != "" { 815 v.Set("marker", opts.Marker) 816 } 817 if opts.EndMarker != "" { 818 v.Set("end_marker", opts.EndMarker) 819 } 820 if opts.Prefix != "" { 821 v.Set("prefix", opts.Prefix) 822 } 823 if opts.Path != "" { 824 v.Set("path", opts.Path) 825 } 826 if opts.Delimiter != 0 { 827 v.Set("delimiter", string(opts.Delimiter)) 828 } 829 h = opts.Headers 830 } 831 return v, h 832} 833 834// ObjectNames returns a slice of names of objects in a given container. 835func (c *Connection) ObjectNames(container string, opts *ObjectsOpts) ([]string, error) { 836 v, h := opts.parse() 837 resp, _, err := c.storage(RequestOpts{ 838 Container: container, 839 Operation: "GET", 840 Parameters: v, 841 ErrorMap: ContainerErrorMap, 842 Headers: h, 843 }) 844 if err != nil { 845 return nil, err 846 } 847 return readLines(resp) 848} 849 850// Object contains information about an object 851type Object struct { 852 Name string `json:"name"` // object name 853 ContentType string `json:"content_type"` // eg application/directory 854 Bytes int64 `json:"bytes"` // size in bytes 855 ServerLastModified string `json:"last_modified"` // Last modified time, eg '2011-06-30T08:20:47.736680' as a string supplied by the server 856 LastModified time.Time // Last modified time converted to a time.Time 857 Hash string `json:"hash"` // MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" 858 PseudoDirectory bool // Set when using delimiter to show that this directory object does not really exist 859 SubDir string `json:"subdir"` // returned only when using delimiter to mark "pseudo directories" 860 ObjectType ObjectType // type of this object 861} 862 863// Objects returns a slice of Object with information about each 864// object in the container. 865// 866// If Delimiter is set in the opts then PseudoDirectory may be set, 867// with ContentType 'application/directory'. These are not real 868// objects but represent directories of objects which haven't had an 869// object created for them. 870func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, error) { 871 v, h := opts.parse() 872 v.Set("format", "json") 873 resp, _, err := c.storage(RequestOpts{ 874 Container: container, 875 Operation: "GET", 876 Parameters: v, 877 ErrorMap: ContainerErrorMap, 878 Headers: h, 879 }) 880 if err != nil { 881 return nil, err 882 } 883 var objects []Object 884 err = readJson(resp, &objects) 885 // Convert Pseudo directories and dates 886 for i := range objects { 887 object := &objects[i] 888 if object.SubDir != "" { 889 object.Name = object.SubDir 890 object.PseudoDirectory = true 891 object.ContentType = "application/directory" 892 } 893 if object.ServerLastModified != "" { 894 // 2012-11-11T14:49:47.887250 895 // 896 // Remove fractional seconds if present. This 897 // then keeps it consistent with Object 898 // which can only return timestamps accurate 899 // to 1 second 900 // 901 // The TimeFormat will parse fractional 902 // seconds if desired though 903 datetime := strings.SplitN(object.ServerLastModified, ".", 2)[0] 904 object.LastModified, err = time.Parse(TimeFormat, datetime) 905 if err != nil { 906 return nil, err 907 } 908 } 909 } 910 return objects, err 911} 912 913// objectsAllOpts makes a copy of opts if set or makes a new one and 914// overrides Limit and Marker 915func objectsAllOpts(opts *ObjectsOpts, Limit int) *ObjectsOpts { 916 var newOpts ObjectsOpts 917 if opts != nil { 918 newOpts = *opts 919 } 920 if newOpts.Limit == 0 { 921 newOpts.Limit = Limit 922 } 923 newOpts.Marker = "" 924 return &newOpts 925} 926 927// A closure defined by the caller to iterate through all objects 928// 929// Call Objects or ObjectNames from here with the *ObjectOpts passed in 930// 931// Do whatever is required with the results then return them 932type ObjectsWalkFn func(*ObjectsOpts) (interface{}, error) 933 934// ObjectsWalk is uses to iterate through all the objects in chunks as 935// returned by Objects or ObjectNames using the Marker and Limit 936// parameters in the ObjectsOpts. 937// 938// Pass in a closure `walkFn` which calls Objects or ObjectNames with 939// the *ObjectsOpts passed to it and does something with the results. 940// 941// Errors will be returned from this function 942// 943// It has a default Limit parameter but you may pass in your own 944func (c *Connection) ObjectsWalk(container string, opts *ObjectsOpts, walkFn ObjectsWalkFn) error { 945 opts = objectsAllOpts(opts, allObjectsChanLimit) 946 for { 947 objects, err := walkFn(opts) 948 if err != nil { 949 return err 950 } 951 var n int 952 var last string 953 switch objects := objects.(type) { 954 case []string: 955 n = len(objects) 956 if n > 0 { 957 last = objects[len(objects)-1] 958 } 959 case []Object: 960 n = len(objects) 961 if n > 0 { 962 last = objects[len(objects)-1].Name 963 } 964 default: 965 panic("Unknown type returned to ObjectsWalk") 966 } 967 if n < opts.Limit { 968 break 969 } 970 opts.Marker = last 971 } 972 return nil 973} 974 975// ObjectsAll is like Objects but it returns an unlimited number of Objects in a slice 976// 977// It calls Objects multiple times using the Marker parameter 978func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) { 979 objects := make([]Object, 0) 980 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 981 newObjects, err := c.Objects(container, opts) 982 if err == nil { 983 objects = append(objects, newObjects...) 984 } 985 return newObjects, err 986 }) 987 return objects, err 988} 989 990// ObjectNamesAll is like ObjectNames but it returns all the Objects 991// 992// It calls ObjectNames multiple times using the Marker parameter 993// 994// It has a default Limit parameter but you may pass in your own 995func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) { 996 objects := make([]string, 0) 997 err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { 998 newObjects, err := c.ObjectNames(container, opts) 999 if err == nil { 1000 objects = append(objects, newObjects...) 1001 } 1002 return newObjects, err 1003 }) 1004 return objects, err 1005} 1006 1007// Account contains information about this account. 1008type Account struct { 1009 BytesUsed int64 // total number of bytes used 1010 Containers int64 // total number of containers 1011 Objects int64 // total number of objects 1012} 1013 1014// getInt64FromHeader is a helper function to decode int64 from header. 1015func getInt64FromHeader(resp *http.Response, header string) (result int64, err error) { 1016 value := resp.Header.Get(header) 1017 result, err = strconv.ParseInt(value, 10, 64) 1018 if err != nil { 1019 err = newErrorf(0, "Bad Header '%s': '%s': %s", header, value, err) 1020 } 1021 return 1022} 1023 1024// Account returns info about the account in an Account struct. 1025func (c *Connection) Account() (info Account, headers Headers, err error) { 1026 var resp *http.Response 1027 resp, headers, err = c.storage(RequestOpts{ 1028 Operation: "HEAD", 1029 ErrorMap: ContainerErrorMap, 1030 NoResponse: true, 1031 }) 1032 if err != nil { 1033 return 1034 } 1035 // Parse the headers into a dict 1036 // 1037 // {'Accept-Ranges': 'bytes', 1038 // 'Content-Length': '0', 1039 // 'Date': 'Tue, 05 Jul 2011 16:37:06 GMT', 1040 // 'X-Account-Bytes-Used': '316598182', 1041 // 'X-Account-Container-Count': '4', 1042 // 'X-Account-Object-Count': '1433'} 1043 if info.BytesUsed, err = getInt64FromHeader(resp, "X-Account-Bytes-Used"); err != nil { 1044 return 1045 } 1046 if info.Containers, err = getInt64FromHeader(resp, "X-Account-Container-Count"); err != nil { 1047 return 1048 } 1049 if info.Objects, err = getInt64FromHeader(resp, "X-Account-Object-Count"); err != nil { 1050 return 1051 } 1052 return 1053} 1054 1055// AccountUpdate adds, replaces or remove account metadata. 1056// 1057// Add or update keys by mentioning them in the Headers. 1058// 1059// Remove keys by setting them to an empty string. 1060func (c *Connection) AccountUpdate(h Headers) error { 1061 _, _, err := c.storage(RequestOpts{ 1062 Operation: "POST", 1063 ErrorMap: ContainerErrorMap, 1064 NoResponse: true, 1065 Headers: h, 1066 }) 1067 return err 1068} 1069 1070// ContainerCreate creates a container. 1071// 1072// If you don't want to add Headers just pass in nil 1073// 1074// No error is returned if it already exists but the metadata if any will be updated. 1075func (c *Connection) ContainerCreate(container string, h Headers) error { 1076 _, _, err := c.storage(RequestOpts{ 1077 Container: container, 1078 Operation: "PUT", 1079 ErrorMap: ContainerErrorMap, 1080 NoResponse: true, 1081 Headers: h, 1082 }) 1083 return err 1084} 1085 1086// ContainerDelete deletes a container. 1087// 1088// May return ContainerDoesNotExist or ContainerNotEmpty 1089func (c *Connection) ContainerDelete(container string) error { 1090 _, _, err := c.storage(RequestOpts{ 1091 Container: container, 1092 Operation: "DELETE", 1093 ErrorMap: ContainerErrorMap, 1094 NoResponse: true, 1095 }) 1096 return err 1097} 1098 1099// Container returns info about a single container including any 1100// metadata in the headers. 1101func (c *Connection) Container(container string) (info Container, headers Headers, err error) { 1102 var resp *http.Response 1103 resp, headers, err = c.storage(RequestOpts{ 1104 Container: container, 1105 Operation: "HEAD", 1106 ErrorMap: ContainerErrorMap, 1107 NoResponse: true, 1108 }) 1109 if err != nil { 1110 return 1111 } 1112 // Parse the headers into the struct 1113 info.Name = container 1114 if info.Bytes, err = getInt64FromHeader(resp, "X-Container-Bytes-Used"); err != nil { 1115 return 1116 } 1117 if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil { 1118 return 1119 } 1120 return 1121} 1122 1123// ContainerUpdate adds, replaces or removes container metadata. 1124// 1125// Add or update keys by mentioning them in the Metadata. 1126// 1127// Remove keys by setting them to an empty string. 1128// 1129// Container metadata can only be read with Container() not with Containers(). 1130func (c *Connection) ContainerUpdate(container string, h Headers) error { 1131 _, _, err := c.storage(RequestOpts{ 1132 Container: container, 1133 Operation: "POST", 1134 ErrorMap: ContainerErrorMap, 1135 NoResponse: true, 1136 Headers: h, 1137 }) 1138 return err 1139} 1140 1141// ------------------------------------------------------------ 1142 1143// ObjectCreateFile represents a swift object open for writing 1144type ObjectCreateFile struct { 1145 checkHash bool // whether we are checking the hash 1146 pipeReader *io.PipeReader // pipe for the caller to use 1147 pipeWriter *io.PipeWriter 1148 hash hash.Hash // hash being build up as we go along 1149 done chan struct{} // signals when the upload has finished 1150 resp *http.Response // valid when done has signalled 1151 err error // ditto 1152 headers Headers // ditto 1153} 1154 1155// Write bytes to the object - see io.Writer 1156func (file *ObjectCreateFile) Write(p []byte) (n int, err error) { 1157 n, err = file.pipeWriter.Write(p) 1158 if err == io.ErrClosedPipe { 1159 if file.err != nil { 1160 return 0, file.err 1161 } 1162 return 0, newError(500, "Write on closed file") 1163 } 1164 if err == nil && file.checkHash { 1165 _, _ = file.hash.Write(p) 1166 } 1167 return 1168} 1169 1170// Close the object and checks the md5sum if it was required. 1171// 1172// Also returns any other errors from the server (eg container not 1173// found) so it is very important to check the errors on this method. 1174func (file *ObjectCreateFile) Close() error { 1175 // Close the body 1176 err := file.pipeWriter.Close() 1177 if err != nil { 1178 return err 1179 } 1180 1181 // Wait for the HTTP operation to complete 1182 <-file.done 1183 1184 // Check errors 1185 if file.err != nil { 1186 return file.err 1187 } 1188 if file.checkHash { 1189 receivedMd5 := strings.ToLower(file.headers["Etag"]) 1190 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1191 if receivedMd5 != calculatedMd5 { 1192 return ObjectCorrupted 1193 } 1194 } 1195 return nil 1196} 1197 1198// Headers returns the response headers from the created object if the upload 1199// has been completed. The Close() method must be called on an ObjectCreateFile 1200// before this method. 1201func (file *ObjectCreateFile) Headers() (Headers, error) { 1202 // error out if upload is not complete. 1203 select { 1204 case <-file.done: 1205 default: 1206 return nil, fmt.Errorf("Cannot get metadata, object upload failed or has not yet completed.") 1207 } 1208 return file.headers, nil 1209} 1210 1211// Check it satisfies the interface 1212var _ io.WriteCloser = &ObjectCreateFile{} 1213 1214// objectPutHeaders create a set of headers for a PUT 1215// 1216// It guesses the contentType from the objectName if it isn't set 1217// 1218// checkHash may be changed 1219func objectPutHeaders(objectName string, checkHash *bool, Hash string, contentType string, h Headers) Headers { 1220 if contentType == "" { 1221 contentType = mime.TypeByExtension(path.Ext(objectName)) 1222 if contentType == "" { 1223 contentType = "application/octet-stream" 1224 } 1225 } 1226 // Meta stuff 1227 extraHeaders := map[string]string{ 1228 "Content-Type": contentType, 1229 } 1230 for key, value := range h { 1231 extraHeaders[key] = value 1232 } 1233 if Hash != "" { 1234 extraHeaders["Etag"] = Hash 1235 *checkHash = false // the server will do it 1236 } 1237 return extraHeaders 1238} 1239 1240// ObjectCreate creates or updates the object in the container. It 1241// returns an io.WriteCloser you should write the contents to. You 1242// MUST call Close() on it and you MUST check the error return from 1243// Close(). 1244// 1245// If checkHash is True then it will calculate the MD5 Hash of the 1246// file as it is being uploaded and check it against that returned 1247// from the server. If it is wrong then it will return 1248// ObjectCorrupted on Close() 1249// 1250// If you know the MD5 hash of the object ahead of time then set the 1251// Hash parameter and it will be sent to the server (as an Etag 1252// header) and the server will check the MD5 itself after the upload, 1253// and this will return ObjectCorrupted on Close() if it is incorrect. 1254// 1255// If you don't want any error protection (not recommended) then set 1256// checkHash to false and Hash to "". 1257// 1258// If contentType is set it will be used, otherwise one will be 1259// guessed from objectName using mime.TypeByExtension 1260func (c *Connection) ObjectCreate(container string, objectName string, checkHash bool, Hash string, contentType string, h Headers) (file *ObjectCreateFile, err error) { 1261 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1262 pipeReader, pipeWriter := io.Pipe() 1263 file = &ObjectCreateFile{ 1264 hash: md5.New(), 1265 checkHash: checkHash, 1266 pipeReader: pipeReader, 1267 pipeWriter: pipeWriter, 1268 done: make(chan struct{}), 1269 } 1270 // Run the PUT in the background piping it data 1271 go func() { 1272 opts := RequestOpts{ 1273 Container: container, 1274 ObjectName: objectName, 1275 Operation: "PUT", 1276 Headers: extraHeaders, 1277 Body: pipeReader, 1278 NoResponse: true, 1279 ErrorMap: objectErrorMap, 1280 } 1281 file.resp, file.headers, file.err = c.storage(opts) 1282 // Signal finished 1283 pipeReader.Close() 1284 close(file.done) 1285 }() 1286 return 1287} 1288 1289func (c *Connection) objectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers, parameters url.Values) (headers Headers, err error) { 1290 extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) 1291 hash := md5.New() 1292 var body io.Reader = contents 1293 if checkHash { 1294 body = io.TeeReader(contents, hash) 1295 } 1296 _, headers, err = c.storage(RequestOpts{ 1297 Container: container, 1298 ObjectName: objectName, 1299 Operation: "PUT", 1300 Headers: extraHeaders, 1301 Body: body, 1302 NoResponse: true, 1303 ErrorMap: objectErrorMap, 1304 Parameters: parameters, 1305 }) 1306 if err != nil { 1307 return 1308 } 1309 if checkHash { 1310 receivedMd5 := strings.ToLower(headers["Etag"]) 1311 calculatedMd5 := fmt.Sprintf("%x", hash.Sum(nil)) 1312 if receivedMd5 != calculatedMd5 { 1313 err = ObjectCorrupted 1314 return 1315 } 1316 } 1317 return 1318} 1319 1320// ObjectPut creates or updates the path in the container from 1321// contents. contents should be an open io.Reader which will have all 1322// its contents read. 1323// 1324// This is a low level interface. 1325// 1326// If checkHash is True then it will calculate the MD5 Hash of the 1327// file as it is being uploaded and check it against that returned 1328// from the server. If it is wrong then it will return 1329// ObjectCorrupted. 1330// 1331// If you know the MD5 hash of the object ahead of time then set the 1332// Hash parameter and it will be sent to the server (as an Etag 1333// header) and the server will check the MD5 itself after the upload, 1334// and this will return ObjectCorrupted if it is incorrect. 1335// 1336// If you don't want any error protection (not recommended) then set 1337// checkHash to false and Hash to "". 1338// 1339// If contentType is set it will be used, otherwise one will be 1340// guessed from objectName using mime.TypeByExtension 1341func (c *Connection) ObjectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers) (headers Headers, err error) { 1342 return c.objectPut(container, objectName, contents, checkHash, Hash, contentType, h, nil) 1343} 1344 1345// ObjectPutBytes creates an object from a []byte in a container. 1346// 1347// This is a simplified interface which checks the MD5. 1348func (c *Connection) ObjectPutBytes(container string, objectName string, contents []byte, contentType string) (err error) { 1349 buf := bytes.NewBuffer(contents) 1350 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1351 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1352 return 1353} 1354 1355// ObjectPutString creates an object from a string in a container. 1356// 1357// This is a simplified interface which checks the MD5 1358func (c *Connection) ObjectPutString(container string, objectName string, contents string, contentType string) (err error) { 1359 buf := strings.NewReader(contents) 1360 h := Headers{"Content-Length": strconv.Itoa(len(contents))} 1361 _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) 1362 return 1363} 1364 1365// ObjectOpenFile represents a swift object open for reading 1366type ObjectOpenFile struct { 1367 connection *Connection // stored copy of Connection used in Open 1368 container string // stored copy of container used in Open 1369 objectName string // stored copy of objectName used in Open 1370 headers Headers // stored copy of headers used in Open 1371 resp *http.Response // http connection 1372 body io.Reader // read data from this 1373 checkHash bool // true if checking MD5 1374 hash hash.Hash // currently accumulating MD5 1375 bytes int64 // number of bytes read on this connection 1376 eof bool // whether we have read end of file 1377 pos int64 // current position when reading 1378 lengthOk bool // whether length is valid 1379 length int64 // length of the object if read 1380 seeked bool // whether we have seeked this file or not 1381 overSeeked bool // set if we have seeked to the end or beyond 1382} 1383 1384// Read bytes from the object - see io.Reader 1385func (file *ObjectOpenFile) Read(p []byte) (n int, err error) { 1386 if file.overSeeked { 1387 return 0, io.EOF 1388 } 1389 n, err = file.body.Read(p) 1390 file.bytes += int64(n) 1391 file.pos += int64(n) 1392 if err == io.EOF { 1393 file.eof = true 1394 } 1395 return 1396} 1397 1398// Seek sets the offset for the next Read to offset, interpreted 1399// according to whence: 0 means relative to the origin of the file, 1 1400// means relative to the current offset, and 2 means relative to the 1401// end. Seek returns the new offset and an Error, if any. 1402// 1403// Seek uses HTTP Range headers which, if the file pointer is moved, 1404// will involve reopening the HTTP connection. 1405// 1406// Note that you can't seek to the end of a file or beyond; HTTP Range 1407// requests don't support the file pointer being outside the data, 1408// unlike os.File 1409// 1410// Seek(0, 1) will return the current file pointer. 1411func (file *ObjectOpenFile) Seek(offset int64, whence int) (newPos int64, err error) { 1412 file.overSeeked = false 1413 switch whence { 1414 case 0: // relative to start 1415 newPos = offset 1416 case 1: // relative to current 1417 newPos = file.pos + offset 1418 case 2: // relative to end 1419 if !file.lengthOk { 1420 return file.pos, newError(0, "Length of file unknown so can't seek from end") 1421 } 1422 newPos = file.length + offset 1423 if offset >= 0 { 1424 file.overSeeked = true 1425 return 1426 } 1427 default: 1428 panic("Unknown whence in ObjectOpenFile.Seek") 1429 } 1430 // If at correct position (quite likely), do nothing 1431 if newPos == file.pos { 1432 return 1433 } 1434 // Close the file... 1435 file.seeked = true 1436 err = file.Close() 1437 if err != nil { 1438 return 1439 } 1440 // ...and re-open with a Range header 1441 if file.headers == nil { 1442 file.headers = Headers{} 1443 } 1444 if newPos > 0 { 1445 file.headers["Range"] = fmt.Sprintf("bytes=%d-", newPos) 1446 } else { 1447 delete(file.headers, "Range") 1448 } 1449 newFile, _, err := file.connection.ObjectOpen(file.container, file.objectName, false, file.headers) 1450 if err != nil { 1451 return 1452 } 1453 // Update the file 1454 file.resp = newFile.resp 1455 file.body = newFile.body 1456 file.checkHash = false 1457 file.pos = newPos 1458 return 1459} 1460 1461// Length gets the objects content length either from a cached copy or 1462// from the server. 1463func (file *ObjectOpenFile) Length() (int64, error) { 1464 if !file.lengthOk { 1465 info, _, err := file.connection.Object(file.container, file.objectName) 1466 file.length = info.Bytes 1467 file.lengthOk = (err == nil) 1468 return file.length, err 1469 } 1470 return file.length, nil 1471} 1472 1473// Close the object and checks the length and md5sum if it was 1474// required and all the object was read 1475func (file *ObjectOpenFile) Close() (err error) { 1476 // Close the body at the end 1477 defer checkClose(file.resp.Body, &err) 1478 1479 // If not end of file or seeked then can't check anything 1480 if !file.eof || file.seeked { 1481 return 1482 } 1483 1484 // Check the MD5 sum if requested 1485 if file.checkHash { 1486 receivedMd5 := strings.ToLower(file.resp.Header.Get("Etag")) 1487 calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) 1488 if receivedMd5 != calculatedMd5 { 1489 err = ObjectCorrupted 1490 return 1491 } 1492 } 1493 1494 // Check to see we read the correct number of bytes 1495 if file.lengthOk && file.length != file.bytes { 1496 err = ObjectCorrupted 1497 return 1498 } 1499 return 1500} 1501 1502// Check it satisfies the interfaces 1503var _ io.ReadCloser = &ObjectOpenFile{} 1504var _ io.Seeker = &ObjectOpenFile{} 1505 1506func (c *Connection) objectOpenBase(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1507 var resp *http.Response 1508 opts := RequestOpts{ 1509 Container: container, 1510 ObjectName: objectName, 1511 Operation: "GET", 1512 ErrorMap: objectErrorMap, 1513 Headers: h, 1514 Parameters: parameters, 1515 } 1516 resp, headers, err = c.storage(opts) 1517 if err != nil { 1518 return 1519 } 1520 // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set 1521 if checkHash && headers.IsLargeObject() { 1522 // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) 1523 checkHash = false 1524 } 1525 file = &ObjectOpenFile{ 1526 connection: c, 1527 container: container, 1528 objectName: objectName, 1529 headers: h, 1530 resp: resp, 1531 checkHash: checkHash, 1532 body: resp.Body, 1533 } 1534 if checkHash { 1535 file.hash = md5.New() 1536 file.body = io.TeeReader(resp.Body, file.hash) 1537 } 1538 // Read Content-Length 1539 if resp.Header.Get("Content-Length") != "" { 1540 file.length, err = getInt64FromHeader(resp, "Content-Length") 1541 file.lengthOk = (err == nil) 1542 } 1543 return 1544} 1545 1546func (c *Connection) objectOpen(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { 1547 err = withLORetry(0, func() (Headers, int64, error) { 1548 file, headers, err = c.objectOpenBase(container, objectName, checkHash, h, parameters) 1549 if err != nil { 1550 return headers, 0, err 1551 } 1552 return headers, file.length, nil 1553 }) 1554 return 1555} 1556 1557// ObjectOpen returns an ObjectOpenFile for reading the contents of 1558// the object. This satisfies the io.ReadCloser and the io.Seeker 1559// interfaces. 1560// 1561// You must call Close() on contents when finished 1562// 1563// Returns the headers of the response. 1564// 1565// If checkHash is true then it will calculate the md5sum of the file 1566// as it is being received and check it against that returned from the 1567// server. If it is wrong then it will return ObjectCorrupted. It 1568// will also check the length returned. No checking will be done if 1569// you don't read all the contents. 1570// 1571// Note that objects with X-Object-Manifest or X-Static-Large-Object 1572// set won't ever have their md5sum's checked as the md5sum reported 1573// on the object is actually the md5sum of the md5sums of the 1574// parts. This isn't very helpful to detect a corrupted download as 1575// the size of the parts aren't known without doing more operations. 1576// If you want to ensure integrity of an object with a manifest then 1577// you will need to download everything in the manifest separately. 1578// 1579// headers["Content-Type"] will give the content type if desired. 1580func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { 1581 return c.objectOpen(container, objectName, checkHash, h, nil) 1582} 1583 1584// ObjectGet gets the object into the io.Writer contents. 1585// 1586// Returns the headers of the response. 1587// 1588// If checkHash is true then it will calculate the md5sum of the file 1589// as it is being received and check it against that returned from the 1590// server. If it is wrong then it will return ObjectCorrupted. 1591// 1592// headers["Content-Type"] will give the content type if desired. 1593func (c *Connection) ObjectGet(container string, objectName string, contents io.Writer, checkHash bool, h Headers) (headers Headers, err error) { 1594 file, headers, err := c.ObjectOpen(container, objectName, checkHash, h) 1595 if err != nil { 1596 return 1597 } 1598 defer checkClose(file, &err) 1599 _, err = io.Copy(contents, file) 1600 return 1601} 1602 1603// ObjectGetBytes returns an object as a []byte. 1604// 1605// This is a simplified interface which checks the MD5 1606func (c *Connection) ObjectGetBytes(container string, objectName string) (contents []byte, err error) { 1607 var buf bytes.Buffer 1608 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1609 contents = buf.Bytes() 1610 return 1611} 1612 1613// ObjectGetString returns an object as a string. 1614// 1615// This is a simplified interface which checks the MD5 1616func (c *Connection) ObjectGetString(container string, objectName string) (contents string, err error) { 1617 var buf bytes.Buffer 1618 _, err = c.ObjectGet(container, objectName, &buf, true, nil) 1619 contents = buf.String() 1620 return 1621} 1622 1623// ObjectDelete deletes the object. 1624// 1625// May return ObjectNotFound if the object isn't found 1626func (c *Connection) ObjectDelete(container string, objectName string) error { 1627 _, _, err := c.storage(RequestOpts{ 1628 Container: container, 1629 ObjectName: objectName, 1630 Operation: "DELETE", 1631 ErrorMap: objectErrorMap, 1632 }) 1633 return err 1634} 1635 1636// ObjectTempUrl returns a temporary URL for an object 1637func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string { 1638 mac := hmac.New(sha1.New, []byte(secretKey)) 1639 prefix, _ := url.Parse(c.StorageUrl) 1640 body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName) 1641 mac.Write([]byte(body)) 1642 sig := hex.EncodeToString(mac.Sum(nil)) 1643 return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix()) 1644} 1645 1646// parseResponseStatus parses string like "200 OK" and returns Error. 1647// 1648// For status codes beween 200 and 299, this returns nil. 1649func parseResponseStatus(resp string, errorMap errorMap) error { 1650 code := 0 1651 reason := resp 1652 t := strings.SplitN(resp, " ", 2) 1653 if len(t) == 2 { 1654 ncode, err := strconv.Atoi(t[0]) 1655 if err == nil { 1656 code = ncode 1657 reason = t[1] 1658 } 1659 } 1660 if errorMap != nil { 1661 if err, ok := errorMap[code]; ok { 1662 return err 1663 } 1664 } 1665 if 200 <= code && code <= 299 { 1666 return nil 1667 } 1668 return newError(code, reason) 1669} 1670 1671// BulkDeleteResult stores results of BulkDelete(). 1672// 1673// Individual errors may (or may not) be returned by Errors. 1674// Errors is a map whose keys are a full path of where the object was 1675// to be deleted, and whose values are Error objects. A full path of 1676// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 1677type BulkDeleteResult struct { 1678 NumberNotFound int64 // # of objects not found. 1679 NumberDeleted int64 // # of deleted objects. 1680 Errors map[string]error // Mapping between object name and an error. 1681 Headers Headers // Response HTTP headers. 1682} 1683 1684func (c *Connection) doBulkDelete(objects []string) (result BulkDeleteResult, err error) { 1685 var buffer bytes.Buffer 1686 for _, s := range objects { 1687 buffer.WriteString(url.QueryEscape(s) + "\n") 1688 } 1689 resp, headers, err := c.storage(RequestOpts{ 1690 Operation: "DELETE", 1691 Parameters: url.Values{"bulk-delete": []string{"1"}}, 1692 Headers: Headers{ 1693 "Accept": "application/json", 1694 "Content-Type": "text/plain", 1695 }, 1696 ErrorMap: ContainerErrorMap, 1697 Body: &buffer, 1698 }) 1699 if err != nil { 1700 return 1701 } 1702 var jsonResult struct { 1703 NotFound int64 `json:"Number Not Found"` 1704 Status string `json:"Response Status"` 1705 Errors [][]string 1706 Deleted int64 `json:"Number Deleted"` 1707 } 1708 err = readJson(resp, &jsonResult) 1709 if err != nil { 1710 return 1711 } 1712 1713 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 1714 result.NumberNotFound = jsonResult.NotFound 1715 result.NumberDeleted = jsonResult.Deleted 1716 result.Headers = headers 1717 el := make(map[string]error, len(jsonResult.Errors)) 1718 for _, t := range jsonResult.Errors { 1719 if len(t) != 2 { 1720 continue 1721 } 1722 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 1723 } 1724 result.Errors = el 1725 return 1726} 1727 1728// BulkDelete deletes multiple objectNames from container in one operation. 1729// 1730// Some servers may not accept bulk-delete requests since bulk-delete is 1731// an optional feature of swift - these will return the Forbidden error. 1732// 1733// See also: 1734// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html 1735// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html 1736func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) { 1737 fullPaths := make([]string, len(objectNames)) 1738 for i, name := range objectNames { 1739 fullPaths[i] = fmt.Sprintf("/%s/%s", container, name) 1740 } 1741 return c.doBulkDelete(fullPaths) 1742} 1743 1744// BulkUploadResult stores results of BulkUpload(). 1745// 1746// Individual errors may (or may not) be returned by Errors. 1747// Errors is a map whose keys are a full path of where an object was 1748// to be created, and whose values are Error objects. A full path of 1749// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". 1750type BulkUploadResult struct { 1751 NumberCreated int64 // # of created objects. 1752 Errors map[string]error // Mapping between object name and an error. 1753 Headers Headers // Response HTTP headers. 1754} 1755 1756// BulkUpload uploads multiple files in one operation. 1757// 1758// uploadPath can be empty, a container name, or a pseudo-directory 1759// within a container. If uploadPath is empty, new containers may be 1760// automatically created. 1761// 1762// Files are read from dataStream. The format of the stream is specified 1763// by the format parameter. Available formats are: 1764// * UploadTar - Plain tar stream. 1765// * UploadTarGzip - Gzip compressed tar stream. 1766// * UploadTarBzip2 - Bzip2 compressed tar stream. 1767// 1768// Some servers may not accept bulk-upload requests since bulk-upload is 1769// an optional feature of swift - these will return the Forbidden error. 1770// 1771// See also: 1772// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-extract-archive.html 1773// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Extract_Archive-d1e2338.html 1774func (c *Connection) BulkUpload(uploadPath string, dataStream io.Reader, format string, h Headers) (result BulkUploadResult, err error) { 1775 extraHeaders := Headers{"Accept": "application/json"} 1776 for key, value := range h { 1777 extraHeaders[key] = value 1778 } 1779 // The following code abuses Container parameter intentionally. 1780 // The best fix might be to rename Container to UploadPath. 1781 resp, headers, err := c.storage(RequestOpts{ 1782 Container: uploadPath, 1783 Operation: "PUT", 1784 Parameters: url.Values{"extract-archive": []string{format}}, 1785 Headers: extraHeaders, 1786 ErrorMap: ContainerErrorMap, 1787 Body: dataStream, 1788 }) 1789 if err != nil { 1790 return 1791 } 1792 // Detect old servers which don't support this feature 1793 if headers["Content-Type"] != "application/json" { 1794 err = Forbidden 1795 return 1796 } 1797 var jsonResult struct { 1798 Created int64 `json:"Number Files Created"` 1799 Status string `json:"Response Status"` 1800 Errors [][]string 1801 } 1802 err = readJson(resp, &jsonResult) 1803 if err != nil { 1804 return 1805 } 1806 1807 err = parseResponseStatus(jsonResult.Status, objectErrorMap) 1808 result.NumberCreated = jsonResult.Created 1809 result.Headers = headers 1810 el := make(map[string]error, len(jsonResult.Errors)) 1811 for _, t := range jsonResult.Errors { 1812 if len(t) != 2 { 1813 continue 1814 } 1815 el[t[0]] = parseResponseStatus(t[1], objectErrorMap) 1816 } 1817 result.Errors = el 1818 return 1819} 1820 1821// Object returns info about a single object including any metadata in the header. 1822// 1823// May return ObjectNotFound. 1824// 1825// Use headers.ObjectMetadata() to read the metadata in the Headers. 1826func (c *Connection) Object(container string, objectName string) (info Object, headers Headers, err error) { 1827 err = withLORetry(0, func() (Headers, int64, error) { 1828 info, headers, err = c.objectBase(container, objectName) 1829 if err != nil { 1830 return headers, 0, err 1831 } 1832 return headers, info.Bytes, nil 1833 }) 1834 return 1835} 1836 1837func (c *Connection) objectBase(container string, objectName string) (info Object, headers Headers, err error) { 1838 var resp *http.Response 1839 resp, headers, err = c.storage(RequestOpts{ 1840 Container: container, 1841 ObjectName: objectName, 1842 Operation: "HEAD", 1843 ErrorMap: objectErrorMap, 1844 NoResponse: true, 1845 }) 1846 if err != nil { 1847 return 1848 } 1849 // Parse the headers into the struct 1850 // HTTP/1.1 200 OK 1851 // Date: Thu, 07 Jun 2010 20:59:39 GMT 1852 // Server: Apache 1853 // Last-Modified: Fri, 12 Jun 2010 13:40:18 GMT 1854 // ETag: 8a964ee2a5e88be344f36c22562a6486 1855 // Content-Length: 512000 1856 // Content-Type: text/plain; charset=UTF-8 1857 // X-Object-Meta-Meat: Bacon 1858 // X-Object-Meta-Fruit: Bacon 1859 // X-Object-Meta-Veggie: Bacon 1860 // X-Object-Meta-Dairy: Bacon 1861 info.Name = objectName 1862 info.ContentType = resp.Header.Get("Content-Type") 1863 if resp.Header.Get("Content-Length") != "" { 1864 if info.Bytes, err = getInt64FromHeader(resp, "Content-Length"); err != nil { 1865 return 1866 } 1867 } 1868 // Currently ceph doesn't return a Last-Modified header for DLO manifests without any segments 1869 // See ceph http://tracker.ceph.com/issues/15812 1870 if resp.Header.Get("Last-Modified") != "" { 1871 info.ServerLastModified = resp.Header.Get("Last-Modified") 1872 if info.LastModified, err = time.Parse(http.TimeFormat, info.ServerLastModified); err != nil { 1873 return 1874 } 1875 } 1876 1877 info.Hash = resp.Header.Get("Etag") 1878 if resp.Header.Get("X-Object-Manifest") != "" { 1879 info.ObjectType = DynamicLargeObjectType 1880 } else if resp.Header.Get("X-Static-Large-Object") != "" { 1881 info.ObjectType = StaticLargeObjectType 1882 } 1883 1884 return 1885} 1886 1887// ObjectUpdate adds, replaces or removes object metadata. 1888// 1889// Add or Update keys by mentioning them in the Metadata. Use 1890// Metadata.ObjectHeaders and Headers.ObjectMetadata to convert your 1891// Metadata to and from normal HTTP headers. 1892// 1893// This removes all metadata previously added to the object and 1894// replaces it with that passed in so to delete keys, just don't 1895// mention them the headers you pass in. 1896// 1897// Object metadata can only be read with Object() not with Objects(). 1898// 1899// This can also be used to set headers not already assigned such as 1900// X-Delete-At or X-Delete-After for expiring objects. 1901// 1902// You cannot use this to change any of the object's other headers 1903// such as Content-Type, ETag, etc. 1904// 1905// Refer to copying an object when you need to update metadata or 1906// other headers such as Content-Type or CORS headers. 1907// 1908// May return ObjectNotFound. 1909func (c *Connection) ObjectUpdate(container string, objectName string, h Headers) error { 1910 _, _, err := c.storage(RequestOpts{ 1911 Container: container, 1912 ObjectName: objectName, 1913 Operation: "POST", 1914 ErrorMap: objectErrorMap, 1915 NoResponse: true, 1916 Headers: h, 1917 }) 1918 return err 1919} 1920 1921// ObjectCopy does a server side copy of an object to a new position 1922// 1923// All metadata is preserved. If metadata is set in the headers then 1924// it overrides the old metadata on the copied object. 1925// 1926// The destination container must exist before the copy. 1927// 1928// You can use this to copy an object to itself - this is the only way 1929// to update the content type of an object. 1930func (c *Connection) ObjectCopy(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string, h Headers) (headers Headers, err error) { 1931 // Meta stuff 1932 extraHeaders := map[string]string{ 1933 "Destination": dstContainer + "/" + dstObjectName, 1934 } 1935 for key, value := range h { 1936 extraHeaders[key] = value 1937 } 1938 _, headers, err = c.storage(RequestOpts{ 1939 Container: srcContainer, 1940 ObjectName: srcObjectName, 1941 Operation: "COPY", 1942 ErrorMap: objectErrorMap, 1943 NoResponse: true, 1944 Headers: extraHeaders, 1945 }) 1946 return 1947} 1948 1949// ObjectMove does a server side move of an object to a new position 1950// 1951// This is a convenience method which calls ObjectCopy then ObjectDelete 1952// 1953// All metadata is preserved. 1954// 1955// The destination container must exist before the copy. 1956func (c *Connection) ObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) (err error) { 1957 _, err = c.ObjectCopy(srcContainer, srcObjectName, dstContainer, dstObjectName, nil) 1958 if err != nil { 1959 return 1960 } 1961 return c.ObjectDelete(srcContainer, srcObjectName) 1962} 1963 1964// ObjectUpdateContentType updates the content type of an object 1965// 1966// This is a convenience method which calls ObjectCopy 1967// 1968// All other metadata is preserved. 1969func (c *Connection) ObjectUpdateContentType(container string, objectName string, contentType string) (err error) { 1970 h := Headers{"Content-Type": contentType} 1971 _, err = c.ObjectCopy(container, objectName, container, objectName, h) 1972 return 1973} 1974 1975// ------------------------------------------------------------ 1976 1977// VersionContainerCreate is a helper method for creating and enabling version controlled containers. 1978// 1979// It builds the current object container, the non-current object version container, and enables versioning. 1980// 1981// If the server doesn't support versioning then it will return 1982// Forbidden however it will have created both the containers at that point. 1983func (c *Connection) VersionContainerCreate(current, version string) error { 1984 if err := c.ContainerCreate(version, nil); err != nil { 1985 return err 1986 } 1987 if err := c.ContainerCreate(current, nil); err != nil { 1988 return err 1989 } 1990 if err := c.VersionEnable(current, version); err != nil { 1991 return err 1992 } 1993 return nil 1994} 1995 1996// VersionEnable enables versioning on the current container with version as the tracking container. 1997// 1998// May return Forbidden if this isn't supported by the server 1999func (c *Connection) VersionEnable(current, version string) error { 2000 h := Headers{"X-Versions-Location": version} 2001 if err := c.ContainerUpdate(current, h); err != nil { 2002 return err 2003 } 2004 // Check to see if the header was set properly 2005 _, headers, err := c.Container(current) 2006 if err != nil { 2007 return err 2008 } 2009 // If failed to set versions header, return Forbidden as the server doesn't support this 2010 if headers["X-Versions-Location"] != version { 2011 return Forbidden 2012 } 2013 return nil 2014} 2015 2016// VersionDisable disables versioning on the current container. 2017func (c *Connection) VersionDisable(current string) error { 2018 h := Headers{"X-Versions-Location": ""} 2019 if err := c.ContainerUpdate(current, h); err != nil { 2020 return err 2021 } 2022 return nil 2023} 2024 2025// VersionObjectList returns a list of older versions of the object. 2026// 2027// Objects are returned in the format <length><object_name>/<timestamp> 2028func (c *Connection) VersionObjectList(version, object string) ([]string, error) { 2029 opts := &ObjectsOpts{ 2030 // <3-character zero-padded hexadecimal character length><object name>/ 2031 Prefix: fmt.Sprintf("%03x", len(object)) + object + "/", 2032 } 2033 return c.ObjectNames(version, opts) 2034} 2035