1// Package box provides an interface to the Box 2// object storage system. 3package box 4 5// FIXME Box only supports file names of 255 characters or less. Names 6// that will not be supported are those that contain non-printable 7// ascii, / or \, names with trailing spaces, and the special names 8// “.” and “..”. 9 10// FIXME box can copy a directory 11 12import ( 13 "context" 14 "crypto/rsa" 15 "encoding/json" 16 "encoding/pem" 17 "fmt" 18 "io" 19 "io/ioutil" 20 "net/http" 21 "net/url" 22 "path" 23 "strconv" 24 "strings" 25 "sync" 26 "sync/atomic" 27 "time" 28 29 "github.com/rclone/rclone/lib/encoder" 30 "github.com/rclone/rclone/lib/env" 31 "github.com/rclone/rclone/lib/jwtutil" 32 33 "github.com/youmark/pkcs8" 34 35 "github.com/pkg/errors" 36 "github.com/rclone/rclone/backend/box/api" 37 "github.com/rclone/rclone/fs" 38 "github.com/rclone/rclone/fs/config" 39 "github.com/rclone/rclone/fs/config/configmap" 40 "github.com/rclone/rclone/fs/config/configstruct" 41 "github.com/rclone/rclone/fs/config/obscure" 42 "github.com/rclone/rclone/fs/fserrors" 43 "github.com/rclone/rclone/fs/fshttp" 44 "github.com/rclone/rclone/fs/hash" 45 "github.com/rclone/rclone/lib/dircache" 46 "github.com/rclone/rclone/lib/oauthutil" 47 "github.com/rclone/rclone/lib/pacer" 48 "github.com/rclone/rclone/lib/rest" 49 "golang.org/x/oauth2" 50 "golang.org/x/oauth2/jws" 51) 52 53const ( 54 rcloneClientID = "d0374ba6pgmaguie02ge15sv1mllndho" 55 rcloneEncryptedClientSecret = "sYbJYm99WB8jzeaLPU0OPDMJKIkZvD2qOn3SyEMfiJr03RdtDt3xcZEIudRhbIDL" 56 minSleep = 10 * time.Millisecond 57 maxSleep = 2 * time.Second 58 decayConstant = 2 // bigger for slower decay, exponential 59 rootURL = "https://api.box.com/2.0" 60 uploadURL = "https://upload.box.com/api/2.0" 61 minUploadCutoff = 50000000 // upload cutoff can be no lower than this 62 defaultUploadCutoff = 50 * 1024 * 1024 63 tokenURL = "https://api.box.com/oauth2/token" 64) 65 66// Globals 67var ( 68 // Description of how to auth for this app 69 oauthConfig = &oauth2.Config{ 70 Scopes: nil, 71 Endpoint: oauth2.Endpoint{ 72 AuthURL: "https://app.box.com/api/oauth2/authorize", 73 TokenURL: "https://app.box.com/api/oauth2/token", 74 }, 75 ClientID: rcloneClientID, 76 ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), 77 RedirectURL: oauthutil.RedirectURL, 78 } 79) 80 81// Register with Fs 82func init() { 83 fs.Register(&fs.RegInfo{ 84 Name: "box", 85 Description: "Box", 86 NewFs: NewFs, 87 Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { 88 jsonFile, ok := m.Get("box_config_file") 89 boxSubType, boxSubTypeOk := m.Get("box_sub_type") 90 boxAccessToken, boxAccessTokenOk := m.Get("access_token") 91 var err error 92 // If using box config.json, use JWT auth 93 if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" { 94 err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m) 95 if err != nil { 96 return nil, errors.Wrap(err, "failed to configure token with jwt authentication") 97 } 98 // Else, if not using an access token, use oauth2 99 } else if boxAccessToken == "" || !boxAccessTokenOk { 100 return oauthutil.ConfigOut("", &oauthutil.Options{ 101 OAuth2Config: oauthConfig, 102 }) 103 } 104 return nil, nil 105 }, 106 Options: append(oauthutil.SharedOptions, []fs.Option{{ 107 Name: "root_folder_id", 108 Help: "Fill in for rclone to use a non root folder as its starting point.", 109 Default: "0", 110 Advanced: true, 111 }, { 112 Name: "box_config_file", 113 Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp, 114 }, { 115 Name: "access_token", 116 Help: "Box App Primary Access Token\n\nLeave blank normally.", 117 }, { 118 Name: "box_sub_type", 119 Default: "user", 120 Examples: []fs.OptionExample{{ 121 Value: "user", 122 Help: "Rclone should act on behalf of a user.", 123 }, { 124 Value: "enterprise", 125 Help: "Rclone should act on behalf of a service account.", 126 }}, 127 }, { 128 Name: "upload_cutoff", 129 Help: "Cutoff for switching to multipart upload (>= 50 MiB).", 130 Default: fs.SizeSuffix(defaultUploadCutoff), 131 Advanced: true, 132 }, { 133 Name: "commit_retries", 134 Help: "Max number of times to try committing a multipart file.", 135 Default: 100, 136 Advanced: true, 137 }, { 138 Name: "list_chunk", 139 Default: 1000, 140 Help: "Size of listing chunk 1-1000.", 141 Advanced: true, 142 }, { 143 Name: "owned_by", 144 Default: "", 145 Help: "Only show items owned by the login (email address) passed in.", 146 Advanced: true, 147 }, { 148 Name: config.ConfigEncoding, 149 Help: config.ConfigEncodingHelp, 150 Advanced: true, 151 // From https://developer.box.com/docs/error-codes#section-400-bad-request : 152 // > Box only supports file or folder names that are 255 characters or less. 153 // > File names containing non-printable ascii, "/" or "\", names with leading 154 // > or trailing spaces, and the special names “.” and “..” are also unsupported. 155 // 156 // Testing revealed names with leading spaces work fine. 157 // Also encode invalid UTF-8 bytes as json doesn't handle them properly. 158 Default: (encoder.Display | 159 encoder.EncodeBackSlash | 160 encoder.EncodeRightSpace | 161 encoder.EncodeInvalidUtf8), 162 }}...), 163 }) 164} 165 166func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, name string, m configmap.Mapper) error { 167 jsonFile = env.ShellExpand(jsonFile) 168 boxConfig, err := getBoxConfig(jsonFile) 169 if err != nil { 170 return errors.Wrap(err, "get box config") 171 } 172 privateKey, err := getDecryptedPrivateKey(boxConfig) 173 if err != nil { 174 return errors.Wrap(err, "get decrypted private key") 175 } 176 claims, err := getClaims(boxConfig, boxSubType) 177 if err != nil { 178 return errors.Wrap(err, "get claims") 179 } 180 signingHeaders := getSigningHeaders(boxConfig) 181 queryParams := getQueryParams(boxConfig) 182 client := fshttp.NewClient(ctx) 183 err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client) 184 return err 185} 186 187func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) { 188 file, err := ioutil.ReadFile(configFile) 189 if err != nil { 190 return nil, errors.Wrap(err, "box: failed to read Box config") 191 } 192 err = json.Unmarshal(file, &boxConfig) 193 if err != nil { 194 return nil, errors.Wrap(err, "box: failed to parse Box config") 195 } 196 return boxConfig, nil 197} 198 199func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) { 200 val, err := jwtutil.RandomHex(20) 201 if err != nil { 202 return nil, errors.Wrap(err, "box: failed to generate random string for jti") 203 } 204 205 claims = &jws.ClaimSet{ 206 Iss: boxConfig.BoxAppSettings.ClientID, 207 Sub: boxConfig.EnterpriseID, 208 Aud: tokenURL, 209 Exp: time.Now().Add(time.Second * 45).Unix(), 210 PrivateClaims: map[string]interface{}{ 211 "box_sub_type": boxSubType, 212 "aud": tokenURL, 213 "jti": val, 214 }, 215 } 216 217 return claims, nil 218} 219 220func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header { 221 signingHeaders := &jws.Header{ 222 Algorithm: "RS256", 223 Typ: "JWT", 224 KeyID: boxConfig.BoxAppSettings.AppAuth.PublicKeyID, 225 } 226 227 return signingHeaders 228} 229 230func getQueryParams(boxConfig *api.ConfigJSON) map[string]string { 231 queryParams := map[string]string{ 232 "client_id": boxConfig.BoxAppSettings.ClientID, 233 "client_secret": boxConfig.BoxAppSettings.ClientSecret, 234 } 235 236 return queryParams 237} 238 239func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) { 240 241 block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey)) 242 if len(rest) > 0 { 243 return nil, errors.Wrap(err, "box: extra data included in private key") 244 } 245 246 rsaKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(boxConfig.BoxAppSettings.AppAuth.Passphrase)) 247 if err != nil { 248 return nil, errors.Wrap(err, "box: failed to decrypt private key") 249 } 250 251 return rsaKey.(*rsa.PrivateKey), nil 252} 253 254// Options defines the configuration for this backend 255type Options struct { 256 UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` 257 CommitRetries int `config:"commit_retries"` 258 Enc encoder.MultiEncoder `config:"encoding"` 259 RootFolderID string `config:"root_folder_id"` 260 AccessToken string `config:"access_token"` 261 ListChunk int `config:"list_chunk"` 262 OwnedBy string `config:"owned_by"` 263} 264 265// Fs represents a remote box 266type Fs struct { 267 name string // name of this remote 268 root string // the path we are working on 269 opt Options // parsed options 270 features *fs.Features // optional features 271 srv *rest.Client // the connection to the one drive server 272 dirCache *dircache.DirCache // Map of directory path to directory id 273 pacer *fs.Pacer // pacer for API calls 274 tokenRenewer *oauthutil.Renew // renew the token on expiry 275 uploadToken *pacer.TokenDispenser // control concurrency 276} 277 278// Object describes a box object 279// 280// Will definitely have info but maybe not meta 281type Object struct { 282 fs *Fs // what this object is part of 283 remote string // The remote path 284 hasMetaData bool // whether info below has been set 285 size int64 // size of the object 286 modTime time.Time // modification time of the object 287 id string // ID of the object 288 publicLink string // Public Link for the object 289 sha1 string // SHA-1 of the object content 290} 291 292// ------------------------------------------------------------ 293 294// Name of the remote (as passed into NewFs) 295func (f *Fs) Name() string { 296 return f.name 297} 298 299// Root of the remote (as passed into NewFs) 300func (f *Fs) Root() string { 301 return f.root 302} 303 304// String converts this Fs to a string 305func (f *Fs) String() string { 306 return fmt.Sprintf("box root '%s'", f.root) 307} 308 309// Features returns the optional features of this Fs 310func (f *Fs) Features() *fs.Features { 311 return f.features 312} 313 314// parsePath parses a box 'url' 315func parsePath(path string) (root string) { 316 root = strings.Trim(path, "/") 317 return 318} 319 320// retryErrorCodes is a slice of error codes that we will retry 321var retryErrorCodes = []int{ 322 429, // Too Many Requests. 323 500, // Internal Server Error 324 502, // Bad Gateway 325 503, // Service Unavailable 326 504, // Gateway Timeout 327 509, // Bandwidth Limit Exceeded 328} 329 330// shouldRetry returns a boolean as to whether this resp and err 331// deserve to be retried. It returns the err as a convenience 332func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 333 if fserrors.ContextError(ctx, &err) { 334 return false, err 335 } 336 authRetry := false 337 338 if resp != nil && resp.StatusCode == 401 && strings.Contains(resp.Header.Get("Www-Authenticate"), "expired_token") { 339 authRetry = true 340 fs.Debugf(nil, "Should retry: %v", err) 341 } 342 343 // Box API errors which should be retries 344 if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "operation_blocked_temporary" { 345 fs.Debugf(nil, "Retrying API error %v", err) 346 return true, err 347 } 348 349 return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 350} 351 352// readMetaDataForPath reads the metadata from the path 353func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) { 354 // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) 355 leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) 356 if err != nil { 357 if err == fs.ErrorDirNotFound { 358 return nil, fs.ErrorObjectNotFound 359 } 360 return nil, err 361 } 362 363 found, err := f.listAll(ctx, directoryID, false, true, true, func(item *api.Item) bool { 364 if strings.EqualFold(item.Name, leaf) { 365 info = item 366 return true 367 } 368 return false 369 }) 370 if err != nil { 371 return nil, err 372 } 373 if !found { 374 return nil, fs.ErrorObjectNotFound 375 } 376 return info, nil 377} 378 379// errorHandler parses a non 2xx error response into an error 380func errorHandler(resp *http.Response) error { 381 // Decode error response 382 errResponse := new(api.Error) 383 err := rest.DecodeJSON(resp, &errResponse) 384 if err != nil { 385 fs.Debugf(nil, "Couldn't decode error response: %v", err) 386 } 387 if errResponse.Code == "" { 388 errResponse.Code = resp.Status 389 } 390 if errResponse.Status == 0 { 391 errResponse.Status = resp.StatusCode 392 } 393 return errResponse 394} 395 396// NewFs constructs an Fs from the path, container:path 397func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 398 // Parse config into Options struct 399 opt := new(Options) 400 err := configstruct.Set(m, opt) 401 if err != nil { 402 return nil, err 403 } 404 405 if opt.UploadCutoff < minUploadCutoff { 406 return nil, errors.Errorf("box: upload cutoff (%v) must be greater than equal to %v", opt.UploadCutoff, fs.SizeSuffix(minUploadCutoff)) 407 } 408 409 root = parsePath(root) 410 411 client := fshttp.NewClient(ctx) 412 var ts *oauthutil.TokenSource 413 // If not using an accessToken, create an oauth client and tokensource 414 if opt.AccessToken == "" { 415 client, ts, err = oauthutil.NewClient(ctx, name, m, oauthConfig) 416 if err != nil { 417 return nil, errors.Wrap(err, "failed to configure Box") 418 } 419 } 420 421 ci := fs.GetConfig(ctx) 422 f := &Fs{ 423 name: name, 424 root: root, 425 opt: *opt, 426 srv: rest.NewClient(client).SetRoot(rootURL), 427 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 428 uploadToken: pacer.NewTokenDispenser(ci.Transfers), 429 } 430 f.features = (&fs.Features{ 431 CaseInsensitive: true, 432 CanHaveEmptyDirectories: true, 433 }).Fill(ctx, f) 434 f.srv.SetErrorHandler(errorHandler) 435 436 // If using an accessToken, set the Authorization header 437 if f.opt.AccessToken != "" { 438 f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken) 439 } 440 441 jsonFile, ok := m.Get("box_config_file") 442 boxSubType, boxSubTypeOk := m.Get("box_sub_type") 443 444 if ts != nil { 445 // If using box config.json and JWT, renewing should just refresh the token and 446 // should do so whether there are uploads pending or not. 447 if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" { 448 f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { 449 err := refreshJWTToken(ctx, jsonFile, boxSubType, name, m) 450 return err 451 }) 452 f.tokenRenewer.Start() 453 } else { 454 // Renew the token in the background 455 f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { 456 _, err := f.readMetaDataForPath(ctx, "") 457 return err 458 }) 459 } 460 } 461 462 // Get rootFolderID 463 rootID := f.opt.RootFolderID 464 f.dirCache = dircache.New(root, rootID, f) 465 466 // Find the current root 467 err = f.dirCache.FindRoot(ctx, false) 468 if err != nil { 469 // Assume it is a file 470 newRoot, remote := dircache.SplitPath(root) 471 tempF := *f 472 tempF.dirCache = dircache.New(newRoot, rootID, &tempF) 473 tempF.root = newRoot 474 // Make new Fs which is the parent 475 err = tempF.dirCache.FindRoot(ctx, false) 476 if err != nil { 477 // No root so return old f 478 return f, nil 479 } 480 _, err := tempF.newObjectWithInfo(ctx, remote, nil) 481 if err != nil { 482 if err == fs.ErrorObjectNotFound { 483 // File doesn't exist so return old f 484 return f, nil 485 } 486 return nil, err 487 } 488 f.features.Fill(ctx, &tempF) 489 // XXX: update the old f here instead of returning tempF, since 490 // `features` were already filled with functions having *f as a receiver. 491 // See https://github.com/rclone/rclone/issues/2182 492 f.dirCache = tempF.dirCache 493 f.root = tempF.root 494 // return an error with an fs which points to the parent 495 return f, fs.ErrorIsFile 496 } 497 return f, nil 498} 499 500// rootSlash returns root with a slash on if it is empty, otherwise empty string 501func (f *Fs) rootSlash() string { 502 if f.root == "" { 503 return f.root 504 } 505 return f.root + "/" 506} 507 508// Return an Object from a path 509// 510// If it can't be found it returns the error fs.ErrorObjectNotFound. 511func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { 512 o := &Object{ 513 fs: f, 514 remote: remote, 515 } 516 var err error 517 if info != nil { 518 // Set info 519 err = o.setMetaData(info) 520 } else { 521 err = o.readMetaData(ctx) // reads info and meta, returning an error 522 } 523 if err != nil { 524 return nil, err 525 } 526 return o, nil 527} 528 529// NewObject finds the Object at remote. If it can't be found 530// it returns the error fs.ErrorObjectNotFound. 531func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 532 return f.newObjectWithInfo(ctx, remote, nil) 533} 534 535// FindLeaf finds a directory of name leaf in the folder with ID pathID 536func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { 537 // Find the leaf in pathID 538 found, err = f.listAll(ctx, pathID, true, false, true, func(item *api.Item) bool { 539 if strings.EqualFold(item.Name, leaf) { 540 pathIDOut = item.ID 541 return true 542 } 543 return false 544 }) 545 return pathIDOut, found, err 546} 547 548// fieldsValue creates a url.Values with fields set to those in api.Item 549func fieldsValue() url.Values { 550 values := url.Values{} 551 values.Set("fields", api.ItemFields) 552 return values 553} 554 555// CreateDir makes a directory with pathID as parent and name leaf 556func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { 557 // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) 558 var resp *http.Response 559 var info *api.Item 560 opts := rest.Opts{ 561 Method: "POST", 562 Path: "/folders", 563 Parameters: fieldsValue(), 564 } 565 mkdir := api.CreateFolder{ 566 Name: f.opt.Enc.FromStandardName(leaf), 567 Parent: api.Parent{ 568 ID: pathID, 569 }, 570 } 571 err = f.pacer.Call(func() (bool, error) { 572 resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) 573 return shouldRetry(ctx, resp, err) 574 }) 575 if err != nil { 576 //fmt.Printf("...Error %v\n", err) 577 return "", err 578 } 579 // fmt.Printf("...Id %q\n", *info.Id) 580 return info.ID, nil 581} 582 583// list the objects into the function supplied 584// 585// If directories is set it only sends directories 586// User function to process a File item from listAll 587// 588// Should return true to finish processing 589type listAllFn func(*api.Item) bool 590 591// Lists the directory required calling the user function on each item found 592// 593// If the user fn ever returns true then it early exits with found = true 594func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, activeOnly bool, fn listAllFn) (found bool, err error) { 595 opts := rest.Opts{ 596 Method: "GET", 597 Path: "/folders/" + dirID + "/items", 598 Parameters: fieldsValue(), 599 } 600 opts.Parameters.Set("limit", strconv.Itoa(f.opt.ListChunk)) 601 opts.Parameters.Set("usemarker", "true") 602 var marker *string 603OUTER: 604 for { 605 if marker != nil { 606 opts.Parameters.Set("marker", *marker) 607 } 608 609 var result api.FolderItems 610 var resp *http.Response 611 err = f.pacer.Call(func() (bool, error) { 612 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 613 return shouldRetry(ctx, resp, err) 614 }) 615 if err != nil { 616 return found, errors.Wrap(err, "couldn't list files") 617 } 618 for i := range result.Entries { 619 item := &result.Entries[i] 620 if item.Type == api.ItemTypeFolder { 621 if filesOnly { 622 continue 623 } 624 } else if item.Type == api.ItemTypeFile { 625 if directoriesOnly { 626 continue 627 } 628 } else { 629 fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) 630 continue 631 } 632 if activeOnly && item.ItemStatus != api.ItemStatusActive { 633 continue 634 } 635 if f.opt.OwnedBy != "" && f.opt.OwnedBy != item.OwnedBy.Login { 636 continue 637 } 638 item.Name = f.opt.Enc.ToStandardName(item.Name) 639 if fn(item) { 640 found = true 641 break OUTER 642 } 643 } 644 marker = result.NextMarker 645 if marker == nil { 646 break 647 } 648 } 649 return 650} 651 652// List the objects and directories in dir into entries. The 653// entries can be returned in any order but should be for a 654// complete directory. 655// 656// dir should be "" to list the root, and should not have 657// trailing slashes. 658// 659// This should return ErrDirNotFound if the directory isn't 660// found. 661func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 662 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 663 if err != nil { 664 return nil, err 665 } 666 var iErr error 667 _, err = f.listAll(ctx, directoryID, false, false, true, func(info *api.Item) bool { 668 remote := path.Join(dir, info.Name) 669 if info.Type == api.ItemTypeFolder { 670 // cache the directory ID for later lookups 671 f.dirCache.Put(remote, info.ID) 672 d := fs.NewDir(remote, info.ModTime()).SetID(info.ID) 673 // FIXME more info from dir? 674 entries = append(entries, d) 675 } else if info.Type == api.ItemTypeFile { 676 o, err := f.newObjectWithInfo(ctx, remote, info) 677 if err != nil { 678 iErr = err 679 return true 680 } 681 entries = append(entries, o) 682 } 683 return false 684 }) 685 if err != nil { 686 return nil, err 687 } 688 if iErr != nil { 689 return nil, iErr 690 } 691 return entries, nil 692} 693 694// Creates from the parameters passed in a half finished Object which 695// must have setMetaData called on it 696// 697// Returns the object, leaf, directoryID and error 698// 699// Used to create new objects 700func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { 701 // Create the directory for the object if it doesn't exist 702 leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) 703 if err != nil { 704 return 705 } 706 // Temporary Object under construction 707 o = &Object{ 708 fs: f, 709 remote: remote, 710 } 711 return o, leaf, directoryID, nil 712} 713 714// preUploadCheck checks to see if a file can be uploaded 715// 716// It returns "", nil if the file is good to go 717// It returns "ID", nil if the file must be updated 718func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) { 719 check := api.PreUploadCheck{ 720 Name: f.opt.Enc.FromStandardName(leaf), 721 Parent: api.Parent{ 722 ID: directoryID, 723 }, 724 } 725 if size >= 0 { 726 check.Size = &size 727 } 728 opts := rest.Opts{ 729 Method: "OPTIONS", 730 Path: "/files/content/", 731 } 732 var result api.PreUploadCheckResponse 733 var resp *http.Response 734 err = f.pacer.Call(func() (bool, error) { 735 resp, err = f.srv.CallJSON(ctx, &opts, &check, &result) 736 return shouldRetry(ctx, resp, err) 737 }) 738 if err != nil { 739 if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "item_name_in_use" { 740 var conflict api.PreUploadCheckConflict 741 err = json.Unmarshal(apiErr.ContextInfo, &conflict) 742 if err != nil { 743 return "", errors.Wrap(err, "pre-upload check: JSON decode failed") 744 } 745 if conflict.Conflicts.Type != api.ItemTypeFile { 746 return "", errors.Wrap(err, "pre-upload check: can't overwrite non file with file") 747 } 748 return conflict.Conflicts.ID, nil 749 } 750 return "", errors.Wrap(err, "pre-upload check") 751 } 752 return "", nil 753} 754 755// Put the object 756// 757// Copy the reader in to the new object which is returned 758// 759// The new object may have been created if an error is returned 760func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 761 // If directory doesn't exist, file doesn't exist so can upload 762 remote := src.Remote() 763 leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false) 764 if err != nil { 765 if err == fs.ErrorDirNotFound { 766 return f.PutUnchecked(ctx, in, src, options...) 767 } 768 return nil, err 769 } 770 771 // Preflight check the upload, which returns the ID if the 772 // object already exists 773 ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size()) 774 if err != nil { 775 return nil, err 776 } 777 if ID == "" { 778 return f.PutUnchecked(ctx, in, src, options...) 779 } 780 781 // If object exists then create a skeleton one with just id 782 o := &Object{ 783 fs: f, 784 remote: remote, 785 id: ID, 786 } 787 return o, o.Update(ctx, in, src, options...) 788} 789 790// PutStream uploads to the remote path with the modTime given of indeterminate size 791func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 792 return f.Put(ctx, in, src, options...) 793} 794 795// PutUnchecked the object into the container 796// 797// This will produce an error if the object already exists 798// 799// Copy the reader in to the new object which is returned 800// 801// The new object may have been created if an error is returned 802func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 803 remote := src.Remote() 804 size := src.Size() 805 modTime := src.ModTime(ctx) 806 807 o, _, _, err := f.createObject(ctx, remote, modTime, size) 808 if err != nil { 809 return nil, err 810 } 811 return o, o.Update(ctx, in, src, options...) 812} 813 814// Mkdir creates the container if it doesn't exist 815func (f *Fs) Mkdir(ctx context.Context, dir string) error { 816 _, err := f.dirCache.FindDir(ctx, dir, true) 817 return err 818} 819 820// deleteObject removes an object by ID 821func (f *Fs) deleteObject(ctx context.Context, id string) error { 822 opts := rest.Opts{ 823 Method: "DELETE", 824 Path: "/files/" + id, 825 NoResponse: true, 826 } 827 return f.pacer.Call(func() (bool, error) { 828 resp, err := f.srv.Call(ctx, &opts) 829 return shouldRetry(ctx, resp, err) 830 }) 831} 832 833// purgeCheck removes the root directory, if check is set then it 834// refuses to do so if it has anything in 835func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 836 root := path.Join(f.root, dir) 837 if root == "" { 838 return errors.New("can't purge root directory") 839 } 840 dc := f.dirCache 841 rootID, err := dc.FindDir(ctx, dir, false) 842 if err != nil { 843 return err 844 } 845 846 opts := rest.Opts{ 847 Method: "DELETE", 848 Path: "/folders/" + rootID, 849 Parameters: url.Values{}, 850 NoResponse: true, 851 } 852 opts.Parameters.Set("recursive", strconv.FormatBool(!check)) 853 var resp *http.Response 854 err = f.pacer.Call(func() (bool, error) { 855 resp, err = f.srv.Call(ctx, &opts) 856 return shouldRetry(ctx, resp, err) 857 }) 858 if err != nil { 859 return errors.Wrap(err, "rmdir failed") 860 } 861 f.dirCache.FlushDir(dir) 862 if err != nil { 863 return err 864 } 865 return nil 866} 867 868// Rmdir deletes the root folder 869// 870// Returns an error if it isn't empty 871func (f *Fs) Rmdir(ctx context.Context, dir string) error { 872 return f.purgeCheck(ctx, dir, true) 873} 874 875// Precision return the precision of this Fs 876func (f *Fs) Precision() time.Duration { 877 return time.Second 878} 879 880// Copy src to this remote using server-side copy operations. 881// 882// This is stored with the remote path given 883// 884// It returns the destination Object and a possible error 885// 886// Will only be called if src.Fs().Name() == f.Name() 887// 888// If it isn't possible then return fs.ErrorCantCopy 889func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 890 srcObj, ok := src.(*Object) 891 if !ok { 892 fs.Debugf(src, "Can't copy - not same remote type") 893 return nil, fs.ErrorCantCopy 894 } 895 err := srcObj.readMetaData(ctx) 896 if err != nil { 897 return nil, err 898 } 899 900 srcPath := srcObj.fs.rootSlash() + srcObj.remote 901 dstPath := f.rootSlash() + remote 902 if strings.ToLower(srcPath) == strings.ToLower(dstPath) { 903 return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) 904 } 905 906 // Create temporary object 907 dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) 908 if err != nil { 909 return nil, err 910 } 911 912 // Copy the object 913 opts := rest.Opts{ 914 Method: "POST", 915 Path: "/files/" + srcObj.id + "/copy", 916 Parameters: fieldsValue(), 917 } 918 copyFile := api.CopyFile{ 919 Name: f.opt.Enc.FromStandardName(leaf), 920 Parent: api.Parent{ 921 ID: directoryID, 922 }, 923 } 924 var resp *http.Response 925 var info *api.Item 926 err = f.pacer.Call(func() (bool, error) { 927 resp, err = f.srv.CallJSON(ctx, &opts, ©File, &info) 928 return shouldRetry(ctx, resp, err) 929 }) 930 if err != nil { 931 return nil, err 932 } 933 err = dstObj.setMetaData(info) 934 if err != nil { 935 return nil, err 936 } 937 return dstObj, nil 938} 939 940// Purge deletes all the files and the container 941// 942// Optional interface: Only implement this if you have a way of 943// deleting all the files quicker than just running Remove() on the 944// result of List() 945func (f *Fs) Purge(ctx context.Context, dir string) error { 946 return f.purgeCheck(ctx, dir, false) 947} 948 949// move a file or folder 950func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (info *api.Item, err error) { 951 // Move the object 952 opts := rest.Opts{ 953 Method: "PUT", 954 Path: endpoint + id, 955 Parameters: fieldsValue(), 956 } 957 move := api.UpdateFileMove{ 958 Name: f.opt.Enc.FromStandardName(leaf), 959 Parent: api.Parent{ 960 ID: directoryID, 961 }, 962 } 963 var resp *http.Response 964 err = f.pacer.Call(func() (bool, error) { 965 resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) 966 return shouldRetry(ctx, resp, err) 967 }) 968 if err != nil { 969 return nil, err 970 } 971 return info, nil 972} 973 974// About gets quota information 975func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { 976 opts := rest.Opts{ 977 Method: "GET", 978 Path: "/users/me", 979 } 980 var user api.User 981 var resp *http.Response 982 err = f.pacer.Call(func() (bool, error) { 983 resp, err = f.srv.CallJSON(ctx, &opts, nil, &user) 984 return shouldRetry(ctx, resp, err) 985 }) 986 if err != nil { 987 return nil, errors.Wrap(err, "failed to read user info") 988 } 989 // FIXME max upload size would be useful to use in Update 990 usage = &fs.Usage{ 991 Used: fs.NewUsageValue(user.SpaceUsed), // bytes in use 992 Total: fs.NewUsageValue(user.SpaceAmount), // bytes total 993 Free: fs.NewUsageValue(user.SpaceAmount - user.SpaceUsed), // bytes free 994 } 995 return usage, nil 996} 997 998// Move src to this remote using server-side move operations. 999// 1000// This is stored with the remote path given 1001// 1002// It returns the destination Object and a possible error 1003// 1004// Will only be called if src.Fs().Name() == f.Name() 1005// 1006// If it isn't possible then return fs.ErrorCantMove 1007func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 1008 srcObj, ok := src.(*Object) 1009 if !ok { 1010 fs.Debugf(src, "Can't move - not same remote type") 1011 return nil, fs.ErrorCantMove 1012 } 1013 1014 // Create temporary object 1015 dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) 1016 if err != nil { 1017 return nil, err 1018 } 1019 1020 // Do the move 1021 info, err := f.move(ctx, "/files/", srcObj.id, leaf, directoryID) 1022 if err != nil { 1023 return nil, err 1024 } 1025 1026 err = dstObj.setMetaData(info) 1027 if err != nil { 1028 return nil, err 1029 } 1030 return dstObj, nil 1031} 1032 1033// DirMove moves src, srcRemote to this remote at dstRemote 1034// using server-side move operations. 1035// 1036// Will only be called if src.Fs().Name() == f.Name() 1037// 1038// If it isn't possible then return fs.ErrorCantDirMove 1039// 1040// If destination exists then return fs.ErrorDirExists 1041func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 1042 srcFs, ok := src.(*Fs) 1043 if !ok { 1044 fs.Debugf(srcFs, "Can't move directory - not same remote type") 1045 return fs.ErrorCantDirMove 1046 } 1047 1048 srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) 1049 if err != nil { 1050 return err 1051 } 1052 1053 // Do the move 1054 _, err = f.move(ctx, "/folders/", srcID, dstLeaf, dstDirectoryID) 1055 if err != nil { 1056 return err 1057 } 1058 srcFs.dirCache.FlushDir(srcRemote) 1059 return nil 1060} 1061 1062// PublicLink adds a "readable by anyone with link" permission on the given file or folder. 1063func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { 1064 id, err := f.dirCache.FindDir(ctx, remote, false) 1065 var opts rest.Opts 1066 if err == nil { 1067 fs.Debugf(f, "attempting to share directory '%s'", remote) 1068 1069 opts = rest.Opts{ 1070 Method: "PUT", 1071 Path: "/folders/" + id, 1072 Parameters: fieldsValue(), 1073 } 1074 } else { 1075 fs.Debugf(f, "attempting to share single file '%s'", remote) 1076 o, err := f.NewObject(ctx, remote) 1077 if err != nil { 1078 return "", err 1079 } 1080 1081 if o.(*Object).publicLink != "" { 1082 return o.(*Object).publicLink, nil 1083 } 1084 1085 opts = rest.Opts{ 1086 Method: "PUT", 1087 Path: "/files/" + o.(*Object).id, 1088 Parameters: fieldsValue(), 1089 } 1090 } 1091 1092 shareLink := api.CreateSharedLink{} 1093 var info api.Item 1094 var resp *http.Response 1095 err = f.pacer.Call(func() (bool, error) { 1096 resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info) 1097 return shouldRetry(ctx, resp, err) 1098 }) 1099 return info.SharedLink.URL, err 1100} 1101 1102// deletePermanently permanently deletes a trashed file 1103func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error { 1104 opts := rest.Opts{ 1105 Method: "DELETE", 1106 NoResponse: true, 1107 } 1108 if itemType == api.ItemTypeFile { 1109 opts.Path = "/files/" + id + "/trash" 1110 } else { 1111 opts.Path = "/folders/" + id + "/trash" 1112 } 1113 return f.pacer.Call(func() (bool, error) { 1114 resp, err := f.srv.Call(ctx, &opts) 1115 return shouldRetry(ctx, resp, err) 1116 }) 1117} 1118 1119// CleanUp empties the trash 1120func (f *Fs) CleanUp(ctx context.Context) (err error) { 1121 var ( 1122 deleteErrors = int64(0) 1123 concurrencyControl = make(chan struct{}, fs.GetConfig(ctx).Checkers) 1124 wg sync.WaitGroup 1125 ) 1126 _, err = f.listAll(ctx, "trash", false, false, false, func(item *api.Item) bool { 1127 if item.Type == api.ItemTypeFolder || item.Type == api.ItemTypeFile { 1128 wg.Add(1) 1129 concurrencyControl <- struct{}{} 1130 go func() { 1131 defer func() { 1132 <-concurrencyControl 1133 wg.Done() 1134 }() 1135 err := f.deletePermanently(ctx, item.Type, item.ID) 1136 if err != nil { 1137 fs.Errorf(f, "failed to delete trash item %q (%q): %v", item.Name, item.ID, err) 1138 atomic.AddInt64(&deleteErrors, 1) 1139 } 1140 }() 1141 } else { 1142 fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) 1143 } 1144 return false 1145 }) 1146 wg.Wait() 1147 if deleteErrors != 0 { 1148 return errors.Errorf("failed to delete %d trash items", deleteErrors) 1149 } 1150 return err 1151} 1152 1153// DirCacheFlush resets the directory cache - used in testing as an 1154// optional interface 1155func (f *Fs) DirCacheFlush() { 1156 f.dirCache.ResetRoot() 1157} 1158 1159// Hashes returns the supported hash sets. 1160func (f *Fs) Hashes() hash.Set { 1161 return hash.Set(hash.SHA1) 1162} 1163 1164// ------------------------------------------------------------ 1165 1166// Fs returns the parent Fs 1167func (o *Object) Fs() fs.Info { 1168 return o.fs 1169} 1170 1171// Return a string version 1172func (o *Object) String() string { 1173 if o == nil { 1174 return "<nil>" 1175 } 1176 return o.remote 1177} 1178 1179// Remote returns the remote path 1180func (o *Object) Remote() string { 1181 return o.remote 1182} 1183 1184// Hash returns the SHA-1 of an object returning a lowercase hex string 1185func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 1186 if t != hash.SHA1 { 1187 return "", hash.ErrUnsupported 1188 } 1189 return o.sha1, nil 1190} 1191 1192// Size returns the size of an object in bytes 1193func (o *Object) Size() int64 { 1194 err := o.readMetaData(context.TODO()) 1195 if err != nil { 1196 fs.Logf(o, "Failed to read metadata: %v", err) 1197 return 0 1198 } 1199 return o.size 1200} 1201 1202// setMetaData sets the metadata from info 1203func (o *Object) setMetaData(info *api.Item) (err error) { 1204 if info.Type == api.ItemTypeFolder { 1205 return fs.ErrorIsDir 1206 } 1207 if info.Type != api.ItemTypeFile { 1208 return errors.Wrapf(fs.ErrorNotAFile, "%q is %q", o.remote, info.Type) 1209 } 1210 o.hasMetaData = true 1211 o.size = int64(info.Size) 1212 o.sha1 = info.SHA1 1213 o.modTime = info.ModTime() 1214 o.id = info.ID 1215 o.publicLink = info.SharedLink.URL 1216 return nil 1217} 1218 1219// readMetaData gets the metadata if it hasn't already been fetched 1220// 1221// it also sets the info 1222func (o *Object) readMetaData(ctx context.Context) (err error) { 1223 if o.hasMetaData { 1224 return nil 1225 } 1226 info, err := o.fs.readMetaDataForPath(ctx, o.remote) 1227 if err != nil { 1228 if apiErr, ok := err.(*api.Error); ok { 1229 if apiErr.Code == "not_found" || apiErr.Code == "trashed" { 1230 return fs.ErrorObjectNotFound 1231 } 1232 } 1233 return err 1234 } 1235 return o.setMetaData(info) 1236} 1237 1238// ModTime returns the modification time of the object 1239// 1240// 1241// It attempts to read the objects mtime and if that isn't present the 1242// LastModified returned in the http headers 1243func (o *Object) ModTime(ctx context.Context) time.Time { 1244 err := o.readMetaData(ctx) 1245 if err != nil { 1246 fs.Logf(o, "Failed to read metadata: %v", err) 1247 return time.Now() 1248 } 1249 return o.modTime 1250} 1251 1252// setModTime sets the modification time of the local fs object 1253func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, error) { 1254 opts := rest.Opts{ 1255 Method: "PUT", 1256 Path: "/files/" + o.id, 1257 Parameters: fieldsValue(), 1258 } 1259 update := api.UpdateFileModTime{ 1260 ContentModifiedAt: api.Time(modTime), 1261 } 1262 var info *api.Item 1263 err := o.fs.pacer.Call(func() (bool, error) { 1264 resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info) 1265 return shouldRetry(ctx, resp, err) 1266 }) 1267 return info, err 1268} 1269 1270// SetModTime sets the modification time of the local fs object 1271func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1272 info, err := o.setModTime(ctx, modTime) 1273 if err != nil { 1274 return err 1275 } 1276 return o.setMetaData(info) 1277} 1278 1279// Storable returns a boolean showing whether this object storable 1280func (o *Object) Storable() bool { 1281 return true 1282} 1283 1284// Open an object for read 1285func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1286 if o.id == "" { 1287 return nil, errors.New("can't download - no id") 1288 } 1289 fs.FixRangeOption(options, o.size) 1290 var resp *http.Response 1291 opts := rest.Opts{ 1292 Method: "GET", 1293 Path: "/files/" + o.id + "/content", 1294 Options: options, 1295 } 1296 err = o.fs.pacer.Call(func() (bool, error) { 1297 resp, err = o.fs.srv.Call(ctx, &opts) 1298 return shouldRetry(ctx, resp, err) 1299 }) 1300 if err != nil { 1301 return nil, err 1302 } 1303 return resp.Body, err 1304} 1305 1306// upload does a single non-multipart upload 1307// 1308// This is recommended for less than 50 MiB of content 1309func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) { 1310 upload := api.UploadFile{ 1311 Name: o.fs.opt.Enc.FromStandardName(leaf), 1312 ContentModifiedAt: api.Time(modTime), 1313 ContentCreatedAt: api.Time(modTime), 1314 Parent: api.Parent{ 1315 ID: directoryID, 1316 }, 1317 } 1318 1319 var resp *http.Response 1320 var result api.FolderItems 1321 opts := rest.Opts{ 1322 Method: "POST", 1323 Body: in, 1324 MultipartMetadataName: "attributes", 1325 MultipartContentName: "contents", 1326 MultipartFileName: upload.Name, 1327 RootURL: uploadURL, 1328 Options: options, 1329 } 1330 // If object has an ID then it is existing so create a new version 1331 if o.id != "" { 1332 opts.Path = "/files/" + o.id + "/content" 1333 } else { 1334 opts.Path = "/files/content" 1335 } 1336 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 1337 resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result) 1338 return shouldRetry(ctx, resp, err) 1339 }) 1340 if err != nil { 1341 return err 1342 } 1343 if result.TotalCount != 1 || len(result.Entries) != 1 { 1344 return errors.Errorf("failed to upload %v - not sure why", o) 1345 } 1346 return o.setMetaData(&result.Entries[0]) 1347} 1348 1349// Update the object with the contents of the io.Reader, modTime and size 1350// 1351// If existing is set then it updates the object rather than creating a new one 1352// 1353// The new object may have been created if an error is returned 1354func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1355 if o.fs.tokenRenewer != nil { 1356 o.fs.tokenRenewer.Start() 1357 defer o.fs.tokenRenewer.Stop() 1358 } 1359 1360 size := src.Size() 1361 modTime := src.ModTime(ctx) 1362 remote := o.Remote() 1363 1364 // Create the directory for the object if it doesn't exist 1365 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) 1366 if err != nil { 1367 return err 1368 } 1369 1370 // Upload with simple or multipart 1371 if size <= int64(o.fs.opt.UploadCutoff) { 1372 err = o.upload(ctx, in, leaf, directoryID, modTime, options...) 1373 } else { 1374 err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...) 1375 } 1376 return err 1377} 1378 1379// Remove an object 1380func (o *Object) Remove(ctx context.Context) error { 1381 return o.fs.deleteObject(ctx, o.id) 1382} 1383 1384// ID returns the ID of the Object if known, or "" if not 1385func (o *Object) ID() string { 1386 return o.id 1387} 1388 1389// Check the interfaces are satisfied 1390var ( 1391 _ fs.Fs = (*Fs)(nil) 1392 _ fs.Purger = (*Fs)(nil) 1393 _ fs.PutStreamer = (*Fs)(nil) 1394 _ fs.Copier = (*Fs)(nil) 1395 _ fs.Abouter = (*Fs)(nil) 1396 _ fs.Mover = (*Fs)(nil) 1397 _ fs.DirMover = (*Fs)(nil) 1398 _ fs.DirCacheFlusher = (*Fs)(nil) 1399 _ fs.PublicLinker = (*Fs)(nil) 1400 _ fs.CleanUpper = (*Fs)(nil) 1401 _ fs.Object = (*Object)(nil) 1402 _ fs.IDer = (*Object)(nil) 1403) 1404