1// Filesystem features and optional interfaces 2 3package fs 4 5import ( 6 "context" 7 "io" 8 "reflect" 9 "strings" 10 "time" 11) 12 13// Features describe the optional features of the Fs 14type Features struct { 15 // Feature flags, whether Fs 16 CaseInsensitive bool // has case insensitive files 17 DuplicateFiles bool // allows duplicate files 18 ReadMimeType bool // can read the mime type of objects 19 WriteMimeType bool // can set the mime type of objects 20 CanHaveEmptyDirectories bool // can have empty directories 21 BucketBased bool // is bucket based (like s3, swift, etc.) 22 BucketBasedRootOK bool // is bucket based and can use from root 23 SetTier bool // allows set tier functionality on objects 24 GetTier bool // allows to retrieve storage tier of objects 25 ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type 26 IsLocal bool // is the local backend 27 SlowModTime bool // if calling ModTime() generally takes an extra transaction 28 SlowHash bool // if calling Hash() generally takes an extra transaction 29 30 // Purge all files in the directory specified 31 // 32 // Implement this if you have a way of deleting all the files 33 // quicker than just running Remove() on the result of List() 34 // 35 // Return an error if it doesn't exist 36 Purge func(ctx context.Context, dir string) error 37 38 // Copy src to this remote using server-side copy operations. 39 // 40 // This is stored with the remote path given 41 // 42 // It returns the destination Object and a possible error 43 // 44 // Will only be called if src.Fs().Name() == f.Name() 45 // 46 // If it isn't possible then return fs.ErrorCantCopy 47 Copy func(ctx context.Context, src Object, remote string) (Object, error) 48 49 // Move src to this remote using server-side move operations. 50 // 51 // This is stored with the remote path given 52 // 53 // It returns the destination Object and a possible error 54 // 55 // Will only be called if src.Fs().Name() == f.Name() 56 // 57 // If it isn't possible then return fs.ErrorCantMove 58 Move func(ctx context.Context, src Object, remote string) (Object, error) 59 60 // DirMove moves src, srcRemote to this remote at dstRemote 61 // using server-side move operations. 62 // 63 // Will only be called if src.Fs().Name() == f.Name() 64 // 65 // If it isn't possible then return fs.ErrorCantDirMove 66 // 67 // If destination exists then return fs.ErrorDirExists 68 DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error 69 70 // ChangeNotify calls the passed function with a path 71 // that has had changes. If the implementation 72 // uses polling, it should adhere to the given interval. 73 ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration) 74 75 // UnWrap returns the Fs that this Fs is wrapping 76 UnWrap func() Fs 77 78 // WrapFs returns the Fs that is wrapping this Fs 79 WrapFs func() Fs 80 81 // SetWrapper sets the Fs that is wrapping this Fs 82 SetWrapper func(f Fs) 83 84 // DirCacheFlush resets the directory cache - used in testing 85 // as an optional interface 86 DirCacheFlush func() 87 88 // PublicLink generates a public link to the remote path (usually readable by anyone) 89 PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) 90 91 // Put in to the remote path with the modTime given of the given size 92 // 93 // May create the object even if it returns an error - if so 94 // will return the object and the error, otherwise will return 95 // nil and the error 96 // 97 // May create duplicates or return errors if src already 98 // exists. 99 PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) 100 101 // PutStream uploads to the remote path with the modTime given of indeterminate size 102 // 103 // May create the object even if it returns an error - if so 104 // will return the object and the error, otherwise will return 105 // nil and the error 106 PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) 107 108 // MergeDirs merges the contents of all the directories passed 109 // in into the first one and rmdirs the other directories. 110 MergeDirs func(ctx context.Context, dirs []Directory) error 111 112 // CleanUp the trash in the Fs 113 // 114 // Implement this if you have a way of emptying the trash or 115 // otherwise cleaning up old versions of files. 116 CleanUp func(ctx context.Context) error 117 118 // ListR lists the objects and directories of the Fs starting 119 // from dir recursively into out. 120 // 121 // dir should be "" to start from the root, and should not 122 // have trailing slashes. 123 // 124 // This should return ErrDirNotFound if the directory isn't 125 // found. 126 // 127 // It should call callback for each tranche of entries read. 128 // These need not be returned in any particular order. If 129 // callback returns an error then the listing will stop 130 // immediately. 131 // 132 // Don't implement this unless you have a more efficient way 133 // of listing recursively that doing a directory traversal. 134 ListR ListRFn 135 136 // About gets quota information from the Fs 137 About func(ctx context.Context) (*Usage, error) 138 139 // OpenWriterAt opens with a handle for random access writes 140 // 141 // Pass in the remote desired and the size if known. 142 // 143 // It truncates any existing object 144 OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error) 145 146 // UserInfo returns info about the connected user 147 UserInfo func(ctx context.Context) (map[string]string, error) 148 149 // Disconnect the current user 150 Disconnect func(ctx context.Context) error 151 152 // Command the backend to run a named command 153 // 154 // The command run is name 155 // args may be used to read arguments from 156 // opts may be used to read optional arguments from 157 // 158 // The result should be capable of being JSON encoded 159 // If it is a string or a []string it will be shown to the user 160 // otherwise it will be JSON encoded and shown to the user like that 161 Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) 162 163 // Shutdown the backend, closing any background tasks and any 164 // cached connections. 165 Shutdown func(ctx context.Context) error 166} 167 168// Disable nil's out the named feature. If it isn't found then it 169// will log a message. 170func (ft *Features) Disable(name string) *Features { 171 v := reflect.ValueOf(ft).Elem() 172 vType := v.Type() 173 for i := 0; i < v.NumField(); i++ { 174 vName := vType.Field(i).Name 175 field := v.Field(i) 176 if strings.EqualFold(name, vName) { 177 if !field.CanSet() { 178 Errorf(nil, "Can't set Feature %q", name) 179 } else { 180 zero := reflect.Zero(field.Type()) 181 field.Set(zero) 182 Debugf(nil, "Reset feature %q", name) 183 } 184 } 185 } 186 return ft 187} 188 189// List returns a slice of all the possible feature names 190func (ft *Features) List() (out []string) { 191 v := reflect.ValueOf(ft).Elem() 192 vType := v.Type() 193 for i := 0; i < v.NumField(); i++ { 194 out = append(out, vType.Field(i).Name) 195 } 196 return out 197} 198 199// Enabled returns a map of features with keys showing whether they 200// are enabled or not 201func (ft *Features) Enabled() (features map[string]bool) { 202 v := reflect.ValueOf(ft).Elem() 203 vType := v.Type() 204 features = make(map[string]bool, v.NumField()) 205 for i := 0; i < v.NumField(); i++ { 206 vName := vType.Field(i).Name 207 field := v.Field(i) 208 if field.Kind() == reflect.Func { 209 // Can't compare functions 210 features[vName] = !field.IsNil() 211 } else { 212 zero := reflect.Zero(field.Type()) 213 features[vName] = field.Interface() != zero.Interface() 214 } 215 } 216 return features 217} 218 219// DisableList nil's out the comma separated list of named features. 220// If it isn't found then it will log a message. 221func (ft *Features) DisableList(list []string) *Features { 222 for _, feature := range list { 223 ft.Disable(strings.TrimSpace(feature)) 224 } 225 return ft 226} 227 228// Fill fills in the function pointers in the Features struct from the 229// optional interfaces. It returns the original updated Features 230// struct passed in. 231func (ft *Features) Fill(ctx context.Context, f Fs) *Features { 232 if do, ok := f.(Purger); ok { 233 ft.Purge = do.Purge 234 } 235 if do, ok := f.(Copier); ok { 236 ft.Copy = do.Copy 237 } 238 if do, ok := f.(Mover); ok { 239 ft.Move = do.Move 240 } 241 if do, ok := f.(DirMover); ok { 242 ft.DirMove = do.DirMove 243 } 244 if do, ok := f.(ChangeNotifier); ok { 245 ft.ChangeNotify = do.ChangeNotify 246 } 247 if do, ok := f.(UnWrapper); ok { 248 ft.UnWrap = do.UnWrap 249 } 250 if do, ok := f.(Wrapper); ok { 251 ft.WrapFs = do.WrapFs 252 ft.SetWrapper = do.SetWrapper 253 } 254 if do, ok := f.(DirCacheFlusher); ok { 255 ft.DirCacheFlush = do.DirCacheFlush 256 } 257 if do, ok := f.(PublicLinker); ok { 258 ft.PublicLink = do.PublicLink 259 } 260 if do, ok := f.(PutUncheckeder); ok { 261 ft.PutUnchecked = do.PutUnchecked 262 } 263 if do, ok := f.(PutStreamer); ok { 264 ft.PutStream = do.PutStream 265 } 266 if do, ok := f.(MergeDirser); ok { 267 ft.MergeDirs = do.MergeDirs 268 } 269 if do, ok := f.(CleanUpper); ok { 270 ft.CleanUp = do.CleanUp 271 } 272 if do, ok := f.(ListRer); ok { 273 ft.ListR = do.ListR 274 } 275 if do, ok := f.(Abouter); ok { 276 ft.About = do.About 277 } 278 if do, ok := f.(OpenWriterAter); ok { 279 ft.OpenWriterAt = do.OpenWriterAt 280 } 281 if do, ok := f.(UserInfoer); ok { 282 ft.UserInfo = do.UserInfo 283 } 284 if do, ok := f.(Disconnecter); ok { 285 ft.Disconnect = do.Disconnect 286 } 287 if do, ok := f.(Commander); ok { 288 ft.Command = do.Command 289 } 290 if do, ok := f.(Shutdowner); ok { 291 ft.Shutdown = do.Shutdown 292 } 293 return ft.DisableList(GetConfig(ctx).DisableFeatures) 294} 295 296// Mask the Features with the Fs passed in 297// 298// Only optional features which are implemented in both the original 299// Fs AND the one passed in will be advertised. Any features which 300// aren't in both will be set to false/nil, except for UnWrap/Wrap which 301// will be left untouched. 302func (ft *Features) Mask(ctx context.Context, f Fs) *Features { 303 mask := f.Features() 304 ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive 305 ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles 306 ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType 307 ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType 308 ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories 309 ft.BucketBased = ft.BucketBased && mask.BucketBased 310 ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK 311 ft.SetTier = ft.SetTier && mask.SetTier 312 ft.GetTier = ft.GetTier && mask.GetTier 313 ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs 314 // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal 315 ft.SlowModTime = ft.SlowModTime && mask.SlowModTime 316 ft.SlowHash = ft.SlowHash && mask.SlowHash 317 318 if mask.Purge == nil { 319 ft.Purge = nil 320 } 321 if mask.Copy == nil { 322 ft.Copy = nil 323 } 324 if mask.Move == nil { 325 ft.Move = nil 326 } 327 if mask.DirMove == nil { 328 ft.DirMove = nil 329 } 330 if mask.ChangeNotify == nil { 331 ft.ChangeNotify = nil 332 } 333 // if mask.UnWrap == nil { 334 // ft.UnWrap = nil 335 // } 336 // if mask.Wrapper == nil { 337 // ft.Wrapper = nil 338 // } 339 if mask.DirCacheFlush == nil { 340 ft.DirCacheFlush = nil 341 } 342 if mask.PublicLink == nil { 343 ft.PublicLink = nil 344 } 345 if mask.PutUnchecked == nil { 346 ft.PutUnchecked = nil 347 } 348 if mask.PutStream == nil { 349 ft.PutStream = nil 350 } 351 if mask.MergeDirs == nil { 352 ft.MergeDirs = nil 353 } 354 if mask.CleanUp == nil { 355 ft.CleanUp = nil 356 } 357 if mask.ListR == nil { 358 ft.ListR = nil 359 } 360 if mask.About == nil { 361 ft.About = nil 362 } 363 if mask.OpenWriterAt == nil { 364 ft.OpenWriterAt = nil 365 } 366 if mask.UserInfo == nil { 367 ft.UserInfo = nil 368 } 369 if mask.Disconnect == nil { 370 ft.Disconnect = nil 371 } 372 // Command is always local so we don't mask it 373 if mask.Shutdown == nil { 374 ft.Shutdown = nil 375 } 376 return ft.DisableList(GetConfig(ctx).DisableFeatures) 377} 378 379// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap 380// method only if available in f. 381func (ft *Features) Wrap(f Fs) *Features { 382 ftCopy := new(Features) 383 *ftCopy = *ft 384 if do, ok := f.(UnWrapper); ok { 385 ftCopy.UnWrap = do.UnWrap 386 } 387 if do, ok := f.(Wrapper); ok { 388 ftCopy.WrapFs = do.WrapFs 389 ftCopy.SetWrapper = do.SetWrapper 390 } 391 return ftCopy 392} 393 394// WrapsFs adds extra information between `f` which wraps `w` 395func (ft *Features) WrapsFs(f Fs, w Fs) *Features { 396 wFeatures := w.Features() 397 if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil { 398 wFeatures.SetWrapper(f) 399 } 400 return ft 401} 402 403// Purger is an optional interfaces for Fs 404type Purger interface { 405 // Purge all files in the directory specified 406 // 407 // Implement this if you have a way of deleting all the files 408 // quicker than just running Remove() on the result of List() 409 // 410 // Return an error if it doesn't exist 411 Purge(ctx context.Context, dir string) error 412} 413 414// Copier is an optional interface for Fs 415type Copier interface { 416 // Copy src to this remote using server-side copy operations. 417 // 418 // This is stored with the remote path given 419 // 420 // It returns the destination Object and a possible error 421 // 422 // Will only be called if src.Fs().Name() == f.Name() 423 // 424 // If it isn't possible then return fs.ErrorCantCopy 425 Copy(ctx context.Context, src Object, remote string) (Object, error) 426} 427 428// Mover is an optional interface for Fs 429type Mover interface { 430 // Move src to this remote using server-side move operations. 431 // 432 // This is stored with the remote path given 433 // 434 // It returns the destination Object and a possible error 435 // 436 // Will only be called if src.Fs().Name() == f.Name() 437 // 438 // If it isn't possible then return fs.ErrorCantMove 439 Move(ctx context.Context, src Object, remote string) (Object, error) 440} 441 442// DirMover is an optional interface for Fs 443type DirMover interface { 444 // DirMove moves src, srcRemote to this remote at dstRemote 445 // using server-side move operations. 446 // 447 // Will only be called if src.Fs().Name() == f.Name() 448 // 449 // If it isn't possible then return fs.ErrorCantDirMove 450 // 451 // If destination exists then return fs.ErrorDirExists 452 DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error 453} 454 455// ChangeNotifier is an optional interface for Fs 456type ChangeNotifier interface { 457 // ChangeNotify calls the passed function with a path 458 // that has had changes. If the implementation 459 // uses polling, it should adhere to the given interval. 460 // At least one value will be written to the channel, 461 // specifying the initial value and updated values might 462 // follow. A 0 Duration should pause the polling. 463 // The ChangeNotify implementation must empty the channel 464 // regularly. When the channel gets closed, the implementation 465 // should stop polling and release resources. 466 ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration) 467} 468 469// EntryType can be associated with remote paths to identify their type 470type EntryType int 471 472// Constants 473const ( 474 // EntryDirectory should be used to classify remote paths in directories 475 EntryDirectory EntryType = iota // 0 476 // EntryObject should be used to classify remote paths in objects 477 EntryObject // 1 478) 479 480// UnWrapper is an optional interfaces for Fs 481type UnWrapper interface { 482 // UnWrap returns the Fs that this Fs is wrapping 483 UnWrap() Fs 484} 485 486// Wrapper is an optional interfaces for Fs 487type Wrapper interface { 488 // Wrap returns the Fs that is wrapping this Fs 489 WrapFs() Fs 490 // SetWrapper sets the Fs that is wrapping this Fs 491 SetWrapper(f Fs) 492} 493 494// DirCacheFlusher is an optional interface for Fs 495type DirCacheFlusher interface { 496 // DirCacheFlush resets the directory cache - used in testing 497 // as an optional interface 498 DirCacheFlush() 499} 500 501// PutUncheckeder is an optional interface for Fs 502type PutUncheckeder interface { 503 // Put in to the remote path with the modTime given of the given size 504 // 505 // May create the object even if it returns an error - if so 506 // will return the object and the error, otherwise will return 507 // nil and the error 508 // 509 // May create duplicates or return errors if src already 510 // exists. 511 PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) 512} 513 514// PutStreamer is an optional interface for Fs 515type PutStreamer interface { 516 // PutStream uploads to the remote path with the modTime given of indeterminate size 517 // 518 // May create the object even if it returns an error - if so 519 // will return the object and the error, otherwise will return 520 // nil and the error 521 PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error) 522} 523 524// PublicLinker is an optional interface for Fs 525type PublicLinker interface { 526 // PublicLink generates a public link to the remote path (usually readable by anyone) 527 PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error) 528} 529 530// MergeDirser is an option interface for Fs 531type MergeDirser interface { 532 // MergeDirs merges the contents of all the directories passed 533 // in into the first one and rmdirs the other directories. 534 MergeDirs(ctx context.Context, dirs []Directory) error 535} 536 537// CleanUpper is an optional interfaces for Fs 538type CleanUpper interface { 539 // CleanUp the trash in the Fs 540 // 541 // Implement this if you have a way of emptying the trash or 542 // otherwise cleaning up old versions of files. 543 CleanUp(ctx context.Context) error 544} 545 546// ListRer is an optional interfaces for Fs 547type ListRer interface { 548 // ListR lists the objects and directories of the Fs starting 549 // from dir recursively into out. 550 // 551 // dir should be "" to start from the root, and should not 552 // have trailing slashes. 553 // 554 // This should return ErrDirNotFound if the directory isn't 555 // found. 556 // 557 // It should call callback for each tranche of entries read. 558 // These need not be returned in any particular order. If 559 // callback returns an error then the listing will stop 560 // immediately. 561 // 562 // Don't implement this unless you have a more efficient way 563 // of listing recursively that doing a directory traversal. 564 ListR(ctx context.Context, dir string, callback ListRCallback) error 565} 566 567// RangeSeeker is the interface that wraps the RangeSeek method. 568// 569// Some of the returns from Object.Open() may optionally implement 570// this method for efficiency purposes. 571type RangeSeeker interface { 572 // RangeSeek behaves like a call to Seek(offset int64, whence 573 // int) with the output wrapped in an io.LimitedReader 574 // limiting the total length to limit. 575 // 576 // RangeSeek with a limit of < 0 is equivalent to a regular Seek. 577 RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error) 578} 579 580// Abouter is an optional interface for Fs 581type Abouter interface { 582 // About gets quota information from the Fs 583 About(ctx context.Context) (*Usage, error) 584} 585 586// OpenWriterAter is an optional interface for Fs 587type OpenWriterAter interface { 588 // OpenWriterAt opens with a handle for random access writes 589 // 590 // Pass in the remote desired and the size if known. 591 // 592 // It truncates any existing object 593 OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error) 594} 595 596// UserInfoer is an optional interface for Fs 597type UserInfoer interface { 598 // UserInfo returns info about the connected user 599 UserInfo(ctx context.Context) (map[string]string, error) 600} 601 602// Disconnecter is an optional interface for Fs 603type Disconnecter interface { 604 // Disconnect the current user 605 Disconnect(ctx context.Context) error 606} 607 608// CommandHelp describes a single backend Command 609// 610// These are automatically inserted in the docs 611type CommandHelp struct { 612 Name string // Name of the command, e.g. "link" 613 Short string // Single line description 614 Long string // Long multi-line description 615 Opts map[string]string // maps option name to a single line help 616} 617 618// Commander is an interface to wrap the Command function 619type Commander interface { 620 // Command the backend to run a named command 621 // 622 // The command run is name 623 // args may be used to read arguments from 624 // opts may be used to read optional arguments from 625 // 626 // The result should be capable of being JSON encoded 627 // If it is a string or a []string it will be shown to the user 628 // otherwise it will be JSON encoded and shown to the user like that 629 Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) 630} 631 632// Shutdowner is an interface to wrap the Shutdown function 633type Shutdowner interface { 634 // Shutdown the backend, closing any background tasks and any 635 // cached connections. 636 Shutdown(ctx context.Context) error 637} 638 639// ObjectsChan is a channel of Objects 640type ObjectsChan chan Object 641 642// Objects is a slice of Object~s 643type Objects []Object 644 645// ObjectPair is a pair of Objects used to describe a potential copy 646// operation. 647type ObjectPair struct { 648 Src, Dst Object 649} 650 651// UnWrapFs unwraps f as much as possible and returns the base Fs 652func UnWrapFs(f Fs) Fs { 653 for { 654 unwrap := f.Features().UnWrap 655 if unwrap == nil { 656 break // not a wrapped Fs, use current 657 } 658 next := unwrap() 659 if next == nil { 660 break // no base Fs found, use current 661 } 662 f = next 663 } 664 return f 665} 666 667// UnWrapObject unwraps o as much as possible and returns the base object 668func UnWrapObject(o Object) Object { 669 for { 670 u, ok := o.(ObjectUnWrapper) 671 if !ok { 672 break // not a wrapped object, use current 673 } 674 next := u.UnWrap() 675 if next == nil { 676 break // no base object found, use current 677 } 678 o = next 679 } 680 return o 681} 682 683// UnWrapObjectInfo returns the underlying Object unwrapped as much as 684// possible or nil. 685func UnWrapObjectInfo(oi ObjectInfo) Object { 686 o, ok := oi.(Object) 687 if !ok { 688 return nil 689 } 690 return UnWrapObject(o) 691} 692