1// Package vfs provides local and remote filesystems support 2package vfs 3 4import ( 5 "errors" 6 "fmt" 7 "io" 8 "net/url" 9 "os" 10 "path" 11 "path/filepath" 12 "runtime" 13 "strings" 14 "time" 15 16 "github.com/eikenb/pipeat" 17 "github.com/pkg/sftp" 18 19 "github.com/drakkan/sftpgo/v2/kms" 20 "github.com/drakkan/sftpgo/v2/logger" 21 "github.com/drakkan/sftpgo/v2/sdk" 22 "github.com/drakkan/sftpgo/v2/util" 23) 24 25const dirMimeType = "inode/directory" 26 27var ( 28 validAzAccessTier = []string{"", "Archive", "Hot", "Cool"} 29 // ErrStorageSizeUnavailable is returned if the storage backend does not support getting the size 30 ErrStorageSizeUnavailable = errors.New("unable to get available size for this storage backend") 31 // ErrVfsUnsupported defines the error for an unsupported VFS operation 32 ErrVfsUnsupported = errors.New("not supported") 33 credentialsDirPath string 34 tempPath string 35 sftpFingerprints []string 36) 37 38// SetCredentialsDirPath sets the credentials dir path 39func SetCredentialsDirPath(credentialsPath string) { 40 credentialsDirPath = credentialsPath 41} 42 43// GetCredentialsDirPath returns the credentials dir path 44func GetCredentialsDirPath() string { 45 return credentialsDirPath 46} 47 48// SetTempPath sets the path for temporary files 49func SetTempPath(fsPath string) { 50 tempPath = fsPath 51} 52 53// GetTempPath returns the path for temporary files 54func GetTempPath() string { 55 return tempPath 56} 57 58// SetSFTPFingerprints sets the SFTP host key fingerprints 59func SetSFTPFingerprints(fp []string) { 60 sftpFingerprints = fp 61} 62 63// Fs defines the interface for filesystem backends 64type Fs interface { 65 Name() string 66 ConnectionID() string 67 Stat(name string) (os.FileInfo, error) 68 Lstat(name string) (os.FileInfo, error) 69 Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) 70 Create(name string, flag int) (File, *PipeWriter, func(), error) 71 Rename(source, target string) error 72 Remove(name string, isDir bool) error 73 Mkdir(name string) error 74 MkdirAll(name string, uid int, gid int) error 75 Symlink(source, target string) error 76 Chown(name string, uid int, gid int) error 77 Chmod(name string, mode os.FileMode) error 78 Chtimes(name string, atime, mtime time.Time) error 79 Truncate(name string, size int64) error 80 ReadDir(dirname string) ([]os.FileInfo, error) 81 Readlink(name string) (string, error) 82 IsUploadResumeSupported() bool 83 IsAtomicUploadSupported() bool 84 CheckRootPath(username string, uid int, gid int) bool 85 ResolvePath(sftpPath string) (string, error) 86 IsNotExist(err error) bool 87 IsPermission(err error) bool 88 IsNotSupported(err error) bool 89 ScanRootDirContents() (int, int64, error) 90 GetDirSize(dirname string) (int, int64, error) 91 GetAtomicUploadPath(name string) string 92 GetRelativePath(name string) string 93 Walk(root string, walkFn filepath.WalkFunc) error 94 Join(elem ...string) string 95 HasVirtualFolders() bool 96 GetMimeType(name string) (string, error) 97 GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) 98 Close() error 99} 100 101// File defines an interface representing a SFTPGo file 102type File interface { 103 io.Reader 104 io.Writer 105 io.Closer 106 io.ReaderAt 107 io.WriterAt 108 io.Seeker 109 Stat() (os.FileInfo, error) 110 Name() string 111 Truncate(size int64) error 112} 113 114// QuotaCheckResult defines the result for a quota check 115type QuotaCheckResult struct { 116 HasSpace bool 117 AllowedSize int64 118 AllowedFiles int 119 UsedSize int64 120 UsedFiles int 121 QuotaSize int64 122 QuotaFiles int 123} 124 125// GetRemainingSize returns the remaining allowed size 126func (q *QuotaCheckResult) GetRemainingSize() int64 { 127 if q.QuotaSize > 0 { 128 return q.QuotaSize - q.UsedSize 129 } 130 return 0 131} 132 133// GetRemainingFiles returns the remaining allowed files 134func (q *QuotaCheckResult) GetRemainingFiles() int { 135 if q.QuotaFiles > 0 { 136 return q.QuotaFiles - q.UsedFiles 137 } 138 return 0 139} 140 141// S3FsConfig defines the configuration for S3 based filesystem 142type S3FsConfig struct { 143 sdk.S3FsConfig 144} 145 146// HideConfidentialData hides confidential data 147func (c *S3FsConfig) HideConfidentialData() { 148 if c.AccessSecret != nil { 149 c.AccessSecret.Hide() 150 } 151} 152 153func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { 154 if c.Bucket != other.Bucket { 155 return false 156 } 157 if c.KeyPrefix != other.KeyPrefix { 158 return false 159 } 160 if c.Region != other.Region { 161 return false 162 } 163 if c.AccessKey != other.AccessKey { 164 return false 165 } 166 if c.Endpoint != other.Endpoint { 167 return false 168 } 169 if c.StorageClass != other.StorageClass { 170 return false 171 } 172 if c.ACL != other.ACL { 173 return false 174 } 175 if c.UploadPartSize != other.UploadPartSize { 176 return false 177 } 178 if c.UploadConcurrency != other.UploadConcurrency { 179 return false 180 } 181 if c.DownloadConcurrency != other.DownloadConcurrency { 182 return false 183 } 184 if c.DownloadPartSize != other.DownloadPartSize { 185 return false 186 } 187 if c.DownloadPartMaxTime != other.DownloadPartMaxTime { 188 return false 189 } 190 if c.ForcePathStyle != other.ForcePathStyle { 191 return false 192 } 193 return c.isSecretEqual(other) 194} 195 196func (c *S3FsConfig) isSecretEqual(other *S3FsConfig) bool { 197 if c.AccessSecret == nil { 198 c.AccessSecret = kms.NewEmptySecret() 199 } 200 if other.AccessSecret == nil { 201 other.AccessSecret = kms.NewEmptySecret() 202 } 203 return c.AccessSecret.IsEqual(other.AccessSecret) 204} 205 206func (c *S3FsConfig) checkCredentials() error { 207 if c.AccessKey == "" && !c.AccessSecret.IsEmpty() { 208 return errors.New("access_key cannot be empty with access_secret not empty") 209 } 210 if c.AccessSecret.IsEmpty() && c.AccessKey != "" { 211 return errors.New("access_secret cannot be empty with access_key not empty") 212 } 213 if c.AccessSecret.IsEncrypted() && !c.AccessSecret.IsValid() { 214 return errors.New("invalid encrypted access_secret") 215 } 216 if !c.AccessSecret.IsEmpty() && !c.AccessSecret.IsValidInput() { 217 return errors.New("invalid access_secret") 218 } 219 return nil 220} 221 222// EncryptCredentials encrypts access secret if it is in plain text 223func (c *S3FsConfig) EncryptCredentials(additionalData string) error { 224 if c.AccessSecret.IsPlain() { 225 c.AccessSecret.SetAdditionalData(additionalData) 226 err := c.AccessSecret.Encrypt() 227 if err != nil { 228 return err 229 } 230 } 231 return nil 232} 233 234func (c *S3FsConfig) checkPartSizeAndConcurrency() error { 235 if c.UploadPartSize != 0 && (c.UploadPartSize < 5 || c.UploadPartSize > 5000) { 236 return errors.New("upload_part_size cannot be != 0, lower than 5 (MB) or greater than 5000 (MB)") 237 } 238 if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 { 239 return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency) 240 } 241 if c.DownloadPartSize != 0 && (c.DownloadPartSize < 5 || c.DownloadPartSize > 5000) { 242 return errors.New("download_part_size cannot be != 0, lower than 5 (MB) or greater than 5000 (MB)") 243 } 244 if c.DownloadConcurrency < 0 || c.DownloadConcurrency > 64 { 245 return fmt.Errorf("invalid download concurrency: %v", c.DownloadConcurrency) 246 } 247 return nil 248} 249 250// Validate returns an error if the configuration is not valid 251func (c *S3FsConfig) Validate() error { 252 if c.AccessSecret == nil { 253 c.AccessSecret = kms.NewEmptySecret() 254 } 255 if c.Bucket == "" { 256 return errors.New("bucket cannot be empty") 257 } 258 if c.Region == "" { 259 return errors.New("region cannot be empty") 260 } 261 if err := c.checkCredentials(); err != nil { 262 return err 263 } 264 if c.KeyPrefix != "" { 265 if strings.HasPrefix(c.KeyPrefix, "/") { 266 return errors.New("key_prefix cannot start with /") 267 } 268 c.KeyPrefix = path.Clean(c.KeyPrefix) 269 if !strings.HasSuffix(c.KeyPrefix, "/") { 270 c.KeyPrefix += "/" 271 } 272 } 273 c.StorageClass = strings.TrimSpace(c.StorageClass) 274 c.ACL = strings.TrimSpace(c.ACL) 275 return c.checkPartSizeAndConcurrency() 276} 277 278// GCSFsConfig defines the configuration for Google Cloud Storage based filesystem 279type GCSFsConfig struct { 280 sdk.GCSFsConfig 281} 282 283// HideConfidentialData hides confidential data 284func (c *GCSFsConfig) HideConfidentialData() { 285 if c.Credentials != nil { 286 c.Credentials.Hide() 287 } 288} 289 290func (c *GCSFsConfig) isEqual(other *GCSFsConfig) bool { 291 if c.Bucket != other.Bucket { 292 return false 293 } 294 if c.KeyPrefix != other.KeyPrefix { 295 return false 296 } 297 if c.AutomaticCredentials != other.AutomaticCredentials { 298 return false 299 } 300 if c.StorageClass != other.StorageClass { 301 return false 302 } 303 if c.ACL != other.ACL { 304 return false 305 } 306 if c.Credentials == nil { 307 c.Credentials = kms.NewEmptySecret() 308 } 309 if other.Credentials == nil { 310 other.Credentials = kms.NewEmptySecret() 311 } 312 return c.Credentials.IsEqual(other.Credentials) 313} 314 315// Validate returns an error if the configuration is not valid 316func (c *GCSFsConfig) Validate(credentialsFilePath string) error { 317 if c.Credentials == nil || c.AutomaticCredentials == 1 { 318 c.Credentials = kms.NewEmptySecret() 319 } 320 if c.Bucket == "" { 321 return errors.New("bucket cannot be empty") 322 } 323 if c.KeyPrefix != "" { 324 if strings.HasPrefix(c.KeyPrefix, "/") { 325 return errors.New("key_prefix cannot start with /") 326 } 327 c.KeyPrefix = path.Clean(c.KeyPrefix) 328 if !strings.HasSuffix(c.KeyPrefix, "/") { 329 c.KeyPrefix += "/" 330 } 331 } 332 if c.Credentials.IsEncrypted() && !c.Credentials.IsValid() { 333 return errors.New("invalid encrypted credentials") 334 } 335 if c.AutomaticCredentials == 0 && !c.Credentials.IsValidInput() { 336 fi, err := os.Stat(credentialsFilePath) 337 if err != nil { 338 return fmt.Errorf("invalid credentials %v", err) 339 } 340 if fi.Size() == 0 { 341 return errors.New("credentials cannot be empty") 342 } 343 } 344 c.StorageClass = strings.TrimSpace(c.StorageClass) 345 c.ACL = strings.TrimSpace(c.ACL) 346 return nil 347} 348 349// AzBlobFsConfig defines the configuration for Azure Blob Storage based filesystem 350type AzBlobFsConfig struct { 351 sdk.AzBlobFsConfig 352} 353 354// HideConfidentialData hides confidential data 355func (c *AzBlobFsConfig) HideConfidentialData() { 356 if c.AccountKey != nil { 357 c.AccountKey.Hide() 358 } 359 if c.SASURL != nil { 360 c.SASURL.Hide() 361 } 362} 363 364func (c *AzBlobFsConfig) isEqual(other *AzBlobFsConfig) bool { 365 if c.Container != other.Container { 366 return false 367 } 368 if c.AccountName != other.AccountName { 369 return false 370 } 371 if c.Endpoint != other.Endpoint { 372 return false 373 } 374 if c.SASURL.IsEmpty() { 375 c.SASURL = kms.NewEmptySecret() 376 } 377 if other.SASURL.IsEmpty() { 378 other.SASURL = kms.NewEmptySecret() 379 } 380 if !c.SASURL.IsEqual(other.SASURL) { 381 return false 382 } 383 if c.KeyPrefix != other.KeyPrefix { 384 return false 385 } 386 if c.UploadPartSize != other.UploadPartSize { 387 return false 388 } 389 if c.UploadConcurrency != other.UploadConcurrency { 390 return false 391 } 392 if c.UseEmulator != other.UseEmulator { 393 return false 394 } 395 if c.AccessTier != other.AccessTier { 396 return false 397 } 398 if c.AccountKey == nil { 399 c.AccountKey = kms.NewEmptySecret() 400 } 401 if other.AccountKey == nil { 402 other.AccountKey = kms.NewEmptySecret() 403 } 404 return c.AccountKey.IsEqual(other.AccountKey) 405} 406 407// EncryptCredentials encrypts access secret if it is in plain text 408func (c *AzBlobFsConfig) EncryptCredentials(additionalData string) error { 409 if c.AccountKey.IsPlain() { 410 c.AccountKey.SetAdditionalData(additionalData) 411 if err := c.AccountKey.Encrypt(); err != nil { 412 return err 413 } 414 } 415 if c.SASURL.IsPlain() { 416 c.SASURL.SetAdditionalData(additionalData) 417 if err := c.SASURL.Encrypt(); err != nil { 418 return err 419 } 420 } 421 return nil 422} 423 424func (c *AzBlobFsConfig) checkCredentials() error { 425 if c.SASURL.IsPlain() { 426 _, err := url.Parse(c.SASURL.GetPayload()) 427 return err 428 } 429 if c.SASURL.IsEncrypted() && !c.SASURL.IsValid() { 430 return errors.New("invalid encrypted sas_url") 431 } 432 if !c.SASURL.IsEmpty() { 433 return nil 434 } 435 if c.AccountName == "" || !c.AccountKey.IsValidInput() { 436 return errors.New("credentials cannot be empty or invalid") 437 } 438 if c.AccountKey.IsEncrypted() && !c.AccountKey.IsValid() { 439 return errors.New("invalid encrypted account_key") 440 } 441 return nil 442} 443 444// Validate returns an error if the configuration is not valid 445func (c *AzBlobFsConfig) Validate() error { 446 if c.AccountKey == nil { 447 c.AccountKey = kms.NewEmptySecret() 448 } 449 if c.SASURL == nil { 450 c.SASURL = kms.NewEmptySecret() 451 } 452 // container could be embedded within SAS URL we check this at runtime 453 if c.SASURL.IsEmpty() && c.Container == "" { 454 return errors.New("container cannot be empty") 455 } 456 if err := c.checkCredentials(); err != nil { 457 return err 458 } 459 if c.KeyPrefix != "" { 460 if strings.HasPrefix(c.KeyPrefix, "/") { 461 return errors.New("key_prefix cannot start with /") 462 } 463 c.KeyPrefix = path.Clean(c.KeyPrefix) 464 if !strings.HasSuffix(c.KeyPrefix, "/") { 465 c.KeyPrefix += "/" 466 } 467 } 468 if c.UploadPartSize < 0 || c.UploadPartSize > 100 { 469 return fmt.Errorf("invalid upload part size: %v", c.UploadPartSize) 470 } 471 if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 { 472 return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency) 473 } 474 if !util.IsStringInSlice(c.AccessTier, validAzAccessTier) { 475 return fmt.Errorf("invalid access tier %#v, valid values: \"''%v\"", c.AccessTier, strings.Join(validAzAccessTier, ", ")) 476 } 477 return nil 478} 479 480// CryptFsConfig defines the configuration to store local files as encrypted 481type CryptFsConfig struct { 482 sdk.CryptFsConfig 483} 484 485// HideConfidentialData hides confidential data 486func (c *CryptFsConfig) HideConfidentialData() { 487 if c.Passphrase != nil { 488 c.Passphrase.Hide() 489 } 490} 491 492func (c *CryptFsConfig) isEqual(other *CryptFsConfig) bool { 493 if c.Passphrase == nil { 494 c.Passphrase = kms.NewEmptySecret() 495 } 496 if other.Passphrase == nil { 497 other.Passphrase = kms.NewEmptySecret() 498 } 499 return c.Passphrase.IsEqual(other.Passphrase) 500} 501 502// EncryptCredentials encrypts access secret if it is in plain text 503func (c *CryptFsConfig) EncryptCredentials(additionalData string) error { 504 if c.Passphrase.IsPlain() { 505 c.Passphrase.SetAdditionalData(additionalData) 506 if err := c.Passphrase.Encrypt(); err != nil { 507 return err 508 } 509 } 510 return nil 511} 512 513// Validate returns an error if the configuration is not valid 514func (c *CryptFsConfig) Validate() error { 515 if c.Passphrase == nil || c.Passphrase.IsEmpty() { 516 return errors.New("invalid passphrase") 517 } 518 if !c.Passphrase.IsValidInput() { 519 return errors.New("passphrase cannot be empty or invalid") 520 } 521 if c.Passphrase.IsEncrypted() && !c.Passphrase.IsValid() { 522 return errors.New("invalid encrypted passphrase") 523 } 524 return nil 525} 526 527// PipeWriter defines a wrapper for pipeat.PipeWriterAt. 528type PipeWriter struct { 529 writer *pipeat.PipeWriterAt 530 err error 531 done chan bool 532} 533 534// NewPipeWriter initializes a new PipeWriter 535func NewPipeWriter(w *pipeat.PipeWriterAt) *PipeWriter { 536 return &PipeWriter{ 537 writer: w, 538 err: nil, 539 done: make(chan bool), 540 } 541} 542 543// Close waits for the upload to end, closes the pipeat.PipeWriterAt and returns an error if any. 544func (p *PipeWriter) Close() error { 545 p.writer.Close() //nolint:errcheck // the returned error is always null 546 <-p.done 547 return p.err 548} 549 550// Done unlocks other goroutines waiting on Close(). 551// It must be called when the upload ends 552func (p *PipeWriter) Done(err error) { 553 p.err = err 554 p.done <- true 555} 556 557// WriteAt is a wrapper for pipeat WriteAt 558func (p *PipeWriter) WriteAt(data []byte, off int64) (int, error) { 559 return p.writer.WriteAt(data, off) 560} 561 562// Write is a wrapper for pipeat Write 563func (p *PipeWriter) Write(data []byte) (int, error) { 564 return p.writer.Write(data) 565} 566 567// IsDirectory checks if a path exists and is a directory 568func IsDirectory(fs Fs, path string) (bool, error) { 569 fileInfo, err := fs.Stat(path) 570 if err != nil { 571 return false, err 572 } 573 return fileInfo.IsDir(), err 574} 575 576// IsLocalOsFs returns true if fs is a local filesystem implementation 577func IsLocalOsFs(fs Fs) bool { 578 return fs.Name() == osFsName 579} 580 581// IsCryptOsFs returns true if fs is an encrypted local filesystem implementation 582func IsCryptOsFs(fs Fs) bool { 583 return fs.Name() == cryptFsName 584} 585 586// IsSFTPFs returns true if fs is an SFTP filesystem 587func IsSFTPFs(fs Fs) bool { 588 return strings.HasPrefix(fs.Name(), sftpFsName) 589} 590 591// IsBufferedSFTPFs returns true if this is a buffered SFTP filesystem 592func IsBufferedSFTPFs(fs Fs) bool { 593 if !IsSFTPFs(fs) { 594 return false 595 } 596 return !fs.IsUploadResumeSupported() 597} 598 599// IsLocalOrUnbufferedSFTPFs returns true if fs is local or SFTP with no buffer 600func IsLocalOrUnbufferedSFTPFs(fs Fs) bool { 601 if IsLocalOsFs(fs) { 602 return true 603 } 604 if IsSFTPFs(fs) { 605 return fs.IsUploadResumeSupported() 606 } 607 return false 608} 609 610// IsLocalOrSFTPFs returns true if fs is local or SFTP 611func IsLocalOrSFTPFs(fs Fs) bool { 612 return IsLocalOsFs(fs) || IsSFTPFs(fs) 613} 614 615// HasOpenRWSupport returns true if the fs can open a file 616// for reading and writing at the same time 617func HasOpenRWSupport(fs Fs) bool { 618 if IsLocalOsFs(fs) { 619 return true 620 } 621 if IsSFTPFs(fs) && fs.IsUploadResumeSupported() { 622 return true 623 } 624 return false 625} 626 627// IsLocalOrCryptoFs returns true if fs is local or local encrypted 628func IsLocalOrCryptoFs(fs Fs) bool { 629 return IsLocalOsFs(fs) || IsCryptOsFs(fs) 630} 631 632// SetPathPermissions calls fs.Chown. 633// It does nothing for local filesystem on windows 634func SetPathPermissions(fs Fs, path string, uid int, gid int) { 635 if uid == -1 && gid == -1 { 636 return 637 } 638 if IsLocalOsFs(fs) { 639 if runtime.GOOS == "windows" { 640 return 641 } 642 } 643 if err := fs.Chown(path, uid, gid); err != nil { 644 fsLog(fs, logger.LevelWarn, "error chowning path %v: %v", path, err) 645 } 646} 647 648func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) { 649 logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...) 650} 651