1#!/usr/bin/env python 2""" 3fs.base 4======= 5 6This module defines the most basic filesystem abstraction, the FS class. 7Instances of FS represent a filesystem containing files and directories 8that can be queried and manipulated. To implement a new kind of filesystem, 9start by sublcassing the base FS class. 10 11For more information regarding implementing a working PyFilesystem interface, see :ref:`implementers`. 12 13""" 14 15from __future__ import with_statement 16 17__all__ = ['DummyLock', 18 'silence_fserrors', 19 'NullFile', 20 'synchronize', 21 'FS', 22 'flags_to_mode', 23 'NoDefaultMeta'] 24 25import os 26import os.path 27import shutil 28import fnmatch 29import datetime 30import time 31import errno 32try: 33 import threading 34except ImportError: 35 import dummy_threading as threading 36 37from fs.path import * 38from fs.errors import * 39from fs.local_functools import wraps 40 41import six 42from six import b 43 44 45class DummyLock(object): 46 """A dummy lock object that doesn't do anything. 47 48 This is used as a placeholder when locking is disabled. We can't 49 directly use the Lock class from the dummy_threading module, since 50 it attempts to sanity-check the sequence of acquire/release calls 51 in a way that breaks when real threading is available. 52 53 """ 54 55 def acquire(self, blocking=1): 56 """Acquiring a DummyLock always succeeds.""" 57 return 1 58 59 def release(self): 60 """Releasing a DummyLock always succeeds.""" 61 pass 62 63 def __enter__(self): 64 return self 65 66 def __exit__(self, exc_type, exc_value, traceback): 67 pass 68 69 70def silence_fserrors(f, *args, **kwargs): 71 """Perform a function call and return ``None`` if an :class:`fs.errors.FSError` is thrown 72 73 :param f: Function to call 74 :param args: Parameters to f 75 :param kwargs: Keyword parameters to f 76 77 """ 78 try: 79 return f(*args, **kwargs) 80 except FSError: 81 return None 82 83 84class NoDefaultMeta(object): 85 """A singleton used to signify that there is no default for getmeta""" 86 pass 87 88 89class NullFile(object): 90 """A NullFile is a file object that has no functionality. 91 92 Null files are returned by the :meth:`fs.base.FS.safeopen` method in FS objects when the 93 file doesn't exist. This can simplify code by negating the need to check 94 if a file exists, or handling exceptions. 95 96 """ 97 def __init__(self): 98 self.closed = False 99 100 def __iter__(self): 101 return self 102 103 def __enter__(self): 104 return self 105 106 def __exit__(self, exc_type, exc_value, traceback): 107 self.closed = True 108 109 def flush(self): 110 pass 111 112 def next(self): 113 raise StopIteration 114 115 def readline(self, *args, **kwargs): 116 return b("") 117 118 def close(self): 119 self.closed = True 120 121 def read(self, size=None): 122 return b("") 123 124 def seek(self, *args, **kwargs): 125 pass 126 127 def tell(self): 128 return 0 129 130 def truncate(self, *args, **kwargs): 131 return 0 132 133 def write(self, data): 134 pass 135 136 def writelines(self, *args, **kwargs): 137 pass 138 139 140def synchronize(func): 141 """Decorator to synchronize a method on self._lock.""" 142 @wraps(func) 143 def acquire_lock(self, *args, **kwargs): 144 self._lock.acquire() 145 try: 146 return func(self, *args, **kwargs) 147 finally: 148 self._lock.release() 149 return acquire_lock 150 151 152class FS(object): 153 """The base class for Filesystem abstraction objects. 154 An instance of a class derived from FS is an abstraction on some kind of filesystem, such as the OS filesystem or a zip file. 155 156 """ 157 158 _meta = {} 159 160 def __init__(self, thread_synchronize=True): 161 """The base class for Filesystem objects. 162 163 :param thread_synconize: If True, a lock object will be created for the object, otherwise a dummy lock will be used. 164 :type thread_synchronize: bool 165 166 """ 167 168 self.closed = False 169 super(FS, self).__init__() 170 self.thread_synchronize = thread_synchronize 171 if thread_synchronize: 172 self._lock = threading.RLock() 173 else: 174 self._lock = DummyLock() 175 176 def __del__(self): 177 if not getattr(self, 'closed', True): 178 try: 179 self.close() 180 except: 181 pass 182 183 def __enter__(self): 184 return self 185 186 def __exit__(self, type, value, traceback): 187 self.close() 188 189 def cachehint(self, enabled): 190 """Recommends the use of caching. Implementations are free to use or 191 ignore this value. 192 193 :param enabled: If True the implementation is permitted to aggressively cache directory 194 structure / file information. Caching such information can speed up many operations, 195 particularly for network based filesystems. The downside of caching is that 196 changes made to directories or files outside of this interface may not be picked up immediately. 197 198 """ 199 pass 200 # Deprecating cache_hint in favour of no underscore version, for consistency 201 cache_hint = cachehint 202 203 def close(self): 204 """Close the filesystem. This will perform any shutdown related 205 operations required. This method will be called automatically when 206 the filesystem object is garbage collected, but it is good practice 207 to call it explicitly so that any attached resourced are freed when they 208 are no longer required. 209 210 """ 211 self.closed = True 212 213 def __getstate__(self): 214 # Locks can't be pickled, so instead we just indicate the 215 # type of lock that should be there. None == no lock, 216 # True == a proper lock, False == a dummy lock. 217 state = self.__dict__.copy() 218 lock = state.get("_lock", None) 219 if lock is not None: 220 if isinstance(lock, threading._RLock): 221 state["_lock"] = True 222 else: 223 state["_lock"] = False 224 return state 225 226 def __setstate__(self, state): 227 self.__dict__.update(state) 228 lock = state.get("_lock") 229 if lock is not None: 230 if lock: 231 self._lock = threading.RLock() 232 else: 233 self._lock = DummyLock() 234 235 def getmeta(self, meta_name, default=NoDefaultMeta): 236 """Retrieve a meta value associated with an FS object. 237 238 Meta values are a way for an FS implementation to report potentially 239 useful information associated with the file system. 240 241 A meta key is a lower case string with no spaces. Meta keys may also 242 be grouped in namespaces in a dotted notation, e.g. 'atomic.namespaces'. 243 FS implementations aren't obliged to return any meta values, but the 244 following are common: 245 246 * *read_only* True if the file system cannot be modified 247 * *thread_safe* True if the implementation is thread safe 248 * *network* True if the file system requires network access 249 * *unicode_paths* True if the file system supports unicode paths 250 * *case_insensitive_paths* True if the file system ignores the case of paths 251 * *atomic.makedir* True if making a directory is an atomic operation 252 * *atomic.rename* True if rename is an atomic operation, (and not implemented as a copy followed by a delete) 253 * *atomic.setcontents* True if the implementation supports setting the contents of a file as an atomic operation (without opening a file) 254 * *free_space* The free space (in bytes) available on the file system 255 * *total_space* The total space (in bytes) available on the file system 256 * *virtual* True if the filesystem defers to other filesystems 257 * *invalid_path_chars* A string containing characters that may not be used in paths 258 259 FS implementations may expose non-generic meta data through a self-named namespace. e.g. ``"somefs.some_meta"`` 260 261 Since no meta value is guaranteed to exist, it is advisable to always supply a 262 default value to ``getmeta``. 263 264 :param meta_name: The name of the meta value to retrieve 265 :param default: An option default to return, if the meta value isn't present 266 :raises `fs.errors.NoMetaError`: If specified meta value is not present, and there is no default 267 268 """ 269 if meta_name not in self._meta: 270 if default is not NoDefaultMeta: 271 return default 272 raise NoMetaError(meta_name=meta_name) 273 return self._meta[meta_name] 274 275 def hasmeta(self, meta_name): 276 """Check that a meta value is supported 277 278 :param meta_name: The name of a meta value to check 279 :rtype: bool 280 281 """ 282 try: 283 self.getmeta(meta_name) 284 except NoMetaError: 285 return False 286 return True 287 288 def validatepath(self, path): 289 """Validate an fs path, throws an :class:`~fs.errors.InvalidPathError` exception if validation fails. 290 291 A path is invalid if it fails to map to a path on the underlaying filesystem. The default 292 implementation checks for the presence of any of the characters in the meta value 'invalid_path_chars', 293 but implementations may have other requirements for paths. 294 295 :param path: an fs path to validatepath 296 :raises `fs.errors.InvalidPathError`: if `path` does not map on to a valid path on this filesystem 297 298 """ 299 invalid_chars = self.getmeta('invalid_path_chars', default=None) 300 if invalid_chars: 301 re_invalid_chars = getattr(self, '_re_invalid_chars', None) 302 if re_invalid_chars is None: 303 self._re_invalid_chars = re_invalid_chars = re.compile('|'.join(re.escape(c) for c in invalid_chars), re.UNICODE) 304 if re_invalid_chars.search(path): 305 raise InvalidCharsInPathError(path) 306 307 def isvalidpath(self, path): 308 """Check if a path is valid on this filesystem 309 310 :param path: an fs path 311 312 """ 313 try: 314 self.validatepath(path) 315 except InvalidPathError: 316 return False 317 else: 318 return True 319 320 def getsyspath(self, path, allow_none=False): 321 """Returns the system path (a path recognized by the OS) if one is present. 322 323 If the path does not map to a system path (and `allow_none` is False) 324 then a NoSysPathError exception is thrown. Otherwise, the system 325 path will be returned as a unicode string. 326 327 :param path: a path within the filesystem 328 :param allow_none: if True, this method will return None when there is no system path, 329 rather than raising NoSysPathError 330 :type allow_none: bool 331 :raises `fs.errors.NoSysPathError`: if the path does not map on to a system path, and allow_none is set to False (default) 332 :rtype: unicode 333 334 """ 335 if not allow_none: 336 raise NoSysPathError(path=path) 337 return None 338 339 def hassyspath(self, path): 340 """Check if the path maps to a system path (a path recognized by the OS). 341 342 :param path: path to check 343 :returns: True if `path` maps to a system path 344 :rtype: bool 345 346 """ 347 return self.getsyspath(path, allow_none=True) is not None 348 349 def getpathurl(self, path, allow_none=False): 350 """Returns a url that corresponds to the given path, if one exists. 351 352 If the path does not have an equivalent URL form (and allow_none is False) 353 then a :class:`~fs.errors.NoPathURLError` exception is thrown. Otherwise the URL will be 354 returns as an unicode string. 355 356 :param path: a path within the filesystem 357 :param allow_none: if true, this method can return None if there is no 358 URL form of the given path 359 :type allow_none: bool 360 :raises `fs.errors.NoPathURLError`: If no URL form exists, and allow_none is False (the default) 361 :rtype: unicode 362 363 """ 364 if not allow_none: 365 raise NoPathURLError(path=path) 366 return None 367 368 def haspathurl(self, path): 369 """Check if the path has an equivalent URL form 370 371 :param path: path to check 372 :returns: True if `path` has a URL form 373 :rtype: bool 374 375 """ 376 return self.getpathurl(path, allow_none=True) is not None 377 378 def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs): 379 """Open a the given path as a file-like object. 380 381 :param path: a path to file that should be opened 382 :type path: string 383 :param mode: mode of file to open, identical to the mode string used 384 in 'file' and 'open' builtins 385 :type mode: string 386 :param kwargs: additional (optional) keyword parameters that may 387 be required to open the file 388 :type kwargs: dict 389 390 :rtype: a file-like object 391 392 :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing 393 :raises `fs.errors.ResourceInvalidError`: if an intermediate directory is an file 394 :raises `fs.errors.ResourceNotFoundError`: if the path is not found 395 396 """ 397 raise UnsupportedError("open file") 398 399 def safeopen(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs): 400 """Like :py:meth:`~fs.base.FS.open`, but returns a 401 :py:class:`~fs.base.NullFile` if the file could not be opened. 402 403 A ``NullFile`` is a dummy file which has all the methods of a file-like object, 404 but contains no data. 405 406 :param path: a path to file that should be opened 407 :type path: string 408 :param mode: mode of file to open, identical to the mode string used 409 in 'file' and 'open' builtins 410 :type mode: string 411 :param kwargs: additional (optional) keyword parameters that may 412 be required to open the file 413 :type kwargs: dict 414 415 :rtype: a file-like object 416 417 """ 418 try: 419 f = self.open(path, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, line_buffering=line_buffering, **kwargs) 420 except ResourceNotFoundError: 421 return NullFile() 422 return f 423 424 def exists(self, path): 425 """Check if a path references a valid resource. 426 427 :param path: A path in the filesystem 428 :type path: string 429 430 :rtype: bool 431 432 """ 433 return self.isfile(path) or self.isdir(path) 434 435 def isdir(self, path): 436 """Check if a path references a directory. 437 438 :param path: a path in the filesystem 439 :type path: string 440 441 :rtype: bool 442 443 """ 444 raise UnsupportedError("check for directory") 445 446 def isfile(self, path): 447 """Check if a path references a file. 448 449 :param path: a path in the filesystem 450 :type path: string 451 452 :rtype: bool 453 454 """ 455 raise UnsupportedError("check for file") 456 457 def __iter__(self): 458 """ Iterates over paths returned by :py:meth:`~fs.base.listdir` method with default params. """ 459 for f in self.listdir(): 460 yield f 461 462 def listdir(self, 463 path="./", 464 wildcard=None, 465 full=False, 466 absolute=False, 467 dirs_only=False, 468 files_only=False): 469 """Lists the the files and directories under a given path. 470 471 The directory contents are returned as a list of unicode paths. 472 473 :param path: root of the path to list 474 :type path: string 475 :param wildcard: Only returns paths that match this wildcard 476 :type wildcard: string containing a wildcard, or a callable that accepts a path and returns a boolean 477 :param full: returns full paths (relative to the root) 478 :type full: bool 479 :param absolute: returns absolute paths (paths beginning with /) 480 :type absolute: bool 481 :param dirs_only: if True, only return directories 482 :type dirs_only: bool 483 :param files_only: if True, only return files 484 :type files_only: bool 485 486 :rtype: iterable of paths 487 488 :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing 489 :raises `fs.errors.ResourceInvalidError`: if the path exists, but is not a directory 490 :raises `fs.errors.ResourceNotFoundError`: if the path is not found 491 492 """ 493 raise UnsupportedError("list directory") 494 495 def listdirinfo(self, 496 path="./", 497 wildcard=None, 498 full=False, 499 absolute=False, 500 dirs_only=False, 501 files_only=False): 502 """Retrieves a list of paths and path info under a given path. 503 504 This method behaves like listdir() but instead of just returning 505 the name of each item in the directory, it returns a tuple of the 506 name and the info dict as returned by getinfo. 507 508 This method may be more efficient than calling 509 :py:meth:`~fs.base.FS.getinfo` on each individual item returned by :py:meth:`~fs.base.FS.listdir`, particularly 510 for network based filesystems. 511 512 :param path: root of the path to list 513 :param wildcard: filter paths that match this wildcard 514 :param dirs_only: only retrieve directories 515 :type dirs_only: bool 516 :param files_only: only retrieve files 517 :type files_only: bool 518 519 :raises `fs.errors.ResourceNotFoundError`: If the path is not found 520 :raises `fs.errors.ResourceInvalidError`: If the path exists, but is not a directory 521 522 """ 523 path = normpath(path) 524 525 def getinfo(p): 526 try: 527 if full or absolute: 528 return self.getinfo(p) 529 else: 530 return self.getinfo(pathjoin(path, p)) 531 except FSError: 532 return {} 533 534 return [(p, getinfo(p)) 535 for p in self.listdir(path, 536 wildcard=wildcard, 537 full=full, 538 absolute=absolute, 539 dirs_only=dirs_only, 540 files_only=files_only)] 541 542 def _listdir_helper(self, 543 path, 544 entries, 545 wildcard=None, 546 full=False, 547 absolute=False, 548 dirs_only=False, 549 files_only=False): 550 """A helper method called by listdir method that applies filtering. 551 552 Given the path to a directory and a list of the names of entries within 553 that directory, this method applies the semantics of the listdir() 554 keyword arguments. An appropriately modified and filtered list of 555 directory entries is returned. 556 557 """ 558 path = normpath(path) 559 if dirs_only and files_only: 560 raise ValueError("dirs_only and files_only can not both be True") 561 562 if wildcard is not None: 563 if not callable(wildcard): 564 wildcard_re = re.compile(fnmatch.translate(wildcard)) 565 wildcard = lambda fn: bool(wildcard_re.match(fn)) 566 entries = [p for p in entries if wildcard(p)] 567 568 if dirs_only: 569 isdir = self.isdir 570 entries = [p for p in entries if isdir(pathcombine(path, p))] 571 elif files_only: 572 isfile = self.isfile 573 entries = [p for p in entries if isfile(pathcombine(path, p))] 574 575 if full: 576 entries = [pathcombine(path, p) for p in entries] 577 elif absolute: 578 path = abspath(path) 579 entries = [(pathcombine(path, p)) for p in entries] 580 581 return entries 582 583 def ilistdir(self, 584 path="./", 585 wildcard=None, 586 full=False, 587 absolute=False, 588 dirs_only=False, 589 files_only=False): 590 """Generator yielding the files and directories under a given path. 591 592 This method behaves identically to :py:meth:`fs.base.FS.listdir` but returns an generator 593 instead of a list. Depending on the filesystem this may be more 594 efficient than calling :py:meth:`fs.base.FS.listdir` and iterating over the resulting list. 595 596 """ 597 return iter(self.listdir(path, 598 wildcard=wildcard, 599 full=full, 600 absolute=absolute, 601 dirs_only=dirs_only, 602 files_only=files_only)) 603 604 def ilistdirinfo(self, 605 path="./", 606 wildcard=None, 607 full=False, 608 absolute=False, 609 dirs_only=False, 610 files_only=False): 611 """Generator yielding paths and path info under a given path. 612 613 This method behaves identically to :py:meth:`~fs.base.listdirinfo` but returns an generator 614 instead of a list. Depending on the filesystem this may be more 615 efficient than calling :py:meth:`~fs.base.listdirinfo` and iterating over the resulting 616 list. 617 618 """ 619 return iter(self.listdirinfo(path, 620 wildcard, 621 full, 622 absolute, 623 dirs_only, 624 files_only)) 625 626 def makedir(self, path, recursive=False, allow_recreate=False): 627 """Make a directory on the filesystem. 628 629 :param path: path of directory 630 :type path: string 631 :param recursive: if True, any intermediate directories will also be created 632 :type recursive: bool 633 :param allow_recreate: if True, re-creating a directory wont be an error 634 :type allow_create: bool 635 636 :raises `fs.errors.DestinationExistsError`: if the path is already a directory, and allow_recreate is False 637 :raises `fs.errors.ParentDirectoryMissingError`: if a containing directory is missing and recursive is False 638 :raises `fs.errors.ResourceInvalidError`: if a path is an existing file 639 :raises `fs.errors.ResourceNotFoundError`: if the path is not found 640 641 """ 642 raise UnsupportedError("make directory") 643 644 def remove(self, path): 645 """Remove a file from the filesystem. 646 647 :param path: Path of the resource to remove 648 :type path: string 649 650 :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing 651 :raises `fs.errors.ResourceInvalidError`: if the path is a directory 652 :raises `fs.errors.ResourceNotFoundError`: if the path does not exist 653 654 """ 655 raise UnsupportedError("remove resource") 656 657 def removedir(self, path, recursive=False, force=False): 658 """Remove a directory from the filesystem 659 660 :param path: path of the directory to remove 661 :type path: string 662 :param recursive: if True, empty parent directories will be removed 663 :type recursive: bool 664 :param force: if True, any directory contents will be removed 665 :type force: bool 666 667 :raises `fs.errors.DirectoryNotEmptyError`: if the directory is not empty and force is False 668 :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing 669 :raises `fs.errors.ResourceInvalidError`: if the path is not a directory 670 :raises `fs.errors.ResourceNotFoundError`: if the path does not exist 671 672 """ 673 raise UnsupportedError("remove directory") 674 675 def rename(self, src, dst): 676 """Renames a file or directory 677 678 :param src: path to rename 679 :type src: string 680 :param dst: new name 681 :type dst: string 682 683 :raises ParentDirectoryMissingError: if a containing directory is missing 684 :raises ResourceInvalidError: if the path or a parent path is not a 685 directory or src is a parent of dst or one of src or dst is a dir 686 and the other don't 687 :raises ResourceNotFoundError: if the src path does not exist 688 689 """ 690 raise UnsupportedError("rename resource") 691 692 @convert_os_errors 693 def settimes(self, path, accessed_time=None, modified_time=None): 694 """Set the accessed time and modified time of a file 695 696 :param path: path to a file 697 :type path: string 698 :param accessed_time: the datetime the file was accessed (defaults to current time) 699 :type accessed_time: datetime 700 :param modified_time: the datetime the file was modified (defaults to current time) 701 :type modified_time: datetime 702 703 """ 704 705 with self._lock: 706 sys_path = self.getsyspath(path, allow_none=True) 707 if sys_path is not None: 708 now = datetime.datetime.now() 709 if accessed_time is None: 710 accessed_time = now 711 if modified_time is None: 712 modified_time = now 713 accessed_time = int(time.mktime(accessed_time.timetuple())) 714 modified_time = int(time.mktime(modified_time.timetuple())) 715 os.utime(sys_path, (accessed_time, modified_time)) 716 return True 717 else: 718 raise UnsupportedError("settimes") 719 720 def getinfo(self, path): 721 """Returns information for a path as a dictionary. The exact content of 722 this dictionary will vary depending on the implementation, but will 723 likely include a few common values. The following values will be found 724 in info dictionaries for most implementations: 725 726 * "size" - Number of bytes used to store the file or directory 727 * "created_time" - A datetime object containing the time the resource was created 728 * "accessed_time" - A datetime object containing the time the resource was last accessed 729 * "modified_time" - A datetime object containing the time the resource was modified 730 731 :param path: a path to retrieve information for 732 :type path: string 733 734 :rtype: dict 735 736 :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate directory is missing 737 :raises `fs.errors.ResourceInvalidError`: if the path is not a directory 738 :raises `fs.errors.ResourceNotFoundError`: if the path does not exist 739 740 """ 741 raise UnsupportedError("get resource info") 742 743 def getinfokeys(self, path, *keys): 744 """Get specified keys from info dict, as returned from `getinfo`. The returned dictionary may 745 not contain all the keys that were asked for, if they aren't available. 746 747 This method allows a filesystem to potentially provide a faster way of retrieving these info values if you 748 are only interested in a subset of them. 749 750 :param path: a path to retrieve information for 751 :param keys: the info keys you would like to retrieve 752 753 :rtype: dict 754 755 """ 756 info = self.getinfo(path) 757 return dict((k, info[k]) for k in keys if k in info) 758 759 def desc(self, path): 760 """Returns short descriptive text regarding a path. Intended mainly as 761 a debugging aid. 762 763 :param path: A path to describe 764 :rtype: str 765 766 """ 767 #if not self.exists(path): 768 # return '' 769 try: 770 sys_path = self.getsyspath(path) 771 except NoSysPathError: 772 return "No description available" 773 return sys_path 774 775 def getcontents(self, path, mode='rb', encoding=None, errors=None, newline=None): 776 """Returns the contents of a file as a string. 777 778 :param path: A path of file to read 779 :param mode: Mode to open file with (should be 'rb' for binary or 't' for text) 780 :param encoding: Encoding to use when reading contents in text mode 781 :param errors: Unicode errors parameter if text mode is use 782 :param newline: Newlines parameter for text mode decoding 783 :rtype: str 784 :returns: file contents 785 786 """ 787 if 'r' not in mode: 788 raise ValueError("mode must contain 'r' to be readable") 789 f = None 790 try: 791 f = self.open(path, mode=mode, encoding=encoding, errors=errors, newline=newline) 792 contents = f.read() 793 return contents 794 finally: 795 if f is not None: 796 f.close() 797 798 def _setcontents(self, 799 path, 800 data, 801 encoding=None, 802 errors=None, 803 chunk_size=1024 * 64, 804 progress_callback=None, 805 finished_callback=None): 806 """Does the work of setcontents. Factored out, so that `setcontents_async` can use it""" 807 if progress_callback is None: 808 progress_callback = lambda bytes_written: None 809 if finished_callback is None: 810 finished_callback = lambda: None 811 812 if not data: 813 progress_callback(0) 814 self.createfile(path, wipe=True) 815 finished_callback() 816 return 0 817 818 bytes_written = 0 819 progress_callback(0) 820 821 if hasattr(data, 'read'): 822 read = data.read 823 chunk = read(chunk_size) 824 if isinstance(chunk, six.text_type): 825 f = self.open(path, 'wt', encoding=encoding, errors=errors) 826 else: 827 f = self.open(path, 'wb') 828 write = f.write 829 try: 830 while chunk: 831 write(chunk) 832 bytes_written += len(chunk) 833 progress_callback(bytes_written) 834 chunk = read(chunk_size) 835 finally: 836 f.close() 837 else: 838 if isinstance(data, six.text_type): 839 with self.open(path, 'wt', encoding=encoding, errors=errors) as f: 840 f.write(data) 841 bytes_written += len(data) 842 else: 843 with self.open(path, 'wb') as f: 844 f.write(data) 845 bytes_written += len(data) 846 progress_callback(bytes_written) 847 848 finished_callback() 849 return bytes_written 850 851 def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=1024 * 64): 852 """A convenience method to create a new file from a string or file-like object 853 854 :param path: a path of the file to create 855 :param data: a string or bytes object containing the contents for the new file 856 :param encoding: if `data` is a file open in text mode, or a text string, then use this `encoding` to write to the destination file 857 :param errors: if `data` is a file open in text mode or a text string, then use `errors` when opening the destination file 858 :param chunk_size: Number of bytes to read in a chunk, if the implementation has to resort to a read / copy loop 859 860 """ 861 return self._setcontents(path, data, encoding=encoding, errors=errors, chunk_size=1024 * 64) 862 863 def setcontents_async(self, 864 path, 865 data, 866 encoding=None, 867 errors=None, 868 chunk_size=1024 * 64, 869 progress_callback=None, 870 finished_callback=None, 871 error_callback=None): 872 """Create a new file from a string or file-like object asynchronously 873 874 This method returns a ``threading.Event`` object. Call the ``wait`` method on the event object 875 to block until all data has been written, or simply ignore it. 876 877 :param path: a path of the file to create 878 :param data: a string or a file-like object containing the contents for the new file 879 :param encoding: if `data` is a file open in text mode, or a text string, then use this `encoding` to write to the destination file 880 :param errors: if `data` is a file open in text mode or a text string, then use `errors` when opening the destination file 881 :param chunk_size: Number of bytes to read and write in a chunk 882 :param progress_callback: A function that is called periodically 883 with the number of bytes written. 884 :param finished_callback: A function that is called when all data has been written 885 :param error_callback: A function that is called with an exception 886 object if any error occurs during the copy process. 887 :returns: An event object that is set when the copy is complete, call 888 the `wait` method of this object to block until the data is written 889 890 """ 891 892 finished_event = threading.Event() 893 894 def do_setcontents(): 895 try: 896 self._setcontents(path, 897 data, 898 encoding=encoding, 899 errors=errors, 900 chunk_size=1024 * 64, 901 progress_callback=progress_callback, 902 finished_callback=finished_callback) 903 except Exception, e: 904 if error_callback is not None: 905 error_callback(e) 906 finally: 907 finished_event.set() 908 909 threading.Thread(target=do_setcontents).start() 910 return finished_event 911 912 def createfile(self, path, wipe=False): 913 """Creates an empty file if it doesn't exist 914 915 :param path: path to the file to create 916 :param wipe: if True, the contents of the file will be erased 917 918 """ 919 with self._lock: 920 if not wipe and self.isfile(path): 921 return 922 f = None 923 try: 924 f = self.open(path, 'wb') 925 finally: 926 if f is not None: 927 f.close() 928 929 def opendir(self, path): 930 """Opens a directory and returns a FS object representing its contents. 931 932 :param path: path to directory to open 933 :type path: string 934 935 :return: the opened dir 936 :rtype: an FS object 937 938 """ 939 940 from fs.wrapfs.subfs import SubFS 941 if not self.exists(path): 942 raise ResourceNotFoundError(path) 943 if not self.isdir(path): 944 raise ResourceInvalidError("path should reference a directory") 945 return SubFS(self, path) 946 947 def walk(self, 948 path="/", 949 wildcard=None, 950 dir_wildcard=None, 951 search="breadth", 952 ignore_errors=False): 953 """Walks a directory tree and yields the root path and contents. 954 Yields a tuple of the path of each directory and a list of its file 955 contents. 956 957 :param path: root path to start walking 958 :type path: string 959 :param wildcard: if given, only return files that match this wildcard 960 :type wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean 961 :param dir_wildcard: if given, only walk directories that match the wildcard 962 :type dir_wildcard: a string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean 963 :param search: a string identifying the method used to walk the directories. There are two such methods: 964 965 * ``"breadth"`` yields paths in the top directories first 966 * ``"depth"`` yields the deepest paths first 967 968 :param ignore_errors: ignore any errors reading the directory 969 :type ignore_errors: bool 970 971 :rtype: iterator of (current_path, paths) 972 973 """ 974 975 path = normpath(path) 976 977 if not self.exists(path): 978 raise ResourceNotFoundError(path) 979 980 def listdir(path, *args, **kwargs): 981 if ignore_errors: 982 try: 983 return self.listdir(path, *args, **kwargs) 984 except: 985 return [] 986 else: 987 return self.listdir(path, *args, **kwargs) 988 989 if wildcard is None: 990 wildcard = lambda f: True 991 elif not callable(wildcard): 992 wildcard_re = re.compile(fnmatch.translate(wildcard)) 993 wildcard = lambda fn: bool(wildcard_re.match(fn)) 994 995 if dir_wildcard is None: 996 dir_wildcard = lambda f: True 997 elif not callable(dir_wildcard): 998 dir_wildcard_re = re.compile(fnmatch.translate(dir_wildcard)) 999 dir_wildcard = lambda fn: bool(dir_wildcard_re.match(fn)) 1000 1001 if search == "breadth": 1002 dirs = [path] 1003 dirs_append = dirs.append 1004 dirs_pop = dirs.pop 1005 isdir = self.isdir 1006 while dirs: 1007 current_path = dirs_pop() 1008 paths = [] 1009 paths_append = paths.append 1010 try: 1011 for filename in listdir(current_path, dirs_only=True): 1012 path = pathcombine(current_path, filename) 1013 if dir_wildcard(path): 1014 dirs_append(path) 1015 for filename in listdir(current_path, files_only=True): 1016 path = pathcombine(current_path, filename) 1017 if wildcard(filename): 1018 paths_append(filename) 1019 except ResourceNotFoundError: 1020 # Could happen if another thread / process deletes something whilst we are walking 1021 pass 1022 1023 yield (current_path, paths) 1024 1025 elif search == "depth": 1026 1027 def recurse(recurse_path): 1028 try: 1029 for path in listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True): 1030 for p in recurse(path): 1031 yield p 1032 except ResourceNotFoundError: 1033 # Could happen if another thread / process deletes something whilst we are walking 1034 pass 1035 yield (recurse_path, listdir(recurse_path, wildcard=wildcard, files_only=True)) 1036 1037 for p in recurse(path): 1038 yield p 1039 1040 else: 1041 raise ValueError("Search should be 'breadth' or 'depth'") 1042 1043 def walkfiles(self, 1044 path="/", 1045 wildcard=None, 1046 dir_wildcard=None, 1047 search="breadth", 1048 ignore_errors=False): 1049 """Like the 'walk' method, but just yields file paths. 1050 1051 :param path: root path to start walking 1052 :type path: string 1053 :param wildcard: if given, only return files that match this wildcard 1054 :type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the file path and returns a boolean 1055 :param dir_wildcard: if given, only walk directories that match the wildcard 1056 :type dir_wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean 1057 :param search: a string identifying the method used to walk the directories. There are two such methods: 1058 1059 * ``"breadth"`` yields paths in the top directories first 1060 * ``"depth"`` yields the deepest paths first 1061 1062 :param ignore_errors: ignore any errors reading the directory 1063 :type ignore_errors: bool 1064 1065 :rtype: iterator of file paths 1066 1067 """ 1068 for path, files in self.walk(normpath(path), wildcard=wildcard, dir_wildcard=dir_wildcard, search=search, ignore_errors=ignore_errors): 1069 for f in files: 1070 yield pathcombine(path, f) 1071 1072 def walkdirs(self, 1073 path="/", 1074 wildcard=None, 1075 search="breadth", 1076 ignore_errors=False): 1077 """Like the 'walk' method but yields directories. 1078 1079 :param path: root path to start walking 1080 :type path: string 1081 :param wildcard: if given, only return directories that match this wildcard 1082 :type wildcard: A string containing a wildcard (e.g. `*.txt`) or a callable that takes the directory name and returns a boolean 1083 :param search: a string identifying the method used to walk the directories. There are two such methods: 1084 1085 * ``"breadth"`` yields paths in the top directories first 1086 * ``"depth"`` yields the deepest paths first 1087 1088 :param ignore_errors: ignore any errors reading the directory 1089 :type ignore_errors: bool 1090 1091 :rtype: iterator of dir paths 1092 1093 """ 1094 for p, _files in self.walk(path, dir_wildcard=wildcard, search=search, ignore_errors=ignore_errors): 1095 yield p 1096 1097 def getsize(self, path): 1098 """Returns the size (in bytes) of a resource. 1099 1100 :param path: a path to the resource 1101 :type path: string 1102 1103 :returns: the size of the file 1104 :rtype: integer 1105 1106 """ 1107 info = self.getinfo(path) 1108 size = info.get('size', None) 1109 if size is None: 1110 raise OperationFailedError("get size of resource", path) 1111 return size 1112 1113 def copy(self, src, dst, overwrite=False, chunk_size=1024 * 64): 1114 """Copies a file from src to dst. 1115 1116 :param src: the source path 1117 :type src: string 1118 :param dst: the destination path 1119 :type dst: string 1120 :param overwrite: if True, then an existing file at the destination may 1121 be overwritten; If False then DestinationExistsError 1122 will be raised. 1123 :type overwrite: bool 1124 :param chunk_size: size of chunks to use if a simple copy is required 1125 (defaults to 64K). 1126 :type chunk_size: bool 1127 1128 """ 1129 with self._lock: 1130 if not self.isfile(src): 1131 if self.isdir(src): 1132 raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") 1133 raise ResourceNotFoundError(src) 1134 if not overwrite and self.exists(dst): 1135 raise DestinationExistsError(dst) 1136 1137 src_syspath = self.getsyspath(src, allow_none=True) 1138 dst_syspath = self.getsyspath(dst, allow_none=True) 1139 1140 if src_syspath is not None and dst_syspath is not None: 1141 self._shutil_copyfile(src_syspath, dst_syspath) 1142 else: 1143 src_file = None 1144 try: 1145 src_file = self.open(src, "rb") 1146 self.setcontents(dst, src_file, chunk_size=chunk_size) 1147 except ResourceNotFoundError: 1148 if self.exists(src) and not self.exists(dirname(dst)): 1149 raise ParentDirectoryMissingError(dst) 1150 finally: 1151 if src_file is not None: 1152 src_file.close() 1153 1154 @classmethod 1155 @convert_os_errors 1156 def _shutil_copyfile(cls, src_syspath, dst_syspath): 1157 try: 1158 shutil.copyfile(src_syspath, dst_syspath) 1159 except IOError, e: 1160 # shutil reports ENOENT when a parent directory is missing 1161 if getattr(e, "errno", None) == errno.ENOENT: 1162 if not os.path.exists(dirname(dst_syspath)): 1163 raise ParentDirectoryMissingError(dst_syspath) 1164 raise 1165 1166 @classmethod 1167 @convert_os_errors 1168 def _shutil_movefile(cls, src_syspath, dst_syspath): 1169 shutil.move(src_syspath, dst_syspath) 1170 1171 1172 def move(self, src, dst, overwrite=False, chunk_size=16384): 1173 """moves a file from one location to another. 1174 1175 :param src: source path 1176 :type src: string 1177 :param dst: destination path 1178 :type dst: string 1179 :param overwrite: When True the destination will be overwritten (if it exists), 1180 otherwise a DestinationExistsError will be thrown 1181 :type overwrite: bool 1182 :param chunk_size: Size of chunks to use when copying, if a simple copy 1183 is required 1184 :type chunk_size: integer 1185 1186 :raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False 1187 1188 """ 1189 1190 with self._lock: 1191 src_syspath = self.getsyspath(src, allow_none=True) 1192 dst_syspath = self.getsyspath(dst, allow_none=True) 1193 1194 # Try to do an os-level rename if possible. 1195 # Otherwise, fall back to copy-and-remove. 1196 if src_syspath is not None and dst_syspath is not None: 1197 if not os.path.isfile(src_syspath): 1198 if os.path.isdir(src_syspath): 1199 raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") 1200 raise ResourceNotFoundError(src) 1201 if not overwrite and os.path.exists(dst_syspath): 1202 raise DestinationExistsError(dst) 1203 try: 1204 os.rename(src_syspath, dst_syspath) 1205 return 1206 except OSError: 1207 pass 1208 self.copy(src, dst, overwrite=overwrite, chunk_size=chunk_size) 1209 self.remove(src) 1210 1211 def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): 1212 """moves a directory from one location to another. 1213 1214 :param src: source directory path 1215 :type src: string 1216 :param dst: destination directory path 1217 :type dst: string 1218 :param overwrite: if True then any existing files in the destination 1219 directory will be overwritten 1220 :type overwrite: bool 1221 :param ignore_errors: if True then this method will ignore FSError 1222 exceptions when moving files 1223 :type ignore_errors: bool 1224 :param chunk_size: size of chunks to use when copying, if a simple copy 1225 is required 1226 :type chunk_size: integer 1227 1228 :raise `fs.errors.DestinationExistsError`: if destination exists and `overwrite` is False 1229 1230 """ 1231 with self._lock: 1232 if not self.isdir(src): 1233 if self.isfile(src): 1234 raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s") 1235 raise ResourceNotFoundError(src) 1236 if not overwrite and self.exists(dst): 1237 raise DestinationExistsError(dst) 1238 1239 src_syspath = self.getsyspath(src, allow_none=True) 1240 dst_syspath = self.getsyspath(dst, allow_none=True) 1241 1242 if src_syspath is not None and dst_syspath is not None: 1243 try: 1244 os.rename(src_syspath, dst_syspath) 1245 return 1246 except OSError: 1247 pass 1248 1249 def movefile_noerrors(src, dst, **kwargs): 1250 try: 1251 return self.move(src, dst, **kwargs) 1252 except FSError: 1253 return 1254 if ignore_errors: 1255 movefile = movefile_noerrors 1256 else: 1257 movefile = self.move 1258 1259 src = abspath(src) 1260 dst = abspath(dst) 1261 1262 if dst: 1263 self.makedir(dst, allow_recreate=overwrite) 1264 1265 for dirname, filenames in self.walk(src, search="depth"): 1266 1267 dst_dirname = relpath(frombase(src, abspath(dirname))) 1268 dst_dirpath = pathjoin(dst, dst_dirname) 1269 self.makedir(dst_dirpath, allow_recreate=True, recursive=True) 1270 1271 for filename in filenames: 1272 1273 src_filename = pathjoin(dirname, filename) 1274 dst_filename = pathjoin(dst_dirpath, filename) 1275 movefile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size) 1276 1277 self.removedir(dirname) 1278 1279 def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): 1280 """copies a directory from one location to another. 1281 1282 :param src: source directory path 1283 :type src: string 1284 :param dst: destination directory path 1285 :type dst: string 1286 :param overwrite: if True then any existing files in the destination 1287 directory will be overwritten 1288 :type overwrite: bool 1289 :param ignore_errors: if True, exceptions when copying will be ignored 1290 :type ignore_errors: bool 1291 :param chunk_size: size of chunks to use when copying, if a simple copy 1292 is required (defaults to 16K) 1293 1294 """ 1295 with self._lock: 1296 if not self.isdir(src): 1297 raise ResourceInvalidError(src, msg="Source is not a directory: %(path)s") 1298 1299 def copyfile_noerrors(src, dst, **kwargs): 1300 try: 1301 return self.copy(src, dst, **kwargs) 1302 except FSError: 1303 return 1304 if ignore_errors: 1305 copyfile = copyfile_noerrors 1306 else: 1307 copyfile = self.copy 1308 1309 src = abspath(src) 1310 dst = abspath(dst) 1311 1312 if not overwrite and self.exists(dst): 1313 raise DestinationExistsError(dst) 1314 1315 if dst: 1316 self.makedir(dst, allow_recreate=True) 1317 1318 for dirname, filenames in self.walk(src): 1319 dst_dirname = relpath(frombase(src, abspath(dirname))) 1320 dst_dirpath = pathjoin(dst, dst_dirname) 1321 self.makedir(dst_dirpath, allow_recreate=True, recursive=True) 1322 for filename in filenames: 1323 src_filename = pathjoin(dirname, filename) 1324 dst_filename = pathjoin(dst_dirpath, filename) 1325 copyfile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size) 1326 1327 def isdirempty(self, path): 1328 """Check if a directory is empty (contains no files or sub-directories) 1329 1330 :param path: a directory path 1331 1332 :rtype: bool 1333 1334 """ 1335 with self._lock: 1336 path = normpath(path) 1337 iter_dir = iter(self.ilistdir(path)) 1338 try: 1339 next(iter_dir) 1340 except StopIteration: 1341 return True 1342 return False 1343 1344 def makeopendir(self, path, recursive=False): 1345 """makes a directory (if it doesn't exist) and returns an FS object for 1346 the newly created directory. 1347 1348 :param path: path to the new directory 1349 :param recursive: if True any intermediate directories will be created 1350 1351 :return: the opened dir 1352 :rtype: an FS object 1353 1354 """ 1355 with self._lock: 1356 self.makedir(path, allow_recreate=True, recursive=recursive) 1357 dir_fs = self.opendir(path) 1358 return dir_fs 1359 1360 def printtree(self, max_levels=5): 1361 """Prints a tree structure of the FS object to the console 1362 1363 :param max_levels: The maximum sub-directories to display, defaults to 1364 5. Set to None for no limit 1365 1366 """ 1367 from fs.utils import print_fs 1368 print_fs(self, max_levels=max_levels) 1369 tree = printtree 1370 1371 def browse(self, hide_dotfiles=False): 1372 """Displays the FS tree in a graphical window (requires wxPython) 1373 1374 :param hide_dotfiles: If True, files and folders that begin with a dot will be hidden 1375 1376 """ 1377 from fs.browsewin import browse 1378 browse(self, hide_dotfiles) 1379 1380 def getmmap(self, path, read_only=False, copy=False): 1381 """Returns a mmap object for this path. 1382 1383 See http://docs.python.org/library/mmap.html for more details on the mmap module. 1384 1385 :param path: A path on this filesystem 1386 :param read_only: If True, the mmap may not be modified 1387 :param copy: If False then changes wont be written back to the file 1388 :raises `fs.errors.NoMMapError`: Only paths that have a syspath can be opened as a mmap 1389 1390 """ 1391 syspath = self.getsyspath(path, allow_none=True) 1392 if syspath is None: 1393 raise NoMMapError(path) 1394 1395 try: 1396 import mmap 1397 except ImportError: 1398 raise NoMMapError(msg="mmap not supported") 1399 1400 if read_only: 1401 f = open(syspath, 'rb') 1402 access = mmap.ACCESS_READ 1403 else: 1404 if copy: 1405 f = open(syspath, 'rb') 1406 access = mmap.ACCESS_COPY 1407 else: 1408 f = open(syspath, 'r+b') 1409 access = mmap.ACCESS_WRITE 1410 1411 m = mmap.mmap(f.fileno(), 0, access=access) 1412 return m 1413 1414 1415def flags_to_mode(flags, binary=True): 1416 """Convert an os.O_* flag bitmask into an FS mode string.""" 1417 if flags & os.O_WRONLY: 1418 if flags & os.O_TRUNC: 1419 mode = "w" 1420 elif flags & os.O_APPEND: 1421 mode = "a" 1422 else: 1423 mode = "r+" 1424 elif flags & os.O_RDWR: 1425 if flags & os.O_TRUNC: 1426 mode = "w+" 1427 elif flags & os.O_APPEND: 1428 mode = "a+" 1429 else: 1430 mode = "r+" 1431 else: 1432 mode = "r" 1433 if flags & os.O_EXCL: 1434 mode += "x" 1435 if binary: 1436 mode += 'b' 1437 else: 1438 mode += 't' 1439 return mode 1440 1441 1442