1""" 2fs.ftpfs 3======== 4 5FTPFS is a filesystem for accessing an FTP server (uses ftplib in standard library) 6 7""" 8 9__all__ = ['FTPFS'] 10 11import sys 12 13import fs 14from fs.base import * 15from fs.errors import * 16from fs.path import pathsplit, abspath, dirname, recursepath, normpath, pathjoin, isbase 17from fs import iotools 18 19from ftplib import FTP, error_perm, error_temp, error_proto, error_reply 20 21try: 22 from ftplib import _GLOBAL_DEFAULT_TIMEOUT 23except ImportError: 24 _GLOBAL_DEFAULT_TIMEOUT = object() 25 26import threading 27import datetime 28import calendar 29 30from socket import error as socket_error 31from fs.local_functools import wraps 32 33import six 34from six import PY3, b 35 36if PY3: 37 from six import BytesIO as StringIO 38else: 39 try: 40 from cStringIO import StringIO 41 except ImportError: 42 from StringIO import StringIO 43 44import time 45 46 47# ----------------------------------------------- 48# Taken from http://www.clapper.org/software/python/grizzled/ 49# ----------------------------------------------- 50 51class Enum(object): 52 def __init__(self, *names): 53 self._names_map = dict((name, i) for i, name in enumerate(names)) 54 55 def __getattr__(self, name): 56 return self._names_map[name] 57 58MONTHS = ('jan', 'feb', 'mar', 'apr', 'may', 'jun', 59 'jul', 'aug', 'sep', 'oct', 'nov', 'dec') 60 61MTIME_TYPE = Enum('UNKNOWN', 'LOCAL', 'REMOTE_MINUTE', 'REMOTE_DAY') 62""" 63``MTIME_TYPE`` identifies how a modification time ought to be interpreted 64(assuming the caller cares). 65 66 - ``LOCAL``: Time is local to the client, granular to (at least) the minute 67 - ``REMOTE_MINUTE``: Time is local to the server and granular to the minute 68 - ``REMOTE_DAY``: Time is local to the server and granular to the day. 69 - ``UNKNOWN``: Time's locale is unknown. 70""" 71 72ID_TYPE = Enum('UNKNOWN', 'FULL') 73""" 74``ID_TYPE`` identifies how a file's identifier should be interpreted. 75 76 - ``FULL``: The ID is known to be complete. 77 - ``UNKNOWN``: The ID is not set or its type is unknown. 78""" 79 80# --------------------------------------------------------------------------- 81# Globals 82# --------------------------------------------------------------------------- 83 84now = time.time() 85current_year = time.localtime().tm_year 86 87# --------------------------------------------------------------------------- 88# Classes 89# --------------------------------------------------------------------------- 90 91class FTPListData(object): 92 """ 93 The `FTPListDataParser` class's ``parse_line()`` method returns an 94 instance of this class, capturing the parsed data. 95 96 :IVariables: 97 name : str 98 The name of the file, if parsable 99 try_cwd : bool 100 ``True`` if the entry might be a directory (i.e., the caller 101 might want to try an FTP ``CWD`` command), ``False`` if it 102 cannot possibly be a directory. 103 try_retr : bool 104 ``True`` if the entry might be a retrievable file (i.e., the caller 105 might want to try an FTP ``RETR`` command), ``False`` if it 106 cannot possibly be a file. 107 size : long 108 The file's size, in bytes 109 mtime : long 110 The file's modification time, as a value that can be passed to 111 ``time.localtime()``. 112 mtime_type : `MTIME_TYPE` 113 How to interpret the modification time. See `MTIME_TYPE`. 114 id : str 115 A unique identifier for the file. The unique identifier is unique 116 on the *server*. On a Unix system, this identifier might be the 117 device number and the file's inode; on other system's, it might 118 be something else. It's also possible for this field to be ``None``. 119 id_type : `ID_TYPE` 120 How to interpret the identifier. See `ID_TYPE`. 121 """ 122 123 def __init__(self, raw_line): 124 self.raw_line = raw_line 125 self.name = None 126 self.try_cwd = False 127 self.try_retr = False 128 self.size = 0 129 self.mtime_type = MTIME_TYPE.UNKNOWN 130 self.mtime = 0 131 self.id_type = ID_TYPE.UNKNOWN 132 self.id = None 133 134class FTPListDataParser(object): 135 """ 136 An ``FTPListDataParser`` object can be used to parse one or more lines 137 that were retrieved by an FTP ``LIST`` command that was sent to a remote 138 server. 139 """ 140 def __init__(self): 141 pass 142 143 def parse_line(self, ftp_list_line): 144 """ 145 Parse a line from an FTP ``LIST`` command. 146 147 :Parameters: 148 ftp_list_line : str 149 The line of output 150 151 :rtype: `FTPListData` 152 :return: An `FTPListData` object describing the parsed line, or 153 ``None`` if the line could not be parsed. Note that it's 154 possible for this method to return a partially-filled 155 `FTPListData` object (e.g., one without a name). 156 """ 157 buf = ftp_list_line 158 159 if len(buf) < 2: # an empty name in EPLF, with no info, could be 2 chars 160 return None 161 162 c = buf[0] 163 if c == '+': 164 return self._parse_EPLF(buf) 165 166 elif c in 'bcdlps-': 167 return self._parse_unix_style(buf) 168 169 i = buf.find(';') 170 if i > 0: 171 return self._parse_multinet(buf, i) 172 173 if c in '0123456789': 174 return self._parse_msdos(buf) 175 176 return None 177 178 # UNIX ls does not show the year for dates in the last six months. 179 # So we have to guess the year. 180 # 181 # Apparently NetWare uses ``twelve months'' instead of ``six months''; ugh. 182 # Some versions of ls also fail to show the year for future dates. 183 184 def _guess_time(self, month, mday, hour=0, minute=0): 185 year = None 186 t = None 187 188 for year in range(current_year - 1, current_year + 100): 189 t = self._get_mtime(year, month, mday, hour, minute) 190 if (now - t) < (350 * 86400): 191 return t 192 193 return 0 194 195 def _get_mtime(self, year, month, mday, hour=0, minute=0, second=0): 196 return time.mktime((year, month, mday, hour, minute, second, 0, 0, -1)) 197 198 def _get_month(self, buf): 199 if len(buf) == 3: 200 for i in range(0, 12): 201 if buf.lower().startswith(MONTHS[i]): 202 return i+1 203 return -1 204 205 def _parse_EPLF(self, buf): 206 result = FTPListData(buf) 207 208 # see http://cr.yp.to/ftp/list/eplf.html 209 #"+i8388621.29609,m824255902,/,\tdev" 210 #"+i8388621.44468,m839956783,r,s10376,\tRFCEPLF" 211 i = 1 212 for j in range(1, len(buf)): 213 if buf[j] == '\t': 214 result.name = buf[j+1:] 215 break 216 217 if buf[j] == ',': 218 c = buf[i] 219 if c == '/': 220 result.try_cwd = True 221 elif c == 'r': 222 result.try_retr = True 223 elif c == 's': 224 result.size = long(buf[i+1:j]) 225 elif c == 'm': 226 result.mtime_type = MTIME_TYPE.LOCAL 227 result.mtime = long(buf[i+1:j]) 228 elif c == 'i': 229 result.id_type = ID_TYPE.FULL 230 result.id = buf[i+1:j-i-1] 231 232 i = j + 1 233 234 return result 235 236 def _parse_unix_style(self, buf): 237 # UNIX-style listing, without inum and without blocks: 238 # "-rw-r--r-- 1 root other 531 Jan 29 03:26 README" 239 # "dr-xr-xr-x 2 root other 512 Apr 8 1994 etc" 240 # "dr-xr-xr-x 2 root 512 Apr 8 1994 etc" 241 # "lrwxrwxrwx 1 root other 7 Jan 25 00:17 bin -> usr/bin" 242 # 243 # Also produced by Microsoft's FTP servers for Windows: 244 # "---------- 1 owner group 1803128 Jul 10 10:18 ls-lR.Z" 245 # "d--------- 1 owner group 0 May 9 19:45 Softlib" 246 # 247 # Also WFTPD for MSDOS: 248 # "-rwxrwxrwx 1 noone nogroup 322 Aug 19 1996 message.ftp" 249 # 250 # Also NetWare: 251 # "d [R----F--] supervisor 512 Jan 16 18:53 login" 252 # "- [R----F--] rhesus 214059 Oct 20 15:27 cx.exe" 253 # 254 # Also NetPresenz for the Mac: 255 # "-------r-- 326 1391972 1392298 Nov 22 1995 MegaPhone.sit" 256 # "drwxrwxr-x folder 2 May 10 1996 network" 257 258 result = FTPListData(buf) 259 260 buflen = len(buf) 261 c = buf[0] 262 if c == 'd': 263 result.try_cwd = True 264 if c == '-': 265 result.try_retr = True 266 if c == 'l': 267 result.try_retr = True 268 result.try_cwd = True 269 270 state = 1 271 i = 0 272 tokens = buf.split() 273 for j in range(1, buflen): 274 if (buf[j] == ' ') and (buf[j - 1] != ' '): 275 if state == 1: # skipping perm 276 state = 2 277 278 elif state == 2: # skipping nlink 279 state = 3 280 if ((j - i) == 6) and (buf[i] == 'f'): # NetPresenz 281 state = 4 282 283 elif state == 3: # skipping UID/GID 284 state = 4 285 286 elif state == 4: # getting tentative size 287 try: 288 size = long(buf[i:j]) 289 except ValueError: 290 pass 291 state = 5 292 293 elif state == 5: # searching for month, else getting tentative size 294 month = self._get_month(buf[i:j]) 295 if month >= 0: 296 state = 6 297 else: 298 size = long(buf[i:j]) 299 300 elif state == 6: # have size and month 301 mday = long(buf[i:j]) 302 state = 7 303 304 elif state == 7: # have size, month, mday 305 if (j - i == 4) and (buf[i+1] == ':'): 306 hour = long(buf[i]) 307 minute = long(buf[i+2:i+4]) 308 result.mtime_type = MTIME_TYPE.REMOTE_MINUTE 309 result.mtime = self._guess_time(month, mday, hour, minute) 310 elif (j - i == 5) and (buf[i+2] == ':'): 311 hour = long(buf[i:i+2]) 312 minute = long(buf[i+3:i+5]) 313 result.mtime_type = MTIME_TYPE.REMOTE_MINUTE 314 result.mtime = self._guess_time(month, mday, hour, minute) 315 elif j - i >= 4: 316 year = long(buf[i:j]) 317 result.mtime_type = MTIME_TYPE.REMOTE_DAY 318 result.mtime = self._get_mtime(year, month, mday) 319 else: 320 break 321 322 result.name = buf[j+1:] 323 state = 8 324 elif state == 8: # twiddling thumbs 325 pass 326 327 i = j + 1 328 while (i < buflen) and (buf[i] == ' '): 329 i += 1 330 331 #if state != 8: 332 #return None 333 334 result.size = size 335 336 if c == 'l': 337 i = 0 338 while (i + 3) < len(result.name): 339 if result.name[i:i+4] == ' -> ': 340 result.target = result.name[i+4:] 341 result.name = result.name[:i] 342 break 343 i += 1 344 345 # eliminate extra NetWare spaces 346 if (buf[1] == ' ') or (buf[1] == '['): 347 namelen = len(result.name) 348 if namelen > 3: 349 result.name = result.name.strip() 350 351 return result 352 353 def _parse_multinet(self, buf, i): 354 355 # MultiNet (some spaces removed from examples) 356 # "00README.TXT;1 2 30-DEC-1996 17:44 [SYSTEM] (RWED,RWED,RE,RE)" 357 # "CORE.DIR;1 1 8-SEP-1996 16:09 [SYSTEM] (RWE,RWE,RE,RE)" 358 # and non-MultiNet VMS: 359 #"CII-MANUAL.TEX;1 213/216 29-JAN-1996 03:33:12 [ANONYMOU,ANONYMOUS] (RWED,RWED,,)" 360 361 result = FTPListData(buf) 362 result.name = buf[:i] 363 buflen = len(buf) 364 365 if i > 4: 366 if buf[i-4:i] == '.DIR': 367 result.name = result.name[0:-4] 368 result.try_cwd = True 369 370 if not result.try_cwd: 371 result.try_retr = True 372 373 try: 374 i = buf.index(' ', i) 375 i = _skip(buf, i, ' ') 376 i = buf.index(' ', i) 377 i = _skip(buf, i, ' ') 378 379 j = i 380 381 j = buf.index('-', j) 382 mday = long(buf[i:j]) 383 384 j = _skip(buf, j, '-') 385 i = j 386 j = buf.index('-', j) 387 month = self._get_month(buf[i:j]) 388 if month < 0: 389 raise IndexError 390 391 j = _skip(buf, j, '-') 392 i = j 393 j = buf.index(' ', j) 394 year = long(buf[i:j]) 395 396 j = _skip(buf, j, ' ') 397 i = j 398 399 j = buf.index(':', j) 400 hour = long(buf[i:j]) 401 j = _skip(buf, j, ':') 402 i = j 403 404 while (buf[j] != ':') and (buf[j] != ' '): 405 j += 1 406 if j == buflen: 407 raise IndexError # abort, abort! 408 409 minute = long(buf[i:j]) 410 411 result.mtime_type = MTIME_TYPE.REMOTE_MINUTE 412 result.mtime = self._get_mtime(year, month, mday, hour, minute) 413 414 except IndexError: 415 pass 416 417 return result 418 419 def _parse_msdos(self, buf): 420 # MSDOS format 421 # 04-27-00 09:09PM <DIR> licensed 422 # 07-18-00 10:16AM <DIR> pub 423 # 04-14-00 03:47PM 589 readme.htm 424 425 buflen = len(buf) 426 i = 0 427 j = 0 428 429 try: 430 result = FTPListData(buf) 431 432 j = buf.index('-', j) 433 month = long(buf[i:j]) 434 435 j = _skip(buf, j, '-') 436 i = j 437 j = buf.index('-', j) 438 mday = long(buf[i:j]) 439 440 j = _skip(buf, j, '-') 441 i = j 442 j = buf.index(' ', j) 443 year = long(buf[i:j]) 444 if year < 50: 445 year += 2000 446 if year < 1000: 447 year += 1900 448 449 j = _skip(buf, j, ' ') 450 i = j 451 j = buf.index(':', j) 452 hour = long(buf[i:j]) 453 j = _skip(buf, j, ':') 454 i = j 455 while not (buf[j] in 'AP'): 456 j += 1 457 if j == buflen: 458 raise IndexError 459 minute = long(buf[i:j]) 460 461 if buf[j] == 'A': 462 j += 1 463 if j == buflen: 464 raise IndexError 465 466 if buf[j] == 'P': 467 hour = (hour + 12) % 24 468 j += 1 469 if j == buflen: 470 raise IndexError 471 472 if buf[j] == 'M': 473 j += 1 474 if j == buflen: 475 raise IndexError 476 477 j = _skip(buf, j, ' ') 478 if buf[j] == '<': 479 result.try_cwd = True 480 j = buf.index(' ', j) 481 else: 482 i = j 483 j = buf.index(' ', j) 484 485 result.size = long(buf[i:j]) 486 result.try_retr = True 487 488 j = _skip(buf, j, ' ') 489 490 result.name = buf[j:] 491 result.mtime_type = MTIME_TYPE.REMOTE_MINUTE 492 result.mtime = self._get_mtime(year, month, mday, hour, minute) 493 except IndexError: 494 pass 495 496 return result 497 498class FTPMlstDataParser(object): 499 """ 500 An ``FTPMlstDataParser`` object can be used to parse one or more lines 501 that were retrieved by an FTP ``MLST`` or ``MLSD`` command that was sent 502 to a remote server. 503 """ 504 def __init__(self): 505 pass 506 507 def parse_line(self, ftp_list_line): 508 """ 509 Parse a line from an FTP ``MLST`` or ``MLSD`` command. 510 511 :Parameters: 512 ftp_list_line : str 513 The line of output 514 515 :rtype: `FTPListData` 516 :return: An `FTPListData` object describing the parsed line, or 517 ``None`` if the line could not be parsed. Note that it's 518 possible for this method to return a partially-filled 519 `FTPListData` object (e.g., one without a mtime). 520 """ 521 result = FTPListData(ftp_list_line) 522 # pull out the name 523 parts = ftp_list_line.partition(' ') 524 result.name = parts[2] 525 526 # parse the facts 527 if parts[0][-1] == ';': 528 for fact in parts[0][:-1].split(';'): 529 parts = fact.partition('=') 530 factname = parts[0].lower() 531 factvalue = parts[2] 532 if factname == 'unique': 533 if factvalue == "0g0" or factvalue == "0g1": 534 # Matrix FTP server sometimes returns bogus "unique" facts 535 result.id_type = ID_TYPE.UNKNOWN 536 else: 537 result.id_type = ID_TYPE.FULL 538 result.id = factvalue 539 elif factname == 'modify': 540 result.mtime_type = MTIME_TYPE.LOCAL 541 result.mtime = calendar.timegm((int(factvalue[0:4]), 542 int(factvalue[4:6]), 543 int(factvalue[6:8]), 544 int(factvalue[8:10]), 545 int(factvalue[10:12]), 546 int(factvalue[12:14]), 547 0, 0, 0)) 548 elif factname == 'size': 549 result.size = long(factvalue) 550 elif factname == 'sizd': 551 # some FTP servers report directory size with sizd 552 result.size = long(factvalue) 553 elif factname == 'type': 554 if factvalue.lower() == 'file': 555 result.try_retr = True 556 elif factvalue.lower() in ['dir', 'cdir', 'pdir']: 557 result.try_cwd = True 558 else: 559 # dunno if it's file or directory 560 result.try_retr = True 561 result.try_cwd = True 562 return result 563 564# --------------------------------------------------------------------------- 565# Public Functions 566# --------------------------------------------------------------------------- 567 568def parse_ftp_list_line(ftp_list_line, is_mlst=False): 569 """ 570 Convenience function that instantiates an `FTPListDataParser` object 571 and passes ``ftp_list_line`` to the object's ``parse_line()`` method, 572 returning the result. 573 574 :Parameters: 575 ftp_list_line : str 576 The line of output 577 578 :rtype: `FTPListData` 579 :return: An `FTPListData` object describing the parsed line, or 580 ``None`` if the line could not be parsed. Note that it's 581 possible for this method to return a partially-filled 582 `FTPListData` object (e.g., one without a name). 583 """ 584 if is_mlst: 585 return FTPMlstDataParser().parse_line(ftp_list_line) 586 else: 587 return FTPListDataParser().parse_line(ftp_list_line) 588 589# --------------------------------------------------------------------------- 590# Private Functions 591# --------------------------------------------------------------------------- 592 593def _skip(s, i, c): 594 while s[i] == c: 595 i += 1 596 if i == len(s): 597 raise IndexError 598 return i 599 600 601def fileftperrors(f): 602 @wraps(f) 603 def deco(self, *args, **kwargs): 604 self._lock.acquire() 605 try: 606 try: 607 ret = f(self, *args, **kwargs) 608 except Exception, e: 609 self.ftpfs._translate_exception(args[0] if args else '', e) 610 finally: 611 self._lock.release() 612 return ret 613 return deco 614 615 616 617class _FTPFile(object): 618 619 """ A file-like that provides access to a file being streamed over ftp.""" 620 621 blocksize = 1024 * 64 622 623 def __init__(self, ftpfs, ftp, path, mode): 624 if not hasattr(self, '_lock'): 625 self._lock = threading.RLock() 626 self.ftpfs = ftpfs 627 self.ftp = ftp 628 self.path = normpath(path) 629 self.mode = mode 630 self.read_pos = 0 631 self.write_pos = 0 632 self.closed = False 633 self.file_size = None 634 if 'r' in mode or 'a' in mode: 635 self.file_size = ftpfs.getsize(path) 636 self.conn = None 637 638 self._start_file(mode, _encode(self.path)) 639 640 @fileftperrors 641 def _start_file(self, mode, path): 642 self.read_pos = 0 643 self.write_pos = 0 644 if 'r' in mode: 645 self.ftp.voidcmd('TYPE I') 646 self.conn = self.ftp.transfercmd('RETR ' + path, None) 647 648 else:#if 'w' in mode or 'a' in mode: 649 self.ftp.voidcmd('TYPE I') 650 if 'a' in mode: 651 self.write_pos = self.file_size 652 self.conn = self.ftp.transfercmd('APPE ' + path) 653 else: 654 self.conn = self.ftp.transfercmd('STOR ' + path) 655 656 @fileftperrors 657 def read(self, size=None): 658 if self.conn is None: 659 return b('') 660 661 chunks = [] 662 if size is None or size < 0: 663 while 1: 664 data = self.conn.recv(self.blocksize) 665 if not data: 666 self.conn.close() 667 self.conn = None 668 self.ftp.voidresp() 669 break 670 chunks.append(data) 671 self.read_pos += len(data) 672 return b('').join(chunks) 673 674 remaining_bytes = size 675 while remaining_bytes: 676 read_size = min(remaining_bytes, self.blocksize) 677 data = self.conn.recv(read_size) 678 if not data: 679 self.conn.close() 680 self.conn = None 681 self.ftp.voidresp() 682 break 683 chunks.append(data) 684 self.read_pos += len(data) 685 remaining_bytes -= len(data) 686 687 return b('').join(chunks) 688 689 @fileftperrors 690 def write(self, data): 691 692 data_pos = 0 693 remaining_data = len(data) 694 695 while remaining_data: 696 chunk_size = min(remaining_data, self.blocksize) 697 self.conn.sendall(data[data_pos:data_pos+chunk_size]) 698 data_pos += chunk_size 699 remaining_data -= chunk_size 700 self.write_pos += chunk_size 701 702 703 def __enter__(self): 704 return self 705 706 def __exit__(self,exc_type,exc_value,traceback): 707 self.close() 708 709 @fileftperrors 710 def flush(self): 711 self.ftpfs._on_file_written(self.path) 712 713 @fileftperrors 714 def seek(self, pos, where=fs.SEEK_SET): 715 # Ftp doesn't support a real seek, so we close the transfer and resume 716 # it at the new position with the REST command 717 # I'm not sure how reliable this method is! 718 if self.file_size is None: 719 raise ValueError("Seek only works with files open for read") 720 721 self._lock.acquire() 722 try: 723 724 current = self.tell() 725 new_pos = None 726 if where == fs.SEEK_SET: 727 new_pos = pos 728 elif where == fs.SEEK_CUR: 729 new_pos = current + pos 730 elif where == fs.SEEK_END: 731 new_pos = self.file_size + pos 732 if new_pos < 0: 733 raise ValueError("Can't seek before start of file") 734 735 if self.conn is not None: 736 self.conn.close() 737 738 finally: 739 self._lock.release() 740 741 self.close() 742 self._lock.acquire() 743 try: 744 self.ftp = self.ftpfs._open_ftp() 745 self.ftp.sendcmd('TYPE I') 746 self.ftp.sendcmd('REST %i' % (new_pos)) 747 self.__init__(self.ftpfs, self.ftp, self.path, self.mode) 748 self.read_pos = new_pos 749 finally: 750 self._lock.release() 751 752 #raise UnsupportedError('ftp seek') 753 754 @fileftperrors 755 def tell(self): 756 if 'r' in self.mode: 757 return self.read_pos 758 else: 759 return self.write_pos 760 761 @fileftperrors 762 def truncate(self, size=None): 763 self.ftpfs._on_file_written(self.path) 764 # Inefficient, but I don't know how else to implement this 765 if size is None: 766 size = self.tell() 767 768 if self.conn is not None: 769 self.conn.close() 770 self.close() 771 772 read_f = None 773 try: 774 read_f = self.ftpfs.open(self.path, 'rb') 775 data = read_f.read(size) 776 finally: 777 if read_f is not None: 778 read_f.close() 779 780 self.ftp = self.ftpfs._open_ftp() 781 self.mode = 'w' 782 self.__init__(self.ftpfs, self.ftp, _encode(self.path), self.mode) 783 #self._start_file(self.mode, self.path) 784 self.write(data) 785 if len(data) < size: 786 self.write('\0' * (size - len(data))) 787 788 789 @fileftperrors 790 def close(self): 791 if 'w' in self.mode or 'a' in self.mode or '+' in self.mode: 792 self.ftpfs._on_file_written(self.path) 793 if self.conn is not None: 794 try: 795 self.conn.close() 796 self.conn = None 797 self.ftp.voidresp() 798 except error_temp, error_perm: 799 pass 800 if self.ftp is not None: 801 try: 802 self.ftp.close() 803 except error_temp, error_perm: 804 pass 805 self.closed = True 806 807 def next(self): 808 return self.readline() 809 810 def readline(self, size=None): 811 return next(iotools.line_iterator(self, size)) 812 813 def __iter__(self): 814 return iotools.line_iterator(self) 815 816 817def ftperrors(f): 818 @wraps(f) 819 def deco(self, *args, **kwargs): 820 self._lock.acquire() 821 try: 822 self._enter_dircache() 823 try: 824 try: 825 ret = f(self, *args, **kwargs) 826 except Exception, e: 827 self._translate_exception(args[0] if args else '', e) 828 finally: 829 self._leave_dircache() 830 finally: 831 self._lock.release() 832 return ret 833 return deco 834 835 836def _encode(s): 837 if isinstance(s, unicode): 838 return s.encode('utf-8') 839 return s 840 841class _DirCache(dict): 842 def __init__(self): 843 super(_DirCache, self).__init__() 844 self.count = 0 845 846 def addref(self): 847 self.count += 1 848 return self.count 849 850 def decref(self): 851 self.count -= 1 852 return self.count 853 854class FTPFS(FS): 855 856 _meta = { 'thread_safe' : True, 857 'network' : True, 858 'virtual': False, 859 'read_only' : False, 860 'unicode_paths' : True, 861 'case_insensitive_paths' : False, 862 'atomic.move' : True, 863 'atomic.copy' : True, 864 'atomic.makedir' : True, 865 'atomic.rename' : True, 866 'atomic.setcontents' : False, 867 'file.read_and_write' : False, 868 } 869 870 def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT, port=21, dircache=True, follow_symlinks=False): 871 """Connect to a FTP server. 872 873 :param host: Host to connect to 874 :param user: Username, or a blank string for anonymous 875 :param passwd: Password, if required 876 :param acct: Accounting information (few servers require this) 877 :param timeout: Timeout in seconds 878 :param port: Port to connection (default is 21) 879 :param dircache: If True then directory information will be cached, 880 speeding up operations such as `getinfo`, `isdir`, `isfile`, but 881 changes to the ftp file structure will not be visible until 882 :meth:`~fs.ftpfs.FTPFS.clear_dircache` is called 883 884 """ 885 886 super(FTPFS, self).__init__() 887 888 self.host = host 889 self.port = port 890 self.user = user 891 self.passwd = passwd 892 self.acct = acct 893 self.timeout = timeout 894 self.default_timeout = timeout is _GLOBAL_DEFAULT_TIMEOUT 895 self.use_dircache = dircache 896 self.follow_symlinks = follow_symlinks 897 898 self.use_mlst = False 899 self._lock = threading.RLock() 900 self._init_dircache() 901 902 self._cache_hint = False 903 try: 904 self.ftp 905 except FSError: 906 self.closed = True 907 raise 908 909 def _init_dircache(self): 910 self.dircache = _DirCache() 911 912 @synchronize 913 def cache_hint(self, enabled): 914 self._cache_hint = bool(enabled) 915 916 def _enter_dircache(self): 917 self.dircache.addref() 918 919 def _leave_dircache(self): 920 self.dircache.decref() 921 if self.use_dircache: 922 if not self.dircache.count and not self._cache_hint: 923 self.clear_dircache() 924 else: 925 self.clear_dircache() 926 assert self.dircache.count >= 0, "dircache count should never be negative" 927 928 @synchronize 929 def _on_file_written(self, path): 930 self.refresh_dircache(dirname(path)) 931 932 @synchronize 933 def _readdir(self, path): 934 path = abspath(normpath(path)) 935 if self.dircache.count: 936 cached_dirlist = self.dircache.get(path) 937 if cached_dirlist is not None: 938 return cached_dirlist 939 dirlist = {} 940 941 def _get_FEAT(ftp): 942 features = dict() 943 try: 944 response = ftp.sendcmd("FEAT") 945 if response[:3] == "211": 946 for line in response.splitlines()[1:]: 947 if line[3] == "211": 948 break 949 if line[0] != ' ': 950 break 951 parts = line[1:].partition(' ') 952 features[parts[0].upper()] = parts[2] 953 except error_perm: 954 # some FTP servers may not support FEAT 955 pass 956 return features 957 958 def on_line(line): 959 if not isinstance(line, unicode): 960 line = line.decode('utf-8') 961 info = parse_ftp_list_line(line, self.use_mlst) 962 if info: 963 info = info.__dict__ 964 if info['name'] not in ('.', '..'): 965 dirlist[info['name']] = info 966 967 try: 968 encoded_path = _encode(path) 969 ftp_features = _get_FEAT(self.ftp) 970 if 'MLST' in ftp_features: 971 self.use_mlst = True 972 try: 973 # only request the facts we need 974 self.ftp.sendcmd("OPTS MLST type;unique;size;modify;") 975 except error_perm: 976 # some FTP servers don't support OPTS MLST 977 pass 978 # need to send MLST first to discover if it's file or dir 979 response = self.ftp.sendcmd("MLST " + encoded_path) 980 lines = response.splitlines() 981 if lines[0][:3] == "250": 982 list_line = lines[1] 983 # MLST line is preceded by space 984 if list_line[0] == ' ': 985 on_line(list_line[1:]) 986 else: # Matrix FTP server has bug 987 on_line(list_line) 988 # if it's a dir, then we can send a MLSD 989 if dirlist[dirlist.keys()[0]]['try_cwd']: 990 dirlist = {} 991 self.ftp.retrlines("MLSD " + encoded_path, on_line) 992 else: 993 self.ftp.dir(encoded_path, on_line) 994 except error_reply: 995 pass 996 self.dircache[path] = dirlist 997 998 def is_symlink(info): 999 return info['try_retr'] and info['try_cwd'] and info.has_key('target') 1000 1001 def resolve_symlink(linkpath): 1002 linkinfo = self.getinfo(linkpath) 1003 if not linkinfo.has_key('resolved'): 1004 linkinfo['resolved'] = linkpath 1005 if is_symlink(linkinfo): 1006 target = linkinfo['target'] 1007 base, fname = pathsplit(linkpath) 1008 return resolve_symlink(pathjoin(base, target)) 1009 else: 1010 return linkinfo 1011 1012 if self.follow_symlinks: 1013 for name in dirlist: 1014 if is_symlink(dirlist[name]): 1015 target = dirlist[name]['target'] 1016 linkinfo = resolve_symlink(pathjoin(path, target)) 1017 for key in linkinfo: 1018 if key != 'name': 1019 dirlist[name][key] = linkinfo[key] 1020 del dirlist[name]['target'] 1021 1022 return dirlist 1023 1024 @synchronize 1025 def clear_dircache(self, *paths): 1026 """ 1027 Clear cached directory information. 1028 1029 :param path: Path of directory to clear cache for, or all directories if 1030 None (the default) 1031 1032 """ 1033 1034 if not paths: 1035 self.dircache.clear() 1036 else: 1037 dircache = self.dircache 1038 paths = [normpath(abspath(path)) for path in paths] 1039 for cached_path in dircache.keys(): 1040 for path in paths: 1041 if isbase(cached_path, path): 1042 dircache.pop(cached_path, None) 1043 break 1044 1045 @synchronize 1046 def refresh_dircache(self, *paths): 1047 for path in paths: 1048 path = abspath(normpath(path)) 1049 self.dircache.pop(path, None) 1050 1051 @synchronize 1052 def _check_path(self, path): 1053 path = normpath(path) 1054 base, fname = pathsplit(abspath(path)) 1055 dirlist = self._readdir(base) 1056 if fname and fname not in dirlist: 1057 raise ResourceNotFoundError(path) 1058 return dirlist, fname 1059 1060 def _get_dirlist(self, path): 1061 path = normpath(path) 1062 base, fname = pathsplit(abspath(path)) 1063 dirlist = self._readdir(base) 1064 return dirlist, fname 1065 1066 1067 @ftperrors 1068 def get_ftp(self): 1069 if self.closed: 1070 return None 1071 if not getattr(self, '_ftp', None): 1072 self._ftp = self._open_ftp() 1073 return self._ftp 1074 1075 ftp = property(get_ftp) 1076 1077 @ftperrors 1078 def _open_ftp(self): 1079 try: 1080 ftp = FTP() 1081 if self.default_timeout or sys.version_info < (2,6,): 1082 ftp.connect(self.host, self.port) 1083 else: 1084 ftp.connect(self.host, self.port, self.timeout) 1085 ftp.login(self.user, self.passwd, self.acct) 1086 except socket_error, e: 1087 raise RemoteConnectionError(str(e), details=e) 1088 return ftp 1089 1090 def __getstate__(self): 1091 state = super(FTPFS, self).__getstate__() 1092 del state['_lock'] 1093 state.pop('_ftp', None) 1094 return state 1095 1096 def __setstate__(self,state): 1097 super(FTPFS, self).__setstate__(state) 1098 self._init_dircache() 1099 self._lock = threading.RLock() 1100 #self._ftp = None 1101 #self.ftp 1102 1103 def __str__(self): 1104 return '<FTPFS %s>' % self.host 1105 1106 def __unicode__(self): 1107 return u'<FTPFS %s>' % self.host 1108 1109 @convert_os_errors 1110 def _translate_exception(self, path, exception): 1111 1112 """ Translates exceptions that my be thrown by the ftp code in to 1113 FS exceptions 1114 1115 TODO: Flesh this out with more specific exceptions 1116 1117 """ 1118 1119 if isinstance(exception, socket_error): 1120 self._ftp = None 1121 raise RemoteConnectionError(str(exception), details=exception) 1122 1123 elif isinstance(exception, error_temp): 1124 code, message = str(exception).split(' ', 1) 1125 self._ftp = None 1126 raise RemoteConnectionError(str(exception), path=path, msg="FTP error: %s" % str(exception), details=exception) 1127 1128 elif isinstance(exception, error_perm): 1129 code, message = str(exception).split(' ', 1) 1130 code = int(code) 1131 if code == 550: 1132 pass 1133 if code == 552: 1134 raise StorageSpaceError 1135 raise PermissionDeniedError(str(exception), path=path, msg="FTP error: %s" % str(exception), details=exception) 1136 1137 raise exception 1138 1139 @ftperrors 1140 def close(self): 1141 if not self.closed: 1142 try: 1143 self.ftp.close() 1144 except FSError: 1145 pass 1146 self.closed = True 1147 1148 def getpathurl(self, path, allow_none=False): 1149 path = normpath(path) 1150 credentials = '%s:%s' % (self.user, self.passwd) 1151 if credentials == ':': 1152 url = 'ftp://%s%s' % (self.host.rstrip('/'), abspath(path)) 1153 else: 1154 url = 'ftp://%s@%s%s' % (credentials, self.host.rstrip('/'), abspath(path)) 1155 return url 1156 1157 @iotools.filelike_to_stream 1158 @ftperrors 1159 def open(self, path, mode, buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs): 1160 path = normpath(path) 1161 mode = mode.lower() 1162 if self.isdir(path): 1163 raise ResourceInvalidError(path) 1164 if 'r' in mode or 'a' in mode: 1165 if not self.isfile(path): 1166 raise ResourceNotFoundError(path) 1167 if 'w' in mode or 'a' in mode or '+' in mode: 1168 self.refresh_dircache(dirname(path)) 1169 ftp = self._open_ftp() 1170 f = _FTPFile(self, ftp, normpath(path), mode) 1171 return f 1172 1173 @ftperrors 1174 def setcontents(self, path, data=b'', encoding=None, errors=None, chunk_size=1024*64): 1175 path = normpath(path) 1176 data = iotools.make_bytes_io(data, encoding=encoding, errors=errors) 1177 self.refresh_dircache(dirname(path)) 1178 self.ftp.storbinary('STOR %s' % _encode(path), data, blocksize=chunk_size) 1179 1180 @ftperrors 1181 def getcontents(self, path, mode="rb", encoding=None, errors=None, newline=None): 1182 path = normpath(path) 1183 contents = StringIO() 1184 self.ftp.retrbinary('RETR %s' % _encode(path), contents.write, blocksize=1024*64) 1185 data = contents.getvalue() 1186 if 'b' in data: 1187 return data 1188 return iotools.decode_binary(data, encoding=encoding, errors=errors) 1189 1190 @ftperrors 1191 def exists(self, path): 1192 path = normpath(path) 1193 if path in ('', '/'): 1194 return True 1195 dirlist, fname = self._get_dirlist(path) 1196 return fname in dirlist 1197 1198 @ftperrors 1199 def isdir(self, path): 1200 path = normpath(path) 1201 if path in ('', '/'): 1202 return True 1203 dirlist, fname = self._get_dirlist(path) 1204 info = dirlist.get(fname) 1205 if info is None: 1206 return False 1207 return info['try_cwd'] 1208 1209 @ftperrors 1210 def isfile(self, path): 1211 path = normpath(path) 1212 if path in ('', '/'): 1213 return False 1214 dirlist, fname = self._get_dirlist(path) 1215 info = dirlist.get(fname) 1216 if info is None: 1217 return False 1218 return not info['try_cwd'] 1219 1220 @ftperrors 1221 def listdir(self, path="./", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): 1222 path = normpath(path) 1223 #self.clear_dircache(path) 1224 if not self.exists(path): 1225 raise ResourceNotFoundError(path) 1226 if not self.isdir(path): 1227 raise ResourceInvalidError(path) 1228 paths = self._readdir(path).keys() 1229 1230 return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only) 1231 1232 @ftperrors 1233 def listdirinfo(self, path="./", 1234 wildcard=None, 1235 full=False, 1236 absolute=False, 1237 dirs_only=False, 1238 files_only=False): 1239 path = normpath(path) 1240 def getinfo(p): 1241 try: 1242 if full or absolute: 1243 return self.getinfo(p) 1244 else: 1245 return self.getinfo(pathjoin(path,p)) 1246 except FSError: 1247 return {} 1248 1249 return [(p, getinfo(p)) 1250 for p in self.listdir(path, 1251 wildcard=wildcard, 1252 full=full, 1253 absolute=absolute, 1254 dirs_only=dirs_only, 1255 files_only=files_only)] 1256 1257 @ftperrors 1258 def makedir(self, path, recursive=False, allow_recreate=False): 1259 path = normpath(path) 1260 if path in ('', '/'): 1261 return 1262 def checkdir(path): 1263 if not self.isdir(path): 1264 self.clear_dircache(dirname(path)) 1265 try: 1266 self.ftp.mkd(_encode(path)) 1267 except error_reply: 1268 return 1269 except error_perm, e: 1270 if recursive or allow_recreate: 1271 return 1272 if str(e).split(' ', 1)[0]=='550': 1273 raise DestinationExistsError(path) 1274 else: 1275 raise 1276 if recursive: 1277 for p in recursepath(path): 1278 checkdir(p) 1279 else: 1280 base = dirname(path) 1281 if not self.exists(base): 1282 raise ParentDirectoryMissingError(path) 1283 1284 if not allow_recreate: 1285 if self.exists(path): 1286 if self.isfile(path): 1287 raise ResourceInvalidError(path) 1288 raise DestinationExistsError(path) 1289 checkdir(path) 1290 1291 @ftperrors 1292 def remove(self, path): 1293 if not self.exists(path): 1294 raise ResourceNotFoundError(path) 1295 if not self.isfile(path): 1296 raise ResourceInvalidError(path) 1297 self.refresh_dircache(dirname(path)) 1298 self.ftp.delete(_encode(path)) 1299 1300 @ftperrors 1301 def removedir(self, path, recursive=False, force=False): 1302 path = abspath(normpath(path)) 1303 if not self.exists(path): 1304 raise ResourceNotFoundError(path) 1305 if self.isfile(path): 1306 raise ResourceInvalidError(path) 1307 if normpath(path) in ('', '/'): 1308 raise RemoveRootError(path) 1309 1310 if not force: 1311 for _checkpath in self.listdir(path): 1312 raise DirectoryNotEmptyError(path) 1313 try: 1314 if force: 1315 for rpath in self.listdir(path, full=True): 1316 try: 1317 if self.isfile(rpath): 1318 self.remove(rpath) 1319 elif self.isdir(rpath): 1320 self.removedir(rpath, force=force) 1321 except FSError: 1322 pass 1323 self.clear_dircache(dirname(path)) 1324 self.ftp.rmd(_encode(path)) 1325 except error_reply: 1326 pass 1327 if recursive: 1328 try: 1329 if dirname(path) not in ('', '/'): 1330 self.removedir(dirname(path), recursive=True) 1331 except DirectoryNotEmptyError: 1332 pass 1333 self.clear_dircache(dirname(path), path) 1334 1335 @ftperrors 1336 def rename(self, src, dst): 1337 try: 1338 self.refresh_dircache(dirname(src), dirname(dst)) 1339 self.ftp.rename(_encode(src), _encode(dst)) 1340 except error_perm, exception: 1341 code, message = str(exception).split(' ', 1) 1342 if code == "550": 1343 if not self.exists(dirname(dst)): 1344 raise ParentDirectoryMissingError(dst) 1345 raise 1346 except error_reply: 1347 pass 1348 1349 @ftperrors 1350 def getinfo(self, path): 1351 dirlist, fname = self._check_path(path) 1352 if not fname: 1353 return {} 1354 info = dirlist[fname].copy() 1355 info['modified_time'] = datetime.datetime.fromtimestamp(info['mtime']) 1356 info['created_time'] = info['modified_time'] 1357 return info 1358 1359 @ftperrors 1360 def getsize(self, path): 1361 1362 size = None 1363 if self.dircache.count: 1364 dirlist, fname = self._check_path(path) 1365 size = dirlist[fname].get('size') 1366 1367 if size is not None: 1368 return size 1369 1370 self.ftp.sendcmd('TYPE I') 1371 size = self.ftp.size(_encode(path)) 1372 if size is None: 1373 dirlist, fname = self._check_path(path) 1374 size = dirlist[fname].get('size') 1375 if size is None: 1376 raise OperationFailedError('getsize', path) 1377 return size 1378 1379 @ftperrors 1380 def desc(self, path): 1381 path = normpath(path) 1382 url = self.getpathurl(path, allow_none=True) 1383 if url: 1384 return url 1385 dirlist, fname = self._check_path(path) 1386 if fname not in dirlist: 1387 raise ResourceNotFoundError(path) 1388 return dirlist[fname].get('raw_line', 'No description available') 1389 1390 @ftperrors 1391 def move(self, src, dst, overwrite=False, chunk_size=16384): 1392 if not overwrite and self.exists(dst): 1393 raise DestinationExistsError(dst) 1394 #self.refresh_dircache(dirname(src), dirname(dst)) 1395 try: 1396 self.rename(src, dst) 1397 except: 1398 self.copy(src, dst, overwrite=overwrite) 1399 self.remove(src) 1400 finally: 1401 self.refresh_dircache(src, dirname(src), dst, dirname(dst)) 1402 1403 @ftperrors 1404 def copy(self, src, dst, overwrite=False, chunk_size=1024*64): 1405 if not self.isfile(src): 1406 if self.isdir(src): 1407 raise ResourceInvalidError(src, msg="Source is not a file: %(path)s") 1408 raise ResourceNotFoundError(src) 1409 if not overwrite and self.exists(dst): 1410 raise DestinationExistsError(dst) 1411 1412 dst = normpath(dst) 1413 src_file = None 1414 try: 1415 src_file = self.open(src, "rb") 1416 ftp = self._open_ftp() 1417 ftp.voidcmd('TYPE I') 1418 ftp.storbinary('STOR %s' % _encode(normpath(dst)), src_file, blocksize=chunk_size) 1419 finally: 1420 self.refresh_dircache(dirname(dst)) 1421 if src_file is not None: 1422 src_file.close() 1423 1424 1425 @ftperrors 1426 def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): 1427 self.clear_dircache(dirname(src), dirname(dst)) 1428 super(FTPFS, self).movedir(src, dst, overwrite, ignore_errors, chunk_size) 1429 1430 @ftperrors 1431 def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384): 1432 self.clear_dircache(dirname(dst)) 1433 super(FTPFS, self).copydir(src, dst, overwrite, ignore_errors, chunk_size) 1434 1435 1436if __name__ == "__main__": 1437 1438 ftp_fs = FTPFS('ftp.ncsa.uiuc.edu') 1439 ftp_fs.cache_hint(True) 1440 from fs.browsewin import browse 1441 browse(ftp_fs) 1442 1443 #ftp_fs = FTPFS('127.0.0.1', 'user', '12345', dircache=True) 1444 #f = ftp_fs.open('testout.txt', 'w') 1445 #f.write("Testing writing to an ftp file!") 1446 #f.write("\nHai!") 1447 #f.close() 1448 1449 #ftp_fs.createfile(u"\N{GREEK CAPITAL LETTER KAPPA}", 'unicode!') 1450 1451 #kappa = u"\N{GREEK CAPITAL LETTER KAPPA}" 1452 #ftp_fs.makedir(kappa) 1453 1454 #print repr(ftp_fs.listdir()) 1455 1456 #print repr(ftp_fs.listdir()) 1457 1458 #ftp_fs.makedir('a/b/c/d', recursive=True) 1459 #print ftp_fs.getsize('/testout.txt') 1460 1461 1462 #print f.read() 1463 #for p in ftp_fs: 1464 # print p 1465 1466 #from fs.utils import print_fs 1467 #print_fs(ftp_fs) 1468 1469 #print ftp_fs.getsize('test.txt') 1470 1471 #from fs.browsewin import browse 1472 #browse(ftp_fs) 1473