1# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*- 2# 3# Copyright 2002 Ben Escoto <ben@emerose.org> 4# Copyright 2007 Kenneth Loafman <kenneth@loafman.com> 5# 6# This file is part of duplicity. 7# 8# Duplicity is free software; you can redistribute it and/or modify it 9# under the terms of the GNU General Public License as published by the 10# Free Software Foundation; either version 2 of the License, or (at your 11# option) any later version. 12# 13# Duplicity is distributed in the hope that it will be useful, but 14# WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16# General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License 19# along with duplicity; if not, write to the Free Software Foundation, 20# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22u""" 23Miscellaneous utilities. 24""" 25 26from __future__ import print_function 27from future import standard_library 28standard_library.install_aliases() 29from builtins import isinstance 30from builtins import map 31from builtins import object 32from builtins import str 33 34import errno 35import json 36import os 37import string 38import sys 39import traceback 40import atexit 41 42from duplicity import tarfile 43import duplicity.config as config 44import duplicity.log as log 45 46try: 47 # For paths, just use path.name/uname rather than converting with these 48 from os import fsencode, fsdecode # pylint: disable=unused-import 49except ImportError: 50 # Most likely Python version < 3.2, so define our own fsencode/fsdecode. 51 # These are functions that encode/decode unicode paths to filesystem encoding, 52 # but the cleverness is that they handle non-unicode characters on Linux 53 # There is a *partial* backport to python available here: 54 # https://github.com/pjdelport/backports.os/blob/master/src/backports/os.py 55 # but if it cannot be trusted for full-circle translation, then we may as well 56 # just read and store the bytes version of the path as path.name before 57 # creating the unicode version (for path matching etc) and ensure that in 58 # real-world usage (as opposed to testing) we create the path objects from a 59 # bytes string. 60 # ToDo: Revisit this once we drop Python 2 support/the backport is complete 61 62 def fsencode(unicode_filename): 63 u"""Convert a unicode filename to a filename encoded in the system encoding""" 64 # For paths, just use path.name rather than converting with this 65 # If we are not doing any cleverness with non-unicode filename bytes, 66 # encoding to system encoding is good enough 67 return unicode_filename.encode(sys.getfilesystemencoding(), u"replace") 68 69 def fsdecode(bytes_filename): 70 u"""Convert a filename encoded in the system encoding to unicode""" 71 # For paths, just use path.uc_name rather than converting with this 72 # If we are not doing any cleverness with non-unicode filename bytes, 73 # decoding using system encoding is good enough. Use "ignore" as 74 # Linux paths can contain non-Unicode characters 75 return bytes_filename.decode(config.fsencoding, u"replace") 76 77 78def exception_traceback(limit=50): 79 u""" 80 @return A string representation in typical Python format of the 81 currently active/raised exception. 82 """ 83 type, value, tb = sys.exc_info() # pylint: disable=redefined-builtin 84 85 lines = traceback.format_tb(tb, limit) 86 lines.extend(traceback.format_exception_only(type, value)) 87 88 msg = u"Traceback (innermost last):\n" 89 if sys.version_info.major >= 3: 90 msg = msg + u"%-20s %s" % (str.join(u"", lines[:-1]), lines[-1]) 91 else: 92 msg = msg + u"%-20s %s" % (string.join(lines[:-1], u""), lines[-1]) 93 94 if sys.version_info.major < 3: 95 return msg.decode(u'unicode-escape', u'replace') 96 return msg 97 98 99def escape(string): 100 u"Convert a (bytes) filename to a format suitable for logging (quoted utf8)" 101 string = fsdecode(string).encode(u'unicode-escape', u'replace') 102 return u"'%s'" % string.decode(u'utf8', u'replace').replace(u"'", u'\\x27') 103 104 105def uindex(index): 106 u"Convert an index (a tuple of path parts) to unicode for printing" 107 if index: 108 return os.path.join(*list(map(fsdecode, index))) 109 else: 110 return u'.' 111 112 113def uexc(e): 114 u"""Returns the exception message in Unicode""" 115 # Exceptions in duplicity often have path names in them, which if they are 116 # non-ascii will cause a UnicodeDecodeError when implicitly decoding to 117 # unicode. So we decode manually, using the filesystem encoding. 118 # 99.99% of the time, this will be a fine encoding to use. 119 if e and e.args: 120 # Find arg that is a string 121 for m in e.args: 122 if isinstance(m, str): 123 # Already unicode 124 return m 125 elif isinstance(m, bytes): 126 # Encoded, likely in filesystem encoding 127 return fsdecode(m) 128 # If the function did not return yet, we did not 129 # succeed in finding a string; return the whole message. 130 # This fails for Python 2, so only do this in Python 3. 131 if sys.version_info[0] > 2: 132 return str(e) 133 # For Python 2, fall back to returning an empty string. 134 else: 135 return u'' 136 else: 137 return u'' 138 139 140def maybe_ignore_errors(fn): 141 u""" 142 Execute fn. If the global configuration setting ignore_errors is 143 set to True, catch errors and log them but do continue (and return 144 None). 145 146 @param fn: A callable. 147 @return Whatever fn returns when called, or None if it failed and ignore_errors is true. 148 """ 149 try: 150 return fn() 151 except Exception as e: 152 if config.ignore_errors: 153 log.Warn(_(u"IGNORED_ERROR: Warning: ignoring error as requested: %s: %s") 154 % (e.__class__.__name__, uexc(e))) 155 return None 156 else: 157 raise 158 159 160class BlackHoleList(list): 161 162 def append(self, x): 163 pass 164 165 166class FakeTarFile(object): 167 debug = 0 168 169 def __iter__(self): 170 return iter([]) 171 172 def close(self): 173 pass 174 175 176def make_tarfile(mode, fp): 177 # We often use 'empty' tarfiles for signatures that haven't been filled out 178 # yet. So we want to ignore ReadError exceptions, which are used to signal 179 # this. 180 try: 181 tf = tarfile.TarFile(u"arbitrary", mode, fp) 182 # Now we cause TarFile to not cache TarInfo objects. It would end up 183 # consuming a lot of memory over the lifetime of our long-lasting 184 # signature files otherwise. 185 tf.members = BlackHoleList() 186 return tf 187 except tarfile.ReadError: 188 return FakeTarFile() 189 190 191def get_tarinfo_name(ti): 192 # Python versions before 2.6 ensure that directories end with /, but 2.6 193 # and later ensure they they *don't* have /. ::shrug:: Internally, we 194 # continue to use pre-2.6 method. 195 if ti.isdir() and not ti.name.endswith(r"/"): 196 return ti.name + r"/" 197 else: 198 return ti.name 199 200 201def ignore_missing(fn, filename): 202 u""" 203 Execute fn on filename. Ignore ENOENT errors, otherwise raise exception. 204 205 @param fn: callable 206 @param filename: string 207 """ 208 try: 209 fn(filename) 210 except OSError as ex: 211 if ex.errno == errno.ENOENT: 212 pass 213 else: 214 raise 215 216 217@atexit.register 218def release_lockfile(): 219 if config.lockfile: 220 log.Debug(_(u"Releasing lockfile %s") % config.lockpath) 221 try: 222 config.lockfile.release() 223 config.lockfile = None 224 os.remove(config.lockpath) 225 config.lockpath = u"" 226 except Exception: 227 log.Error(u"Could not release lockfile: %s", str(e)) 228 pass 229 230 231def copyfileobj(infp, outfp, byte_count=-1): 232 u"""Copy byte_count bytes from infp to outfp, or all if byte_count < 0 233 234 Returns the number of bytes actually written (may be less than 235 byte_count if find eof. Does not close either fileobj. 236 237 """ 238 blocksize = 64 * 1024 239 bytes_written = 0 240 if byte_count < 0: 241 while 1: 242 buf = infp.read(blocksize) 243 if not buf: 244 break 245 bytes_written += len(buf) 246 outfp.write(buf) 247 else: 248 while bytes_written + blocksize <= byte_count: 249 buf = infp.read(blocksize) 250 if not buf: 251 break 252 bytes_written += len(buf) 253 outfp.write(buf) 254 buf = infp.read(byte_count - bytes_written) 255 bytes_written += len(buf) 256 outfp.write(buf) 257 return bytes_written 258 259 260def which(program): 261 u""" 262 Return absolute path for program name. 263 Returns None if program not found. 264 """ 265 266 def is_exe(fpath): 267 return os.path.isfile(fpath) and os.path.isabs(fpath) and os.access(fpath, os.X_OK) 268 269 fpath, fname = os.path.split(program) 270 if fpath: 271 if is_exe(program): 272 return program 273 else: 274 for path in os.getenv(u"PATH").split(os.pathsep): 275 path = path.strip(u'"') 276 exe_file = os.path.abspath(os.path.join(path, program)) 277 if is_exe(exe_file): 278 return exe_file 279 280 return None 281 282 283def start_debugger(remote=False): 284 if (not os.getenv(u'DEBUG_RUNNING', None) and (u'--pydevd' in sys.argv or os.getenv(u'PYDEVD', None))): 285 if remote: 286 # modify this for your configuration. 287 # client = base path in machine that Liclipse is on 288 # server = base path in machine that duplicity is on 289 client = u'/Users/ken/workspace/duplicity-testfiles' 290 server = u'/home/ken/workspace/duplicity-testfiles' 291 292 # relative paths under duplicity root 293 duppaths = [ 294 u'', 295 u'bin', 296 u'duplicity', 297 u'duplicity/backends', 298 u'testing', 299 u'testing/functional', 300 u'testing/unit', 301 ] 302 pathlist = [(os.path.normpath(os.path.join(client, p)), 303 os.path.normpath(os.path.join(server, p))) for p in duppaths] 304 os.environ[u'PATHS_FROM_ECLIPSE_TO_PYTHON'] = json.dumps(pathlist) 305 306 import pydevd # pylint: disable=import-error 307 pydevd.settrace(u'dione.local', port=5678, stdoutToServer=True, stderrToServer=True) 308 309 # In a dev environment the path is screwed so fix it. 310 base = sys.path.pop(0) 311 base = base.split(os.path.sep)[:-1] 312 base = os.path.sep.join(base) 313 sys.path.insert(0, base) 314 315 os.environ[u'DEBUG_RUNNING'] = u'yes' 316