1# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*-
2#
3# Copyright 2002 Ben Escoto <ben@emerose.org>
4# Copyright 2007 Kenneth Loafman <kenneth@loafman.com>
5#
6# This file is part of duplicity.
7#
8# Duplicity is free software; you can redistribute it and/or modify it
9# under the terms of the GNU General Public License as published by the
10# Free Software Foundation; either version 2 of the License, or (at your
11# option) any later version.
12#
13# Duplicity is distributed in the hope that it will be useful, but
14# WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16# General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with duplicity; if not, write to the Free Software Foundation,
20# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21
22u"""
23Miscellaneous utilities.
24"""
25
26from __future__ import print_function
27from future import standard_library
28standard_library.install_aliases()
29from builtins import isinstance
30from builtins import map
31from builtins import object
32from builtins import str
33
34import errno
35import json
36import os
37import string
38import sys
39import traceback
40import atexit
41
42from duplicity import tarfile
43import duplicity.config as config
44import duplicity.log as log
45
46try:
47    # For paths, just use path.name/uname rather than converting with these
48    from os import fsencode, fsdecode  # pylint: disable=unused-import
49except ImportError:
50    # Most likely Python version < 3.2, so define our own fsencode/fsdecode.
51    # These are functions that encode/decode unicode paths to filesystem encoding,
52    # but the cleverness is that they handle non-unicode characters on Linux
53    # There is a *partial* backport to python available here:
54    # https://github.com/pjdelport/backports.os/blob/master/src/backports/os.py
55    # but if it cannot be trusted for full-circle translation, then we may as well
56    # just read and store the bytes version of the path as path.name before
57    # creating the unicode version (for path matching etc) and ensure that in
58    # real-world usage (as opposed to testing) we create the path objects from a
59    # bytes string.
60    # ToDo: Revisit this once we drop Python 2 support/the backport is complete
61
62    def fsencode(unicode_filename):
63        u"""Convert a unicode filename to a filename encoded in the system encoding"""
64        # For paths, just use path.name rather than converting with this
65        # If we are not doing any cleverness with non-unicode filename bytes,
66        # encoding to system encoding is good enough
67        return unicode_filename.encode(sys.getfilesystemencoding(), u"replace")
68
69    def fsdecode(bytes_filename):
70        u"""Convert a filename encoded in the system encoding to unicode"""
71        # For paths, just use path.uc_name rather than converting with this
72        # If we are not doing any cleverness with non-unicode filename bytes,
73        # decoding using system encoding is good enough. Use "ignore" as
74        # Linux paths can contain non-Unicode characters
75        return bytes_filename.decode(config.fsencoding, u"replace")
76
77
78def exception_traceback(limit=50):
79    u"""
80    @return A string representation in typical Python format of the
81            currently active/raised exception.
82    """
83    type, value, tb = sys.exc_info()  # pylint: disable=redefined-builtin
84
85    lines = traceback.format_tb(tb, limit)
86    lines.extend(traceback.format_exception_only(type, value))
87
88    msg = u"Traceback (innermost last):\n"
89    if sys.version_info.major >= 3:
90        msg = msg + u"%-20s %s" % (str.join(u"", lines[:-1]), lines[-1])
91    else:
92        msg = msg + u"%-20s %s" % (string.join(lines[:-1], u""), lines[-1])
93
94    if sys.version_info.major < 3:
95        return msg.decode(u'unicode-escape', u'replace')
96    return msg
97
98
99def escape(string):
100    u"Convert a (bytes) filename to a format suitable for logging (quoted utf8)"
101    string = fsdecode(string).encode(u'unicode-escape', u'replace')
102    return u"'%s'" % string.decode(u'utf8', u'replace').replace(u"'", u'\\x27')
103
104
105def uindex(index):
106    u"Convert an index (a tuple of path parts) to unicode for printing"
107    if index:
108        return os.path.join(*list(map(fsdecode, index)))
109    else:
110        return u'.'
111
112
113def uexc(e):
114    u"""Returns the exception message in Unicode"""
115    # Exceptions in duplicity often have path names in them, which if they are
116    # non-ascii will cause a UnicodeDecodeError when implicitly decoding to
117    # unicode.  So we decode manually, using the filesystem encoding.
118    # 99.99% of the time, this will be a fine encoding to use.
119    if e and e.args:
120        # Find arg that is a string
121        for m in e.args:
122            if isinstance(m, str):
123                # Already unicode
124                return m
125            elif isinstance(m, bytes):
126                # Encoded, likely in filesystem encoding
127                return fsdecode(m)
128        # If the function did not return yet, we did not
129        # succeed in finding a string; return the whole message.
130        # This fails for Python 2, so only do this in Python 3.
131        if sys.version_info[0] > 2:
132            return str(e)
133        # For Python 2, fall back to returning an empty string.
134        else:
135            return u''
136    else:
137        return u''
138
139
140def maybe_ignore_errors(fn):
141    u"""
142    Execute fn. If the global configuration setting ignore_errors is
143    set to True, catch errors and log them but do continue (and return
144    None).
145
146    @param fn: A callable.
147    @return Whatever fn returns when called, or None if it failed and ignore_errors is true.
148    """
149    try:
150        return fn()
151    except Exception as e:
152        if config.ignore_errors:
153            log.Warn(_(u"IGNORED_ERROR: Warning: ignoring error as requested: %s: %s")
154                     % (e.__class__.__name__, uexc(e)))
155            return None
156        else:
157            raise
158
159
160class BlackHoleList(list):
161
162    def append(self, x):
163        pass
164
165
166class FakeTarFile(object):
167    debug = 0
168
169    def __iter__(self):
170        return iter([])
171
172    def close(self):
173        pass
174
175
176def make_tarfile(mode, fp):
177    # We often use 'empty' tarfiles for signatures that haven't been filled out
178    # yet.  So we want to ignore ReadError exceptions, which are used to signal
179    # this.
180    try:
181        tf = tarfile.TarFile(u"arbitrary", mode, fp)
182        # Now we cause TarFile to not cache TarInfo objects.  It would end up
183        # consuming a lot of memory over the lifetime of our long-lasting
184        # signature files otherwise.
185        tf.members = BlackHoleList()
186        return tf
187    except tarfile.ReadError:
188        return FakeTarFile()
189
190
191def get_tarinfo_name(ti):
192    # Python versions before 2.6 ensure that directories end with /, but 2.6
193    # and later ensure they they *don't* have /.  ::shrug::  Internally, we
194    # continue to use pre-2.6 method.
195    if ti.isdir() and not ti.name.endswith(r"/"):
196        return ti.name + r"/"
197    else:
198        return ti.name
199
200
201def ignore_missing(fn, filename):
202    u"""
203    Execute fn on filename.  Ignore ENOENT errors, otherwise raise exception.
204
205    @param fn: callable
206    @param filename: string
207    """
208    try:
209        fn(filename)
210    except OSError as ex:
211        if ex.errno == errno.ENOENT:
212            pass
213        else:
214            raise
215
216
217@atexit.register
218def release_lockfile():
219    if config.lockfile:
220        log.Debug(_(u"Releasing lockfile %s") % config.lockpath)
221        try:
222            config.lockfile.release()
223            config.lockfile = None
224            os.remove(config.lockpath)
225            config.lockpath = u""
226        except Exception:
227            log.Error(u"Could not release lockfile: %s", str(e))
228            pass
229
230
231def copyfileobj(infp, outfp, byte_count=-1):
232    u"""Copy byte_count bytes from infp to outfp, or all if byte_count < 0
233
234    Returns the number of bytes actually written (may be less than
235    byte_count if find eof.  Does not close either fileobj.
236
237    """
238    blocksize = 64 * 1024
239    bytes_written = 0
240    if byte_count < 0:
241        while 1:
242            buf = infp.read(blocksize)
243            if not buf:
244                break
245            bytes_written += len(buf)
246            outfp.write(buf)
247    else:
248        while bytes_written + blocksize <= byte_count:
249            buf = infp.read(blocksize)
250            if not buf:
251                break
252            bytes_written += len(buf)
253            outfp.write(buf)
254        buf = infp.read(byte_count - bytes_written)
255        bytes_written += len(buf)
256        outfp.write(buf)
257    return bytes_written
258
259
260def which(program):
261    u"""
262    Return absolute path for program name.
263    Returns None if program not found.
264    """
265
266    def is_exe(fpath):
267        return os.path.isfile(fpath) and os.path.isabs(fpath) and os.access(fpath, os.X_OK)
268
269    fpath, fname = os.path.split(program)
270    if fpath:
271        if is_exe(program):
272            return program
273    else:
274        for path in os.getenv(u"PATH").split(os.pathsep):
275            path = path.strip(u'"')
276            exe_file = os.path.abspath(os.path.join(path, program))
277            if is_exe(exe_file):
278                return exe_file
279
280    return None
281
282
283def start_debugger(remote=False):
284    if (not os.getenv(u'DEBUG_RUNNING', None) and (u'--pydevd' in sys.argv or os.getenv(u'PYDEVD', None))):
285        if remote:
286            # modify this for your configuration.
287            # client = base path in machine that Liclipse is on
288            # server = base path in machine that duplicity is on
289            client = u'/Users/ken/workspace/duplicity-testfiles'
290            server = u'/home/ken/workspace/duplicity-testfiles'
291
292            # relative paths under duplicity root
293            duppaths = [
294                u'',
295                u'bin',
296                u'duplicity',
297                u'duplicity/backends',
298                u'testing',
299                u'testing/functional',
300                u'testing/unit',
301            ]
302            pathlist = [(os.path.normpath(os.path.join(client, p)),
303                         os.path.normpath(os.path.join(server, p))) for p in duppaths]
304            os.environ[u'PATHS_FROM_ECLIPSE_TO_PYTHON'] = json.dumps(pathlist)
305
306        import pydevd  # pylint: disable=import-error
307        pydevd.settrace(u'dione.local', port=5678, stdoutToServer=True, stderrToServer=True)
308
309        # In a dev environment the path is screwed so fix it.
310        base = sys.path.pop(0)
311        base = base.split(os.path.sep)[:-1]
312        base = os.path.sep.join(base)
313        sys.path.insert(0, base)
314
315        os.environ[u'DEBUG_RUNNING'] = u'yes'
316