1# util.py
2# Basic dnf utils.
3#
4# Copyright (C) 2012-2016 Red Hat, Inc.
5#
6# This copyrighted material is made available to anyone wishing to use,
7# modify, copy, or redistribute it subject to the terms and conditions of
8# the GNU General Public License v.2, or (at your option) any later version.
9# This program is distributed in the hope that it will be useful, but WITHOUT
10# ANY WARRANTY expressed or implied, including the implied warranties of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
12# Public License for more details.  You should have received a copy of the
13# GNU General Public License along with this program; if not, write to the
14# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
16# source code or documentation are not subject to the GNU General Public
17# License and may only be used or replicated with the express permission of
18# Red Hat, Inc.
19#
20
21from __future__ import print_function
22from __future__ import absolute_import
23from __future__ import unicode_literals
24
25from .pycomp import PY3, basestring
26from dnf.i18n import _, ucd
27import argparse
28import dnf
29import dnf.callback
30import dnf.const
31import dnf.pycomp
32import errno
33import functools
34import hawkey
35import itertools
36import locale
37import logging
38import os
39import pwd
40import shutil
41import sys
42import tempfile
43import time
44import libdnf.repo
45import libdnf.transaction
46
47logger = logging.getLogger('dnf')
48
49MAIN_PROG = argparse.ArgumentParser().prog if argparse.ArgumentParser().prog == "yum" else "dnf"
50MAIN_PROG_UPPER = MAIN_PROG.upper()
51
52"""DNF Utilities."""
53
54
55def _parse_specs(namespace, values):
56    """
57    Categorize :param values list into packages, groups and filenames
58
59    :param namespace: argparse.Namespace, where specs will be stored
60    :param values: list of specs, whether packages ('foo') or groups/modules ('@bar')
61                   or filenames ('*.rmp', 'http://*', ...)
62
63    To access packages use: specs.pkg_specs,
64    to access groups use: specs.grp_specs,
65    to access filenames use: specs.filenames
66    """
67
68    setattr(namespace, "filenames", [])
69    setattr(namespace, "grp_specs", [])
70    setattr(namespace, "pkg_specs", [])
71    tmp_set = set()
72    for value in values:
73        if value in tmp_set:
74            continue
75        tmp_set.add(value)
76        schemes = dnf.pycomp.urlparse.urlparse(value)[0]
77        if value.endswith('.rpm'):
78            namespace.filenames.append(value)
79        elif schemes and schemes in ('http', 'ftp', 'file', 'https'):
80            namespace.filenames.append(value)
81        elif value.startswith('@'):
82            namespace.grp_specs.append(value[1:])
83        else:
84            namespace.pkg_specs.append(value)
85
86
87def _urlopen_progress(url, conf, progress=None):
88    if progress is None:
89        progress = dnf.callback.NullDownloadProgress()
90    pload = dnf.repo.RemoteRPMPayload(url, conf, progress)
91    est_remote_size = sum([pload.download_size])
92    progress.start(1, est_remote_size)
93    targets = [pload._librepo_target()]
94    try:
95        libdnf.repo.PackageTarget.downloadPackages(libdnf.repo.VectorPPackageTarget(targets), True)
96    except RuntimeError as e:
97        if conf.strict:
98            raise IOError(str(e))
99        logger.error(str(e))
100    return pload.local_path
101
102def _urlopen(url, conf=None, repo=None, mode='w+b', **kwargs):
103    """
104    Open the specified absolute url, return a file object
105    which respects proxy setting even for non-repo downloads
106    """
107    if PY3 and 'b' not in mode:
108        kwargs.setdefault('encoding', 'utf-8')
109    fo = tempfile.NamedTemporaryFile(mode, **kwargs)
110
111    try:
112        if repo:
113            repo._repo.downloadUrl(url, fo.fileno())
114        else:
115            libdnf.repo.Downloader.downloadURL(conf._config if conf else None, url, fo.fileno())
116    except RuntimeError as e:
117        raise IOError(str(e))
118
119    fo.seek(0)
120    return fo
121
122def rtrim(s, r):
123    if s.endswith(r):
124        s = s[:-len(r)]
125    return s
126
127
128def am_i_root():
129    # used by ansible (lib/ansible/modules/packaging/os/dnf.py)
130    return os.geteuid() == 0
131
132def clear_dir(path):
133    """Remove all files and dirs under `path`
134
135    Also see rm_rf()
136
137    """
138    for entry in os.listdir(path):
139        contained_path = os.path.join(path, entry)
140        rm_rf(contained_path)
141
142def ensure_dir(dname):
143    # used by ansible (lib/ansible/modules/packaging/os/dnf.py)
144    try:
145        os.makedirs(dname, mode=0o755)
146    except OSError as e:
147        if e.errno != errno.EEXIST or not os.path.isdir(dname):
148            raise e
149
150
151def split_path(path):
152    """
153    Split path by path separators.
154    Use os.path.join() to join the path back to string.
155    """
156    result = []
157
158    head = path
159    while True:
160        head, tail = os.path.split(head)
161        if not tail:
162            if head or not result:
163                # if not result: make sure result is [""] so os.path.join(*result) can be called
164                result.insert(0, head)
165            break
166        result.insert(0, tail)
167
168    return result
169
170
171def empty(iterable):
172    try:
173        l = len(iterable)
174    except TypeError:
175        l = len(list(iterable))
176    return l == 0
177
178def first(iterable):
179    """Returns the first item from an iterable or None if it has no elements."""
180    it = iter(iterable)
181    try:
182        return next(it)
183    except StopIteration:
184        return None
185
186
187def first_not_none(iterable):
188    it = iter(iterable)
189    try:
190        return next(item for item in it if item is not None)
191    except StopIteration:
192        return None
193
194
195def file_age(fn):
196    return time.time() - file_timestamp(fn)
197
198def file_timestamp(fn):
199    return os.stat(fn).st_mtime
200
201def get_effective_login():
202    try:
203        return pwd.getpwuid(os.geteuid())[0]
204    except KeyError:
205        return "UID: %s" % os.geteuid()
206
207def get_in(dct, keys, not_found):
208    """Like dict.get() for nested dicts."""
209    for k in keys:
210        dct = dct.get(k)
211        if dct is None:
212            return not_found
213    return dct
214
215def group_by_filter(fn, iterable):
216    def splitter(acc, item):
217        acc[not bool(fn(item))].append(item)
218        return acc
219    return functools.reduce(splitter, iterable, ([], []))
220
221def insert_if(item, iterable, condition):
222    """Insert an item into an iterable by a condition."""
223    for original_item in iterable:
224        if condition(original_item):
225            yield item
226        yield original_item
227
228def is_exhausted(iterator):
229    """Test whether an iterator is exhausted."""
230    try:
231        next(iterator)
232    except StopIteration:
233        return True
234    else:
235        return False
236
237def is_glob_pattern(pattern):
238    if is_string_type(pattern):
239        pattern = [pattern]
240    return (isinstance(pattern, list) and any(set(p) & set("*[?") for p in pattern))
241
242def is_string_type(obj):
243    if PY3:
244        return isinstance(obj, str)
245    else:
246        return isinstance(obj, basestring)
247
248def lazyattr(attrname):
249    """Decorator to get lazy attribute initialization.
250
251    Composes with @property. Force reinitialization by deleting the <attrname>.
252    """
253    def get_decorated(fn):
254        def cached_getter(obj):
255            try:
256                return getattr(obj, attrname)
257            except AttributeError:
258                val = fn(obj)
259                setattr(obj, attrname, val)
260                return val
261        return cached_getter
262    return get_decorated
263
264
265def mapall(fn, *seq):
266    """Like functools.map(), but return a list instead of an iterator.
267
268    This means all side effects of fn take place even without iterating the
269    result.
270
271    """
272    return list(map(fn, *seq))
273
274def normalize_time(timestamp):
275    """Convert time into locale aware datetime string object."""
276    t = time.strftime("%c", time.localtime(timestamp))
277    if not dnf.pycomp.PY3:
278        current_locale_setting = locale.getlocale()[1]
279        if current_locale_setting:
280            t = t.decode(current_locale_setting)
281    return t
282
283def on_ac_power():
284    """Decide whether we are on line power.
285
286    Returns True if we are on line power, False if not, None if it can not be
287    decided.
288
289    """
290    try:
291        ps_folder = "/sys/class/power_supply"
292        ac_nodes = [node for node in os.listdir(ps_folder) if node.startswith("AC")]
293        if len(ac_nodes) > 0:
294            ac_node = ac_nodes[0]
295            with open("{}/{}/online".format(ps_folder, ac_node)) as ac_status:
296                data = ac_status.read()
297                return int(data) == 1
298        return None
299    except (IOError, ValueError):
300        return None
301
302
303def on_metered_connection():
304    """Decide whether we are on metered connection.
305
306    Returns:
307      True: if on metered connection
308      False: if not
309      None: if it can not be decided
310    """
311    try:
312        import dbus
313    except ImportError:
314        return None
315    try:
316        bus = dbus.SystemBus()
317        proxy = bus.get_object("org.freedesktop.NetworkManager",
318                               "/org/freedesktop/NetworkManager")
319        iface = dbus.Interface(proxy, "org.freedesktop.DBus.Properties")
320        metered = iface.Get("org.freedesktop.NetworkManager", "Metered")
321    except dbus.DBusException:
322        return None
323    if metered == 0: # NM_METERED_UNKNOWN
324        return None
325    elif metered in (1, 3): # NM_METERED_YES, NM_METERED_GUESS_YES
326        return True
327    elif metered in (2, 4): # NM_METERED_NO, NM_METERED_GUESS_NO
328        return False
329    else: # Something undocumented (at least at this moment)
330        raise ValueError("Unknown value for metered property: %r", metered)
331
332def partition(pred, iterable):
333    """Use a predicate to partition entries into false entries and true entries.
334
335    Credit: Python library itertools' documentation.
336
337    """
338    t1, t2 = itertools.tee(iterable)
339    return dnf.pycomp.filterfalse(pred, t1), filter(pred, t2)
340
341def rm_rf(path):
342    try:
343        shutil.rmtree(path)
344    except OSError:
345        pass
346
347def split_by(iterable, condition):
348    """Split an iterable into tuples by a condition.
349
350    Inserts a separator before each item which meets the condition and then
351    cuts the iterable by these separators.
352
353    """
354    separator = object()  # A unique object.
355    # Create a function returning tuple of objects before the separator.
356    def next_subsequence(it):
357        return tuple(itertools.takewhile(lambda e: e != separator, it))
358
359    # Mark each place where the condition is met by the separator.
360    marked = insert_if(separator, iterable, condition)
361
362    # The 1st subsequence may be empty if the 1st item meets the condition.
363    yield next_subsequence(marked)
364
365    while True:
366        subsequence = next_subsequence(marked)
367        if not subsequence:
368            break
369        yield subsequence
370
371def strip_prefix(s, prefix):
372    if s.startswith(prefix):
373        return s[len(prefix):]
374    return None
375
376
377def touch(path, no_create=False):
378    """Create an empty file if it doesn't exist or bump it's timestamps.
379
380    If no_create is True only bumps the timestamps.
381    """
382    if no_create or os.access(path, os.F_OK):
383        return os.utime(path, None)
384    with open(path, 'a'):
385        pass
386
387
388def _terminal_messenger(tp='write', msg="", out=sys.stdout):
389    try:
390        if tp == 'write':
391            out.write(msg)
392        elif tp == 'flush':
393            out.flush()
394        elif tp == 'write_flush':
395            out.write(msg)
396            out.flush()
397        elif tp == 'print':
398            print(msg, file=out)
399        else:
400            raise ValueError('Unsupported type: ' + tp)
401    except IOError as e:
402        logger.critical('{}: {}'.format(type(e).__name__, ucd(e)))
403        pass
404
405
406def _format_resolve_problems(resolve_problems):
407    """
408    Format string about problems in resolve
409
410    :param resolve_problems: list with list of strings (output of goal.problem_rules())
411    :return: string
412    """
413    msg = ""
414    count_problems = (len(resolve_problems) > 1)
415    for i, rs in enumerate(resolve_problems, start=1):
416        if count_problems:
417            msg += "\n " + _("Problem") + " %d: " % i
418        else:
419            msg += "\n " + _("Problem") + ": "
420        msg += "\n  - ".join(rs)
421    return msg
422
423
424def _te_nevra(te):
425    nevra = te.N() + '-'
426    if te.E() is not None and te.E() != '0':
427        nevra += te.E() + ':'
428    return nevra + te.V() + '-' + te.R() + '.' + te.A()
429
430
431def _log_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
432    logger.debug("Logging transaction elements")
433    for rpm_el in rpm_transaction:
434        tsi = rpm_el.Key()
435        tsi_state = None
436        if tsi is not None:
437            tsi_state = tsi.state
438        msg = "RPM element: '{}', Key(): '{}', Key state: '{}', Failed() '{}': ".format(
439            _te_nevra(rpm_el), tsi, tsi_state, rpm_el.Failed())
440        logger.debug(msg)
441    for tsi in swdb_transaction:
442        msg = "SWDB element: '{}', State: '{}', Action: '{}', From repo: '{}', Reason: '{}', " \
443              "Get reason: '{}'".format(str(tsi), tsi.state, tsi.action, tsi.from_repo, tsi.reason,
444                                        tsi.get_reason())
445        logger.debug(msg)
446
447
448def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
449    revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED,
450                      libdnf.transaction.TransactionItemAction_OBSOLETED,
451                      libdnf.transaction.TransactionItemAction_REMOVE,
452                      libdnf.transaction.TransactionItemAction_UPGRADED,
453                      libdnf.transaction.TransactionItemAction_REINSTALLED}
454    cached_tsi = [tsi for tsi in swdb_transaction]
455    el_not_found = False
456    error = False
457    for rpm_el in rpm_transaction:
458        te_nevra = _te_nevra(rpm_el)
459        tsi = rpm_el.Key()
460        if tsi is None or not hasattr(tsi, "pkg"):
461            for tsi_candidate in cached_tsi:
462                if tsi_candidate.state != libdnf.transaction.TransactionItemState_UNKNOWN:
463                    continue
464                if tsi_candidate.action not in revert_actions:
465                    continue
466                if str(tsi_candidate) == te_nevra:
467                    tsi = tsi_candidate
468                    break
469        if tsi is None or not hasattr(tsi, "pkg"):
470            logger.critical(_("TransactionItem not found for key: {}").format(te_nevra))
471            el_not_found = True
472            continue
473        if rpm_el.Failed():
474            tsi.state = libdnf.transaction.TransactionItemState_ERROR
475            error = True
476        else:
477            tsi.state = libdnf.transaction.TransactionItemState_DONE
478    for tsi in cached_tsi:
479        if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
480            logger.critical(_("TransactionSWDBItem not found for key: {}").format(str(tsi)))
481            el_not_found = True
482    if error:
483        logger.debug(_('Errors occurred during transaction.'))
484    if el_not_found:
485        _log_rpm_trans_with_swdb(rpm_transaction, cached_tsi)
486
487
488class tmpdir(object):
489    # used by subscription-manager (src/dnf-plugins/product-id.py)
490    def __init__(self):
491        prefix = '%s-' % dnf.const.PREFIX
492        self.path = tempfile.mkdtemp(prefix=prefix)
493
494    def __enter__(self):
495        return self.path
496
497    def __exit__(self, exc_type, exc_value, traceback):
498        rm_rf(self.path)
499
500class Bunch(dict):
501    """Dictionary with attribute accessing syntax.
502
503    In DNF, prefer using this over dnf.yum.misc.GenericHolder.
504
505    Credit: Alex Martelli, Doug Hudgeon
506
507    """
508    def __init__(self, *args, **kwds):
509         super(Bunch, self).__init__(*args, **kwds)
510         self.__dict__ = self
511
512    def __hash__(self):
513        return id(self)
514
515
516class MultiCallList(list):
517    def __init__(self, iterable):
518        super(MultiCallList, self).__init__()
519        self.extend(iterable)
520
521    def __getattr__(self, what):
522        def fn(*args, **kwargs):
523            def call_what(v):
524                method = getattr(v, what)
525                return method(*args, **kwargs)
526            return list(map(call_what, self))
527        return fn
528
529    def __setattr__(self, what, val):
530        def setter(item):
531            setattr(item, what, val)
532        return list(map(setter, self))
533
534
535def _make_lists(transaction):
536    b = Bunch({
537        'downgraded': [],
538        'erased': [],
539        'erased_clean': [],
540        'erased_dep': [],
541        'installed': [],
542        'installed_group': [],
543        'installed_dep': [],
544        'installed_weak': [],
545        'reinstalled': [],
546        'upgraded': [],
547        'failed': [],
548    })
549
550    for tsi in transaction:
551        if tsi.state == libdnf.transaction.TransactionItemState_ERROR:
552            b.failed.append(tsi)
553        elif tsi.action == libdnf.transaction.TransactionItemAction_DOWNGRADE:
554            b.downgraded.append(tsi)
555        elif tsi.action == libdnf.transaction.TransactionItemAction_INSTALL:
556            if tsi.reason == libdnf.transaction.TransactionItemReason_GROUP:
557                b.installed_group.append(tsi)
558            elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY:
559                b.installed_dep.append(tsi)
560            elif tsi.reason == libdnf.transaction.TransactionItemReason_WEAK_DEPENDENCY:
561                b.installed_weak.append(tsi)
562            else:
563                # TransactionItemReason_USER
564                b.installed.append(tsi)
565        elif tsi.action == libdnf.transaction.TransactionItemAction_REINSTALL:
566            b.reinstalled.append(tsi)
567        elif tsi.action == libdnf.transaction.TransactionItemAction_REMOVE:
568            if tsi.reason == libdnf.transaction.TransactionItemReason_CLEAN:
569                b.erased_clean.append(tsi)
570            elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY:
571                b.erased_dep.append(tsi)
572            else:
573                b.erased.append(tsi)
574        elif tsi.action == libdnf.transaction.TransactionItemAction_UPGRADE:
575            b.upgraded.append(tsi)
576
577    return b
578
579
580def _post_transaction_output(base, transaction, action_callback):
581    """Returns a human-readable summary of the results of the
582    transaction.
583
584    :param action_callback: function generating output for specific action. It
585       takes two parameters - action as a string and list of affected packages for
586       this action
587    :return: a list of lines containing a human-readable summary of the
588       results of the transaction
589    """
590    def _tsi_or_pkg_nevra_cmp(item1, item2):
591        """Compares two transaction items or packages by nevra.
592           Used as a fallback when tsi does not contain package object.
593        """
594        ret = (item1.name > item2.name) - (item1.name < item2.name)
595        if ret != 0:
596            return ret
597        nevra1 = hawkey.NEVRA(name=item1.name, epoch=item1.epoch, version=item1.version,
598                              release=item1.release, arch=item1.arch)
599        nevra2 = hawkey.NEVRA(name=item2.name, epoch=item2.epoch, version=item2.version,
600                              release=item2.release, arch=item2.arch)
601        ret = nevra1.evr_cmp(nevra2, base.sack)
602        if ret != 0:
603            return ret
604        return (item1.arch > item2.arch) - (item1.arch < item2.arch)
605
606    list_bunch = dnf.util._make_lists(transaction)
607
608    skipped_conflicts, skipped_broken = base._skipped_packages(
609        report_problems=False, transaction=transaction)
610    skipped = skipped_conflicts.union(skipped_broken)
611
612    out = []
613    for (action, tsis) in [(_('Upgraded'), list_bunch.upgraded),
614                           (_('Downgraded'), list_bunch.downgraded),
615                           (_('Installed'), list_bunch.installed +
616                            list_bunch.installed_group +
617                            list_bunch.installed_weak +
618                            list_bunch.installed_dep),
619                           (_('Reinstalled'), list_bunch.reinstalled),
620                           (_('Skipped'), skipped),
621                           (_('Removed'), list_bunch.erased +
622                               list_bunch.erased_dep +
623                               list_bunch.erased_clean),
624                           (_('Failed'), list_bunch.failed)]:
625        out.extend(action_callback(
626            action, sorted(tsis, key=functools.cmp_to_key(_tsi_or_pkg_nevra_cmp))))
627
628    return out
629
630
631def _name_unset_wrapper(input_name):
632    # returns <name-unset> for everything that evaluates to False (None, empty..)
633    return input_name if input_name else _("<name-unset>")
634