1# util.py 2# Basic dnf utils. 3# 4# Copyright (C) 2012-2016 Red Hat, Inc. 5# 6# This copyrighted material is made available to anyone wishing to use, 7# modify, copy, or redistribute it subject to the terms and conditions of 8# the GNU General Public License v.2, or (at your option) any later version. 9# This program is distributed in the hope that it will be useful, but WITHOUT 10# ANY WARRANTY expressed or implied, including the implied warranties of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 12# Public License for more details. You should have received a copy of the 13# GNU General Public License along with this program; if not, write to the 14# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 15# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the 16# source code or documentation are not subject to the GNU General Public 17# License and may only be used or replicated with the express permission of 18# Red Hat, Inc. 19# 20 21from __future__ import print_function 22from __future__ import absolute_import 23from __future__ import unicode_literals 24 25from .pycomp import PY3, basestring 26from dnf.i18n import _, ucd 27import argparse 28import dnf 29import dnf.callback 30import dnf.const 31import dnf.pycomp 32import errno 33import functools 34import hawkey 35import itertools 36import locale 37import logging 38import os 39import pwd 40import shutil 41import sys 42import tempfile 43import time 44import libdnf.repo 45import libdnf.transaction 46 47logger = logging.getLogger('dnf') 48 49MAIN_PROG = argparse.ArgumentParser().prog if argparse.ArgumentParser().prog == "yum" else "dnf" 50MAIN_PROG_UPPER = MAIN_PROG.upper() 51 52"""DNF Utilities.""" 53 54 55def _parse_specs(namespace, values): 56 """ 57 Categorize :param values list into packages, groups and filenames 58 59 :param namespace: argparse.Namespace, where specs will be stored 60 :param values: list of specs, whether packages ('foo') or groups/modules ('@bar') 61 or filenames ('*.rmp', 'http://*', ...) 62 63 To access packages use: specs.pkg_specs, 64 to access groups use: specs.grp_specs, 65 to access filenames use: specs.filenames 66 """ 67 68 setattr(namespace, "filenames", []) 69 setattr(namespace, "grp_specs", []) 70 setattr(namespace, "pkg_specs", []) 71 tmp_set = set() 72 for value in values: 73 if value in tmp_set: 74 continue 75 tmp_set.add(value) 76 schemes = dnf.pycomp.urlparse.urlparse(value)[0] 77 if value.endswith('.rpm'): 78 namespace.filenames.append(value) 79 elif schemes and schemes in ('http', 'ftp', 'file', 'https'): 80 namespace.filenames.append(value) 81 elif value.startswith('@'): 82 namespace.grp_specs.append(value[1:]) 83 else: 84 namespace.pkg_specs.append(value) 85 86 87def _urlopen_progress(url, conf, progress=None): 88 if progress is None: 89 progress = dnf.callback.NullDownloadProgress() 90 pload = dnf.repo.RemoteRPMPayload(url, conf, progress) 91 est_remote_size = sum([pload.download_size]) 92 progress.start(1, est_remote_size) 93 targets = [pload._librepo_target()] 94 try: 95 libdnf.repo.PackageTarget.downloadPackages(libdnf.repo.VectorPPackageTarget(targets), True) 96 except RuntimeError as e: 97 if conf.strict: 98 raise IOError(str(e)) 99 logger.error(str(e)) 100 return pload.local_path 101 102def _urlopen(url, conf=None, repo=None, mode='w+b', **kwargs): 103 """ 104 Open the specified absolute url, return a file object 105 which respects proxy setting even for non-repo downloads 106 """ 107 if PY3 and 'b' not in mode: 108 kwargs.setdefault('encoding', 'utf-8') 109 fo = tempfile.NamedTemporaryFile(mode, **kwargs) 110 111 try: 112 if repo: 113 repo._repo.downloadUrl(url, fo.fileno()) 114 else: 115 libdnf.repo.Downloader.downloadURL(conf._config if conf else None, url, fo.fileno()) 116 except RuntimeError as e: 117 raise IOError(str(e)) 118 119 fo.seek(0) 120 return fo 121 122def rtrim(s, r): 123 if s.endswith(r): 124 s = s[:-len(r)] 125 return s 126 127 128def am_i_root(): 129 # used by ansible (lib/ansible/modules/packaging/os/dnf.py) 130 return os.geteuid() == 0 131 132def clear_dir(path): 133 """Remove all files and dirs under `path` 134 135 Also see rm_rf() 136 137 """ 138 for entry in os.listdir(path): 139 contained_path = os.path.join(path, entry) 140 rm_rf(contained_path) 141 142def ensure_dir(dname): 143 # used by ansible (lib/ansible/modules/packaging/os/dnf.py) 144 try: 145 os.makedirs(dname, mode=0o755) 146 except OSError as e: 147 if e.errno != errno.EEXIST or not os.path.isdir(dname): 148 raise e 149 150 151def split_path(path): 152 """ 153 Split path by path separators. 154 Use os.path.join() to join the path back to string. 155 """ 156 result = [] 157 158 head = path 159 while True: 160 head, tail = os.path.split(head) 161 if not tail: 162 if head or not result: 163 # if not result: make sure result is [""] so os.path.join(*result) can be called 164 result.insert(0, head) 165 break 166 result.insert(0, tail) 167 168 return result 169 170 171def empty(iterable): 172 try: 173 l = len(iterable) 174 except TypeError: 175 l = len(list(iterable)) 176 return l == 0 177 178def first(iterable): 179 """Returns the first item from an iterable or None if it has no elements.""" 180 it = iter(iterable) 181 try: 182 return next(it) 183 except StopIteration: 184 return None 185 186 187def first_not_none(iterable): 188 it = iter(iterable) 189 try: 190 return next(item for item in it if item is not None) 191 except StopIteration: 192 return None 193 194 195def file_age(fn): 196 return time.time() - file_timestamp(fn) 197 198def file_timestamp(fn): 199 return os.stat(fn).st_mtime 200 201def get_effective_login(): 202 try: 203 return pwd.getpwuid(os.geteuid())[0] 204 except KeyError: 205 return "UID: %s" % os.geteuid() 206 207def get_in(dct, keys, not_found): 208 """Like dict.get() for nested dicts.""" 209 for k in keys: 210 dct = dct.get(k) 211 if dct is None: 212 return not_found 213 return dct 214 215def group_by_filter(fn, iterable): 216 def splitter(acc, item): 217 acc[not bool(fn(item))].append(item) 218 return acc 219 return functools.reduce(splitter, iterable, ([], [])) 220 221def insert_if(item, iterable, condition): 222 """Insert an item into an iterable by a condition.""" 223 for original_item in iterable: 224 if condition(original_item): 225 yield item 226 yield original_item 227 228def is_exhausted(iterator): 229 """Test whether an iterator is exhausted.""" 230 try: 231 next(iterator) 232 except StopIteration: 233 return True 234 else: 235 return False 236 237def is_glob_pattern(pattern): 238 if is_string_type(pattern): 239 pattern = [pattern] 240 return (isinstance(pattern, list) and any(set(p) & set("*[?") for p in pattern)) 241 242def is_string_type(obj): 243 if PY3: 244 return isinstance(obj, str) 245 else: 246 return isinstance(obj, basestring) 247 248def lazyattr(attrname): 249 """Decorator to get lazy attribute initialization. 250 251 Composes with @property. Force reinitialization by deleting the <attrname>. 252 """ 253 def get_decorated(fn): 254 def cached_getter(obj): 255 try: 256 return getattr(obj, attrname) 257 except AttributeError: 258 val = fn(obj) 259 setattr(obj, attrname, val) 260 return val 261 return cached_getter 262 return get_decorated 263 264 265def mapall(fn, *seq): 266 """Like functools.map(), but return a list instead of an iterator. 267 268 This means all side effects of fn take place even without iterating the 269 result. 270 271 """ 272 return list(map(fn, *seq)) 273 274def normalize_time(timestamp): 275 """Convert time into locale aware datetime string object.""" 276 t = time.strftime("%c", time.localtime(timestamp)) 277 if not dnf.pycomp.PY3: 278 current_locale_setting = locale.getlocale()[1] 279 if current_locale_setting: 280 t = t.decode(current_locale_setting) 281 return t 282 283def on_ac_power(): 284 """Decide whether we are on line power. 285 286 Returns True if we are on line power, False if not, None if it can not be 287 decided. 288 289 """ 290 try: 291 ps_folder = "/sys/class/power_supply" 292 ac_nodes = [node for node in os.listdir(ps_folder) if node.startswith("AC")] 293 if len(ac_nodes) > 0: 294 ac_node = ac_nodes[0] 295 with open("{}/{}/online".format(ps_folder, ac_node)) as ac_status: 296 data = ac_status.read() 297 return int(data) == 1 298 return None 299 except (IOError, ValueError): 300 return None 301 302 303def on_metered_connection(): 304 """Decide whether we are on metered connection. 305 306 Returns: 307 True: if on metered connection 308 False: if not 309 None: if it can not be decided 310 """ 311 try: 312 import dbus 313 except ImportError: 314 return None 315 try: 316 bus = dbus.SystemBus() 317 proxy = bus.get_object("org.freedesktop.NetworkManager", 318 "/org/freedesktop/NetworkManager") 319 iface = dbus.Interface(proxy, "org.freedesktop.DBus.Properties") 320 metered = iface.Get("org.freedesktop.NetworkManager", "Metered") 321 except dbus.DBusException: 322 return None 323 if metered == 0: # NM_METERED_UNKNOWN 324 return None 325 elif metered in (1, 3): # NM_METERED_YES, NM_METERED_GUESS_YES 326 return True 327 elif metered in (2, 4): # NM_METERED_NO, NM_METERED_GUESS_NO 328 return False 329 else: # Something undocumented (at least at this moment) 330 raise ValueError("Unknown value for metered property: %r", metered) 331 332def partition(pred, iterable): 333 """Use a predicate to partition entries into false entries and true entries. 334 335 Credit: Python library itertools' documentation. 336 337 """ 338 t1, t2 = itertools.tee(iterable) 339 return dnf.pycomp.filterfalse(pred, t1), filter(pred, t2) 340 341def rm_rf(path): 342 try: 343 shutil.rmtree(path) 344 except OSError: 345 pass 346 347def split_by(iterable, condition): 348 """Split an iterable into tuples by a condition. 349 350 Inserts a separator before each item which meets the condition and then 351 cuts the iterable by these separators. 352 353 """ 354 separator = object() # A unique object. 355 # Create a function returning tuple of objects before the separator. 356 def next_subsequence(it): 357 return tuple(itertools.takewhile(lambda e: e != separator, it)) 358 359 # Mark each place where the condition is met by the separator. 360 marked = insert_if(separator, iterable, condition) 361 362 # The 1st subsequence may be empty if the 1st item meets the condition. 363 yield next_subsequence(marked) 364 365 while True: 366 subsequence = next_subsequence(marked) 367 if not subsequence: 368 break 369 yield subsequence 370 371def strip_prefix(s, prefix): 372 if s.startswith(prefix): 373 return s[len(prefix):] 374 return None 375 376 377def touch(path, no_create=False): 378 """Create an empty file if it doesn't exist or bump it's timestamps. 379 380 If no_create is True only bumps the timestamps. 381 """ 382 if no_create or os.access(path, os.F_OK): 383 return os.utime(path, None) 384 with open(path, 'a'): 385 pass 386 387 388def _terminal_messenger(tp='write', msg="", out=sys.stdout): 389 try: 390 if tp == 'write': 391 out.write(msg) 392 elif tp == 'flush': 393 out.flush() 394 elif tp == 'write_flush': 395 out.write(msg) 396 out.flush() 397 elif tp == 'print': 398 print(msg, file=out) 399 else: 400 raise ValueError('Unsupported type: ' + tp) 401 except IOError as e: 402 logger.critical('{}: {}'.format(type(e).__name__, ucd(e))) 403 pass 404 405 406def _format_resolve_problems(resolve_problems): 407 """ 408 Format string about problems in resolve 409 410 :param resolve_problems: list with list of strings (output of goal.problem_rules()) 411 :return: string 412 """ 413 msg = "" 414 count_problems = (len(resolve_problems) > 1) 415 for i, rs in enumerate(resolve_problems, start=1): 416 if count_problems: 417 msg += "\n " + _("Problem") + " %d: " % i 418 else: 419 msg += "\n " + _("Problem") + ": " 420 msg += "\n - ".join(rs) 421 return msg 422 423 424def _te_nevra(te): 425 nevra = te.N() + '-' 426 if te.E() is not None and te.E() != '0': 427 nevra += te.E() + ':' 428 return nevra + te.V() + '-' + te.R() + '.' + te.A() 429 430 431def _log_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): 432 logger.debug("Logging transaction elements") 433 for rpm_el in rpm_transaction: 434 tsi = rpm_el.Key() 435 tsi_state = None 436 if tsi is not None: 437 tsi_state = tsi.state 438 msg = "RPM element: '{}', Key(): '{}', Key state: '{}', Failed() '{}': ".format( 439 _te_nevra(rpm_el), tsi, tsi_state, rpm_el.Failed()) 440 logger.debug(msg) 441 for tsi in swdb_transaction: 442 msg = "SWDB element: '{}', State: '{}', Action: '{}', From repo: '{}', Reason: '{}', " \ 443 "Get reason: '{}'".format(str(tsi), tsi.state, tsi.action, tsi.from_repo, tsi.reason, 444 tsi.get_reason()) 445 logger.debug(msg) 446 447 448def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction): 449 revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED, 450 libdnf.transaction.TransactionItemAction_OBSOLETED, 451 libdnf.transaction.TransactionItemAction_REMOVE, 452 libdnf.transaction.TransactionItemAction_UPGRADED, 453 libdnf.transaction.TransactionItemAction_REINSTALLED} 454 cached_tsi = [tsi for tsi in swdb_transaction] 455 el_not_found = False 456 error = False 457 for rpm_el in rpm_transaction: 458 te_nevra = _te_nevra(rpm_el) 459 tsi = rpm_el.Key() 460 if tsi is None or not hasattr(tsi, "pkg"): 461 for tsi_candidate in cached_tsi: 462 if tsi_candidate.state != libdnf.transaction.TransactionItemState_UNKNOWN: 463 continue 464 if tsi_candidate.action not in revert_actions: 465 continue 466 if str(tsi_candidate) == te_nevra: 467 tsi = tsi_candidate 468 break 469 if tsi is None or not hasattr(tsi, "pkg"): 470 logger.critical(_("TransactionItem not found for key: {}").format(te_nevra)) 471 el_not_found = True 472 continue 473 if rpm_el.Failed(): 474 tsi.state = libdnf.transaction.TransactionItemState_ERROR 475 error = True 476 else: 477 tsi.state = libdnf.transaction.TransactionItemState_DONE 478 for tsi in cached_tsi: 479 if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN: 480 logger.critical(_("TransactionSWDBItem not found for key: {}").format(str(tsi))) 481 el_not_found = True 482 if error: 483 logger.debug(_('Errors occurred during transaction.')) 484 if el_not_found: 485 _log_rpm_trans_with_swdb(rpm_transaction, cached_tsi) 486 487 488class tmpdir(object): 489 # used by subscription-manager (src/dnf-plugins/product-id.py) 490 def __init__(self): 491 prefix = '%s-' % dnf.const.PREFIX 492 self.path = tempfile.mkdtemp(prefix=prefix) 493 494 def __enter__(self): 495 return self.path 496 497 def __exit__(self, exc_type, exc_value, traceback): 498 rm_rf(self.path) 499 500class Bunch(dict): 501 """Dictionary with attribute accessing syntax. 502 503 In DNF, prefer using this over dnf.yum.misc.GenericHolder. 504 505 Credit: Alex Martelli, Doug Hudgeon 506 507 """ 508 def __init__(self, *args, **kwds): 509 super(Bunch, self).__init__(*args, **kwds) 510 self.__dict__ = self 511 512 def __hash__(self): 513 return id(self) 514 515 516class MultiCallList(list): 517 def __init__(self, iterable): 518 super(MultiCallList, self).__init__() 519 self.extend(iterable) 520 521 def __getattr__(self, what): 522 def fn(*args, **kwargs): 523 def call_what(v): 524 method = getattr(v, what) 525 return method(*args, **kwargs) 526 return list(map(call_what, self)) 527 return fn 528 529 def __setattr__(self, what, val): 530 def setter(item): 531 setattr(item, what, val) 532 return list(map(setter, self)) 533 534 535def _make_lists(transaction): 536 b = Bunch({ 537 'downgraded': [], 538 'erased': [], 539 'erased_clean': [], 540 'erased_dep': [], 541 'installed': [], 542 'installed_group': [], 543 'installed_dep': [], 544 'installed_weak': [], 545 'reinstalled': [], 546 'upgraded': [], 547 'failed': [], 548 }) 549 550 for tsi in transaction: 551 if tsi.state == libdnf.transaction.TransactionItemState_ERROR: 552 b.failed.append(tsi) 553 elif tsi.action == libdnf.transaction.TransactionItemAction_DOWNGRADE: 554 b.downgraded.append(tsi) 555 elif tsi.action == libdnf.transaction.TransactionItemAction_INSTALL: 556 if tsi.reason == libdnf.transaction.TransactionItemReason_GROUP: 557 b.installed_group.append(tsi) 558 elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: 559 b.installed_dep.append(tsi) 560 elif tsi.reason == libdnf.transaction.TransactionItemReason_WEAK_DEPENDENCY: 561 b.installed_weak.append(tsi) 562 else: 563 # TransactionItemReason_USER 564 b.installed.append(tsi) 565 elif tsi.action == libdnf.transaction.TransactionItemAction_REINSTALL: 566 b.reinstalled.append(tsi) 567 elif tsi.action == libdnf.transaction.TransactionItemAction_REMOVE: 568 if tsi.reason == libdnf.transaction.TransactionItemReason_CLEAN: 569 b.erased_clean.append(tsi) 570 elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: 571 b.erased_dep.append(tsi) 572 else: 573 b.erased.append(tsi) 574 elif tsi.action == libdnf.transaction.TransactionItemAction_UPGRADE: 575 b.upgraded.append(tsi) 576 577 return b 578 579 580def _post_transaction_output(base, transaction, action_callback): 581 """Returns a human-readable summary of the results of the 582 transaction. 583 584 :param action_callback: function generating output for specific action. It 585 takes two parameters - action as a string and list of affected packages for 586 this action 587 :return: a list of lines containing a human-readable summary of the 588 results of the transaction 589 """ 590 def _tsi_or_pkg_nevra_cmp(item1, item2): 591 """Compares two transaction items or packages by nevra. 592 Used as a fallback when tsi does not contain package object. 593 """ 594 ret = (item1.name > item2.name) - (item1.name < item2.name) 595 if ret != 0: 596 return ret 597 nevra1 = hawkey.NEVRA(name=item1.name, epoch=item1.epoch, version=item1.version, 598 release=item1.release, arch=item1.arch) 599 nevra2 = hawkey.NEVRA(name=item2.name, epoch=item2.epoch, version=item2.version, 600 release=item2.release, arch=item2.arch) 601 ret = nevra1.evr_cmp(nevra2, base.sack) 602 if ret != 0: 603 return ret 604 return (item1.arch > item2.arch) - (item1.arch < item2.arch) 605 606 list_bunch = dnf.util._make_lists(transaction) 607 608 skipped_conflicts, skipped_broken = base._skipped_packages( 609 report_problems=False, transaction=transaction) 610 skipped = skipped_conflicts.union(skipped_broken) 611 612 out = [] 613 for (action, tsis) in [(_('Upgraded'), list_bunch.upgraded), 614 (_('Downgraded'), list_bunch.downgraded), 615 (_('Installed'), list_bunch.installed + 616 list_bunch.installed_group + 617 list_bunch.installed_weak + 618 list_bunch.installed_dep), 619 (_('Reinstalled'), list_bunch.reinstalled), 620 (_('Skipped'), skipped), 621 (_('Removed'), list_bunch.erased + 622 list_bunch.erased_dep + 623 list_bunch.erased_clean), 624 (_('Failed'), list_bunch.failed)]: 625 out.extend(action_callback( 626 action, sorted(tsis, key=functools.cmp_to_key(_tsi_or_pkg_nevra_cmp)))) 627 628 return out 629 630 631def _name_unset_wrapper(input_name): 632 # returns <name-unset> for everything that evaluates to False (None, empty..) 633 return input_name if input_name else _("<name-unset>") 634