1# comps.py 2# Interface to libcomps. 3# 4# Copyright (C) 2013-2018 Red Hat, Inc. 5# 6# This copyrighted material is made available to anyone wishing to use, 7# modify, copy, or redistribute it subject to the terms and conditions of 8# the GNU General Public License v.2, or (at your option) any later version. 9# This program is distributed in the hope that it will be useful, but WITHOUT 10# ANY WARRANTY expressed or implied, including the implied warranties of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 12# Public License for more details. You should have received a copy of the 13# GNU General Public License along with this program; if not, write to the 14# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 15# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the 16# source code or documentation are not subject to the GNU General Public 17# License and may only be used or replicated with the express permission of 18# Red Hat, Inc. 19# 20 21from __future__ import absolute_import 22from __future__ import print_function 23from __future__ import unicode_literals 24 25import libdnf.transaction 26 27from dnf.exceptions import CompsError 28from dnf.i18n import _, ucd 29from functools import reduce 30 31import dnf.i18n 32import dnf.util 33import fnmatch 34import gettext 35import itertools 36import libcomps 37import locale 38import logging 39import operator 40import re 41import sys 42 43logger = logging.getLogger("dnf") 44 45# :api :binformat 46CONDITIONAL = libdnf.transaction.CompsPackageType_CONDITIONAL 47DEFAULT = libdnf.transaction.CompsPackageType_DEFAULT 48MANDATORY = libdnf.transaction.CompsPackageType_MANDATORY 49OPTIONAL = libdnf.transaction.CompsPackageType_OPTIONAL 50 51ALL_TYPES = CONDITIONAL | DEFAULT | MANDATORY | OPTIONAL 52 53 54def _internal_comps_length(comps): 55 collections = (comps.categories, comps.groups, comps.environments) 56 return reduce(operator.__add__, map(len, collections)) 57 58 59def _first_if_iterable(seq): 60 if seq is None: 61 return None 62 return dnf.util.first(seq) 63 64 65def _by_pattern(pattern, case_sensitive, sqn): 66 """Return items from sqn matching either exactly or glob-wise.""" 67 68 pattern = dnf.i18n.ucd(pattern) 69 exact = {g for g in sqn if g.name == pattern or g.id == pattern} 70 if exact: 71 return exact 72 73 if case_sensitive: 74 match = re.compile(fnmatch.translate(pattern)).match 75 else: 76 match = re.compile(fnmatch.translate(pattern), flags=re.I).match 77 78 ret = set() 79 for g in sqn: 80 if match(g.id): 81 ret.add(g) 82 elif g.name is not None and match(g.name): 83 ret.add(g) 84 elif g.ui_name is not None and match(g.ui_name): 85 ret.add(g) 86 87 return ret 88 89 90def _fn_display_order(group): 91 return sys.maxsize if group.display_order is None else group.display_order 92 93 94def install_or_skip(install_fnc, grp_or_env_id, types, exclude=None, 95 strict=True, exclude_groups=None): 96 """Either mark in persistor as installed given `grp_or_env` (group 97 or environment) or skip it (if it's already installed). 98 `install_fnc` has to be Solver._group_install 99 or Solver._environment_install. 100 """ 101 try: 102 return install_fnc(grp_or_env_id, types, exclude, strict, exclude_groups) 103 except dnf.comps.CompsError as e: 104 logger.warning("%s, %s", ucd(e)[:-1], _("skipping.")) 105 106 107class _Langs(object): 108 109 """Get all usable abbreviations for the current language.""" 110 111 def __init__(self): 112 self.last_locale = None 113 self.cache = None 114 115 @staticmethod 116 def _dotted_locale_str(): 117 lcl = locale.getlocale(locale.LC_MESSAGES) 118 if lcl == (None, None): 119 return 'C' 120 return '.'.join(lcl) 121 122 def get(self): 123 current_locale = self._dotted_locale_str() 124 if self.last_locale == current_locale: 125 return self.cache 126 127 self.cache = [] 128 locales = [current_locale] 129 if current_locale != 'C': 130 locales.append('C') 131 for l in locales: 132 for nlang in gettext._expand_lang(l): 133 if nlang not in self.cache: 134 self.cache.append(nlang) 135 136 self.last_locale = current_locale 137 return self.cache 138 139 140class CompsQuery(object): 141 142 AVAILABLE = 1 143 INSTALLED = 2 144 145 ENVIRONMENTS = 1 146 GROUPS = 2 147 148 def __init__(self, comps, history, kinds, status): 149 self.comps = comps 150 self.history = history 151 self.kinds = kinds 152 self.status = status 153 154 def _get_groups(self, available, installed): 155 result = set() 156 if self.status & self.AVAILABLE: 157 result.update({i.id for i in available}) 158 if self.status & self.INSTALLED: 159 for i in installed: 160 group = i.getCompsGroupItem() 161 if not group: 162 continue 163 result.add(group.getGroupId()) 164 return result 165 166 def _get_envs(self, available, installed): 167 result = set() 168 if self.status & self.AVAILABLE: 169 result.update({i.id for i in available}) 170 if self.status & self.INSTALLED: 171 for i in installed: 172 env = i.getCompsEnvironmentItem() 173 if not env: 174 continue 175 result.add(env.getEnvironmentId()) 176 return result 177 178 def get(self, *patterns): 179 res = dnf.util.Bunch() 180 res.environments = [] 181 res.groups = [] 182 for pat in patterns: 183 envs = grps = [] 184 if self.kinds & self.ENVIRONMENTS: 185 available = self.comps.environments_by_pattern(pat) 186 installed = self.history.env.search_by_pattern(pat) 187 envs = self._get_envs(available, installed) 188 res.environments.extend(envs) 189 if self.kinds & self.GROUPS: 190 available = self.comps.groups_by_pattern(pat) 191 installed = self.history.group.search_by_pattern(pat) 192 grps = self._get_groups(available, installed) 193 res.groups.extend(grps) 194 if not envs and not grps: 195 if self.status == self.INSTALLED: 196 msg = _("Module or Group '%s' is not installed.") % ucd(pat) 197 elif self.status == self.AVAILABLE: 198 msg = _("Module or Group '%s' is not available.") % ucd(pat) 199 else: 200 msg = _("Module or Group '%s' does not exist.") % ucd(pat) 201 raise CompsError(msg) 202 return res 203 204 205class Forwarder(object): 206 def __init__(self, iobj, langs): 207 self._i = iobj 208 self._langs = langs 209 210 def __getattr__(self, name): 211 return getattr(self._i, name) 212 213 def _ui_text(self, default, dct): 214 for l in self._langs.get(): 215 t = dct.get(l) 216 if t is not None: 217 return t 218 return default 219 220 @property 221 def ui_description(self): 222 return self._ui_text(self.desc, self.desc_by_lang) 223 224 @property 225 def ui_name(self): 226 return self._ui_text(self.name, self.name_by_lang) 227 228class Category(Forwarder): 229 # :api 230 def __init__(self, iobj, langs, group_factory): 231 super(Category, self).__init__(iobj, langs) 232 self._group_factory = group_factory 233 234 def _build_group(self, grp_id): 235 grp = self._group_factory(grp_id.name) 236 if grp is None: 237 msg = "no group '%s' from category '%s'" 238 raise ValueError(msg % (grp_id.name, self.id)) 239 return grp 240 241 def groups_iter(self): 242 for grp_id in self.group_ids: 243 yield self._build_group(grp_id) 244 245 @property 246 def groups(self): 247 return list(self.groups_iter()) 248 249class Environment(Forwarder): 250 # :api 251 252 def __init__(self, iobj, langs, group_factory): 253 super(Environment, self).__init__(iobj, langs) 254 self._group_factory = group_factory 255 256 def _build_group(self, grp_id): 257 grp = self._group_factory(grp_id.name) 258 if grp is None: 259 msg = "no group '%s' from environment '%s'" 260 raise ValueError(msg % (grp_id.name, self.id)) 261 return grp 262 263 def _build_groups(self, ids): 264 groups = [] 265 for gi in ids: 266 try: 267 groups.append(self._build_group(gi)) 268 except ValueError as e: 269 logger.error(e) 270 271 return groups 272 273 def groups_iter(self): 274 for grp_id in itertools.chain(self.group_ids, self.option_ids): 275 try: 276 yield self._build_group(grp_id) 277 except ValueError as e: 278 logger.error(e) 279 280 @property 281 def mandatory_groups(self): 282 return self._build_groups(self.group_ids) 283 284 @property 285 def optional_groups(self): 286 return self._build_groups(self.option_ids) 287 288class Group(Forwarder): 289 # :api 290 def __init__(self, iobj, langs, pkg_factory): 291 super(Group, self).__init__(iobj, langs) 292 self._pkg_factory = pkg_factory 293 self.selected = iobj.default 294 295 def _packages_of_type(self, type_): 296 return [pkg for pkg in self.packages if pkg.type == type_] 297 298 @property 299 def conditional_packages(self): 300 return self._packages_of_type(libcomps.PACKAGE_TYPE_CONDITIONAL) 301 302 @property 303 def default_packages(self): 304 return self._packages_of_type(libcomps.PACKAGE_TYPE_DEFAULT) 305 306 def packages_iter(self): 307 # :api 308 return map(self._pkg_factory, self.packages) 309 310 @property 311 def mandatory_packages(self): 312 return self._packages_of_type(libcomps.PACKAGE_TYPE_MANDATORY) 313 314 @property 315 def optional_packages(self): 316 return self._packages_of_type(libcomps.PACKAGE_TYPE_OPTIONAL) 317 318 @property 319 def visible(self): 320 return self._i.uservisible 321 322class Package(Forwarder): 323 """Represents comps package data. :api""" 324 325 _OPT_MAP = { 326 libcomps.PACKAGE_TYPE_CONDITIONAL : CONDITIONAL, 327 libcomps.PACKAGE_TYPE_DEFAULT : DEFAULT, 328 libcomps.PACKAGE_TYPE_MANDATORY : MANDATORY, 329 libcomps.PACKAGE_TYPE_OPTIONAL : OPTIONAL, 330 } 331 332 def __init__(self, ipkg): 333 self._i = ipkg 334 335 @property 336 def name(self): 337 # :api 338 return self._i.name 339 340 @property 341 def option_type(self): 342 # :api 343 return self._OPT_MAP[self.type] 344 345class Comps(object): 346 # :api 347 348 def __init__(self): 349 self._i = libcomps.Comps() 350 self._langs = _Langs() 351 352 def __len__(self): 353 return _internal_comps_length(self._i) 354 355 def _build_category(self, icategory): 356 return Category(icategory, self._langs, self._group_by_id) 357 358 def _build_environment(self, ienvironment): 359 return Environment(ienvironment, self._langs, self._group_by_id) 360 361 def _build_group(self, igroup): 362 return Group(igroup, self._langs, self._build_package) 363 364 def _build_package(self, ipkg): 365 return Package(ipkg) 366 367 def _add_from_xml_filename(self, fn): 368 comps = libcomps.Comps() 369 try: 370 comps.fromxml_f(fn) 371 except libcomps.ParserError: 372 errors = comps.get_last_errors() 373 raise CompsError(' '.join(errors)) 374 self._i += comps 375 376 @property 377 def categories(self): 378 # :api 379 return list(self.categories_iter()) 380 381 def category_by_pattern(self, pattern, case_sensitive=False): 382 # :api 383 assert dnf.util.is_string_type(pattern) 384 cats = self.categories_by_pattern(pattern, case_sensitive) 385 return _first_if_iterable(cats) 386 387 def categories_by_pattern(self, pattern, case_sensitive=False): 388 # :api 389 assert dnf.util.is_string_type(pattern) 390 return _by_pattern(pattern, case_sensitive, self.categories) 391 392 def categories_iter(self): 393 # :api 394 return (self._build_category(c) for c in self._i.categories) 395 396 @property 397 def environments(self): 398 # :api 399 return sorted(self.environments_iter(), key=_fn_display_order) 400 401 def _environment_by_id(self, id): 402 assert dnf.util.is_string_type(id) 403 return dnf.util.first(g for g in self.environments_iter() if g.id == id) 404 405 def environment_by_pattern(self, pattern, case_sensitive=False): 406 # :api 407 assert dnf.util.is_string_type(pattern) 408 envs = self.environments_by_pattern(pattern, case_sensitive) 409 return _first_if_iterable(envs) 410 411 def environments_by_pattern(self, pattern, case_sensitive=False): 412 # :api 413 assert dnf.util.is_string_type(pattern) 414 envs = list(self.environments_iter()) 415 found_envs = _by_pattern(pattern, case_sensitive, envs) 416 return sorted(found_envs, key=_fn_display_order) 417 418 def environments_iter(self): 419 # :api 420 return (self._build_environment(e) for e in self._i.environments) 421 422 @property 423 def groups(self): 424 # :api 425 return sorted(self.groups_iter(), key=_fn_display_order) 426 427 def _group_by_id(self, id_): 428 assert dnf.util.is_string_type(id_) 429 return dnf.util.first(g for g in self.groups_iter() if g.id == id_) 430 431 def group_by_pattern(self, pattern, case_sensitive=False): 432 # :api 433 assert dnf.util.is_string_type(pattern) 434 grps = self.groups_by_pattern(pattern, case_sensitive) 435 return _first_if_iterable(grps) 436 437 def groups_by_pattern(self, pattern, case_sensitive=False): 438 # :api 439 assert dnf.util.is_string_type(pattern) 440 grps = _by_pattern(pattern, case_sensitive, list(self.groups_iter())) 441 return sorted(grps, key=_fn_display_order) 442 443 def groups_iter(self): 444 # :api 445 return (self._build_group(g) for g in self._i.groups) 446 447class CompsTransPkg(object): 448 def __init__(self, pkg_or_name): 449 if dnf.util.is_string_type(pkg_or_name): 450 # from package name 451 self.basearchonly = False 452 self.name = pkg_or_name 453 self.optional = True 454 self.requires = None 455 elif isinstance(pkg_or_name, libdnf.transaction.CompsGroupPackage): 456 # from swdb package 457 # TODO: 458 self.basearchonly = False 459 # self.basearchonly = pkg_or_name.basearchonly 460 self.name = pkg_or_name.getName() 461 self.optional = pkg_or_name.getPackageType() & libcomps.PACKAGE_TYPE_OPTIONAL 462 # TODO: 463 self.requires = None 464 # self.requires = pkg_or_name.requires 465 else: 466 # from comps package 467 self.basearchonly = pkg_or_name.basearchonly 468 self.name = pkg_or_name.name 469 self.optional = pkg_or_name.type & libcomps.PACKAGE_TYPE_OPTIONAL 470 self.requires = pkg_or_name.requires 471 472 def __eq__(self, other): 473 return (self.name == other.name and 474 self.basearchonly == other.basearchonly and 475 self.optional == other.optional and 476 self.requires == other.requires) 477 478 def __str__(self): 479 return self.name 480 481 def __hash__(self): 482 return hash((self.name, 483 self.basearchonly, 484 self.optional, 485 self.requires)) 486 487class TransactionBunch(object): 488 def __init__(self): 489 self._install = set() 490 self._install_opt = set() 491 self._remove = set() 492 self._upgrade = set() 493 494 def __iadd__(self, other): 495 self._install.update(other._install) 496 self._install_opt.update(other._install_opt) 497 self._upgrade.update(other._upgrade) 498 self._remove = (self._remove | other._remove) - \ 499 self._install - self._install_opt - self._upgrade 500 return self 501 502 def __len__(self): 503 return len(self.install) + len(self.install_opt) + len(self.upgrade) + len(self.remove) 504 505 @staticmethod 506 def _set_value(param, val): 507 for item in val: 508 if isinstance(item, CompsTransPkg): 509 param.add(item) 510 else: 511 param.add(CompsTransPkg(item)) 512 513 @property 514 def install(self): 515 """ 516 Packages to be installed with strict=True - transaction will 517 fail if they cannot be installed due to dependency errors etc. 518 """ 519 return self._install 520 521 @install.setter 522 def install(self, value): 523 self._set_value(self._install, value) 524 525 @property 526 def install_opt(self): 527 """ 528 Packages to be installed with strict=False - they will be 529 skipped if they cannot be installed 530 """ 531 return self._install_opt 532 533 @install_opt.setter 534 def install_opt(self, value): 535 self._set_value(self._install_opt, value) 536 537 @property 538 def remove(self): 539 return self._remove 540 541 @remove.setter 542 def remove(self, value): 543 self._set_value(self._remove, value) 544 545 @property 546 def upgrade(self): 547 return self._upgrade 548 549 @upgrade.setter 550 def upgrade(self, value): 551 self._set_value(self._upgrade, value) 552 553 554class Solver(object): 555 def __init__(self, history, comps, reason_fn): 556 self.history = history 557 self.comps = comps 558 self._reason_fn = reason_fn 559 560 @staticmethod 561 def _mandatory_group_set(env): 562 return {grp.id for grp in env.mandatory_groups} 563 564 @staticmethod 565 def _full_package_set(grp): 566 return {pkg.getName() for pkg in grp.mandatory_packages + 567 grp.default_packages + grp.optional_packages + 568 grp.conditional_packages} 569 570 @staticmethod 571 def _pkgs_of_type(group, pkg_types, exclude=[]): 572 def filter(pkgs): 573 return [pkg for pkg in pkgs 574 if pkg.name not in exclude] 575 576 pkgs = set() 577 if pkg_types & MANDATORY: 578 pkgs.update(filter(group.mandatory_packages)) 579 if pkg_types & DEFAULT: 580 pkgs.update(filter(group.default_packages)) 581 if pkg_types & OPTIONAL: 582 pkgs.update(filter(group.optional_packages)) 583 if pkg_types & CONDITIONAL: 584 pkgs.update(filter(group.conditional_packages)) 585 return pkgs 586 587 def _removable_pkg(self, pkg_name): 588 assert dnf.util.is_string_type(pkg_name) 589 return self.history.group.is_removable_pkg(pkg_name) 590 591 def _removable_grp(self, group_id): 592 assert dnf.util.is_string_type(group_id) 593 return self.history.env.is_removable_group(group_id) 594 595 def _environment_install(self, env_id, pkg_types, exclude, strict=True, exclude_groups=None): 596 assert dnf.util.is_string_type(env_id) 597 comps_env = self.comps._environment_by_id(env_id) 598 if not comps_env: 599 raise CompsError(_("Environment id '%s' does not exist.") % ucd(env_id)) 600 601 swdb_env = self.history.env.new(env_id, comps_env.name, comps_env.ui_name, pkg_types) 602 self.history.env.install(swdb_env) 603 604 trans = TransactionBunch() 605 for comps_group in comps_env.mandatory_groups: 606 if exclude_groups and comps_group.id in exclude_groups: 607 continue 608 trans += self._group_install(comps_group.id, pkg_types, exclude, strict) 609 swdb_env.addGroup(comps_group.id, True, MANDATORY) 610 611 for comps_group in comps_env.optional_groups: 612 if exclude_groups and comps_group.id in exclude_groups: 613 continue 614 swdb_env.addGroup(comps_group.id, False, OPTIONAL) 615 # TODO: if a group is already installed, mark it as installed? 616 return trans 617 618 def _environment_remove(self, env_id): 619 assert dnf.util.is_string_type(env_id) is True 620 swdb_env = self.history.env.get(env_id) 621 if not swdb_env: 622 raise CompsError(_("Environment id '%s' is not installed.") % env_id) 623 624 self.history.env.remove(swdb_env) 625 626 trans = TransactionBunch() 627 group_ids = set([i.getGroupId() for i in swdb_env.getGroups()]) 628 for group_id in group_ids: 629 if not self._removable_grp(group_id): 630 continue 631 trans += self._group_remove(group_id) 632 return trans 633 634 def _environment_upgrade(self, env_id): 635 assert dnf.util.is_string_type(env_id) 636 comps_env = self.comps._environment_by_id(env_id) 637 swdb_env = self.history.env.get(env_id) 638 if not swdb_env: 639 raise CompsError(_("Environment '%s' is not installed.") % env_id) 640 if not comps_env: 641 raise CompsError(_("Environment '%s' is not available.") % env_id) 642 643 old_set = set([i.getGroupId() for i in swdb_env.getGroups()]) 644 pkg_types = swdb_env.getPackageTypes() 645 646 # create a new record for current transaction 647 swdb_env = self.history.env.new(comps_env.id, comps_env.name, comps_env.ui_name, pkg_types) 648 649 trans = TransactionBunch() 650 for comps_group in comps_env.mandatory_groups: 651 if comps_group.id in old_set: 652 if self.history.group.get(comps_group.id): 653 # upgrade installed group 654 trans += self._group_upgrade(comps_group.id) 655 else: 656 # install new group 657 trans += self._group_install(comps_group.id, pkg_types) 658 swdb_env.addGroup(comps_group.id, True, MANDATORY) 659 660 for comps_group in comps_env.optional_groups: 661 if comps_group.id in old_set and self.history.group.get(comps_group.id): 662 # upgrade installed group 663 trans += self._group_upgrade(comps_group.id) 664 swdb_env.addGroup(comps_group.id, False, OPTIONAL) 665 # TODO: if a group is already installed, mark it as installed? 666 self.history.env.upgrade(swdb_env) 667 return trans 668 669 def _group_install(self, group_id, pkg_types, exclude=None, strict=True, exclude_groups=None): 670 assert dnf.util.is_string_type(group_id) 671 comps_group = self.comps._group_by_id(group_id) 672 if not comps_group: 673 raise CompsError(_("Group id '%s' does not exist.") % ucd(group_id)) 674 675 swdb_group = self.history.group.new(group_id, comps_group.name, comps_group.ui_name, pkg_types) 676 for i in comps_group.packages_iter(): 677 swdb_group.addPackage(i.name, False, Package._OPT_MAP[i.type]) 678 self.history.group.install(swdb_group) 679 680 trans = TransactionBunch() 681 # TODO: remove exclude 682 if strict: 683 trans.install.update(self._pkgs_of_type(comps_group, pkg_types, exclude=[])) 684 else: 685 trans.install_opt.update(self._pkgs_of_type(comps_group, pkg_types, exclude=[])) 686 return trans 687 688 def _group_remove(self, group_id): 689 assert dnf.util.is_string_type(group_id) 690 swdb_group = self.history.group.get(group_id) 691 if not swdb_group: 692 raise CompsError(_("Module or Group '%s' is not installed.") % group_id) 693 self.history.group.remove(swdb_group) 694 trans = TransactionBunch() 695 trans.remove = {pkg for pkg in swdb_group.getPackages() if self._removable_pkg(pkg.getName())} 696 return trans 697 698 def _group_upgrade(self, group_id): 699 assert dnf.util.is_string_type(group_id) 700 comps_group = self.comps._group_by_id(group_id) 701 swdb_group = self.history.group.get(group_id) 702 exclude = [] 703 704 if not swdb_group: 705 argument = comps_group.ui_name if comps_group else group_id 706 raise CompsError(_("Module or Group '%s' is not installed.") % argument) 707 if not comps_group: 708 raise CompsError(_("Module or Group '%s' is not available.") % group_id) 709 pkg_types = swdb_group.getPackageTypes() 710 old_set = set([i.getName() for i in swdb_group.getPackages()]) 711 new_set = self._pkgs_of_type(comps_group, pkg_types, exclude) 712 713 # create a new record for current transaction 714 swdb_group = self.history.group.new(group_id, comps_group.name, comps_group.ui_name, pkg_types) 715 for i in comps_group.packages_iter(): 716 swdb_group.addPackage(i.name, False, Package._OPT_MAP[i.type]) 717 self.history.group.upgrade(swdb_group) 718 719 trans = TransactionBunch() 720 trans.install = {pkg for pkg in new_set if pkg.name not in old_set} 721 trans.remove = {name for name in old_set 722 if name not in [pkg.name for pkg in new_set]} 723 trans.upgrade = {pkg for pkg in new_set if pkg.name in old_set} 724 return trans 725 726 def _exclude_packages_from_installed_groups(self, base): 727 for group in self.persistor.groups: 728 p_grp = self.persistor.group(group) 729 if p_grp.installed: 730 installed_pkg_names = \ 731 set(p_grp.full_list) - set(p_grp.pkg_exclude) 732 installed_pkgs = base.sack.query().installed().filterm(name=installed_pkg_names) 733 for pkg in installed_pkgs: 734 base._goal.install(pkg) 735