1# Copyright (C) 2020 Red Hat, Inc. 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation; either version 2 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU Library General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import unicode_literals 20 21import libdnf 22import hawkey 23 24from dnf.i18n import _ 25import dnf.exceptions 26 27import json 28 29 30VERSION_MAJOR = 0 31VERSION_MINOR = 0 32VERSION = "%s.%s" % (VERSION_MAJOR, VERSION_MINOR) 33""" 34The version of the stored transaction. 35 36MAJOR version denotes backwards incompatible changes (old dnf won't work with 37new transaction JSON). 38 39MINOR version denotes extending the format without breaking backwards 40compatibility (old dnf can work with new transaction JSON). Forwards 41compatibility needs to be handled by being able to process the old format as 42well as the new one. 43""" 44 45 46class TransactionError(dnf.exceptions.Error): 47 def __init__(self, msg): 48 super(TransactionError, self).__init__(msg) 49 50 51class TransactionReplayError(dnf.exceptions.Error): 52 def __init__(self, filename, errors): 53 """ 54 :param filename: The name of the transaction file being replayed 55 :param errors: a list of error classes or a string with an error description 56 """ 57 58 # store args in case someone wants to read them from a caught exception 59 self.filename = filename 60 if isinstance(errors, (list, tuple)): 61 self.errors = errors 62 else: 63 self.errors = [errors] 64 65 if filename: 66 msg = _('The following problems occurred while replaying the transaction from file "{filename}":').format(filename=filename) 67 else: 68 msg = _('The following problems occurred while running a transaction:') 69 70 for error in self.errors: 71 msg += "\n " + str(error) 72 73 super(TransactionReplayError, self).__init__(msg) 74 75 76class IncompatibleTransactionVersionError(TransactionReplayError): 77 def __init__(self, filename, msg): 78 super(IncompatibleTransactionVersionError, self).__init__(filename, msg) 79 80 81def _check_version(version, filename): 82 major, minor = version.split('.') 83 84 try: 85 major = int(major) 86 except ValueError as e: 87 raise TransactionReplayError( 88 filename, 89 _('Invalid major version "{major}", number expected.').format(major=major) 90 ) 91 92 try: 93 int(minor) # minor is unused, just check it's a number 94 except ValueError as e: 95 raise TransactionReplayError( 96 filename, 97 _('Invalid minor version "{minor}", number expected.').format(minor=minor) 98 ) 99 100 if major != VERSION_MAJOR: 101 raise IncompatibleTransactionVersionError( 102 filename, 103 _('Incompatible major version "{major}", supported major version is "{major_supp}".') 104 .format(major=major, major_supp=VERSION_MAJOR) 105 ) 106 107 108def serialize_transaction(transaction): 109 """ 110 Serializes a transaction to a data structure that is equivalent to the stored JSON format. 111 :param transaction: the transaction to serialize (an instance of dnf.db.history.TransactionWrapper) 112 """ 113 114 data = { 115 "version": VERSION, 116 } 117 rpms = [] 118 groups = [] 119 environments = [] 120 121 if transaction is None: 122 return data 123 124 for tsi in transaction.packages(): 125 if tsi.is_package(): 126 rpms.append({ 127 "action": tsi.action_name, 128 "nevra": tsi.nevra, 129 "reason": libdnf.transaction.TransactionItemReasonToString(tsi.reason), 130 "repo_id": tsi.from_repo 131 }) 132 133 elif tsi.is_group(): 134 group = tsi.get_group() 135 136 group_data = { 137 "action": tsi.action_name, 138 "id": group.getGroupId(), 139 "packages": [], 140 "package_types": libdnf.transaction.compsPackageTypeToString(group.getPackageTypes()) 141 } 142 143 for pkg in group.getPackages(): 144 group_data["packages"].append({ 145 "name": pkg.getName(), 146 "installed": pkg.getInstalled(), 147 "package_type": libdnf.transaction.compsPackageTypeToString(pkg.getPackageType()) 148 }) 149 150 groups.append(group_data) 151 152 elif tsi.is_environment(): 153 env = tsi.get_environment() 154 155 env_data = { 156 "action": tsi.action_name, 157 "id": env.getEnvironmentId(), 158 "groups": [], 159 "package_types": libdnf.transaction.compsPackageTypeToString(env.getPackageTypes()) 160 } 161 162 for grp in env.getGroups(): 163 env_data["groups"].append({ 164 "id": grp.getGroupId(), 165 "installed": grp.getInstalled(), 166 "group_type": libdnf.transaction.compsPackageTypeToString(grp.getGroupType()) 167 }) 168 169 environments.append(env_data) 170 171 if rpms: 172 data["rpms"] = rpms 173 174 if groups: 175 data["groups"] = groups 176 177 if environments: 178 data["environments"] = environments 179 180 return data 181 182 183class TransactionReplay(object): 184 """ 185 A class that encapsulates replaying a transaction. The transaction data are 186 loaded and stored when the class is initialized. The transaction is run by 187 calling the `run()` method, after the transaction is created (but before it is 188 performed), the `post_transaction()` method needs to be called to verify no 189 extra packages were pulled in and also to fix the reasons. 190 """ 191 192 def __init__( 193 self, 194 base, 195 filename="", 196 data=None, 197 ignore_extras=False, 198 ignore_installed=False, 199 skip_unavailable=False 200 ): 201 """ 202 :param base: the dnf base 203 :param filename: the filename to load the transaction from (conflicts with the 'data' argument) 204 :param data: the dictionary to load the transaction from (conflicts with the 'filename' argument) 205 :param ignore_extras: whether to ignore extra package pulled into the transaction 206 :param ignore_installed: whether to ignore installed versions of packages 207 :param skip_unavailable: whether to skip transaction packages that aren't available 208 """ 209 210 self._base = base 211 self._filename = filename 212 self._ignore_installed = ignore_installed 213 self._ignore_extras = ignore_extras 214 self._skip_unavailable = skip_unavailable 215 216 if not self._base.conf.strict: 217 self._skip_unavailable = True 218 219 self._nevra_cache = set() 220 self._nevra_reason_cache = {} 221 self._warnings = [] 222 223 if filename and data: 224 raise ValueError(_("Conflicting TransactionReplay arguments have been specified: filename, data")) 225 elif filename: 226 self._load_from_file(filename) 227 else: 228 self._load_from_data(data) 229 230 231 def _load_from_file(self, fn): 232 self._filename = fn 233 with open(fn, "r") as f: 234 try: 235 replay_data = json.load(f) 236 except json.decoder.JSONDecodeError as e: 237 raise TransactionReplayError(fn, str(e) + ".") 238 239 try: 240 self._load_from_data(replay_data) 241 except TransactionError as e: 242 raise TransactionReplayError(fn, e) 243 244 def _load_from_data(self, data): 245 self._replay_data = data 246 self._verify_toplevel_json(self._replay_data) 247 248 self._rpms = self._replay_data.get("rpms", []) 249 self._assert_type(self._rpms, list, "rpms", "array") 250 251 self._groups = self._replay_data.get("groups", []) 252 self._assert_type(self._groups, list, "groups", "array") 253 254 self._environments = self._replay_data.get("environments", []) 255 self._assert_type(self._environments, list, "environments", "array") 256 257 def _raise_or_warn(self, warn_only, msg): 258 if warn_only: 259 self._warnings.append(msg) 260 else: 261 raise TransactionError(msg) 262 263 def _assert_type(self, value, t, id, expected): 264 if not isinstance(value, t): 265 raise TransactionError(_('Unexpected type of "{id}", {exp} expected.').format(id=id, exp=expected)) 266 267 def _verify_toplevel_json(self, replay_data): 268 fn = self._filename 269 270 if "version" not in replay_data: 271 raise TransactionReplayError(fn, _('Missing key "{key}".'.format(key="version"))) 272 273 self._assert_type(replay_data["version"], str, "version", "string") 274 275 _check_version(replay_data["version"], fn) 276 277 def _replay_pkg_action(self, pkg_data): 278 try: 279 action = pkg_data["action"] 280 nevra = pkg_data["nevra"] 281 repo_id = pkg_data["repo_id"] 282 reason = libdnf.transaction.StringToTransactionItemReason(pkg_data["reason"]) 283 except KeyError as e: 284 raise TransactionError( 285 _('Missing object key "{key}" in an rpm.').format(key=e.args[0]) 286 ) 287 except IndexError as e: 288 raise TransactionError( 289 _('Unexpected value of package reason "{reason}" for rpm nevra "{nevra}".') 290 .format(reason=pkg_data["reason"], nevra=nevra) 291 ) 292 293 subj = hawkey.Subject(nevra) 294 parsed_nevras = subj.get_nevra_possibilities(forms=[hawkey.FORM_NEVRA]) 295 296 if len(parsed_nevras) != 1: 297 raise TransactionError(_('Cannot parse NEVRA for package "{nevra}".').format(nevra=nevra)) 298 299 parsed_nevra = parsed_nevras[0] 300 na = "%s.%s" % (parsed_nevra.name, parsed_nevra.arch) 301 302 query_na = self._base.sack.query().filter(name=parsed_nevra.name, arch=parsed_nevra.arch) 303 304 epoch = parsed_nevra.epoch if parsed_nevra.epoch is not None else 0 305 query = query_na.filter(epoch=epoch, version=parsed_nevra.version, release=parsed_nevra.release) 306 307 # In case the package is found in the same repo as in the original 308 # transaction, limit the query to that plus installed packages. IOW 309 # remove packages with the same NEVRA in case they are found in 310 # multiple repos and the repo the package came from originally is one 311 # of them. 312 # This can e.g. make a difference in the system-upgrade plugin, in case 313 # the same NEVRA is in two repos, this makes sure the same repo is used 314 # for both download and upgrade steps of the plugin. 315 if repo_id: 316 query_repo = query.filter(reponame=repo_id) 317 if query_repo: 318 query = query_repo.union(query.installed()) 319 320 if not query: 321 self._raise_or_warn(self._skip_unavailable, _('Cannot find rpm nevra "{nevra}".').format(nevra=nevra)) 322 return 323 324 # a cache to check no extra packages were pulled into the transaction 325 if action != "Reason Change": 326 self._nevra_cache.add(nevra) 327 328 # store reasons for forward actions and "Removed", the rest of the 329 # actions reasons should stay as they were determined by the transaction 330 if action in ("Install", "Upgrade", "Downgrade", "Reinstall", "Removed"): 331 self._nevra_reason_cache[nevra] = reason 332 333 if action in ("Install", "Upgrade", "Downgrade"): 334 if action == "Install" and query_na.installed() and not self._base._get_installonly_query(query_na): 335 self._raise_or_warn(self._ignore_installed, 336 _('Package "{na}" is already installed for action "{action}".').format(na=na, action=action)) 337 338 sltr = dnf.selector.Selector(self._base.sack).set(pkg=query) 339 self._base.goal.install(select=sltr, optional=not self._base.conf.strict) 340 elif action == "Reinstall": 341 query = query.available() 342 343 if not query: 344 self._raise_or_warn(self._skip_unavailable, 345 _('Package nevra "{nevra}" not available in repositories for action "{action}".') 346 .format(nevra=nevra, action=action)) 347 return 348 349 sltr = dnf.selector.Selector(self._base.sack).set(pkg=query) 350 self._base.goal.install(select=sltr, optional=not self._base.conf.strict) 351 elif action in ("Upgraded", "Downgraded", "Reinstalled", "Removed", "Obsoleted"): 352 query = query.installed() 353 354 if not query: 355 self._raise_or_warn(self._ignore_installed, 356 _('Package nevra "{nevra}" not installed for action "{action}".').format(nevra=nevra, action=action)) 357 return 358 359 # erasing the original version (the reverse part of an action like 360 # e.g. upgrade) is more robust, but we can't do it if 361 # skip_unavailable is True, because if the forward part of the 362 # action is skipped, we would simply remove the package here 363 if not self._skip_unavailable or action == "Removed": 364 for pkg in query: 365 self._base.goal.erase(pkg, clean_deps=False) 366 elif action == "Reason Change": 367 self._base.history.set_reason(query[0], reason) 368 else: 369 raise TransactionError( 370 _('Unexpected value of package action "{action}" for rpm nevra "{nevra}".') 371 .format(action=action, nevra=nevra) 372 ) 373 374 def _create_swdb_group(self, group_id, pkg_types, pkgs): 375 comps_group = self._base.comps._group_by_id(group_id) 376 if not comps_group: 377 self._raise_or_warn(self._skip_unavailable, _("Group id '%s' is not available.") % group_id) 378 return None 379 380 swdb_group = self._base.history.group.new(group_id, comps_group.name, comps_group.ui_name, pkg_types) 381 382 try: 383 for pkg in pkgs: 384 name = pkg["name"] 385 self._assert_type(name, str, "groups.packages.name", "string") 386 installed = pkg["installed"] 387 self._assert_type(installed, bool, "groups.packages.installed", "boolean") 388 package_type = pkg["package_type"] 389 self._assert_type(package_type, str, "groups.packages.package_type", "string") 390 391 try: 392 swdb_group.addPackage(name, installed, libdnf.transaction.stringToCompsPackageType(package_type)) 393 except libdnf.error.Error as e: 394 raise TransactionError(str(e)) 395 396 except KeyError as e: 397 raise TransactionError( 398 _('Missing object key "{key}" in groups.packages.').format(key=e.args[0]) 399 ) 400 401 return swdb_group 402 403 def _swdb_group_install(self, group_id, pkg_types, pkgs): 404 swdb_group = self._create_swdb_group(group_id, pkg_types, pkgs) 405 406 if swdb_group is not None: 407 self._base.history.group.install(swdb_group) 408 409 def _swdb_group_upgrade(self, group_id, pkg_types, pkgs): 410 if not self._base.history.group.get(group_id): 411 self._raise_or_warn( self._ignore_installed, _("Group id '%s' is not installed.") % group_id) 412 return 413 414 swdb_group = self._create_swdb_group(group_id, pkg_types, pkgs) 415 416 if swdb_group is not None: 417 self._base.history.group.upgrade(swdb_group) 418 419 def _swdb_group_remove(self, group_id, pkg_types, pkgs): 420 if not self._base.history.group.get(group_id): 421 self._raise_or_warn(self._ignore_installed, _("Group id '%s' is not installed.") % group_id) 422 return 423 424 swdb_group = self._create_swdb_group(group_id, pkg_types, pkgs) 425 426 if swdb_group is not None: 427 self._base.history.group.remove(swdb_group) 428 429 def _create_swdb_environment(self, env_id, pkg_types, groups): 430 comps_env = self._base.comps._environment_by_id(env_id) 431 if not comps_env: 432 self._raise_or_warn(self._skip_unavailable, _("Environment id '%s' is not available.") % env_id) 433 return None 434 435 swdb_env = self._base.history.env.new(env_id, comps_env.name, comps_env.ui_name, pkg_types) 436 437 try: 438 for grp in groups: 439 id = grp["id"] 440 self._assert_type(id, str, "environments.groups.id", "string") 441 installed = grp["installed"] 442 self._assert_type(installed, bool, "environments.groups.installed", "boolean") 443 group_type = grp["group_type"] 444 self._assert_type(group_type, str, "environments.groups.group_type", "string") 445 446 try: 447 group_type = libdnf.transaction.stringToCompsPackageType(group_type) 448 except libdnf.error.Error as e: 449 raise TransactionError(str(e)) 450 451 if group_type not in ( 452 libdnf.transaction.CompsPackageType_MANDATORY, 453 libdnf.transaction.CompsPackageType_OPTIONAL 454 ): 455 raise TransactionError( 456 _('Invalid value "{group_type}" of environments.groups.group_type, ' 457 'only "mandatory" or "optional" is supported.' 458 ).format(group_type=grp["group_type"]) 459 ) 460 461 swdb_env.addGroup(id, installed, group_type) 462 except KeyError as e: 463 raise TransactionError( 464 _('Missing object key "{key}" in environments.groups.').format(key=e.args[0]) 465 ) 466 467 return swdb_env 468 469 def _swdb_environment_install(self, env_id, pkg_types, groups): 470 swdb_env = self._create_swdb_environment(env_id, pkg_types, groups) 471 472 if swdb_env is not None: 473 self._base.history.env.install(swdb_env) 474 475 def _swdb_environment_upgrade(self, env_id, pkg_types, groups): 476 if not self._base.history.env.get(env_id): 477 self._raise_or_warn(self._ignore_installed,_("Environment id '%s' is not installed.") % env_id) 478 return 479 480 swdb_env = self._create_swdb_environment(env_id, pkg_types, groups) 481 482 if swdb_env is not None: 483 self._base.history.env.upgrade(swdb_env) 484 485 def _swdb_environment_remove(self, env_id, pkg_types, groups): 486 if not self._base.history.env.get(env_id): 487 self._raise_or_warn(self._ignore_installed, _("Environment id '%s' is not installed.") % env_id) 488 return 489 490 swdb_env = self._create_swdb_environment(env_id, pkg_types, groups) 491 492 if swdb_env is not None: 493 self._base.history.env.remove(swdb_env) 494 495 def get_data(self): 496 """ 497 :returns: the loaded data of the transaction 498 """ 499 500 return self._replay_data 501 502 def get_warnings(self): 503 """ 504 :returns: an array of warnings gathered during the transaction replay 505 """ 506 507 return self._warnings 508 509 def run(self): 510 """ 511 Replays the transaction. 512 """ 513 514 fn = self._filename 515 errors = [] 516 517 for pkg_data in self._rpms: 518 try: 519 self._replay_pkg_action(pkg_data) 520 except TransactionError as e: 521 errors.append(e) 522 523 for group_data in self._groups: 524 try: 525 action = group_data["action"] 526 group_id = group_data["id"] 527 528 try: 529 pkg_types = libdnf.transaction.stringToCompsPackageType(group_data["package_types"]) 530 except libdnf.error.Error as e: 531 errors.append(TransactionError(str(e))) 532 continue 533 534 if action == "Install": 535 self._swdb_group_install(group_id, pkg_types, group_data["packages"]) 536 elif action == "Upgrade": 537 self._swdb_group_upgrade(group_id, pkg_types, group_data["packages"]) 538 elif action == "Removed": 539 self._swdb_group_remove(group_id, pkg_types, group_data["packages"]) 540 else: 541 errors.append(TransactionError( 542 _('Unexpected value of group action "{action}" for group "{group}".') 543 .format(action=action, group=group_id) 544 )) 545 except KeyError as e: 546 errors.append(TransactionError( 547 _('Missing object key "{key}" in a group.').format(key=e.args[0]) 548 )) 549 except TransactionError as e: 550 errors.append(e) 551 552 for env_data in self._environments: 553 try: 554 action = env_data["action"] 555 env_id = env_data["id"] 556 557 try: 558 pkg_types = libdnf.transaction.stringToCompsPackageType(env_data["package_types"]) 559 except libdnf.error.Error as e: 560 errors.append(TransactionError(str(e))) 561 continue 562 563 if action == "Install": 564 self._swdb_environment_install(env_id, pkg_types, env_data["groups"]) 565 elif action == "Upgrade": 566 self._swdb_environment_upgrade(env_id, pkg_types, env_data["groups"]) 567 elif action == "Removed": 568 self._swdb_environment_remove(env_id, pkg_types, env_data["groups"]) 569 else: 570 errors.append(TransactionError( 571 _('Unexpected value of environment action "{action}" for environment "{env}".') 572 .format(action=action, env=env_id) 573 )) 574 except KeyError as e: 575 errors.append(TransactionError( 576 _('Missing object key "{key}" in an environment.').format(key=e.args[0]) 577 )) 578 except TransactionError as e: 579 errors.append(e) 580 581 if errors: 582 raise TransactionReplayError(fn, errors) 583 584 def post_transaction(self): 585 """ 586 Sets reasons in the transaction history to values from the stored transaction. 587 588 Also serves to check whether additional packages were pulled in by the 589 transaction, which results in an error (unless ignore_extras is True). 590 """ 591 592 if not self._base.transaction: 593 return 594 595 errors = [] 596 597 for tsi in self._base.transaction: 598 try: 599 pkg = tsi.pkg 600 except KeyError as e: 601 # the transaction item has no package, happens for action == "Reason Change" 602 continue 603 604 nevra = str(pkg) 605 606 if nevra not in self._nevra_cache: 607 # if ignore_installed is True, we don't want to check for 608 # Upgraded/Downgraded/Reinstalled extras in the transaction, 609 # basically those may be installed and we are ignoring them 610 if not self._ignore_installed or not tsi.action in ( 611 libdnf.transaction.TransactionItemAction_UPGRADED, 612 libdnf.transaction.TransactionItemAction_DOWNGRADED, 613 libdnf.transaction.TransactionItemAction_REINSTALLED 614 ): 615 msg = _('Package nevra "{nevra}", which is not present in the transaction file, was pulled ' 616 'into the transaction.' 617 ).format(nevra=nevra) 618 619 if not self._ignore_extras: 620 errors.append(TransactionError(msg)) 621 else: 622 self._warnings.append(msg) 623 624 try: 625 replay_reason = self._nevra_reason_cache[nevra] 626 627 if tsi.action in ( 628 libdnf.transaction.TransactionItemAction_INSTALL, 629 libdnf.transaction.TransactionItemAction_REMOVE 630 ) or libdnf.transaction.TransactionItemReasonCompare(replay_reason, tsi.reason) > 0: 631 tsi.reason = replay_reason 632 except KeyError as e: 633 # if the pkg nevra wasn't found, we don't want to change the reason 634 pass 635 636 if errors: 637 raise TransactionReplayError(self._filename, errors) 638