1# Copyright (C) 2009-2010 Canonical Ltd. 2# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. 3# 4# Author: Scott Moser <scott.moser@canonical.com> 5# Author: Juerg Haefliger <juerg.haefliger@hp.com> 6# 7# This file is part of cloud-init. See LICENSE file for license information. 8 9"""Apt Configure: Configure apt for the user.""" 10 11import glob 12import os 13import re 14import pathlib 15from textwrap import dedent 16 17from cloudinit.config.schema import ( 18 get_schema_doc, validate_cloudconfig_schema) 19from cloudinit import gpg 20from cloudinit import log as logging 21from cloudinit import subp 22from cloudinit import templater 23from cloudinit import util 24from cloudinit.settings import PER_INSTANCE 25 26LOG = logging.getLogger(__name__) 27 28# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') 29ADD_APT_REPO_MATCH = r"^[\w-]+:\w" 30 31APT_LOCAL_KEYS = '/etc/apt/trusted.gpg' 32APT_TRUSTED_GPG_DIR = '/etc/apt/trusted.gpg.d/' 33CLOUD_INIT_GPG_DIR = '/etc/apt/cloud-init.gpg.d/' 34 35frequency = PER_INSTANCE 36distros = ["ubuntu", "debian"] 37mirror_property = { 38 'type': 'array', 39 'items': { 40 'type': 'object', 41 'additionalProperties': False, 42 'required': ['arches'], 43 'properties': { 44 'arches': { 45 'type': 'array', 46 'items': { 47 'type': 'string' 48 }, 49 'minItems': 1 50 }, 51 'uri': { 52 'type': 'string', 53 'format': 'uri' 54 }, 55 'search': { 56 'type': 'array', 57 'items': { 58 'type': 'string', 59 'format': 'uri' 60 }, 61 'minItems': 1 62 }, 63 'search_dns': { 64 'type': 'boolean', 65 }, 66 'keyid': { 67 'type': 'string' 68 }, 69 'key': { 70 'type': 'string' 71 }, 72 'keyserver': { 73 'type': 'string' 74 } 75 } 76 } 77} 78schema = { 79 'id': 'cc_apt_configure', 80 'name': 'Apt Configure', 81 'title': 'Configure apt for the user', 82 'description': dedent("""\ 83 This module handles both configuration of apt options and adding 84 source lists. There are configuration options such as 85 ``apt_get_wrapper`` and ``apt_get_command`` that control how 86 cloud-init invokes apt-get. These configuration options are 87 handled on a per-distro basis, so consult documentation for 88 cloud-init's distro support for instructions on using 89 these config options. 90 91 .. note:: 92 To ensure that apt configuration is valid yaml, any strings 93 containing special characters, especially ``:`` should be quoted. 94 95 .. note:: 96 For more information about apt configuration, see the 97 ``Additional apt configuration`` example."""), 98 'distros': distros, 99 'examples': [dedent("""\ 100 apt: 101 preserve_sources_list: false 102 disable_suites: 103 - $RELEASE-updates 104 - backports 105 - $RELEASE 106 - mysuite 107 primary: 108 - arches: 109 - amd64 110 - i386 111 - default 112 uri: 'http://us.archive.ubuntu.com/ubuntu' 113 search: 114 - 'http://cool.but-sometimes-unreachable.com/ubuntu' 115 - 'http://us.archive.ubuntu.com/ubuntu' 116 search_dns: false 117 - arches: 118 - s390x 119 - arm64 120 uri: 'http://archive-to-use-for-arm64.example.com/ubuntu' 121 122 security: 123 - arches: 124 - default 125 search_dns: true 126 sources_list: | 127 deb $MIRROR $RELEASE main restricted 128 deb-src $MIRROR $RELEASE main restricted 129 deb $PRIMARY $RELEASE universe restricted 130 deb $SECURITY $RELEASE-security multiverse 131 debconf_selections: 132 set1: the-package the-package/some-flag boolean true 133 conf: | 134 APT { 135 Get { 136 Assume-Yes 'true'; 137 Fix-Broken 'true'; 138 } 139 } 140 proxy: 'http://[[user][:pass]@]host[:port]/' 141 http_proxy: 'http://[[user][:pass]@]host[:port]/' 142 ftp_proxy: 'ftp://[[user][:pass]@]host[:port]/' 143 https_proxy: 'https://[[user][:pass]@]host[:port]/' 144 sources: 145 source1: 146 keyid: 'keyid' 147 keyserver: 'keyserverurl' 148 source: 'deb [signed-by=$KEY_FILE] http://<url>/ xenial main' 149 source2: 150 source: 'ppa:<ppa-name>' 151 source3: 152 source: 'deb $MIRROR $RELEASE multiverse' 153 key: | 154 ------BEGIN PGP PUBLIC KEY BLOCK------- 155 <key data> 156 ------END PGP PUBLIC KEY BLOCK-------""")], 157 'frequency': frequency, 158 'type': 'object', 159 'properties': { 160 'apt': { 161 'type': 'object', 162 'additionalProperties': False, 163 'properties': { 164 'preserve_sources_list': { 165 'type': 'boolean', 166 'default': False, 167 'description': dedent("""\ 168 By default, cloud-init will generate a new sources 169 list in ``/etc/apt/sources.list.d`` based on any 170 changes specified in cloud config. To disable this 171 behavior and preserve the sources list from the 172 pristine image, set ``preserve_sources_list`` 173 to ``true``. 174 175 The ``preserve_sources_list`` option overrides 176 all other config keys that would alter 177 ``sources.list`` or ``sources.list.d``, 178 **except** for additional sources to be added 179 to ``sources.list.d``.""") 180 }, 181 'disable_suites': { 182 'type': 'array', 183 'items': { 184 'type': 'string' 185 }, 186 'uniqueItems': True, 187 'description': dedent("""\ 188 Entries in the sources list can be disabled using 189 ``disable_suites``, which takes a list of suites 190 to be disabled. If the string ``$RELEASE`` is 191 present in a suite in the ``disable_suites`` list, 192 it will be replaced with the release name. If a 193 suite specified in ``disable_suites`` is not 194 present in ``sources.list`` it will be ignored. 195 For convenience, several aliases are provided for 196 ``disable_suites``: 197 198 - ``updates`` => ``$RELEASE-updates`` 199 - ``backports`` => ``$RELEASE-backports`` 200 - ``security`` => ``$RELEASE-security`` 201 - ``proposed`` => ``$RELEASE-proposed`` 202 - ``release`` => ``$RELEASE``. 203 204 When a suite is disabled using ``disable_suites``, 205 its entry in ``sources.list`` is not deleted; it 206 is just commented out.""") 207 }, 208 'primary': { 209 **mirror_property, 210 'description': dedent("""\ 211 The primary and security archive mirrors can 212 be specified using the ``primary`` and 213 ``security`` keys, respectively. Both the 214 ``primary`` and ``security`` keys take a list 215 of configs, allowing mirrors to be specified 216 on a per-architecture basis. Each config is a 217 dictionary which must have an entry for 218 ``arches``, specifying which architectures 219 that config entry is for. The keyword 220 ``default`` applies to any architecture not 221 explicitly listed. The mirror url can be specified 222 with the ``uri`` key, or a list of mirrors to 223 check can be provided in order, with the first 224 mirror that can be resolved being selected. This 225 allows the same configuration to be used in 226 different environment, with different hosts used 227 for a local apt mirror. If no mirror is provided 228 by ``uri`` or ``search``, ``search_dns`` may be 229 used to search for dns names in the format 230 ``<distro>-mirror`` in each of the following: 231 232 - fqdn of this host per cloud metadata, 233 - localdomain, 234 - domains listed in ``/etc/resolv.conf``. 235 236 If there is a dns entry for ``<distro>-mirror``, 237 then it is assumed that there is a distro mirror 238 at ``http://<distro>-mirror.<domain>/<distro>``. 239 If the ``primary`` key is defined, but not the 240 ``security`` key, then then configuration for 241 ``primary`` is also used for ``security``. 242 If ``search_dns`` is used for the ``security`` 243 key, the search pattern will be 244 ``<distro>-security-mirror``. 245 246 Each mirror may also specify a key to import via 247 any of the following optional keys: 248 249 - ``keyid``: a key to import via shortid or \ 250 fingerprint. 251 - ``key``: a raw PGP key. 252 - ``keyserver``: alternate keyserver to pull \ 253 ``keyid`` key from. 254 255 If no mirrors are specified, or all lookups fail, 256 then default mirrors defined in the datasource 257 are used. If none are present in the datasource 258 either the following defaults are used: 259 260 - ``primary`` => \ 261 ``http://archive.ubuntu.com/ubuntu``. 262 - ``security`` => \ 263 ``http://security.ubuntu.com/ubuntu`` 264 """) 265 }, 266 'security': { 267 **mirror_property, 268 'description': dedent("""\ 269 Please refer to the primary config documentation""") 270 }, 271 'add_apt_repo_match': { 272 'type': 'string', 273 'default': ADD_APT_REPO_MATCH, 274 'description': dedent("""\ 275 All source entries in ``apt-sources`` that match 276 regex in ``add_apt_repo_match`` will be added to 277 the system using ``add-apt-repository``. If 278 ``add_apt_repo_match`` is not specified, it 279 defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) 280 }, 281 'debconf_selections': { 282 'type': 'object', 283 'items': {'type': 'string'}, 284 'description': dedent("""\ 285 Debconf additional configurations can be specified as a 286 dictionary under the ``debconf_selections`` config 287 key, with each key in the dict representing a 288 different set of configurations. The value of each key 289 must be a string containing all the debconf 290 configurations that must be applied. We will bundle 291 all of the values and pass them to 292 ``debconf-set-selections``. Therefore, each value line 293 must be a valid entry for ``debconf-set-selections``, 294 meaning that they must possess for distinct fields: 295 296 ``pkgname question type answer`` 297 298 Where: 299 300 - ``pkgname`` is the name of the package. 301 - ``question`` the name of the questions. 302 - ``type`` is the type of question. 303 - ``answer`` is the value used to ansert the \ 304 question. 305 306 For example: \ 307 ``ippackage ippackage/ip string 127.0.01`` 308 """) 309 }, 310 'sources_list': { 311 'type': 'string', 312 'description': dedent("""\ 313 Specifies a custom template for rendering 314 ``sources.list`` . If no ``sources_list`` template 315 is given, cloud-init will use sane default. Within 316 this template, the following strings will be 317 replaced with the appropriate values: 318 319 - ``$MIRROR`` 320 - ``$RELEASE`` 321 - ``$PRIMARY`` 322 - ``$SECURITY`` 323 - ``$KEY_FILE``""") 324 }, 325 'conf': { 326 'type': 'string', 327 'description': dedent("""\ 328 Specify configuration for apt, such as proxy 329 configuration. This configuration is specified as a 330 string. For multiline apt configuration, make sure 331 to follow yaml syntax.""") 332 }, 333 'https_proxy': { 334 'type': 'string', 335 'description': dedent("""\ 336 More convenient way to specify https apt proxy. 337 https proxy url is specified in the format 338 ``https://[[user][:pass]@]host[:port]/``.""") 339 }, 340 'http_proxy': { 341 'type': 'string', 342 'description': dedent("""\ 343 More convenient way to specify http apt proxy. 344 http proxy url is specified in the format 345 ``http://[[user][:pass]@]host[:port]/``.""") 346 }, 347 'proxy': { 348 'type': 'string', 349 'description': 'Alias for defining a http apt proxy.' 350 }, 351 'ftp_proxy': { 352 'type': 'string', 353 'description': dedent("""\ 354 More convenient way to specify ftp apt proxy. 355 ftp proxy url is specified in the format 356 ``ftp://[[user][:pass]@]host[:port]/``.""") 357 }, 358 'sources': { 359 'type': 'object', 360 'items': {'type': 'string'}, 361 'description': dedent("""\ 362 Source list entries can be specified as a 363 dictionary under the ``sources`` config key, with 364 each key in the dict representing a different source 365 file. The key of each source entry will be used 366 as an id that can be referenced in other config 367 entries, as well as the filename for the source's 368 configuration under ``/etc/apt/sources.list.d``. 369 If the name does not end with ``.list``, it will 370 be appended. If there is no configuration for a 371 key in ``sources``, no file will be written, but 372 the key may still be referred to as an id in other 373 ``sources`` entries. 374 375 Each entry under ``sources`` is a dictionary which 376 may contain any of the following optional keys: 377 378 - ``source``: a sources.list entry \ 379 (some variable replacements apply). 380 - ``keyid``: a key to import via shortid or \ 381 fingerprint. 382 - ``key``: a raw PGP key. 383 - ``keyserver``: alternate keyserver to pull \ 384 ``keyid`` key from. 385 - ``filename``: specify the name of the .list file 386 387 The ``source`` key supports variable 388 replacements for the following strings: 389 390 - ``$MIRROR`` 391 - ``$PRIMARY`` 392 - ``$SECURITY`` 393 - ``$RELEASE`` 394 - ``$KEY_FILE``""") 395 } 396 } 397 } 398 } 399} 400 401__doc__ = get_schema_doc(schema) 402 403 404# place where apt stores cached repository data 405APT_LISTS = "/var/lib/apt/lists" 406 407# Files to store proxy information 408APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" 409APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy" 410 411# Default keyserver to use 412DEFAULT_KEYSERVER = "keyserver.ubuntu.com" 413 414# Default archive mirrors 415PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", 416 "SECURITY": "http://security.ubuntu.com/ubuntu/"} 417PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", 418 "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} 419PRIMARY_ARCHES = ['amd64', 'i386'] 420PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el', 'riscv64'] 421 422 423def get_default_mirrors(arch=None, target=None): 424 """returns the default mirrors for the target. These depend on the 425 architecture, for more see: 426 https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" 427 if arch is None: 428 arch = util.get_dpkg_architecture(target) 429 if arch in PRIMARY_ARCHES: 430 return PRIMARY_ARCH_MIRRORS.copy() 431 if arch in PORTS_ARCHES: 432 return PORTS_MIRRORS.copy() 433 raise ValueError("No default mirror known for arch %s" % arch) 434 435 436def handle(name, ocfg, cloud, log, _): 437 """process the config for apt_config. This can be called from 438 curthooks if a global apt config was provided or via the "apt" 439 standalone command.""" 440 # keeping code close to curtin codebase via entry handler 441 target = None 442 if log is not None: 443 global LOG 444 LOG = log 445 # feed back converted config, but only work on the subset under 'apt' 446 ocfg = convert_to_v3_apt_format(ocfg) 447 cfg = ocfg.get('apt', {}) 448 449 if not isinstance(cfg, dict): 450 raise ValueError( 451 "Expected dictionary for 'apt' config, found {config_type}".format( 452 config_type=type(cfg))) 453 454 validate_cloudconfig_schema(cfg, schema) 455 apply_debconf_selections(cfg, target) 456 apply_apt(cfg, cloud, target) 457 458 459def _should_configure_on_empty_apt(): 460 # if no config was provided, should apt configuration be done? 461 if util.system_is_snappy(): 462 return False, "system is snappy." 463 if not (subp.which('apt-get') or subp.which('apt')): 464 return False, "no apt commands." 465 return True, "Apt is available." 466 467 468def apply_apt(cfg, cloud, target): 469 # cfg is the 'apt' top level dictionary already in 'v3' format. 470 if not cfg: 471 should_config, msg = _should_configure_on_empty_apt() 472 if not should_config: 473 LOG.debug("Nothing to do: No apt config and %s", msg) 474 return 475 476 LOG.debug("handling apt config: %s", cfg) 477 478 release = util.lsb_release(target=target)['codename'] 479 arch = util.get_dpkg_architecture(target) 480 mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) 481 LOG.debug("Apt Mirror info: %s", mirrors) 482 483 if util.is_false(cfg.get('preserve_sources_list', False)): 484 add_mirror_keys(cfg, target) 485 generate_sources_list(cfg, release, mirrors, cloud) 486 rename_apt_lists(mirrors, target, arch) 487 488 try: 489 apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) 490 except (IOError, OSError): 491 LOG.exception("Failed to apply proxy or apt config info:") 492 493 # Process 'apt_source -> sources {dict}' 494 if 'sources' in cfg: 495 params = mirrors 496 params['RELEASE'] = release 497 params['MIRROR'] = mirrors["MIRROR"] 498 499 matcher = None 500 matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) 501 if matchcfg: 502 matcher = re.compile(matchcfg).search 503 504 add_apt_sources(cfg['sources'], cloud, target=target, 505 template_params=params, aa_repo_match=matcher) 506 507 508def debconf_set_selections(selections, target=None): 509 if not selections.endswith(b'\n'): 510 selections += b'\n' 511 subp.subp(['debconf-set-selections'], data=selections, target=target, 512 capture=True) 513 514 515def dpkg_reconfigure(packages, target=None): 516 # For any packages that are already installed, but have preseed data 517 # we populate the debconf database, but the filesystem configuration 518 # would be preferred on a subsequent dpkg-reconfigure. 519 # so, what we have to do is "know" information about certain packages 520 # to unconfigure them. 521 unhandled = [] 522 to_config = [] 523 for pkg in packages: 524 if pkg in CONFIG_CLEANERS: 525 LOG.debug("unconfiguring %s", pkg) 526 CONFIG_CLEANERS[pkg](target) 527 to_config.append(pkg) 528 else: 529 unhandled.append(pkg) 530 531 if len(unhandled): 532 LOG.warning("The following packages were installed and preseeded, " 533 "but cannot be unconfigured: %s", unhandled) 534 535 if len(to_config): 536 subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + 537 list(to_config), data=None, target=target, capture=True) 538 539 540def apply_debconf_selections(cfg, target=None): 541 """apply_debconf_selections - push content to debconf""" 542 # debconf_selections: 543 # set1: | 544 # cloud-init cloud-init/datasources multiselect MAAS 545 # set2: pkg pkg/value string bar 546 selsets = cfg.get('debconf_selections') 547 if not selsets: 548 LOG.debug("debconf_selections was not set in config") 549 return 550 551 selections = '\n'.join( 552 [selsets[key] for key in sorted(selsets.keys())]) 553 debconf_set_selections(selections.encode(), target=target) 554 555 # get a complete list of packages listed in input 556 pkgs_cfgd = set() 557 for _key, content in selsets.items(): 558 for line in content.splitlines(): 559 if line.startswith("#"): 560 continue 561 pkg = re.sub(r"[:\s].*", "", line) 562 pkgs_cfgd.add(pkg) 563 564 pkgs_installed = util.get_installed_packages(target) 565 566 LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) 567 need_reconfig = pkgs_cfgd.intersection(pkgs_installed) 568 569 if len(need_reconfig) == 0: 570 LOG.debug("no need for reconfig") 571 return 572 573 dpkg_reconfigure(need_reconfig, target=target) 574 575 576def clean_cloud_init(target): 577 """clean out any local cloud-init config""" 578 flist = glob.glob( 579 subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) 580 581 LOG.debug("cleaning cloud-init config from: %s", flist) 582 for dpkg_cfg in flist: 583 os.unlink(dpkg_cfg) 584 585 586def mirrorurl_to_apt_fileprefix(mirror): 587 """mirrorurl_to_apt_fileprefix 588 Convert a mirror url to the file prefix used by apt on disk to 589 store cache information for that mirror. 590 To do so do: 591 - take off ???:// 592 - drop tailing / 593 - convert in string / to _""" 594 string = mirror 595 if string.endswith("/"): 596 string = string[0:-1] 597 pos = string.find("://") 598 if pos >= 0: 599 string = string[pos + 3:] 600 string = string.replace("/", "_") 601 return string 602 603 604def rename_apt_lists(new_mirrors, target, arch): 605 """rename_apt_lists - rename apt lists to preserve old cache data""" 606 default_mirrors = get_default_mirrors(arch) 607 608 pre = subp.target_path(target, APT_LISTS) 609 for (name, omirror) in default_mirrors.items(): 610 nmirror = new_mirrors.get(name) 611 if not nmirror: 612 continue 613 614 oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror) 615 nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror) 616 if oprefix == nprefix: 617 continue 618 olen = len(oprefix) 619 for filename in glob.glob("%s_*" % oprefix): 620 newname = "%s%s" % (nprefix, filename[olen:]) 621 LOG.debug("Renaming apt list %s to %s", filename, newname) 622 try: 623 os.rename(filename, newname) 624 except OSError: 625 # since this is a best effort task, warn with but don't fail 626 LOG.warning("Failed to rename apt list:", exc_info=True) 627 628 629def mirror_to_placeholder(tmpl, mirror, placeholder): 630 """mirror_to_placeholder 631 replace the specified mirror in a template with a placeholder string 632 Checks for existance of the expected mirror and warns if not found""" 633 if mirror not in tmpl: 634 LOG.warning("Expected mirror '%s' not found in: %s", mirror, tmpl) 635 return tmpl.replace(mirror, placeholder) 636 637 638def map_known_suites(suite): 639 """there are a few default names which will be auto-extended. 640 This comes at the inability to use those names literally as suites, 641 but on the other hand increases readability of the cfg quite a lot""" 642 mapping = {'updates': '$RELEASE-updates', 643 'backports': '$RELEASE-backports', 644 'security': '$RELEASE-security', 645 'proposed': '$RELEASE-proposed', 646 'release': '$RELEASE'} 647 try: 648 retsuite = mapping[suite] 649 except KeyError: 650 retsuite = suite 651 return retsuite 652 653 654def disable_suites(disabled, src, release): 655 """reads the config for suites to be disabled and removes those 656 from the template""" 657 if not disabled: 658 return src 659 660 retsrc = src 661 for suite in disabled: 662 suite = map_known_suites(suite) 663 releasesuite = templater.render_string(suite, {'RELEASE': release}) 664 LOG.debug("Disabling suite %s as %s", suite, releasesuite) 665 666 newsrc = "" 667 for line in retsrc.splitlines(True): 668 if line.startswith("#"): 669 newsrc += line 670 continue 671 672 # sources.list allow options in cols[1] which can have spaces 673 # so the actual suite can be [2] or later. example: 674 # deb [ arch=amd64,armel k=v ] http://example.com/debian 675 cols = line.split() 676 if len(cols) > 1: 677 pcol = 2 678 if cols[1].startswith("["): 679 for col in cols[1:]: 680 pcol += 1 681 if col.endswith("]"): 682 break 683 684 if cols[pcol] == releasesuite: 685 line = '# suite disabled by cloud-init: %s' % line 686 newsrc += line 687 retsrc = newsrc 688 689 return retsrc 690 691 692def add_mirror_keys(cfg, target): 693 """Adds any keys included in the primary/security mirror clauses""" 694 for key in ('primary', 'security'): 695 for mirror in cfg.get(key, []): 696 add_apt_key(mirror, target, file_name=key) 697 698 699def generate_sources_list(cfg, release, mirrors, cloud): 700 """generate_sources_list 701 create a source.list file based on a custom or default template 702 by replacing mirrors and release in the template""" 703 aptsrc = "/etc/apt/sources.list" 704 params = {'RELEASE': release, 'codename': release} 705 for k in mirrors: 706 params[k] = mirrors[k] 707 params[k.lower()] = mirrors[k] 708 709 tmpl = cfg.get('sources_list', None) 710 if tmpl is None: 711 LOG.info("No custom template provided, fall back to builtin") 712 template_fn = cloud.get_template_filename('sources.list.%s' % 713 (cloud.distro.name)) 714 if not template_fn: 715 template_fn = cloud.get_template_filename('sources.list') 716 if not template_fn: 717 LOG.warning("No template found, " 718 "not rendering /etc/apt/sources.list") 719 return 720 tmpl = util.load_file(template_fn) 721 722 rendered = templater.render_string(tmpl, params) 723 disabled = disable_suites(cfg.get('disable_suites'), rendered, release) 724 util.write_file(aptsrc, disabled, mode=0o644) 725 726 727def add_apt_key_raw(key, file_name, hardened=False, target=None): 728 """ 729 actual adding of a key as defined in key argument 730 to the system 731 """ 732 LOG.debug("Adding key:\n'%s'", key) 733 try: 734 name = pathlib.Path(file_name).stem 735 return apt_key('add', output_file=name, data=key, hardened=hardened) 736 except subp.ProcessExecutionError: 737 LOG.exception("failed to add apt GPG Key to apt keyring") 738 raise 739 740 741def add_apt_key(ent, target=None, hardened=False, file_name=None): 742 """ 743 Add key to the system as defined in ent (if any). 744 Supports raw keys or keyid's 745 The latter will as a first step fetched to get the raw key 746 """ 747 if 'keyid' in ent and 'key' not in ent: 748 keyserver = DEFAULT_KEYSERVER 749 if 'keyserver' in ent: 750 keyserver = ent['keyserver'] 751 752 ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) 753 754 if 'key' in ent: 755 return add_apt_key_raw( 756 ent['key'], 757 file_name or ent['filename'], 758 hardened=hardened) 759 760 761def update_packages(cloud): 762 cloud.distro.update_package_sources() 763 764 765def add_apt_sources(srcdict, cloud, target=None, template_params=None, 766 aa_repo_match=None): 767 """ 768 install keys and repo source .list files defined in 'sources' 769 770 for each 'source' entry in the config: 771 1. expand template variables and write source .list file in 772 /etc/apt/sources.list.d/ 773 2. install defined keys 774 3. update packages via distro-specific method (i.e. apt-key update) 775 776 777 @param srcdict: a dict containing elements required 778 @param cloud: cloud instance object 779 780 Example srcdict value: 781 { 782 'rio-grande-repo': { 783 'source': 'deb [signed-by=$KEY_FILE] $MIRROR $RELEASE main', 784 'keyid': 'B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77', 785 'keyserver': 'pgp.mit.edu' 786 } 787 } 788 789 Note: Deb822 format is not supported 790 """ 791 if template_params is None: 792 template_params = {} 793 794 if aa_repo_match is None: 795 raise ValueError('did not get a valid repo matcher') 796 797 if not isinstance(srcdict, dict): 798 raise TypeError('unknown apt format: %s' % (srcdict)) 799 800 for filename in srcdict: 801 ent = srcdict[filename] 802 LOG.debug("adding source/key '%s'", ent) 803 if 'filename' not in ent: 804 ent['filename'] = filename 805 806 if 'source' in ent and '$KEY_FILE' in ent['source']: 807 key_file = add_apt_key(ent, target, hardened=True) 808 template_params['KEY_FILE'] = key_file 809 else: 810 key_file = add_apt_key(ent, target) 811 812 if 'source' not in ent: 813 continue 814 source = ent['source'] 815 source = templater.render_string(source, template_params) 816 817 if not ent['filename'].startswith("/"): 818 ent['filename'] = os.path.join("/etc/apt/sources.list.d/", 819 ent['filename']) 820 if not ent['filename'].endswith(".list"): 821 ent['filename'] += ".list" 822 823 if aa_repo_match(source): 824 try: 825 subp.subp(["add-apt-repository", source], target=target) 826 except subp.ProcessExecutionError: 827 LOG.exception("add-apt-repository failed.") 828 raise 829 continue 830 831 sourcefn = subp.target_path(target, ent['filename']) 832 try: 833 contents = "%s\n" % (source) 834 util.write_file(sourcefn, contents, omode="a") 835 except IOError as detail: 836 LOG.exception("failed write to file %s: %s", sourcefn, detail) 837 raise 838 839 update_packages(cloud) 840 841 return 842 843 844def convert_v1_to_v2_apt_format(srclist): 845 """convert v1 apt format to v2 (dict in apt_sources)""" 846 srcdict = {} 847 if isinstance(srclist, list): 848 LOG.debug("apt config: convert V1 to V2 format (source list to dict)") 849 for srcent in srclist: 850 if 'filename' not in srcent: 851 # file collides for multiple !filename cases for compatibility 852 # yet we need them all processed, so not same dictionary key 853 srcent['filename'] = "cloud_config_sources.list" 854 key = util.rand_dict_key(srcdict, "cloud_config_sources.list") 855 else: 856 # all with filename use that as key (matching new format) 857 key = srcent['filename'] 858 srcdict[key] = srcent 859 elif isinstance(srclist, dict): 860 srcdict = srclist 861 else: 862 raise ValueError("unknown apt_sources format") 863 864 return srcdict 865 866 867def convert_key(oldcfg, aptcfg, oldkey, newkey): 868 """convert an old key to the new one if the old one exists 869 returns true if a key was found and converted""" 870 if oldcfg.get(oldkey, None) is not None: 871 aptcfg[newkey] = oldcfg.get(oldkey) 872 del oldcfg[oldkey] 873 return True 874 return False 875 876 877def convert_mirror(oldcfg, aptcfg): 878 """convert old apt_mirror keys into the new more advanced mirror spec""" 879 keymap = [('apt_mirror', 'uri'), 880 ('apt_mirror_search', 'search'), 881 ('apt_mirror_search_dns', 'search_dns')] 882 converted = False 883 newmcfg = {'arches': ['default']} 884 for oldkey, newkey in keymap: 885 if convert_key(oldcfg, newmcfg, oldkey, newkey): 886 converted = True 887 888 # only insert new style config if anything was converted 889 if converted: 890 aptcfg['primary'] = [newmcfg] 891 892 893def convert_v2_to_v3_apt_format(oldcfg): 894 """convert old to new keys and adapt restructured mirror spec""" 895 mapoldkeys = {'apt_sources': 'sources', 896 'apt_mirror': None, 897 'apt_mirror_search': None, 898 'apt_mirror_search_dns': None, 899 'apt_proxy': 'proxy', 900 'apt_http_proxy': 'http_proxy', 901 'apt_ftp_proxy': 'https_proxy', 902 'apt_https_proxy': 'ftp_proxy', 903 'apt_preserve_sources_list': 'preserve_sources_list', 904 'apt_custom_sources_list': 'sources_list', 905 'add_apt_repo_match': 'add_apt_repo_match'} 906 needtoconvert = [] 907 for oldkey in mapoldkeys: 908 if oldkey in oldcfg: 909 if oldcfg[oldkey] in (None, ""): 910 del oldcfg[oldkey] 911 else: 912 needtoconvert.append(oldkey) 913 914 # no old config, so no new one to be created 915 if not needtoconvert: 916 return oldcfg 917 LOG.debug("apt config: convert V2 to V3 format for keys '%s'", 918 ", ".join(needtoconvert)) 919 920 # if old AND new config are provided, prefer the new one (LP #1616831) 921 newaptcfg = oldcfg.get('apt', None) 922 if newaptcfg is not None: 923 LOG.debug("apt config: V1/2 and V3 format specified, preferring V3") 924 for oldkey in needtoconvert: 925 newkey = mapoldkeys[oldkey] 926 verify = oldcfg[oldkey] # drop, but keep a ref for verification 927 del oldcfg[oldkey] 928 if newkey is None or newaptcfg.get(newkey, None) is None: 929 # no simple mapping or no collision on this particular key 930 continue 931 if verify != newaptcfg[newkey]: 932 raise ValueError("Old and New apt format defined with unequal " 933 "values %s vs %s @ %s" % (verify, 934 newaptcfg[newkey], 935 oldkey)) 936 # return conf after clearing conflicting V1/2 keys 937 return oldcfg 938 939 # create new format from old keys 940 aptcfg = {} 941 942 # simple renames / moves under the apt key 943 for oldkey in mapoldkeys: 944 if mapoldkeys[oldkey] is not None: 945 convert_key(oldcfg, aptcfg, oldkey, mapoldkeys[oldkey]) 946 947 # mirrors changed in a more complex way 948 convert_mirror(oldcfg, aptcfg) 949 950 for oldkey in mapoldkeys: 951 if oldcfg.get(oldkey, None) is not None: 952 raise ValueError("old apt key '%s' left after conversion" % oldkey) 953 954 # insert new format into config and return full cfg with only v3 content 955 oldcfg['apt'] = aptcfg 956 return oldcfg 957 958 959def convert_to_v3_apt_format(cfg): 960 """convert the old list based format to the new dict based one. After that 961 convert the old dict keys/format to v3 a.k.a 'new apt config'""" 962 # V1 -> V2, the apt_sources entry from list to dict 963 apt_sources = cfg.get('apt_sources', None) 964 if apt_sources is not None: 965 cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources) 966 967 # V2 -> V3, move all former globals under the "apt" key 968 # Restructure into new key names and mirror hierarchy 969 cfg = convert_v2_to_v3_apt_format(cfg) 970 971 return cfg 972 973 974def search_for_mirror_dns(configured, mirrortype, cfg, cloud): 975 """ 976 Try to resolve a list of predefines DNS names to pick mirrors 977 """ 978 mirror = None 979 980 if configured: 981 mydom = "" 982 doms = [] 983 984 if mirrortype == "primary": 985 mirrordns = "mirror" 986 elif mirrortype == "security": 987 mirrordns = "security-mirror" 988 else: 989 raise ValueError("unknown mirror type") 990 991 # if we have a fqdn, then search its domain portion first 992 (_, fqdn) = util.get_hostname_fqdn(cfg, cloud) 993 mydom = ".".join(fqdn.split(".")[1:]) 994 if mydom: 995 doms.append(".%s" % mydom) 996 997 doms.extend((".localdomain", "",)) 998 999 mirror_list = [] 1000 distro = cloud.distro.name 1001 mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrordns, "%s", distro) 1002 for post in doms: 1003 mirror_list.append(mirrorfmt % (post)) 1004 1005 mirror = util.search_for_mirror(mirror_list) 1006 1007 return mirror 1008 1009 1010def update_mirror_info(pmirror, smirror, arch, cloud): 1011 """sets security mirror to primary if not defined. 1012 returns defaults if no mirrors are defined""" 1013 if pmirror is not None: 1014 if smirror is None: 1015 smirror = pmirror 1016 return {'PRIMARY': pmirror, 1017 'SECURITY': smirror} 1018 1019 # None specified at all, get default mirrors from cloud 1020 mirror_info = cloud.datasource.get_package_mirror_info() 1021 if mirror_info: 1022 # get_package_mirror_info() returns a dictionary with 1023 # arbitrary key/value pairs including 'primary' and 'security' keys. 1024 # caller expects dict with PRIMARY and SECURITY. 1025 m = mirror_info.copy() 1026 m['PRIMARY'] = m['primary'] 1027 m['SECURITY'] = m['security'] 1028 1029 return m 1030 1031 # if neither apt nor cloud configured mirrors fall back to 1032 return get_default_mirrors(arch) 1033 1034 1035def get_arch_mirrorconfig(cfg, mirrortype, arch): 1036 """out of a list of potential mirror configurations select 1037 and return the one matching the architecture (or default)""" 1038 # select the mirror specification (if-any) 1039 mirror_cfg_list = cfg.get(mirrortype, None) 1040 if mirror_cfg_list is None: 1041 return None 1042 1043 # select the specification matching the target arch 1044 default = None 1045 for mirror_cfg_elem in mirror_cfg_list: 1046 arches = mirror_cfg_elem.get("arches") or [] 1047 if arch in arches: 1048 return mirror_cfg_elem 1049 if "default" in arches: 1050 default = mirror_cfg_elem 1051 return default 1052 1053 1054def get_mirror(cfg, mirrortype, arch, cloud): 1055 """pass the three potential stages of mirror specification 1056 returns None is neither of them found anything otherwise the first 1057 hit is returned""" 1058 mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) 1059 if mcfg is None: 1060 return None 1061 1062 # directly specified 1063 mirror = mcfg.get("uri", None) 1064 1065 # fallback to search if specified 1066 if mirror is None: 1067 # list of mirrors to try to resolve 1068 mirror = util.search_for_mirror(mcfg.get("search", None)) 1069 1070 # fallback to search_dns if specified 1071 if mirror is None: 1072 # list of mirrors to try to resolve 1073 mirror = search_for_mirror_dns(mcfg.get("search_dns", None), 1074 mirrortype, cfg, cloud) 1075 1076 return mirror 1077 1078 1079def find_apt_mirror_info(cfg, cloud, arch=None): 1080 """find_apt_mirror_info 1081 find an apt_mirror given the cfg provided. 1082 It can check for separate config of primary and security mirrors 1083 If only primary is given security is assumed to be equal to primary 1084 If the generic apt_mirror is given that is defining for both 1085 """ 1086 1087 if arch is None: 1088 arch = util.get_dpkg_architecture() 1089 LOG.debug("got arch for mirror selection: %s", arch) 1090 pmirror = get_mirror(cfg, "primary", arch, cloud) 1091 LOG.debug("got primary mirror: %s", pmirror) 1092 smirror = get_mirror(cfg, "security", arch, cloud) 1093 LOG.debug("got security mirror: %s", smirror) 1094 1095 mirror_info = update_mirror_info(pmirror, smirror, arch, cloud) 1096 1097 # less complex replacements use only MIRROR, derive from primary 1098 mirror_info["MIRROR"] = mirror_info["PRIMARY"] 1099 1100 return mirror_info 1101 1102 1103def apply_apt_config(cfg, proxy_fname, config_fname): 1104 """apply_apt_config 1105 Applies any apt*proxy config from if specified 1106 """ 1107 # Set up any apt proxy 1108 cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), 1109 ('http_proxy', 'Acquire::http::Proxy "%s";'), 1110 ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), 1111 ('https_proxy', 'Acquire::https::Proxy "%s";')) 1112 1113 proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] 1114 if len(proxies): 1115 LOG.debug("write apt proxy info to %s", proxy_fname) 1116 util.write_file(proxy_fname, '\n'.join(proxies) + '\n') 1117 elif os.path.isfile(proxy_fname): 1118 util.del_file(proxy_fname) 1119 LOG.debug("no apt proxy configured, removed %s", proxy_fname) 1120 1121 if cfg.get('conf', None): 1122 LOG.debug("write apt config info to %s", config_fname) 1123 util.write_file(config_fname, cfg.get('conf')) 1124 elif os.path.isfile(config_fname): 1125 util.del_file(config_fname) 1126 LOG.debug("no apt config configured, removed %s", config_fname) 1127 1128 1129def apt_key(command, output_file=None, data=None, hardened=False, 1130 human_output=True): 1131 """apt-key replacement 1132 1133 commands implemented: 'add', 'list', 'finger' 1134 1135 @param output_file: name of output gpg file (without .gpg or .asc) 1136 @param data: key contents 1137 @param human_output: list keys formatted for human parsing 1138 @param hardened: write keys to to /etc/apt/cloud-init.gpg.d/ (referred to 1139 with [signed-by] in sources file) 1140 """ 1141 1142 def _get_key_files(): 1143 """return all apt keys 1144 1145 /etc/apt/trusted.gpg (if it exists) and all keyfiles (and symlinks to 1146 keyfiles) in /etc/apt/trusted.gpg.d/ are returned 1147 1148 based on apt-key implementation 1149 """ 1150 key_files = [APT_LOCAL_KEYS] if os.path.isfile(APT_LOCAL_KEYS) else [] 1151 1152 for file in os.listdir(APT_TRUSTED_GPG_DIR): 1153 if file.endswith('.gpg') or file.endswith('.asc'): 1154 key_files.append(APT_TRUSTED_GPG_DIR + file) 1155 return key_files if key_files else '' 1156 1157 def apt_key_add(): 1158 """apt-key add <file> 1159 1160 returns filepath to new keyring, or '/dev/null' when an error occurs 1161 """ 1162 file_name = '/dev/null' 1163 if not output_file: 1164 util.logexc( 1165 LOG, 'Unknown filename, failed to add key: "{}"'.format(data)) 1166 else: 1167 try: 1168 key_dir = \ 1169 CLOUD_INIT_GPG_DIR if hardened else APT_TRUSTED_GPG_DIR 1170 stdout = gpg.dearmor(data) 1171 file_name = '{}{}.gpg'.format(key_dir, output_file) 1172 util.write_file(file_name, stdout) 1173 except subp.ProcessExecutionError: 1174 util.logexc(LOG, 'Gpg error, failed to add key: {}'.format( 1175 data)) 1176 except UnicodeDecodeError: 1177 util.logexc(LOG, 'Decode error, failed to add key: {}'.format( 1178 data)) 1179 return file_name 1180 1181 def apt_key_list(): 1182 """apt-key list 1183 1184 returns string of all trusted keys (in /etc/apt/trusted.gpg and 1185 /etc/apt/trusted.gpg.d/) 1186 """ 1187 key_list = [] 1188 for key_file in _get_key_files(): 1189 try: 1190 key_list.append(gpg.list(key_file, human_output=human_output)) 1191 except subp.ProcessExecutionError as error: 1192 LOG.warning('Failed to list key "%s": %s', key_file, error) 1193 return '\n'.join(key_list) 1194 1195 if command == 'add': 1196 return apt_key_add() 1197 elif command == 'finger' or command == 'list': 1198 return apt_key_list() 1199 else: 1200 raise ValueError( 1201 'apt_key() commands add, list, and finger are currently supported') 1202 1203 1204CONFIG_CLEANERS = { 1205 'cloud-init': clean_cloud_init, 1206} 1207 1208# vi: ts=4 expandtab 1209