1# Copyright (C) 2012 Canonical Ltd. 2# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. 3# Copyright (C) 2012 Yahoo! Inc. 4# 5# Author: Scott Moser <scott.moser@canonical.com> 6# Author: Juerg Haefliger <juerg.haefliger@hp.com> 7# Author: Joshua Harlow <harlowja@yahoo-inc.com> 8# Author: Ben Howard <ben.howard@canonical.com> 9# 10# This file is part of cloud-init. See LICENSE file for license information. 11 12import abc 13import os 14import re 15import stat 16import string 17import urllib.parse 18from io import StringIO 19from typing import Any, Mapping # noqa: F401 20 21from cloudinit import importer 22from cloudinit import log as logging 23from cloudinit import net 24from cloudinit.net import activators 25from cloudinit.net import eni 26from cloudinit.net import network_state 27from cloudinit.net import renderers 28from cloudinit.net.network_state import parse_net_config_data 29from cloudinit import persistence 30from cloudinit import ssh_util 31from cloudinit import type_utils 32from cloudinit import subp 33from cloudinit import util 34 35from cloudinit.features import \ 36 ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES 37 38from cloudinit.distros.parsers import hosts 39from .networking import LinuxNetworking 40 41 42# Used when a cloud-config module can be run on all cloud-init distibutions. 43# The value 'all' is surfaced in module documentation for distro support. 44ALL_DISTROS = 'all' 45 46OSFAMILIES = { 47 'alpine': ['alpine'], 48 'arch': ['arch'], 49 'debian': ['debian', 'ubuntu'], 50 'freebsd': ['freebsd'], 51 'gentoo': ['gentoo'], 52 'redhat': ['almalinux', 'amazon', 'centos', 'cloudlinux', 'eurolinux', 53 'fedora', 'openEuler', 'photon', 'rhel', 'rocky', 'virtuozzo'], 54 'suse': ['opensuse', 'sles'], 55} 56 57LOG = logging.getLogger(__name__) 58 59# This is a best guess regex, based on current EC2 AZs on 2017-12-11. 60# It could break when Amazon adds new regions and new AZs. 61_EC2_AZ_RE = re.compile('^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$') 62 63# Default NTP Client Configurations 64PREFERRED_NTP_CLIENTS = ['chrony', 'systemd-timesyncd', 'ntp', 'ntpdate'] 65 66# Letters/Digits/Hyphen characters, for use in domain name validation 67LDH_ASCII_CHARS = string.ascii_letters + string.digits + "-" 68 69 70class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): 71 72 usr_lib_exec = "/usr/lib" 73 hosts_fn = "/etc/hosts" 74 ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" 75 hostname_conf_fn = "/etc/hostname" 76 tz_zone_dir = "/usr/share/zoneinfo" 77 init_cmd = ['service'] # systemctl, service etc 78 renderer_configs = {} # type: Mapping[str, Mapping[str, Any]] 79 _preferred_ntp_clients = None 80 networking_cls = LinuxNetworking 81 # This is used by self.shutdown_command(), and can be overridden in 82 # subclasses 83 shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} 84 85 _ci_pkl_version = 1 86 prefer_fqdn = False 87 resolve_conf_fn = "/etc/resolv.conf" 88 89 def __init__(self, name, cfg, paths): 90 self._paths = paths 91 self._cfg = cfg 92 self.name = name 93 self.networking = self.networking_cls() 94 95 def _unpickle(self, ci_pkl_version: int) -> None: 96 """Perform deserialization fixes for Distro.""" 97 if "networking" not in self.__dict__ or not self.networking.__dict__: 98 # This is either a Distro pickle with no networking attribute OR 99 # this is a Distro pickle with a networking attribute but from 100 # before ``Networking`` had any state (meaning that 101 # Networking.__setstate__ will not be called). In either case, we 102 # want to ensure that `self.networking` is freshly-instantiated: 103 # either because it isn't present at all, or because it will be 104 # missing expected instance state otherwise. 105 self.networking = self.networking_cls() 106 107 @abc.abstractmethod 108 def install_packages(self, pkglist): 109 raise NotImplementedError() 110 111 def _write_network(self, settings): 112 """Deprecated. Remove if/when arch and gentoo support renderers.""" 113 raise NotImplementedError( 114 "Legacy function '_write_network' was called in distro '%s'.\n" 115 "_write_network_config needs implementation.\n" % self.name) 116 117 def _write_network_state(self, network_state): 118 priority = util.get_cfg_by_path( 119 self._cfg, ('network', 'renderers'), None) 120 121 name, render_cls = renderers.select(priority=priority) 122 LOG.debug("Selected renderer '%s' from priority list: %s", 123 name, priority) 124 renderer = render_cls(config=self.renderer_configs.get(name)) 125 renderer.render_network_state(network_state) 126 127 def _find_tz_file(self, tz): 128 tz_file = os.path.join(self.tz_zone_dir, str(tz)) 129 if not os.path.isfile(tz_file): 130 raise IOError(("Invalid timezone %s," 131 " no file found at %s") % (tz, tz_file)) 132 return tz_file 133 134 def get_option(self, opt_name, default=None): 135 return self._cfg.get(opt_name, default) 136 137 def set_option(self, opt_name, value=None): 138 self._cfg[opt_name] = value 139 140 def set_hostname(self, hostname, fqdn=None): 141 writeable_hostname = self._select_hostname(hostname, fqdn) 142 self._write_hostname(writeable_hostname, self.hostname_conf_fn) 143 self._apply_hostname(writeable_hostname) 144 145 def uses_systemd(self): 146 """Wrapper to report whether this distro uses systemd or sysvinit.""" 147 return uses_systemd() 148 149 @abc.abstractmethod 150 def package_command(self, command, args=None, pkgs=None): 151 raise NotImplementedError() 152 153 @abc.abstractmethod 154 def update_package_sources(self): 155 raise NotImplementedError() 156 157 def get_primary_arch(self): 158 arch = os.uname()[4] 159 if arch in ("i386", "i486", "i586", "i686"): 160 return "i386" 161 return arch 162 163 def _get_arch_package_mirror_info(self, arch=None): 164 mirror_info = self.get_option("package_mirrors", []) 165 if not arch: 166 arch = self.get_primary_arch() 167 return _get_arch_package_mirror_info(mirror_info, arch) 168 169 def get_package_mirror_info(self, arch=None, data_source=None): 170 # This resolves the package_mirrors config option 171 # down to a single dict of {mirror_name: mirror_url} 172 arch_info = self._get_arch_package_mirror_info(arch) 173 return _get_package_mirror_info(data_source=data_source, 174 mirror_info=arch_info) 175 176 def apply_network(self, settings, bring_up=True): 177 """Deprecated. Remove if/when arch and gentoo support renderers.""" 178 # this applies network where 'settings' is interfaces(5) style 179 # it is obsolete compared to apply_network_config 180 # Write it out 181 182 # pylint: disable=assignment-from-no-return 183 # We have implementations in arch and gentoo still 184 dev_names = self._write_network(settings) 185 # pylint: enable=assignment-from-no-return 186 # Now try to bring them up 187 if bring_up: 188 return self._bring_up_interfaces(dev_names) 189 return False 190 191 def _apply_network_from_network_config(self, netconfig, bring_up=True): 192 """Deprecated. Remove if/when arch and gentoo support renderers.""" 193 distro = self.__class__ 194 LOG.warning("apply_network_config is not currently implemented " 195 "for distribution '%s'. Attempting to use apply_network", 196 distro) 197 header = '\n'.join([ 198 "# Converted from network_config for distro %s" % distro, 199 "# Implementation of _write_network_config is needed." 200 ]) 201 ns = network_state.parse_net_config_data(netconfig) 202 contents = eni.network_state_to_eni( 203 ns, header=header, render_hwaddress=True) 204 return self.apply_network(contents, bring_up=bring_up) 205 206 def generate_fallback_config(self): 207 return net.generate_fallback_config() 208 209 def apply_network_config(self, netconfig, bring_up=False) -> bool: 210 """Apply the network config. 211 212 If bring_up is True, attempt to bring up the passed in devices. If 213 devices is None, attempt to bring up devices returned by 214 _write_network_config. 215 216 Returns True if any devices failed to come up, otherwise False. 217 """ 218 # This method is preferred to apply_network which only takes 219 # a much less complete network config format (interfaces(5)). 220 network_state = parse_net_config_data(netconfig) 221 try: 222 self._write_network_state(network_state) 223 except NotImplementedError: 224 # backwards compat until all distros have apply_network_config 225 return self._apply_network_from_network_config( 226 netconfig, bring_up=bring_up) 227 228 # Now try to bring them up 229 if bring_up: 230 LOG.debug('Bringing up newly configured network interfaces') 231 network_activator = activators.select_activator() 232 network_activator.bring_up_all_interfaces(network_state) 233 else: 234 LOG.debug("Not bringing up newly configured network interfaces") 235 return False 236 237 def apply_network_config_names(self, netconfig): 238 net.apply_network_config_names(netconfig) 239 240 @abc.abstractmethod 241 def apply_locale(self, locale, out_fn=None): 242 raise NotImplementedError() 243 244 @abc.abstractmethod 245 def set_timezone(self, tz): 246 raise NotImplementedError() 247 248 def _get_localhost_ip(self): 249 return "127.0.0.1" 250 251 def get_locale(self): 252 raise NotImplementedError() 253 254 @abc.abstractmethod 255 def _read_hostname(self, filename, default=None): 256 raise NotImplementedError() 257 258 @abc.abstractmethod 259 def _write_hostname(self, hostname, filename): 260 raise NotImplementedError() 261 262 @abc.abstractmethod 263 def _read_system_hostname(self): 264 raise NotImplementedError() 265 266 def _apply_hostname(self, hostname): 267 # This really only sets the hostname 268 # temporarily (until reboot so it should 269 # not be depended on). Use the write 270 # hostname functions for 'permanent' adjustments. 271 LOG.debug("Non-persistently setting the system hostname to %s", 272 hostname) 273 try: 274 subp.subp(['hostname', hostname]) 275 except subp.ProcessExecutionError: 276 util.logexc(LOG, "Failed to non-persistently adjust the system " 277 "hostname to %s", hostname) 278 279 def _select_hostname(self, hostname, fqdn): 280 # Prefer the short hostname over the long 281 # fully qualified domain name 282 if util.get_cfg_option_bool(self._cfg, "prefer_fqdn_over_hostname", 283 self.prefer_fqdn) and fqdn: 284 return fqdn 285 if not hostname: 286 return fqdn 287 return hostname 288 289 @staticmethod 290 def expand_osfamily(family_list): 291 distros = [] 292 for family in family_list: 293 if family not in OSFAMILIES: 294 raise ValueError( 295 "No distributions found for osfamily {}".format(family) 296 ) 297 distros.extend(OSFAMILIES[family]) 298 return distros 299 300 def update_hostname(self, hostname, fqdn, prev_hostname_fn): 301 applying_hostname = hostname 302 303 # Determine what the actual written hostname should be 304 hostname = self._select_hostname(hostname, fqdn) 305 306 # If the previous hostname file exists lets see if we 307 # can get a hostname from it 308 if prev_hostname_fn and os.path.exists(prev_hostname_fn): 309 prev_hostname = self._read_hostname(prev_hostname_fn) 310 else: 311 prev_hostname = None 312 313 # Lets get where we should write the system hostname 314 # and what the system hostname is 315 (sys_fn, sys_hostname) = self._read_system_hostname() 316 update_files = [] 317 318 # If there is no previous hostname or it differs 319 # from what we want, lets update it or create the 320 # file in the first place 321 if not prev_hostname or prev_hostname != hostname: 322 update_files.append(prev_hostname_fn) 323 324 # If the system hostname is different than the previous 325 # one or the desired one lets update it as well 326 if ((not sys_hostname) or (sys_hostname == prev_hostname and 327 sys_hostname != hostname)): 328 update_files.append(sys_fn) 329 330 # If something else has changed the hostname after we set it 331 # initially, we should not overwrite those changes (we should 332 # only be setting the hostname once per instance) 333 if (sys_hostname and prev_hostname and 334 sys_hostname != prev_hostname): 335 LOG.info("%s differs from %s, assuming user maintained hostname.", 336 prev_hostname_fn, sys_fn) 337 return 338 339 # Remove duplicates (incase the previous config filename) 340 # is the same as the system config filename, don't bother 341 # doing it twice 342 update_files = set([f for f in update_files if f]) 343 LOG.debug("Attempting to update hostname to %s in %s files", 344 hostname, len(update_files)) 345 346 for fn in update_files: 347 try: 348 self._write_hostname(hostname, fn) 349 except IOError: 350 util.logexc(LOG, "Failed to write hostname %s to %s", hostname, 351 fn) 352 353 # If the system hostname file name was provided set the 354 # non-fqdn as the transient hostname. 355 if sys_fn in update_files: 356 self._apply_hostname(applying_hostname) 357 358 def update_etc_hosts(self, hostname, fqdn): 359 header = '' 360 if os.path.exists(self.hosts_fn): 361 eh = hosts.HostsConf(util.load_file(self.hosts_fn)) 362 else: 363 eh = hosts.HostsConf('') 364 header = util.make_header(base="added") 365 local_ip = self._get_localhost_ip() 366 prev_info = eh.get_entry(local_ip) 367 need_change = False 368 if not prev_info: 369 eh.add_entry(local_ip, fqdn, hostname) 370 need_change = True 371 else: 372 need_change = True 373 for entry in prev_info: 374 entry_fqdn = None 375 entry_aliases = [] 376 if len(entry) >= 1: 377 entry_fqdn = entry[0] 378 if len(entry) >= 2: 379 entry_aliases = entry[1:] 380 if entry_fqdn is not None and entry_fqdn == fqdn: 381 if hostname in entry_aliases: 382 # Exists already, leave it be 383 need_change = False 384 if need_change: 385 # Doesn't exist, add that entry in... 386 new_entries = list(prev_info) 387 new_entries.append([fqdn, hostname]) 388 eh.del_entries(local_ip) 389 for entry in new_entries: 390 if len(entry) == 1: 391 eh.add_entry(local_ip, entry[0]) 392 elif len(entry) >= 2: 393 eh.add_entry(local_ip, *entry) 394 if need_change: 395 contents = StringIO() 396 if header: 397 contents.write("%s\n" % (header)) 398 contents.write("%s\n" % (eh)) 399 util.write_file(self.hosts_fn, contents.getvalue(), mode=0o644) 400 401 @property 402 def preferred_ntp_clients(self): 403 """Allow distro to determine the preferred ntp client list""" 404 if not self._preferred_ntp_clients: 405 self._preferred_ntp_clients = list(PREFERRED_NTP_CLIENTS) 406 407 return self._preferred_ntp_clients 408 409 def _bring_up_interface(self, device_name): 410 """Deprecated. Remove if/when arch and gentoo support renderers.""" 411 raise NotImplementedError 412 413 def _bring_up_interfaces(self, device_names): 414 """Deprecated. Remove if/when arch and gentoo support renderers.""" 415 am_failed = 0 416 for d in device_names: 417 if not self._bring_up_interface(d): 418 am_failed += 1 419 if am_failed == 0: 420 return True 421 return False 422 423 def get_default_user(self): 424 return self.get_option('default_user') 425 426 def add_user(self, name, **kwargs): 427 """ 428 Add a user to the system using standard GNU tools 429 430 This should be overriden on distros where useradd is not desirable or 431 not available. 432 """ 433 # XXX need to make add_user idempotent somehow as we 434 # still want to add groups or modify SSH keys on pre-existing 435 # users in the image. 436 if util.is_user(name): 437 LOG.info("User %s already exists, skipping.", name) 438 return 439 440 if 'create_groups' in kwargs: 441 create_groups = kwargs.pop('create_groups') 442 else: 443 create_groups = True 444 445 useradd_cmd = ['useradd', name] 446 log_useradd_cmd = ['useradd', name] 447 if util.system_is_snappy(): 448 useradd_cmd.append('--extrausers') 449 log_useradd_cmd.append('--extrausers') 450 451 # Since we are creating users, we want to carefully validate the 452 # inputs. If something goes wrong, we can end up with a system 453 # that nobody can login to. 454 useradd_opts = { 455 "gecos": '--comment', 456 "homedir": '--home', 457 "primary_group": '--gid', 458 "uid": '--uid', 459 "groups": '--groups', 460 "passwd": '--password', 461 "shell": '--shell', 462 "expiredate": '--expiredate', 463 "inactive": '--inactive', 464 "selinux_user": '--selinux-user', 465 } 466 467 useradd_flags = { 468 "no_user_group": '--no-user-group', 469 "system": '--system', 470 "no_log_init": '--no-log-init', 471 } 472 473 redact_opts = ['passwd'] 474 475 # support kwargs having groups=[list] or groups="g1,g2" 476 groups = kwargs.get('groups') 477 if groups: 478 if isinstance(groups, str): 479 groups = groups.split(",") 480 481 # remove any white spaces in group names, most likely 482 # that came in as a string like: groups: group1, group2 483 groups = [g.strip() for g in groups] 484 485 # kwargs.items loop below wants a comma delimeted string 486 # that can go right through to the command. 487 kwargs['groups'] = ",".join(groups) 488 489 primary_group = kwargs.get('primary_group') 490 if primary_group: 491 groups.append(primary_group) 492 493 if create_groups and groups: 494 for group in groups: 495 if not util.is_group(group): 496 self.create_group(group) 497 LOG.debug("created group '%s' for user '%s'", group, name) 498 499 # Check the values and create the command 500 for key, val in sorted(kwargs.items()): 501 502 if key in useradd_opts and val and isinstance(val, str): 503 useradd_cmd.extend([useradd_opts[key], val]) 504 505 # Redact certain fields from the logs 506 if key in redact_opts: 507 log_useradd_cmd.extend([useradd_opts[key], 'REDACTED']) 508 else: 509 log_useradd_cmd.extend([useradd_opts[key], val]) 510 511 elif key in useradd_flags and val: 512 useradd_cmd.append(useradd_flags[key]) 513 log_useradd_cmd.append(useradd_flags[key]) 514 515 # Don't create the home directory if directed so or if the user is a 516 # system user 517 if kwargs.get('no_create_home') or kwargs.get('system'): 518 useradd_cmd.append('-M') 519 log_useradd_cmd.append('-M') 520 else: 521 useradd_cmd.append('-m') 522 log_useradd_cmd.append('-m') 523 524 # Run the command 525 LOG.debug("Adding user %s", name) 526 try: 527 subp.subp(useradd_cmd, logstring=log_useradd_cmd) 528 except Exception as e: 529 util.logexc(LOG, "Failed to create user %s", name) 530 raise e 531 532 def add_snap_user(self, name, **kwargs): 533 """ 534 Add a snappy user to the system using snappy tools 535 """ 536 537 snapuser = kwargs.get('snapuser') 538 known = kwargs.get('known', False) 539 create_user_cmd = ["snap", "create-user", "--sudoer", "--json"] 540 if known: 541 create_user_cmd.append("--known") 542 create_user_cmd.append(snapuser) 543 544 # Run the command 545 LOG.debug("Adding snap user %s", name) 546 try: 547 (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd, 548 capture=True) 549 LOG.debug("snap create-user returned: %s:%s", out, err) 550 jobj = util.load_json(out) 551 username = jobj.get('username', None) 552 except Exception as e: 553 util.logexc(LOG, "Failed to create snap user %s", name) 554 raise e 555 556 return username 557 558 def create_user(self, name, **kwargs): 559 """ 560 Creates or partially updates the ``name`` user in the system. 561 562 This defers the actual user creation to ``self.add_user`` or 563 ``self.add_snap_user``, and most of the keys in ``kwargs`` will be 564 processed there if and only if the user does not already exist. 565 566 Once the existence of the ``name`` user has been ensured, this method 567 then processes these keys (for both just-created and pre-existing 568 users): 569 570 * ``plain_text_passwd`` 571 * ``hashed_passwd`` 572 * ``lock_passwd`` 573 * ``sudo`` 574 * ``ssh_authorized_keys`` 575 * ``ssh_redirect_user`` 576 """ 577 578 # Add a snap user, if requested 579 if 'snapuser' in kwargs: 580 return self.add_snap_user(name, **kwargs) 581 582 # Add the user 583 self.add_user(name, **kwargs) 584 585 # Set password if plain-text password provided and non-empty 586 if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: 587 self.set_passwd(name, kwargs['plain_text_passwd']) 588 589 # Set password if hashed password is provided and non-empty 590 if 'hashed_passwd' in kwargs and kwargs['hashed_passwd']: 591 self.set_passwd(name, kwargs['hashed_passwd'], hashed=True) 592 593 # Default locking down the account. 'lock_passwd' defaults to True. 594 # lock account unless lock_password is False. 595 if kwargs.get('lock_passwd', True): 596 self.lock_passwd(name) 597 598 # Configure sudo access 599 if 'sudo' in kwargs and kwargs['sudo'] is not False: 600 self.write_sudo_rules(name, kwargs['sudo']) 601 602 # Import SSH keys 603 if 'ssh_authorized_keys' in kwargs: 604 # Try to handle this in a smart manner. 605 keys = kwargs['ssh_authorized_keys'] 606 if isinstance(keys, str): 607 keys = [keys] 608 elif isinstance(keys, dict): 609 keys = list(keys.values()) 610 if keys is not None: 611 if not isinstance(keys, (tuple, list, set)): 612 LOG.warning("Invalid type '%s' detected for" 613 " 'ssh_authorized_keys', expected list," 614 " string, dict, or set.", type(keys)) 615 keys = [] 616 else: 617 keys = set(keys) or [] 618 ssh_util.setup_user_keys(set(keys), name) 619 if 'ssh_redirect_user' in kwargs: 620 cloud_keys = kwargs.get('cloud_public_ssh_keys', []) 621 if not cloud_keys: 622 LOG.warning( 623 'Unable to disable SSH logins for %s given' 624 ' ssh_redirect_user: %s. No cloud public-keys present.', 625 name, kwargs['ssh_redirect_user']) 626 else: 627 redirect_user = kwargs['ssh_redirect_user'] 628 disable_option = ssh_util.DISABLE_USER_OPTS 629 disable_option = disable_option.replace('$USER', redirect_user) 630 disable_option = disable_option.replace('$DISABLE_USER', name) 631 ssh_util.setup_user_keys( 632 set(cloud_keys), name, options=disable_option) 633 return True 634 635 def lock_passwd(self, name): 636 """ 637 Lock the password of a user, i.e., disable password logins 638 """ 639 # passwd must use short '-l' due to SLES11 lacking long form '--lock' 640 lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) 641 try: 642 cmd = next(tool for tool in lock_tools if subp.which(tool[0])) 643 except StopIteration as e: 644 raise RuntimeError(( 645 "Unable to lock user account '%s'. No tools available. " 646 " Tried: %s.") % (name, [c[0] for c in lock_tools]) 647 ) from e 648 try: 649 subp.subp(cmd) 650 except Exception as e: 651 util.logexc(LOG, 'Failed to disable password for user %s', name) 652 raise e 653 654 def expire_passwd(self, user): 655 try: 656 subp.subp(['passwd', '--expire', user]) 657 except Exception as e: 658 util.logexc(LOG, "Failed to set 'expire' for %s", user) 659 raise e 660 661 def set_passwd(self, user, passwd, hashed=False): 662 pass_string = '%s:%s' % (user, passwd) 663 cmd = ['chpasswd'] 664 665 if hashed: 666 # Need to use the short option name '-e' instead of '--encrypted' 667 # (which would be more descriptive) since SLES 11 doesn't know 668 # about long names. 669 cmd.append('-e') 670 671 try: 672 subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user) 673 except Exception as e: 674 util.logexc(LOG, "Failed to set password for %s", user) 675 raise e 676 677 return True 678 679 def ensure_sudo_dir(self, path, sudo_base='/etc/sudoers'): 680 # Ensure the dir is included and that 681 # it actually exists as a directory 682 sudoers_contents = '' 683 base_exists = False 684 if os.path.exists(sudo_base): 685 sudoers_contents = util.load_file(sudo_base) 686 base_exists = True 687 found_include = False 688 for line in sudoers_contents.splitlines(): 689 line = line.strip() 690 include_match = re.search(r"^[#|@]includedir\s+(.*)$", line) 691 if not include_match: 692 continue 693 included_dir = include_match.group(1).strip() 694 if not included_dir: 695 continue 696 included_dir = os.path.abspath(included_dir) 697 if included_dir == path: 698 found_include = True 699 break 700 if not found_include: 701 try: 702 if not base_exists: 703 lines = [('# See sudoers(5) for more information' 704 ' on "#include" directives:'), '', 705 util.make_header(base="added"), 706 "#includedir %s" % (path), ''] 707 sudoers_contents = "\n".join(lines) 708 util.write_file(sudo_base, sudoers_contents, 0o440) 709 else: 710 lines = ['', util.make_header(base="added"), 711 "#includedir %s" % (path), ''] 712 sudoers_contents = "\n".join(lines) 713 util.append_file(sudo_base, sudoers_contents) 714 LOG.debug("Added '#includedir %s' to %s", path, sudo_base) 715 except IOError as e: 716 util.logexc(LOG, "Failed to write %s", sudo_base) 717 raise e 718 util.ensure_dir(path, 0o750) 719 720 def write_sudo_rules(self, user, rules, sudo_file=None): 721 if not sudo_file: 722 sudo_file = self.ci_sudoers_fn 723 724 lines = [ 725 '', 726 "# User rules for %s" % user, 727 ] 728 if isinstance(rules, (list, tuple)): 729 for rule in rules: 730 lines.append("%s %s" % (user, rule)) 731 elif isinstance(rules, str): 732 lines.append("%s %s" % (user, rules)) 733 else: 734 msg = "Can not create sudoers rule addition with type %r" 735 raise TypeError(msg % (type_utils.obj_name(rules))) 736 content = "\n".join(lines) 737 content += "\n" # trailing newline 738 739 self.ensure_sudo_dir(os.path.dirname(sudo_file)) 740 if not os.path.exists(sudo_file): 741 contents = [ 742 util.make_header(), 743 content, 744 ] 745 try: 746 util.write_file(sudo_file, "\n".join(contents), 0o440) 747 except IOError as e: 748 util.logexc(LOG, "Failed to write sudoers file %s", sudo_file) 749 raise e 750 else: 751 try: 752 util.append_file(sudo_file, content) 753 except IOError as e: 754 util.logexc(LOG, "Failed to append sudoers file %s", sudo_file) 755 raise e 756 757 def create_group(self, name, members=None): 758 group_add_cmd = ['groupadd', name] 759 if util.system_is_snappy(): 760 group_add_cmd.append('--extrausers') 761 if not members: 762 members = [] 763 764 # Check if group exists, and then add it doesn't 765 if util.is_group(name): 766 LOG.warning("Skipping creation of existing group '%s'", name) 767 else: 768 try: 769 subp.subp(group_add_cmd) 770 LOG.info("Created new group %s", name) 771 except Exception: 772 util.logexc(LOG, "Failed to create group %s", name) 773 774 # Add members to the group, if so defined 775 if len(members) > 0: 776 for member in members: 777 if not util.is_user(member): 778 LOG.warning("Unable to add group member '%s' to group '%s'" 779 "; user does not exist.", member, name) 780 continue 781 782 subp.subp(['usermod', '-a', '-G', name, member]) 783 LOG.info("Added user '%s' to group '%s'", member, name) 784 785 def shutdown_command(self, *, mode, delay, message): 786 # called from cc_power_state_change.load_power_state 787 command = ["shutdown", self.shutdown_options_map[mode]] 788 try: 789 if delay != "now": 790 delay = "+%d" % int(delay) 791 except ValueError as e: 792 raise TypeError( 793 "power_state[delay] must be 'now' or '+m' (minutes)." 794 " found '%s'." % (delay,) 795 ) from e 796 args = command + [delay] 797 if message: 798 args.append(message) 799 return args 800 801 def manage_service(self, action, service): 802 """ 803 Perform the requested action on a service. This handles the common 804 'systemctl' and 'service' cases and may be overridden in subclasses 805 as necessary. 806 May raise ProcessExecutionError 807 """ 808 init_cmd = self.init_cmd 809 if self.uses_systemd() or 'systemctl' in init_cmd: 810 init_cmd = ['systemctl'] 811 cmds = {'stop': ['stop', service], 812 'start': ['start', service], 813 'enable': ['enable', service], 814 'restart': ['restart', service], 815 'reload': ['reload-or-restart', service], 816 'try-reload': ['reload-or-try-restart', service], 817 } 818 else: 819 cmds = {'stop': [service, 'stop'], 820 'start': [service, 'start'], 821 'enable': [service, 'start'], 822 'restart': [service, 'restart'], 823 'reload': [service, 'restart'], 824 'try-reload': [service, 'restart'], 825 } 826 cmd = list(init_cmd) + list(cmds[action]) 827 return subp.subp(cmd, capture=True) 828 829 830def _apply_hostname_transformations_to_url(url: str, transformations: list): 831 """ 832 Apply transformations to a URL's hostname, return transformed URL. 833 834 This is a separate function because unwrapping and rewrapping only the 835 hostname portion of a URL is complex. 836 837 :param url: 838 The URL to operate on. 839 :param transformations: 840 A list of ``(str) -> Optional[str]`` functions, which will be applied 841 in order to the hostname portion of the URL. If any function 842 (regardless of ordering) returns None, ``url`` will be returned without 843 any modification. 844 845 :return: 846 A string whose value is ``url`` with the hostname ``transformations`` 847 applied, or ``None`` if ``url`` is unparseable. 848 """ 849 try: 850 parts = urllib.parse.urlsplit(url) 851 except ValueError: 852 # If we can't even parse the URL, we shouldn't use it for anything 853 return None 854 new_hostname = parts.hostname 855 if new_hostname is None: 856 # The URL given doesn't have a hostname component, so (a) we can't 857 # transform it, and (b) it won't work as a mirror; return None. 858 return None 859 860 for transformation in transformations: 861 new_hostname = transformation(new_hostname) 862 if new_hostname is None: 863 # If a transformation returns None, that indicates we should abort 864 # processing and return `url` unmodified 865 return url 866 867 new_netloc = new_hostname 868 if parts.port is not None: 869 new_netloc = "{}:{}".format(new_netloc, parts.port) 870 return urllib.parse.urlunsplit(parts._replace(netloc=new_netloc)) 871 872 873def _sanitize_mirror_url(url: str): 874 """ 875 Given a mirror URL, replace or remove any invalid URI characters. 876 877 This performs the following actions on the URL's hostname: 878 * Checks if it is an IP address, returning the URL immediately if it is 879 * Converts it to its IDN form (see below for details) 880 * Replaces any non-Letters/Digits/Hyphen (LDH) characters in it with 881 hyphens 882 * Removes any leading/trailing hyphens from each domain name label 883 884 Before we replace any invalid domain name characters, we first need to 885 ensure that any valid non-ASCII characters in the hostname will not be 886 replaced, by ensuring the hostname is in its Internationalized domain name 887 (IDN) representation (see RFC 5890). This conversion has to be applied to 888 the whole hostname (rather than just the substitution variables), because 889 the Punycode algorithm used by IDNA transcodes each part of the hostname as 890 a whole string (rather than encoding individual characters). It cannot be 891 applied to the whole URL, because (a) the Punycode algorithm expects to 892 operate on domain names so doesn't output a valid URL, and (b) non-ASCII 893 characters in non-hostname parts of the URL aren't encoded via Punycode. 894 895 To put this in RFC 5890's terminology: before we remove or replace any 896 characters from our domain name (which we do to ensure that each label is a 897 valid LDH Label), we first ensure each label is in its A-label form. 898 899 (Note that Python's builtin idna encoding is actually IDNA2003, not 900 IDNA2008. This changes the specifics of how some characters are encoded to 901 ASCII, but doesn't affect the logic here.) 902 903 :param url: 904 The URL to operate on. 905 906 :return: 907 A sanitized version of the URL, which will have been IDNA encoded if 908 necessary, or ``None`` if the generated string is not a parseable URL. 909 """ 910 # Acceptable characters are LDH characters, plus "." to separate each label 911 acceptable_chars = LDH_ASCII_CHARS + "." 912 transformations = [ 913 # This is an IP address, not a hostname, so no need to apply the 914 # transformations 915 lambda hostname: None if net.is_ip_address(hostname) else hostname, 916 917 # Encode with IDNA to get the correct characters (as `bytes`), then 918 # decode with ASCII so we return a `str` 919 lambda hostname: hostname.encode('idna').decode('ascii'), 920 921 # Replace any unacceptable characters with "-" 922 lambda hostname: ''.join( 923 c if c in acceptable_chars else "-" for c in hostname 924 ), 925 926 # Drop leading/trailing hyphens from each part of the hostname 927 lambda hostname: '.'.join( 928 part.strip('-') for part in hostname.split('.') 929 ), 930 ] 931 932 return _apply_hostname_transformations_to_url(url, transformations) 933 934 935def _get_package_mirror_info(mirror_info, data_source=None, 936 mirror_filter=util.search_for_mirror): 937 # given a arch specific 'mirror_info' entry (from package_mirrors) 938 # search through the 'search' entries, and fallback appropriately 939 # return a dict with only {name: mirror} entries. 940 if not mirror_info: 941 mirror_info = {} 942 943 subst = {} 944 if data_source and data_source.availability_zone: 945 subst['availability_zone'] = data_source.availability_zone 946 947 # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) 948 # the region is us-east-1. so region = az[0:-1] 949 if _EC2_AZ_RE.match(data_source.availability_zone): 950 ec2_region = data_source.availability_zone[0:-1] 951 952 if ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES: 953 subst['ec2_region'] = "%s" % ec2_region 954 elif data_source.platform_type == "ec2": 955 subst['ec2_region'] = "%s" % ec2_region 956 957 if data_source and data_source.region: 958 subst['region'] = data_source.region 959 960 results = {} 961 for (name, mirror) in mirror_info.get('failsafe', {}).items(): 962 results[name] = mirror 963 964 for (name, searchlist) in mirror_info.get('search', {}).items(): 965 mirrors = [] 966 for tmpl in searchlist: 967 try: 968 mirror = tmpl % subst 969 except KeyError: 970 continue 971 972 mirror = _sanitize_mirror_url(mirror) 973 if mirror is not None: 974 mirrors.append(mirror) 975 976 found = mirror_filter(mirrors) 977 if found: 978 results[name] = found 979 980 LOG.debug("filtered distro mirror info: %s", results) 981 982 return results 983 984 985def _get_arch_package_mirror_info(package_mirrors, arch): 986 # pull out the specific arch from a 'package_mirrors' config option 987 default = None 988 for item in package_mirrors: 989 arches = item.get("arches") 990 if arch in arches: 991 return item 992 if "default" in arches: 993 default = item 994 return default 995 996 997def fetch(name): 998 locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) 999 if not locs: 1000 raise ImportError("No distribution found for distro %s (searched %s)" 1001 % (name, looked_locs)) 1002 mod = importer.import_module(locs[0]) 1003 cls = getattr(mod, 'Distro') 1004 return cls 1005 1006 1007def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", 1008 tz_local="/etc/localtime"): 1009 util.write_file(tz_conf, str(tz).rstrip() + "\n") 1010 # This ensures that the correct tz will be used for the system 1011 if tz_local and tz_file: 1012 # use a symlink if there exists a symlink or tz_local is not present 1013 islink = os.path.islink(tz_local) 1014 if islink or not os.path.exists(tz_local): 1015 if islink: 1016 util.del_file(tz_local) 1017 os.symlink(tz_file, tz_local) 1018 else: 1019 util.copy(tz_file, tz_local) 1020 return 1021 1022 1023def uses_systemd(): 1024 try: 1025 res = os.lstat('/run/systemd/system') 1026 return stat.S_ISDIR(res.st_mode) 1027 except Exception: 1028 return False 1029 1030 1031# vi: ts=4 expandtab 1032