1# Copyright (C) 2012 Canonical Ltd. 2# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. 3# Copyright (C) 2012 Yahoo! Inc. 4# 5# Author: Scott Moser <scott.moser@canonical.com> 6# Author: Juerg Haefliger <juerg.haefliger@hp.com> 7# Author: Joshua Harlow <harlowja@yahoo-inc.com> 8# 9# This file is part of cloud-init. See LICENSE file for license information. 10 11import contextlib 12import copy as obj_copy 13import email 14import glob 15import grp 16import gzip 17import hashlib 18import io 19import json 20import os 21import os.path 22import platform 23import pwd 24import random 25import re 26import shlex 27import shutil 28import socket 29import stat 30import string 31import subprocess 32import sys 33import time 34from base64 import b64decode, b64encode 35from errno import ENOENT 36from functools import lru_cache 37from urllib import parse 38from typing import List 39 40from cloudinit import importer 41from cloudinit import log as logging 42from cloudinit import subp 43from cloudinit import ( 44 mergers, 45 safeyaml, 46 temp_utils, 47 type_utils, 48 url_helper, 49 version, 50) 51from cloudinit.settings import CFG_BUILTIN 52 53_DNS_REDIRECT_IP = None 54LOG = logging.getLogger(__name__) 55 56# Helps cleanup filenames to ensure they aren't FS incompatible 57FN_REPLACEMENTS = { 58 os.sep: '_', 59} 60FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) 61 62TRUE_STRINGS = ('true', '1', 'on', 'yes') 63FALSE_STRINGS = ('off', '0', 'no', 'false') 64 65 66def kernel_version(): 67 return tuple(map(int, os.uname().release.split('.')[:2])) 68 69 70@lru_cache() 71def get_dpkg_architecture(target=None): 72 """Return the sanitized string output by `dpkg --print-architecture`. 73 74 N.B. This function is wrapped in functools.lru_cache, so repeated calls 75 won't shell out every time. 76 """ 77 out, _ = subp.subp(['dpkg', '--print-architecture'], capture=True, 78 target=target) 79 return out.strip() 80 81 82@lru_cache() 83def lsb_release(target=None): 84 fmap = {'Codename': 'codename', 'Description': 'description', 85 'Distributor ID': 'id', 'Release': 'release'} 86 87 data = {} 88 try: 89 out, _ = subp.subp(['lsb_release', '--all'], capture=True, 90 target=target) 91 for line in out.splitlines(): 92 fname, _, val = line.partition(":") 93 if fname in fmap: 94 data[fmap[fname]] = val.strip() 95 missing = [k for k in fmap.values() if k not in data] 96 if len(missing): 97 LOG.warning("Missing fields in lsb_release --all output: %s", 98 ','.join(missing)) 99 100 except subp.ProcessExecutionError as err: 101 LOG.warning("Unable to get lsb_release --all: %s", err) 102 data = dict((v, "UNAVAILABLE") for v in fmap.values()) 103 104 return data 105 106 107def decode_binary(blob, encoding='utf-8'): 108 # Converts a binary type into a text type using given encoding. 109 if isinstance(blob, str): 110 return blob 111 return blob.decode(encoding) 112 113 114def encode_text(text, encoding='utf-8'): 115 # Converts a text string into a binary type using given encoding. 116 if isinstance(text, bytes): 117 return text 118 return text.encode(encoding) 119 120 121def b64d(source): 122 # Base64 decode some data, accepting bytes or unicode/str, and returning 123 # str/unicode if the result is utf-8 compatible, otherwise returning bytes. 124 decoded = b64decode(source) 125 try: 126 return decoded.decode('utf-8') 127 except UnicodeDecodeError: 128 return decoded 129 130 131def b64e(source): 132 # Base64 encode some data, accepting bytes or unicode/str, and returning 133 # str/unicode if the result is utf-8 compatible, otherwise returning bytes. 134 if not isinstance(source, bytes): 135 source = source.encode('utf-8') 136 return b64encode(source).decode('utf-8') 137 138 139def fully_decoded_payload(part): 140 # In Python 3, decoding the payload will ironically hand us a bytes object. 141 # 'decode' means to decode according to Content-Transfer-Encoding, not 142 # according to any charset in the Content-Type. So, if we end up with 143 # bytes, first try to decode to str via CT charset, and failing that, try 144 # utf-8 using surrogate escapes. 145 cte_payload = part.get_payload(decode=True) 146 if (part.get_content_maintype() == 'text' and 147 isinstance(cte_payload, bytes)): 148 charset = part.get_charset() 149 if charset and charset.input_codec: 150 encoding = charset.input_codec 151 else: 152 encoding = 'utf-8' 153 return cte_payload.decode(encoding, 'surrogateescape') 154 return cte_payload 155 156 157class SeLinuxGuard(object): 158 def __init__(self, path, recursive=False): 159 # Late import since it might not always 160 # be possible to use this 161 try: 162 self.selinux = importer.import_module('selinux') 163 except ImportError: 164 self.selinux = None 165 self.path = path 166 self.recursive = recursive 167 168 def __enter__(self): 169 if self.selinux and self.selinux.is_selinux_enabled(): 170 return True 171 else: 172 return False 173 174 def __exit__(self, excp_type, excp_value, excp_traceback): 175 if not self.selinux or not self.selinux.is_selinux_enabled(): 176 return 177 if not os.path.lexists(self.path): 178 return 179 180 path = os.path.realpath(self.path) 181 try: 182 stats = os.lstat(path) 183 self.selinux.matchpathcon(path, stats[stat.ST_MODE]) 184 except OSError: 185 return 186 187 LOG.debug("Restoring selinux mode for %s (recursive=%s)", 188 path, self.recursive) 189 try: 190 self.selinux.restorecon(path, recursive=self.recursive) 191 except OSError as e: 192 LOG.warning('restorecon failed on %s,%s maybe badness? %s', 193 path, self.recursive, e) 194 195 196class MountFailedError(Exception): 197 pass 198 199 200class DecompressionError(Exception): 201 pass 202 203 204def fork_cb(child_cb, *args, **kwargs): 205 fid = os.fork() 206 if fid == 0: 207 try: 208 child_cb(*args, **kwargs) 209 os._exit(0) 210 except Exception: 211 logexc(LOG, "Failed forking and calling callback %s", 212 type_utils.obj_name(child_cb)) 213 os._exit(1) 214 else: 215 LOG.debug("Forked child %s who will run callback %s", 216 fid, type_utils.obj_name(child_cb)) 217 218 219def is_true(val, addons=None): 220 if isinstance(val, (bool)): 221 return val is True 222 check_set = TRUE_STRINGS 223 if addons: 224 check_set = list(check_set) + addons 225 if str(val).lower().strip() in check_set: 226 return True 227 return False 228 229 230def is_false(val, addons=None): 231 if isinstance(val, (bool)): 232 return val is False 233 check_set = FALSE_STRINGS 234 if addons: 235 check_set = list(check_set) + addons 236 if str(val).lower().strip() in check_set: 237 return True 238 return False 239 240 241def translate_bool(val, addons=None): 242 if not val: 243 # This handles empty lists and false and 244 # other things that python believes are false 245 return False 246 # If its already a boolean skip 247 if isinstance(val, (bool)): 248 return val 249 return is_true(val, addons) 250 251 252def rand_str(strlen=32, select_from=None): 253 r = random.SystemRandom() 254 if not select_from: 255 select_from = string.ascii_letters + string.digits 256 return "".join([r.choice(select_from) for _x in range(0, strlen)]) 257 258 259def rand_dict_key(dictionary, postfix=None): 260 if not postfix: 261 postfix = "" 262 while True: 263 newkey = rand_str(strlen=8) + "_" + postfix 264 if newkey not in dictionary: 265 break 266 return newkey 267 268 269def read_conf(fname): 270 try: 271 return load_yaml(load_file(fname), default={}) 272 except IOError as e: 273 if e.errno == ENOENT: 274 return {} 275 else: 276 raise 277 278 279# Merges X lists, and then keeps the 280# unique ones, but orders by sort order 281# instead of by the original order 282def uniq_merge_sorted(*lists): 283 return sorted(uniq_merge(*lists)) 284 285 286# Merges X lists and then iterates over those 287# and only keeps the unique items (order preserving) 288# and returns that merged and uniqued list as the 289# final result. 290# 291# Note: if any entry is a string it will be 292# split on commas and empty entries will be 293# evicted and merged in accordingly. 294def uniq_merge(*lists): 295 combined_list = [] 296 for a_list in lists: 297 if isinstance(a_list, str): 298 a_list = a_list.strip().split(",") 299 # Kickout the empty ones 300 a_list = [a for a in a_list if a] 301 combined_list.extend(a_list) 302 return uniq_list(combined_list) 303 304 305def clean_filename(fn): 306 for (k, v) in FN_REPLACEMENTS.items(): 307 fn = fn.replace(k, v) 308 removals = [] 309 for k in fn: 310 if k not in FN_ALLOWED: 311 removals.append(k) 312 for k in removals: 313 fn = fn.replace(k, '') 314 fn = fn.strip() 315 return fn 316 317 318def decomp_gzip(data, quiet=True, decode=True): 319 try: 320 buf = io.BytesIO(encode_text(data)) 321 with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: 322 # E1101 is https://github.com/PyCQA/pylint/issues/1444 323 if decode: 324 return decode_binary(gh.read()) # pylint: disable=E1101 325 else: 326 return gh.read() # pylint: disable=E1101 327 except Exception as e: 328 if quiet: 329 return data 330 else: 331 raise DecompressionError(str(e)) from e 332 333 334def extract_usergroup(ug_pair): 335 if not ug_pair: 336 return (None, None) 337 ug_parted = ug_pair.split(':', 1) 338 u = ug_parted[0].strip() 339 if len(ug_parted) == 2: 340 g = ug_parted[1].strip() 341 else: 342 g = None 343 if not u or u == "-1" or u.lower() == "none": 344 u = None 345 if not g or g == "-1" or g.lower() == "none": 346 g = None 347 return (u, g) 348 349 350def find_modules(root_dir): 351 entries = dict() 352 for fname in glob.glob(os.path.join(root_dir, "*.py")): 353 if not os.path.isfile(fname): 354 continue 355 modname = os.path.basename(fname)[0:-3] 356 modname = modname.strip() 357 if modname and modname.find(".") == -1: 358 entries[fname] = modname 359 return entries 360 361 362def multi_log(text, console=True, stderr=True, 363 log=None, log_level=logging.DEBUG, fallback_to_stdout=True): 364 if stderr: 365 sys.stderr.write(text) 366 if console: 367 conpath = "/dev/console" 368 if os.path.exists(conpath): 369 with open(conpath, 'w') as wfh: 370 wfh.write(text) 371 wfh.flush() 372 elif fallback_to_stdout: 373 # A container may lack /dev/console (arguably a container bug). If 374 # it does not exist, then write output to stdout. this will result 375 # in duplicate stderr and stdout messages if stderr was True. 376 # 377 # even though upstart or systemd might have set up output to go to 378 # /dev/console, the user may have configured elsewhere via 379 # cloud-config 'output'. If there is /dev/console, messages will 380 # still get there. 381 sys.stdout.write(text) 382 if log: 383 if text[-1] == "\n": 384 log.log(log_level, text[:-1]) 385 else: 386 log.log(log_level, text) 387 388 389@lru_cache() 390def is_Linux(): 391 return 'Linux' in platform.system() 392 393 394@lru_cache() 395def is_BSD(): 396 if 'BSD' in platform.system(): 397 return True 398 if platform.system() == 'DragonFly': 399 return True 400 return False 401 402 403@lru_cache() 404def is_FreeBSD(): 405 return system_info()['variant'] == "freebsd" 406 407 408@lru_cache() 409def is_DragonFlyBSD(): 410 return system_info()['variant'] == "dragonfly" 411 412 413@lru_cache() 414def is_NetBSD(): 415 return system_info()['variant'] == "netbsd" 416 417 418@lru_cache() 419def is_OpenBSD(): 420 return system_info()['variant'] == "openbsd" 421 422 423def get_cfg_option_bool(yobj, key, default=False): 424 if key not in yobj: 425 return default 426 return translate_bool(yobj[key]) 427 428 429def get_cfg_option_str(yobj, key, default=None): 430 if key not in yobj: 431 return default 432 val = yobj[key] 433 if not isinstance(val, str): 434 val = str(val) 435 return val 436 437 438def get_cfg_option_int(yobj, key, default=0): 439 return int(get_cfg_option_str(yobj, key, default=default)) 440 441 442def _parse_redhat_release(release_file=None): 443 """Return a dictionary of distro info fields from /etc/redhat-release. 444 445 Dict keys will align with /etc/os-release keys: 446 ID, VERSION_ID, VERSION_CODENAME 447 """ 448 449 if not release_file: 450 release_file = '/etc/redhat-release' 451 if not os.path.exists(release_file): 452 return {} 453 redhat_release = load_file(release_file) 454 redhat_regex = ( 455 r'(?P<name>.+) release (?P<version>[\d\.]+) ' 456 r'\((?P<codename>[^)]+)\)') 457 458 # Virtuozzo deviates here 459 if "Virtuozzo" in redhat_release: 460 redhat_regex = r'(?P<name>.+) release (?P<version>[\d\.]+)' 461 462 match = re.match(redhat_regex, redhat_release) 463 if match: 464 group = match.groupdict() 465 466 # Virtuozzo has no codename in this file 467 if "Virtuozzo" in group['name']: 468 group['codename'] = group['name'] 469 470 group['name'] = group['name'].lower().partition(' linux')[0] 471 if group['name'] == 'red hat enterprise': 472 group['name'] = 'redhat' 473 return {'ID': group['name'], 'VERSION_ID': group['version'], 474 'VERSION_CODENAME': group['codename']} 475 return {} 476 477 478@lru_cache() 479def get_linux_distro(): 480 distro_name = '' 481 distro_version = '' 482 flavor = '' 483 os_release = {} 484 os_release_rhel = False 485 if os.path.exists('/etc/os-release'): 486 os_release = load_shell_content(load_file('/etc/os-release')) 487 if not os_release: 488 os_release_rhel = True 489 os_release = _parse_redhat_release() 490 if os_release: 491 distro_name = os_release.get('ID', '') 492 distro_version = os_release.get('VERSION_ID', '') 493 if 'sles' in distro_name or 'suse' in distro_name: 494 # RELEASE_BLOCKER: We will drop this sles divergent behavior in 495 # the future so that get_linux_distro returns a named tuple 496 # which will include both version codename and architecture 497 # on all distributions. 498 flavor = platform.machine() 499 elif distro_name == 'photon': 500 flavor = os_release.get('PRETTY_NAME', '') 501 elif distro_name == 'virtuozzo' and not os_release_rhel: 502 # Only use this if the redhat file is not parsed 503 flavor = os_release.get('PRETTY_NAME', '') 504 else: 505 flavor = os_release.get('VERSION_CODENAME', '') 506 if not flavor: 507 match = re.match(r'[^ ]+ \((?P<codename>[^)]+)\)', 508 os_release.get('VERSION', '')) 509 if match: 510 flavor = match.groupdict()['codename'] 511 if distro_name == 'rhel': 512 distro_name = 'redhat' 513 elif is_BSD(): 514 distro_name = platform.system().lower() 515 distro_version = platform.release() 516 else: 517 dist = ('', '', '') 518 try: 519 # Was removed in 3.8 520 dist = platform.dist() # pylint: disable=W1505,E1101 521 except Exception: 522 pass 523 finally: 524 found = None 525 for entry in dist: 526 if entry: 527 found = 1 528 if not found: 529 LOG.warning('Unable to determine distribution, template ' 530 'expansion may have unexpected results') 531 return dist 532 533 return (distro_name, distro_version, flavor) 534 535 536@lru_cache() 537def system_info(): 538 info = { 539 'platform': platform.platform(), 540 'system': platform.system(), 541 'release': platform.release(), 542 'python': platform.python_version(), 543 'uname': list(platform.uname()), 544 'dist': get_linux_distro() 545 } 546 system = info['system'].lower() 547 var = 'unknown' 548 if system == "linux": 549 linux_dist = info['dist'][0].lower() 550 if linux_dist in ( 551 'almalinux', 'alpine', 'arch', 'centos', 'cloudlinux', 552 'debian', 'eurolinux', 'fedora', 'openEuler', 'photon', 553 'rhel', 'rocky', 'suse', 'virtuozzo'): 554 var = linux_dist 555 elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): 556 var = 'ubuntu' 557 elif linux_dist == 'redhat': 558 var = 'rhel' 559 elif linux_dist in ( 560 'opensuse', 'opensuse-tumbleweed', 'opensuse-leap', 561 'sles', 'sle_hpc'): 562 var = 'suse' 563 else: 564 var = 'linux' 565 elif system in ( 566 'windows', 'darwin', "freebsd", "netbsd", 567 "openbsd", "dragonfly"): 568 var = system 569 570 info['variant'] = var 571 572 return info 573 574 575def get_cfg_option_list(yobj, key, default=None): 576 """ 577 Gets the C{key} config option from C{yobj} as a list of strings. If the 578 key is present as a single string it will be returned as a list with one 579 string arg. 580 581 @param yobj: The configuration object. 582 @param key: The configuration key to get. 583 @param default: The default to return if key is not found. 584 @return: The configuration option as a list of strings or default if key 585 is not found. 586 """ 587 if key not in yobj: 588 return default 589 if yobj[key] is None: 590 return [] 591 val = yobj[key] 592 if isinstance(val, (list)): 593 cval = [v for v in val] 594 return cval 595 if not isinstance(val, str): 596 val = str(val) 597 return [val] 598 599 600# get a cfg entry by its path array 601# for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) 602def get_cfg_by_path(yobj, keyp, default=None): 603 """Return the value of the item at path C{keyp} in C{yobj}. 604 605 example: 606 get_cfg_by_path({'a': {'b': {'num': 4}}}, 'a/b/num') == 4 607 get_cfg_by_path({'a': {'b': {'num': 4}}}, 'c/d') == None 608 609 @param yobj: A dictionary. 610 @param keyp: A path inside yobj. it can be a '/' delimited string, 611 or an iterable. 612 @param default: The default to return if the path does not exist. 613 @return: The value of the item at keyp." 614 is not found.""" 615 616 if isinstance(keyp, str): 617 keyp = keyp.split("/") 618 cur = yobj 619 for tok in keyp: 620 if tok not in cur: 621 return default 622 cur = cur[tok] 623 return cur 624 625 626def fixup_output(cfg, mode): 627 (outfmt, errfmt) = get_output_cfg(cfg, mode) 628 redirect_output(outfmt, errfmt) 629 return (outfmt, errfmt) 630 631 632# redirect_output(outfmt, errfmt, orig_out, orig_err) 633# replace orig_out and orig_err with filehandles specified in outfmt or errfmt 634# fmt can be: 635# > FILEPATH 636# >> FILEPATH 637# | program [ arg1 [ arg2 [ ... ] ] ] 638# 639# with a '|', arguments are passed to shell, so one level of 640# shell escape is required. 641# 642# if _CLOUD_INIT_SAVE_STDOUT is set in environment to a non empty and true 643# value then output input will not be closed (useful for debugging). 644# 645def redirect_output(outfmt, errfmt, o_out=None, o_err=None): 646 647 if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDOUT")): 648 LOG.debug("Not redirecting output due to _CLOUD_INIT_SAVE_STDOUT") 649 return 650 651 if not o_out: 652 o_out = sys.stdout 653 if not o_err: 654 o_err = sys.stderr 655 656 # pylint: disable=subprocess-popen-preexec-fn 657 def set_subprocess_umask_and_gid(): 658 """Reconfigure umask and group ID to create output files securely. 659 660 This is passed to subprocess.Popen as preexec_fn, so it is executed in 661 the context of the newly-created process. It: 662 663 * sets the umask of the process so created files aren't world-readable 664 * if an adm group exists in the system, sets that as the process' GID 665 (so that the created file(s) are owned by root:adm) 666 """ 667 os.umask(0o037) 668 try: 669 group_id = grp.getgrnam("adm").gr_gid 670 except KeyError: 671 # No adm group, don't set a group 672 pass 673 else: 674 os.setgid(group_id) 675 676 if outfmt: 677 LOG.debug("Redirecting %s to %s", o_out, outfmt) 678 (mode, arg) = outfmt.split(" ", 1) 679 if mode == ">" or mode == ">>": 680 owith = "ab" 681 if mode == ">": 682 owith = "wb" 683 new_fp = open(arg, owith) 684 elif mode == "|": 685 proc = subprocess.Popen( 686 arg, 687 shell=True, 688 stdin=subprocess.PIPE, 689 preexec_fn=set_subprocess_umask_and_gid, 690 ) 691 new_fp = proc.stdin 692 else: 693 raise TypeError("Invalid type for output format: %s" % outfmt) 694 695 if o_out: 696 os.dup2(new_fp.fileno(), o_out.fileno()) 697 698 if errfmt == outfmt: 699 LOG.debug("Redirecting %s to %s", o_err, outfmt) 700 os.dup2(new_fp.fileno(), o_err.fileno()) 701 return 702 703 if errfmt: 704 LOG.debug("Redirecting %s to %s", o_err, errfmt) 705 (mode, arg) = errfmt.split(" ", 1) 706 if mode == ">" or mode == ">>": 707 owith = "ab" 708 if mode == ">": 709 owith = "wb" 710 new_fp = open(arg, owith) 711 elif mode == "|": 712 proc = subprocess.Popen( 713 arg, 714 shell=True, 715 stdin=subprocess.PIPE, 716 preexec_fn=set_subprocess_umask_and_gid, 717 ) 718 new_fp = proc.stdin 719 else: 720 raise TypeError("Invalid type for error format: %s" % errfmt) 721 722 if o_err: 723 os.dup2(new_fp.fileno(), o_err.fileno()) 724 725 726def make_url(scheme, host, port=None, 727 path='', params='', query='', fragment=''): 728 729 pieces = [scheme or ''] 730 731 netloc = '' 732 if host: 733 netloc = str(host) 734 735 if port is not None: 736 netloc += ":" + "%s" % (port) 737 738 pieces.append(netloc or '') 739 pieces.append(path or '') 740 pieces.append(params or '') 741 pieces.append(query or '') 742 pieces.append(fragment or '') 743 744 return parse.urlunparse(pieces) 745 746 747def mergemanydict(srcs, reverse=False): 748 if reverse: 749 srcs = reversed(srcs) 750 merged_cfg = {} 751 for cfg in srcs: 752 if cfg: 753 # Figure out which mergers to apply... 754 mergers_to_apply = mergers.dict_extract_mergers(cfg) 755 if not mergers_to_apply: 756 mergers_to_apply = mergers.default_mergers() 757 merger = mergers.construct(mergers_to_apply) 758 merged_cfg = merger.merge(merged_cfg, cfg) 759 return merged_cfg 760 761 762@contextlib.contextmanager 763def chdir(ndir): 764 curr = os.getcwd() 765 try: 766 os.chdir(ndir) 767 yield ndir 768 finally: 769 os.chdir(curr) 770 771 772@contextlib.contextmanager 773def umask(n_msk): 774 old = os.umask(n_msk) 775 try: 776 yield old 777 finally: 778 os.umask(old) 779 780 781def center(text, fill, max_len): 782 return '{0:{fill}{align}{size}}'.format(text, fill=fill, 783 align="^", size=max_len) 784 785 786def del_dir(path): 787 LOG.debug("Recursively deleting %s", path) 788 shutil.rmtree(path) 789 790 791# read_optional_seed 792# returns boolean indicating success or failure (presense of files) 793# if files are present, populates 'fill' dictionary with 'user-data' and 794# 'meta-data' entries 795def read_optional_seed(fill, base="", ext="", timeout=5): 796 try: 797 (md, ud, vd) = read_seeded(base, ext, timeout) 798 fill['user-data'] = ud 799 fill['vendor-data'] = vd 800 fill['meta-data'] = md 801 return True 802 except url_helper.UrlError as e: 803 if e.code == url_helper.NOT_FOUND: 804 return False 805 raise 806 807 808def fetch_ssl_details(paths=None): 809 ssl_details = {} 810 # Lookup in these locations for ssl key/cert files 811 ssl_cert_paths = [ 812 '/var/lib/cloud/data/ssl', 813 '/var/lib/cloud/instance/data/ssl', 814 ] 815 if paths: 816 ssl_cert_paths.extend([ 817 os.path.join(paths.get_ipath_cur('data'), 'ssl'), 818 os.path.join(paths.get_cpath('data'), 'ssl'), 819 ]) 820 ssl_cert_paths = uniq_merge(ssl_cert_paths) 821 ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] 822 cert_file = None 823 for d in ssl_cert_paths: 824 if os.path.isfile(os.path.join(d, 'cert.pem')): 825 cert_file = os.path.join(d, 'cert.pem') 826 break 827 key_file = None 828 for d in ssl_cert_paths: 829 if os.path.isfile(os.path.join(d, 'key.pem')): 830 key_file = os.path.join(d, 'key.pem') 831 break 832 if cert_file and key_file: 833 ssl_details['cert_file'] = cert_file 834 ssl_details['key_file'] = key_file 835 elif cert_file: 836 ssl_details['cert_file'] = cert_file 837 return ssl_details 838 839 840def load_yaml(blob, default=None, allowed=(dict,)): 841 loaded = default 842 blob = decode_binary(blob) 843 try: 844 LOG.debug("Attempting to load yaml from string " 845 "of length %s with allowed root types %s", 846 len(blob), allowed) 847 converted = safeyaml.load(blob) 848 if converted is None: 849 LOG.debug("loaded blob returned None, returning default.") 850 converted = default 851 elif not isinstance(converted, allowed): 852 # Yes this will just be caught, but thats ok for now... 853 raise TypeError(("Yaml load allows %s root types," 854 " but got %s instead") % 855 (allowed, type_utils.obj_name(converted))) 856 loaded = converted 857 except (safeyaml.YAMLError, TypeError, ValueError) as e: 858 msg = 'Failed loading yaml blob' 859 mark = None 860 if hasattr(e, 'context_mark') and getattr(e, 'context_mark'): 861 mark = getattr(e, 'context_mark') 862 elif hasattr(e, 'problem_mark') and getattr(e, 'problem_mark'): 863 mark = getattr(e, 'problem_mark') 864 if mark: 865 msg += ( 866 '. Invalid format at line {line} column {col}: "{err}"'.format( 867 line=mark.line + 1, col=mark.column + 1, err=e)) 868 else: 869 msg += '. {err}'.format(err=e) 870 LOG.warning(msg) 871 return loaded 872 873 874def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): 875 if base.find("%s") >= 0: 876 ud_url = base % ("user-data" + ext) 877 vd_url = base % ("vendor-data" + ext) 878 md_url = base % ("meta-data" + ext) 879 else: 880 ud_url = "%s%s%s" % (base, "user-data", ext) 881 vd_url = "%s%s%s" % (base, "vendor-data", ext) 882 md_url = "%s%s%s" % (base, "meta-data", ext) 883 884 md_resp = url_helper.read_file_or_url(md_url, timeout=timeout, 885 retries=retries) 886 md = None 887 if md_resp.ok(): 888 md = load_yaml(decode_binary(md_resp.contents), default={}) 889 890 ud_resp = url_helper.read_file_or_url(ud_url, timeout=timeout, 891 retries=retries) 892 ud = None 893 if ud_resp.ok(): 894 ud = ud_resp.contents 895 896 vd = None 897 try: 898 vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, 899 retries=retries) 900 except url_helper.UrlError as e: 901 LOG.debug("Error in vendor-data response: %s", e) 902 else: 903 if vd_resp.ok(): 904 vd = vd_resp.contents 905 else: 906 LOG.debug("Error in vendor-data response") 907 908 return (md, ud, vd) 909 910 911def read_conf_d(confd): 912 # Get reverse sorted list (later trumps newer) 913 confs = sorted(os.listdir(confd), reverse=True) 914 915 # Remove anything not ending in '.cfg' 916 confs = [f for f in confs if f.endswith(".cfg")] 917 918 # Remove anything not a file 919 confs = [f for f in confs 920 if os.path.isfile(os.path.join(confd, f))] 921 922 # Load them all so that they can be merged 923 cfgs = [] 924 for fn in confs: 925 cfgs.append(read_conf(os.path.join(confd, fn))) 926 927 return mergemanydict(cfgs) 928 929 930def read_conf_with_confd(cfgfile): 931 cfg = read_conf(cfgfile) 932 933 confd = False 934 if "conf_d" in cfg: 935 confd = cfg['conf_d'] 936 if confd: 937 if not isinstance(confd, str): 938 raise TypeError(("Config file %s contains 'conf_d' " 939 "with non-string type %s") % 940 (cfgfile, type_utils.obj_name(confd))) 941 else: 942 confd = str(confd).strip() 943 elif os.path.isdir("%s.d" % cfgfile): 944 confd = "%s.d" % cfgfile 945 946 if not confd or not os.path.isdir(confd): 947 return cfg 948 949 # Conf.d settings override input configuration 950 confd_cfg = read_conf_d(confd) 951 return mergemanydict([confd_cfg, cfg]) 952 953 954def read_conf_from_cmdline(cmdline=None): 955 # return a dictionary of config on the cmdline or None 956 return load_yaml(read_cc_from_cmdline(cmdline=cmdline)) 957 958 959def read_cc_from_cmdline(cmdline=None): 960 # this should support reading cloud-config information from 961 # the kernel command line. It is intended to support content of the 962 # format: 963 # cc: <yaml content here|urlencoded yaml content> [end_cc] 964 # this would include: 965 # cc: ssh_import_id: [smoser, kirkland]\\n 966 # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc 967 # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc 968 # cc:ssh_import_id: %5Bsmoser%5D end_cc 969 if cmdline is None: 970 cmdline = get_cmdline() 971 972 tag_begin = "cc:" 973 tag_end = "end_cc" 974 begin_l = len(tag_begin) 975 end_l = len(tag_end) 976 clen = len(cmdline) 977 tokens = [] 978 begin = cmdline.find(tag_begin) 979 while begin >= 0: 980 end = cmdline.find(tag_end, begin + begin_l) 981 if end < 0: 982 end = clen 983 tokens.append( 984 parse.unquote( 985 cmdline[begin + begin_l:end].lstrip()).replace("\\n", "\n")) 986 begin = cmdline.find(tag_begin, end + end_l) 987 988 return '\n'.join(tokens) 989 990 991def dos2unix(contents): 992 # find first end of line 993 pos = contents.find('\n') 994 if pos <= 0 or contents[pos - 1] != '\r': 995 return contents 996 return contents.replace('\r\n', '\n') 997 998 999def get_hostname_fqdn(cfg, cloud, metadata_only=False): 1000 """Get hostname and fqdn from config if present and fallback to cloud. 1001 1002 @param cfg: Dictionary of merged user-data configuration (from init.cfg). 1003 @param cloud: Cloud instance from init.cloudify(). 1004 @param metadata_only: Boolean, set True to only query cloud meta-data, 1005 returning None if not present in meta-data. 1006 @return: a Tuple of strings <hostname>, <fqdn>. Values can be none when 1007 metadata_only is True and no cfg or metadata provides hostname info. 1008 """ 1009 if "fqdn" in cfg: 1010 # user specified a fqdn. Default hostname then is based off that 1011 fqdn = cfg['fqdn'] 1012 hostname = get_cfg_option_str(cfg, "hostname", fqdn.split('.')[0]) 1013 else: 1014 if "hostname" in cfg and cfg['hostname'].find('.') > 0: 1015 # user specified hostname, and it had '.' in it 1016 # be nice to them. set fqdn and hostname from that 1017 fqdn = cfg['hostname'] 1018 hostname = cfg['hostname'][:fqdn.find('.')] 1019 else: 1020 # no fqdn set, get fqdn from cloud. 1021 # get hostname from cfg if available otherwise cloud 1022 fqdn = cloud.get_hostname(fqdn=True, metadata_only=metadata_only) 1023 if "hostname" in cfg: 1024 hostname = cfg['hostname'] 1025 else: 1026 hostname = cloud.get_hostname(metadata_only=metadata_only) 1027 return (hostname, fqdn) 1028 1029 1030def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): 1031 """ 1032 For each host a single line should be present with 1033 the following information: 1034 1035 IP_address canonical_hostname [aliases...] 1036 1037 Fields of the entry are separated by any number of blanks and/or tab 1038 characters. Text from a "#" character until the end of the line is a 1039 comment, and is ignored. Host names may contain only alphanumeric 1040 characters, minus signs ("-"), and periods ("."). They must begin with 1041 an alphabetic character and end with an alphanumeric character. 1042 Optional aliases provide for name changes, alternate spellings, shorter 1043 hostnames, or generic hostnames (for example, localhost). 1044 """ 1045 fqdn = None 1046 try: 1047 for line in load_file(filename).splitlines(): 1048 hashpos = line.find("#") 1049 if hashpos >= 0: 1050 line = line[0:hashpos] 1051 line = line.strip() 1052 if not line: 1053 continue 1054 1055 # If there there is less than 3 entries 1056 # (IP_address, canonical_hostname, alias) 1057 # then ignore this line 1058 toks = line.split() 1059 if len(toks) < 3: 1060 continue 1061 1062 if hostname in toks[2:]: 1063 fqdn = toks[1] 1064 break 1065 except IOError: 1066 pass 1067 return fqdn 1068 1069 1070def is_resolvable(name): 1071 """determine if a url is resolvable, return a boolean 1072 This also attempts to be resilent against dns redirection. 1073 1074 Note, that normal nsswitch resolution is used here. So in order 1075 to avoid any utilization of 'search' entries in /etc/resolv.conf 1076 we have to append '.'. 1077 1078 The top level 'invalid' domain is invalid per RFC. And example.com 1079 should also not exist. The '__cloud_init_expected_not_found__' entry will 1080 be resolved inside the search list. 1081 """ 1082 global _DNS_REDIRECT_IP 1083 if _DNS_REDIRECT_IP is None: 1084 badips = set() 1085 badnames = ("does-not-exist.example.com.", "example.invalid.", 1086 "__cloud_init_expected_not_found__") 1087 badresults = {} 1088 for iname in badnames: 1089 try: 1090 result = socket.getaddrinfo(iname, None, 0, 0, 1091 socket.SOCK_STREAM, 1092 socket.AI_CANONNAME) 1093 badresults[iname] = [] 1094 for (_fam, _stype, _proto, cname, sockaddr) in result: 1095 badresults[iname].append("%s: %s" % (cname, sockaddr[0])) 1096 badips.add(sockaddr[0]) 1097 except (socket.gaierror, socket.error): 1098 pass 1099 _DNS_REDIRECT_IP = badips 1100 if badresults: 1101 LOG.debug("detected dns redirection: %s", badresults) 1102 1103 try: 1104 result = socket.getaddrinfo(name, None) 1105 # check first result's sockaddr field 1106 addr = result[0][4][0] 1107 if addr in _DNS_REDIRECT_IP: 1108 return False 1109 return True 1110 except (socket.gaierror, socket.error): 1111 return False 1112 1113 1114def get_hostname(): 1115 hostname = socket.gethostname() 1116 return hostname 1117 1118 1119def gethostbyaddr(ip): 1120 try: 1121 return socket.gethostbyaddr(ip)[0] 1122 except socket.herror: 1123 return None 1124 1125 1126def is_resolvable_url(url): 1127 """determine if this url is resolvable (existing or ip).""" 1128 return log_time(logfunc=LOG.debug, msg="Resolving URL: " + url, 1129 func=is_resolvable, 1130 args=(parse.urlparse(url).hostname,)) 1131 1132 1133def search_for_mirror(candidates): 1134 """ 1135 Search through a list of mirror urls for one that works 1136 This needs to return quickly. 1137 """ 1138 if candidates is None: 1139 return None 1140 1141 LOG.debug("search for mirror in candidates: '%s'", candidates) 1142 for cand in candidates: 1143 try: 1144 if is_resolvable_url(cand): 1145 LOG.debug("found working mirror: '%s'", cand) 1146 return cand 1147 except Exception: 1148 pass 1149 return None 1150 1151 1152def close_stdin(): 1153 """ 1154 reopen stdin as /dev/null so even subprocesses or other os level things get 1155 /dev/null as input. 1156 1157 if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty and true 1158 value then input will not be closed (useful for debugging). 1159 """ 1160 if is_true(os.environ.get("_CLOUD_INIT_SAVE_STDIN")): 1161 return 1162 with open(os.devnull) as fp: 1163 os.dup2(fp.fileno(), sys.stdin.fileno()) 1164 1165 1166def find_devs_with_freebsd(criteria=None, oformat='device', 1167 tag=None, no_cache=False, path=None): 1168 devlist = [] 1169 if not criteria: 1170 return glob.glob("/dev/msdosfs/*") + glob.glob("/dev/iso9660/*") 1171 if criteria.startswith("LABEL="): 1172 label = criteria.lstrip("LABEL=") 1173 devlist = [ 1174 p for p in ['/dev/msdosfs/' + label, '/dev/iso9660/' + label] 1175 if os.path.exists(p)] 1176 elif criteria == "TYPE=vfat": 1177 devlist = glob.glob("/dev/msdosfs/*") 1178 elif criteria == "TYPE=iso9660": 1179 devlist = glob.glob("/dev/iso9660/*") 1180 return devlist 1181 1182 1183def find_devs_with_netbsd(criteria=None, oformat='device', 1184 tag=None, no_cache=False, path=None): 1185 devlist = [] 1186 label = None 1187 _type = None 1188 if criteria: 1189 if criteria.startswith("LABEL="): 1190 label = criteria.lstrip("LABEL=") 1191 if criteria.startswith("TYPE="): 1192 _type = criteria.lstrip("TYPE=") 1193 out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) 1194 for dev in out.split(): 1195 if label or _type: 1196 mscdlabel_out, _ = subp.subp(['mscdlabel', dev], rcs=[0, 1]) 1197 if label and not ('label "%s"' % label) in mscdlabel_out: 1198 continue 1199 if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: 1200 continue 1201 if _type == "vfat" and "ISO filesystem" in mscdlabel_out: 1202 continue 1203 devlist.append('/dev/' + dev) 1204 return devlist 1205 1206 1207def find_devs_with_openbsd(criteria=None, oformat='device', 1208 tag=None, no_cache=False, path=None): 1209 out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) 1210 devlist = [] 1211 for entry in out.split(','): 1212 if not entry.endswith(':'): 1213 # ffs partition with a serial, not a config-drive 1214 continue 1215 if entry == 'fd0:': 1216 continue 1217 part_id = 'a' if entry.startswith('cd') else 'i' 1218 devlist.append(entry[:-1] + part_id) 1219 if criteria == "TYPE=iso9660": 1220 devlist = [i for i in devlist if i.startswith('cd')] 1221 elif criteria in ["LABEL=CONFIG-2", "TYPE=vfat"]: 1222 devlist = [i for i in devlist if not i.startswith('cd')] 1223 elif criteria: 1224 LOG.debug("Unexpected criteria: %s", criteria) 1225 return ['/dev/' + i for i in devlist] 1226 1227 1228def find_devs_with_dragonflybsd(criteria=None, oformat='device', 1229 tag=None, no_cache=False, path=None): 1230 out, _err = subp.subp(['sysctl', '-n', 'kern.disks'], rcs=[0]) 1231 devlist = [i for i in sorted(out.split(), reverse=True) 1232 if not i.startswith("md") and not i.startswith("vn")] 1233 1234 if criteria == "TYPE=iso9660": 1235 devlist = [i for i in devlist 1236 if i.startswith('cd') or i.startswith('acd')] 1237 elif criteria in ["LABEL=CONFIG-2", "TYPE=vfat"]: 1238 devlist = [i for i in devlist 1239 if not (i.startswith('cd') or i.startswith('acd'))] 1240 elif criteria: 1241 LOG.debug("Unexpected criteria: %s", criteria) 1242 return ['/dev/' + i for i in devlist] 1243 1244 1245def find_devs_with(criteria=None, oformat='device', 1246 tag=None, no_cache=False, path=None): 1247 """ 1248 find devices matching given criteria (via blkid) 1249 criteria can be *one* of: 1250 TYPE=<filesystem> 1251 LABEL=<label> 1252 UUID=<uuid> 1253 """ 1254 if is_FreeBSD(): 1255 return find_devs_with_freebsd(criteria, oformat, 1256 tag, no_cache, path) 1257 elif is_NetBSD(): 1258 return find_devs_with_netbsd(criteria, oformat, 1259 tag, no_cache, path) 1260 elif is_OpenBSD(): 1261 return find_devs_with_openbsd(criteria, oformat, 1262 tag, no_cache, path) 1263 elif is_DragonFlyBSD(): 1264 return find_devs_with_dragonflybsd(criteria, oformat, 1265 tag, no_cache, path) 1266 1267 blk_id_cmd = ['blkid'] 1268 options = [] 1269 if criteria: 1270 # Search for block devices with tokens named NAME that 1271 # have the value 'value' and display any devices which are found. 1272 # Common values for NAME include TYPE, LABEL, and UUID. 1273 # If there are no devices specified on the command line, 1274 # all block devices will be searched; otherwise, 1275 # only search the devices specified by the user. 1276 options.append("-t%s" % (criteria)) 1277 if tag: 1278 # For each (specified) device, show only the tags that match tag. 1279 options.append("-s%s" % (tag)) 1280 if no_cache: 1281 # If you want to start with a clean cache 1282 # (i.e. don't report devices previously scanned 1283 # but not necessarily available at this time), specify /dev/null. 1284 options.extend(["-c", "/dev/null"]) 1285 if oformat: 1286 # Display blkid's output using the specified format. 1287 # The format parameter may be: 1288 # full, value, list, device, udev, export 1289 options.append('-o%s' % (oformat)) 1290 if path: 1291 options.append(path) 1292 cmd = blk_id_cmd + options 1293 # See man blkid for why 2 is added 1294 try: 1295 (out, _err) = subp.subp(cmd, rcs=[0, 2]) 1296 except subp.ProcessExecutionError as e: 1297 if e.errno == ENOENT: 1298 # blkid not found... 1299 out = "" 1300 else: 1301 raise 1302 entries = [] 1303 for line in out.splitlines(): 1304 line = line.strip() 1305 if line: 1306 entries.append(line) 1307 return entries 1308 1309 1310def blkid(devs=None, disable_cache=False): 1311 """Get all device tags details from blkid. 1312 1313 @param devs: Optional list of device paths you wish to query. 1314 @param disable_cache: Bool, set True to start with clean cache. 1315 1316 @return: Dict of key value pairs of info for the device. 1317 """ 1318 if devs is None: 1319 devs = [] 1320 else: 1321 devs = list(devs) 1322 1323 cmd = ['blkid', '-o', 'full'] 1324 if disable_cache: 1325 cmd.extend(['-c', '/dev/null']) 1326 cmd.extend(devs) 1327 1328 # we have to decode with 'replace' as shelx.split (called by 1329 # load_shell_content) can't take bytes. So this is potentially 1330 # lossy of non-utf-8 chars in blkid output. 1331 out, _ = subp.subp(cmd, capture=True, decode="replace") 1332 ret = {} 1333 for line in out.splitlines(): 1334 dev, _, data = line.partition(":") 1335 ret[dev] = load_shell_content(data) 1336 ret[dev]["DEVNAME"] = dev 1337 1338 return ret 1339 1340 1341def peek_file(fname, max_bytes): 1342 LOG.debug("Peeking at %s (max_bytes=%s)", fname, max_bytes) 1343 with open(fname, 'rb') as ifh: 1344 return ifh.read(max_bytes) 1345 1346 1347def uniq_list(in_list): 1348 out_list = [] 1349 for i in in_list: 1350 if i in out_list: 1351 continue 1352 else: 1353 out_list.append(i) 1354 return out_list 1355 1356 1357def load_file(fname, read_cb=None, quiet=False, decode=True): 1358 LOG.debug("Reading from %s (quiet=%s)", fname, quiet) 1359 ofh = io.BytesIO() 1360 try: 1361 with open(fname, 'rb') as ifh: 1362 pipe_in_out(ifh, ofh, chunk_cb=read_cb) 1363 except IOError as e: 1364 if not quiet: 1365 raise 1366 if e.errno != ENOENT: 1367 raise 1368 contents = ofh.getvalue() 1369 LOG.debug("Read %s bytes from %s", len(contents), fname) 1370 if decode: 1371 return decode_binary(contents) 1372 else: 1373 return contents 1374 1375 1376@lru_cache() 1377def _get_cmdline(): 1378 if is_container(): 1379 try: 1380 contents = load_file("/proc/1/cmdline") 1381 # replace nulls with space and drop trailing null 1382 cmdline = contents.replace("\x00", " ")[:-1] 1383 except Exception as e: 1384 LOG.warning("failed reading /proc/1/cmdline: %s", e) 1385 cmdline = "" 1386 else: 1387 try: 1388 cmdline = load_file("/proc/cmdline").strip() 1389 except Exception: 1390 cmdline = "" 1391 1392 return cmdline 1393 1394 1395def get_cmdline(): 1396 if 'DEBUG_PROC_CMDLINE' in os.environ: 1397 return os.environ["DEBUG_PROC_CMDLINE"] 1398 1399 return _get_cmdline() 1400 1401 1402def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): 1403 bytes_piped = 0 1404 while True: 1405 data = in_fh.read(chunk_size) 1406 if len(data) == 0: 1407 break 1408 else: 1409 out_fh.write(data) 1410 bytes_piped += len(data) 1411 if chunk_cb: 1412 chunk_cb(bytes_piped) 1413 out_fh.flush() 1414 return bytes_piped 1415 1416 1417def chownbyid(fname, uid=None, gid=None): 1418 if uid in [None, -1] and gid in [None, -1]: 1419 # Nothing to do 1420 return 1421 LOG.debug("Changing the ownership of %s to %s:%s", fname, uid, gid) 1422 os.chown(fname, uid, gid) 1423 1424 1425def chownbyname(fname, user=None, group=None): 1426 uid = -1 1427 gid = -1 1428 try: 1429 if user: 1430 uid = pwd.getpwnam(user).pw_uid 1431 if group: 1432 gid = grp.getgrnam(group).gr_gid 1433 except KeyError as e: 1434 raise OSError("Unknown user or group: %s" % (e)) from e 1435 chownbyid(fname, uid, gid) 1436 1437 1438# Always returns well formated values 1439# cfg is expected to have an entry 'output' in it, which is a dictionary 1440# that includes entries for 'init', 'config', 'final' or 'all' 1441# init: /var/log/cloud.out 1442# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] 1443# final: 1444# output: "| logger -p" 1445# error: "> /dev/null" 1446# this returns the specific 'mode' entry, cleanly formatted, with value 1447def get_output_cfg(cfg, mode): 1448 ret = [None, None] 1449 if not cfg or 'output' not in cfg: 1450 return ret 1451 1452 outcfg = cfg['output'] 1453 if mode in outcfg: 1454 modecfg = outcfg[mode] 1455 else: 1456 if 'all' not in outcfg: 1457 return ret 1458 # if there is a 'all' item in the output list 1459 # then it applies to all users of this (init, config, final) 1460 modecfg = outcfg['all'] 1461 1462 # if value is a string, it specifies stdout and stderr 1463 if isinstance(modecfg, str): 1464 ret = [modecfg, modecfg] 1465 1466 # if its a list, then we expect (stdout, stderr) 1467 if isinstance(modecfg, list): 1468 if len(modecfg) > 0: 1469 ret[0] = modecfg[0] 1470 if len(modecfg) > 1: 1471 ret[1] = modecfg[1] 1472 1473 # if it is a dictionary, expect 'out' and 'error' 1474 # items, which indicate out and error 1475 if isinstance(modecfg, dict): 1476 if 'output' in modecfg: 1477 ret[0] = modecfg['output'] 1478 if 'error' in modecfg: 1479 ret[1] = modecfg['error'] 1480 1481 # if err's entry == "&1", then make it same as stdout 1482 # as in shell syntax of "echo foo >/dev/null 2>&1" 1483 if ret[1] == "&1": 1484 ret[1] = ret[0] 1485 1486 swlist = [">>", ">", "|"] 1487 for i in range(len(ret)): 1488 if not ret[i]: 1489 continue 1490 val = ret[i].lstrip() 1491 found = False 1492 for s in swlist: 1493 if val.startswith(s): 1494 val = "%s %s" % (s, val[len(s):].strip()) 1495 found = True 1496 break 1497 if not found: 1498 # default behavior is append 1499 val = "%s %s" % (">>", val.strip()) 1500 ret[i] = val 1501 1502 return ret 1503 1504 1505def get_config_logfiles(cfg): 1506 """Return a list of log file paths from the configuration dictionary. 1507 1508 @param cfg: The cloud-init merged configuration dictionary. 1509 """ 1510 logs = [] 1511 if not cfg or not isinstance(cfg, dict): 1512 return logs 1513 default_log = cfg.get('def_log_file') 1514 if default_log: 1515 logs.append(default_log) 1516 for fmt in get_output_cfg(cfg, None): 1517 if not fmt: 1518 continue 1519 match = re.match(r'(?P<type>\||>+)\s*(?P<target>.*)', fmt) 1520 if not match: 1521 continue 1522 target = match.group('target') 1523 parts = target.split() 1524 if len(parts) == 1: 1525 logs.append(target) 1526 elif ['tee', '-a'] == parts[:2]: 1527 logs.append(parts[2]) 1528 return list(set(logs)) 1529 1530 1531def logexc(log, msg, *args): 1532 # Setting this here allows this to change 1533 # levels easily (not always error level) 1534 # or even desirable to have that much junk 1535 # coming out to a non-debug stream 1536 if msg: 1537 log.warning(msg, *args) 1538 # Debug gets the full trace. However, nose has a bug whereby its 1539 # logcapture plugin doesn't properly handle the case where there is no 1540 # actual exception. To avoid tracebacks during the test suite then, we'll 1541 # do the actual exc_info extraction here, and if there is no exception in 1542 # flight, we'll just pass in None. 1543 exc_info = sys.exc_info() 1544 if exc_info == (None, None, None): 1545 exc_info = None 1546 log.debug(msg, exc_info=exc_info, *args) 1547 1548 1549def hash_blob(blob, routine, mlen=None): 1550 hasher = hashlib.new(routine) 1551 hasher.update(encode_text(blob)) 1552 digest = hasher.hexdigest() 1553 # Don't get to long now 1554 if mlen is not None: 1555 return digest[0:mlen] 1556 else: 1557 return digest 1558 1559 1560def is_user(name): 1561 try: 1562 if pwd.getpwnam(name): 1563 return True 1564 except KeyError: 1565 return False 1566 1567 1568def is_group(name): 1569 try: 1570 if grp.getgrnam(name): 1571 return True 1572 except KeyError: 1573 return False 1574 1575 1576def rename(src, dest): 1577 LOG.debug("Renaming %s to %s", src, dest) 1578 # TODO(harlowja) use a se guard here?? 1579 os.rename(src, dest) 1580 1581 1582def ensure_dirs(dirlist, mode=0o755): 1583 for d in dirlist: 1584 ensure_dir(d, mode) 1585 1586 1587def load_json(text, root_types=(dict,)): 1588 decoded = json.loads(decode_binary(text)) 1589 if not isinstance(decoded, tuple(root_types)): 1590 expected_types = ", ".join([str(t) for t in root_types]) 1591 raise TypeError("(%s) root types expected, got %s instead" 1592 % (expected_types, type(decoded))) 1593 return decoded 1594 1595 1596def json_serialize_default(_obj): 1597 """Handler for types which aren't json serializable.""" 1598 try: 1599 return 'ci-b64:{0}'.format(b64e(_obj)) 1600 except AttributeError: 1601 return 'Warning: redacted unserializable type {0}'.format(type(_obj)) 1602 1603 1604def json_preserialize_binary(data): 1605 """Preserialize any discovered binary values to avoid json.dumps issues. 1606 1607 Used only on python 2.7 where default type handling is not honored for 1608 failure to encode binary data. LP: #1801364. 1609 TODO(Drop this function when py2.7 support is dropped from cloud-init) 1610 """ 1611 data = obj_copy.deepcopy(data) 1612 for key, value in data.items(): 1613 if isinstance(value, (dict)): 1614 data[key] = json_preserialize_binary(value) 1615 if isinstance(value, bytes): 1616 data[key] = 'ci-b64:{0}'.format(b64e(value)) 1617 return data 1618 1619 1620def json_dumps(data): 1621 """Return data in nicely formatted json.""" 1622 try: 1623 return json.dumps( 1624 data, indent=1, sort_keys=True, separators=(',', ': '), 1625 default=json_serialize_default) 1626 except UnicodeDecodeError: 1627 if sys.version_info[:2] == (2, 7): 1628 data = json_preserialize_binary(data) 1629 return json.dumps(data) 1630 raise 1631 1632 1633def ensure_dir(path, mode=None): 1634 if not os.path.isdir(path): 1635 # Make the dir and adjust the mode 1636 with SeLinuxGuard(os.path.dirname(path), recursive=True): 1637 os.makedirs(path) 1638 chmod(path, mode) 1639 else: 1640 # Just adjust the mode 1641 chmod(path, mode) 1642 1643 1644@contextlib.contextmanager 1645def unmounter(umount): 1646 try: 1647 yield umount 1648 finally: 1649 if umount: 1650 umount_cmd = ["umount", umount] 1651 subp.subp(umount_cmd) 1652 1653 1654def mounts(): 1655 mounted = {} 1656 try: 1657 # Go through mounts to see what is already mounted 1658 if os.path.exists("/proc/mounts"): 1659 mount_locs = load_file("/proc/mounts").splitlines() 1660 method = 'proc' 1661 else: 1662 (mountoutput, _err) = subp.subp("mount") 1663 mount_locs = mountoutput.splitlines() 1664 method = 'mount' 1665 mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$' 1666 for mpline in mount_locs: 1667 # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered) 1668 # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates) 1669 try: 1670 if method == 'proc': 1671 (dev, mp, fstype, opts, _freq, _passno) = mpline.split() 1672 else: 1673 m = re.search(mountre, mpline) 1674 dev = m.group(1) 1675 mp = m.group(2) 1676 fstype = m.group(3) 1677 opts = m.group(4) 1678 except Exception: 1679 continue 1680 # If the name of the mount point contains spaces these 1681 # can be escaped as '\040', so undo that.. 1682 mp = mp.replace("\\040", " ") 1683 mounted[dev] = { 1684 'fstype': fstype, 1685 'mountpoint': mp, 1686 'opts': opts, 1687 } 1688 LOG.debug("Fetched %s mounts from %s", mounted, method) 1689 except (IOError, OSError): 1690 logexc(LOG, "Failed fetching mount points") 1691 return mounted 1692 1693 1694def mount_cb(device, callback, data=None, mtype=None, 1695 update_env_for_mount=None): 1696 """ 1697 Mount the device, call method 'callback' passing the directory 1698 in which it was mounted, then unmount. Return whatever 'callback' 1699 returned. If data != None, also pass data to callback. 1700 1701 mtype is a filesystem type. it may be a list, string (a single fsname) 1702 or a list of fsnames. 1703 """ 1704 1705 if isinstance(mtype, str): 1706 mtypes = [mtype] 1707 elif isinstance(mtype, (list, tuple)): 1708 mtypes = list(mtype) 1709 elif mtype is None: 1710 mtypes = None 1711 else: 1712 raise TypeError( 1713 'Unsupported type provided for mtype parameter: {_type}'.format( 1714 _type=type(mtype))) 1715 1716 # clean up 'mtype' input a bit based on platform. 1717 if is_Linux(): 1718 if mtypes is None: 1719 mtypes = ["auto"] 1720 elif is_BSD(): 1721 if mtypes is None: 1722 mtypes = ['ufs', 'cd9660', 'msdos'] 1723 for index, mtype in enumerate(mtypes): 1724 if mtype == "iso9660": 1725 mtypes[index] = "cd9660" 1726 if mtype in ["vfat", "msdosfs"]: 1727 mtypes[index] = "msdos" 1728 else: 1729 # we cannot do a smart "auto", so just call 'mount' once with no -t 1730 mtypes = [''] 1731 1732 mounted = mounts() 1733 with temp_utils.tempdir() as tmpd: 1734 umount = False 1735 if os.path.realpath(device) in mounted: 1736 mountpoint = mounted[os.path.realpath(device)]['mountpoint'] 1737 else: 1738 failure_reason = None 1739 for mtype in mtypes: 1740 mountpoint = None 1741 try: 1742 mountcmd = ['mount', '-o', 'ro'] 1743 if mtype: 1744 mountcmd.extend(['-t', mtype]) 1745 mountcmd.append(device) 1746 mountcmd.append(tmpd) 1747 subp.subp(mountcmd, update_env=update_env_for_mount) 1748 umount = tmpd # This forces it to be unmounted (when set) 1749 mountpoint = tmpd 1750 break 1751 except (IOError, OSError) as exc: 1752 LOG.debug("Failed mount of '%s' as '%s': %s", 1753 device, mtype, exc) 1754 failure_reason = exc 1755 if not mountpoint: 1756 raise MountFailedError("Failed mounting %s to %s due to: %s" % 1757 (device, tmpd, failure_reason)) 1758 1759 # Be nice and ensure it ends with a slash 1760 if not mountpoint.endswith("/"): 1761 mountpoint += "/" 1762 with unmounter(umount): 1763 if data is None: 1764 ret = callback(mountpoint) 1765 else: 1766 ret = callback(mountpoint, data) 1767 return ret 1768 1769 1770def get_builtin_cfg(): 1771 # Deep copy so that others can't modify 1772 return obj_copy.deepcopy(CFG_BUILTIN) 1773 1774 1775def is_link(path): 1776 LOG.debug("Testing if a link exists for %s", path) 1777 return os.path.islink(path) 1778 1779 1780def sym_link(source, link, force=False): 1781 LOG.debug("Creating symbolic link from %r => %r", link, source) 1782 if force and os.path.exists(link): 1783 del_file(link) 1784 os.symlink(source, link) 1785 1786 1787def del_file(path): 1788 LOG.debug("Attempting to remove %s", path) 1789 try: 1790 os.unlink(path) 1791 except OSError as e: 1792 if e.errno != ENOENT: 1793 raise e 1794 1795 1796def copy(src, dest): 1797 LOG.debug("Copying %s to %s", src, dest) 1798 shutil.copy(src, dest) 1799 1800 1801def time_rfc2822(): 1802 try: 1803 ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) 1804 except Exception: 1805 ts = "??" 1806 return ts 1807 1808 1809@lru_cache() 1810def boottime(): 1811 """Use sysctlbyname(3) via ctypes to find kern.boottime 1812 1813 kern.boottime is of type struct timeval. Here we create a 1814 private class to easier unpack it. 1815 1816 @return boottime: float to be compatible with linux 1817 """ 1818 import ctypes 1819 import ctypes.util 1820 1821 NULL_BYTES = b"\x00" 1822 1823 class timeval(ctypes.Structure): 1824 _fields_ = [ 1825 ("tv_sec", ctypes.c_int64), 1826 ("tv_usec", ctypes.c_int64) 1827 ] 1828 libc = ctypes.CDLL(ctypes.util.find_library('c')) 1829 size = ctypes.c_size_t() 1830 size.value = ctypes.sizeof(timeval) 1831 buf = timeval() 1832 if libc.sysctlbyname(b"kern.boottime" + NULL_BYTES, ctypes.byref(buf), 1833 ctypes.byref(size), None, 0) != -1: 1834 return buf.tv_sec + buf.tv_usec / 1000000.0 1835 raise RuntimeError("Unable to retrieve kern.boottime on this system") 1836 1837 1838def uptime(): 1839 uptime_str = '??' 1840 method = 'unknown' 1841 try: 1842 if os.path.exists("/proc/uptime"): 1843 method = '/proc/uptime' 1844 contents = load_file("/proc/uptime") 1845 if contents: 1846 uptime_str = contents.split()[0] 1847 else: 1848 method = 'ctypes' 1849 # This is the *BSD codepath 1850 uptime_str = str(time.time() - boottime()) 1851 1852 except Exception: 1853 logexc(LOG, "Unable to read uptime using method: %s" % method) 1854 return uptime_str 1855 1856 1857def append_file(path, content): 1858 write_file(path, content, omode="ab", mode=None) 1859 1860 1861def ensure_file( 1862 path, mode: int = 0o644, *, preserve_mode: bool = False 1863) -> None: 1864 write_file( 1865 path, content='', omode="ab", mode=mode, preserve_mode=preserve_mode 1866 ) 1867 1868 1869def safe_int(possible_int): 1870 try: 1871 return int(possible_int) 1872 except (ValueError, TypeError): 1873 return None 1874 1875 1876def chmod(path, mode): 1877 real_mode = safe_int(mode) 1878 if path and real_mode: 1879 with SeLinuxGuard(path): 1880 os.chmod(path, real_mode) 1881 1882 1883def get_group_id(grp_name: str) -> int: 1884 """ 1885 Returns the group id of a group name, or -1 if no group exists 1886 1887 @param grp_name: the name of the group 1888 """ 1889 gid = -1 1890 try: 1891 gid = grp.getgrnam(grp_name).gr_gid 1892 except KeyError: 1893 LOG.debug("Group %s is not a valid group name", grp_name) 1894 return gid 1895 1896 1897def get_permissions(path: str) -> int: 1898 """ 1899 Returns the octal permissions of the file/folder pointed by the path, 1900 encoded as an int. 1901 1902 @param path: The full path of the file/folder. 1903 """ 1904 1905 return stat.S_IMODE(os.stat(path).st_mode) 1906 1907 1908def get_owner(path: str) -> str: 1909 """ 1910 Returns the owner of the file/folder pointed by the path. 1911 1912 @param path: The full path of the file/folder. 1913 """ 1914 st = os.stat(path) 1915 return pwd.getpwuid(st.st_uid).pw_name 1916 1917 1918def get_group(path: str) -> str: 1919 """ 1920 Returns the group of the file/folder pointed by the path. 1921 1922 @param path: The full path of the file/folder. 1923 """ 1924 st = os.stat(path) 1925 return grp.getgrgid(st.st_gid).gr_name 1926 1927 1928def get_user_groups(username: str) -> List[str]: 1929 """ 1930 Returns a list of all groups to which the user belongs 1931 1932 @param username: the user we want to check 1933 """ 1934 groups = [] 1935 for group in grp.getgrall(): 1936 if username in group.gr_mem: 1937 groups.append(group.gr_name) 1938 1939 gid = pwd.getpwnam(username).pw_gid 1940 groups.append(grp.getgrgid(gid).gr_name) 1941 return groups 1942 1943 1944def write_file( 1945 filename, 1946 content, 1947 mode=0o644, 1948 omode="wb", 1949 preserve_mode=False, 1950 *, 1951 ensure_dir_exists=True 1952): 1953 """ 1954 Writes a file with the given content and sets the file mode as specified. 1955 Restores the SELinux context if possible. 1956 1957 @param filename: The full path of the file to write. 1958 @param content: The content to write to the file. 1959 @param mode: The filesystem mode to set on the file. 1960 @param omode: The open mode used when opening the file (w, wb, a, etc.) 1961 @param preserve_mode: If True and `filename` exists, preserve `filename`s 1962 current mode instead of applying `mode`. 1963 @param ensure_dir_exists: If True (the default), ensure that the directory 1964 containing `filename` exists before writing to 1965 the file. 1966 """ 1967 1968 if preserve_mode: 1969 try: 1970 mode = get_permissions(filename) 1971 except OSError: 1972 pass 1973 1974 if ensure_dir_exists: 1975 ensure_dir(os.path.dirname(filename)) 1976 if 'b' in omode.lower(): 1977 content = encode_text(content) 1978 write_type = 'bytes' 1979 else: 1980 content = decode_binary(content) 1981 write_type = 'characters' 1982 try: 1983 mode_r = "%o" % mode 1984 except TypeError: 1985 mode_r = "%r" % mode 1986 LOG.debug("Writing to %s - %s: [%s] %s %s", 1987 filename, omode, mode_r, len(content), write_type) 1988 with SeLinuxGuard(path=filename): 1989 with open(filename, omode) as fh: 1990 fh.write(content) 1991 fh.flush() 1992 chmod(filename, mode) 1993 1994 1995def delete_dir_contents(dirname): 1996 """ 1997 Deletes all contents of a directory without deleting the directory itself. 1998 1999 @param dirname: The directory whose contents should be deleted. 2000 """ 2001 for node in os.listdir(dirname): 2002 node_fullpath = os.path.join(dirname, node) 2003 if os.path.isdir(node_fullpath): 2004 del_dir(node_fullpath) 2005 else: 2006 del_file(node_fullpath) 2007 2008 2009def make_header(comment_char="#", base='created'): 2010 ci_ver = version.version_string() 2011 header = str(comment_char) 2012 header += " %s by cloud-init v. %s" % (base.title(), ci_ver) 2013 header += " on %s" % time_rfc2822() 2014 return header 2015 2016 2017def abs_join(base, *paths): 2018 return os.path.abspath(os.path.join(base, *paths)) 2019 2020 2021# shellify, takes a list of commands 2022# for each entry in the list 2023# if it is an array, shell protect it (with single ticks) 2024# if it is a string, do nothing 2025def shellify(cmdlist, add_header=True): 2026 if not isinstance(cmdlist, (tuple, list)): 2027 raise TypeError( 2028 "Input to shellify was type '%s'. Expected list or tuple." % 2029 (type_utils.obj_name(cmdlist))) 2030 2031 content = '' 2032 if add_header: 2033 content += "#!/bin/sh\n" 2034 escaped = "%s%s%s%s" % ("'", '\\', "'", "'") 2035 cmds_made = 0 2036 for args in cmdlist: 2037 # If the item is a list, wrap all items in single tick. 2038 # If its not, then just write it directly. 2039 if isinstance(args, (list, tuple)): 2040 fixed = [] 2041 for f in args: 2042 fixed.append("'%s'" % (str(f).replace("'", escaped))) 2043 content = "%s%s\n" % (content, ' '.join(fixed)) 2044 cmds_made += 1 2045 elif isinstance(args, str): 2046 content = "%s%s\n" % (content, args) 2047 cmds_made += 1 2048 # Yaml parsing of a comment results in None 2049 elif args is None: 2050 pass 2051 else: 2052 raise TypeError( 2053 "Unable to shellify type '%s'. Expected list, string, tuple. " 2054 "Got: %s" % (type_utils.obj_name(args), args)) 2055 2056 LOG.debug("Shellified %s commands.", cmds_made) 2057 return content 2058 2059 2060def strip_prefix_suffix(line, prefix=None, suffix=None): 2061 if prefix and line.startswith(prefix): 2062 line = line[len(prefix):] 2063 if suffix and line.endswith(suffix): 2064 line = line[:-len(suffix)] 2065 return line 2066 2067 2068def _cmd_exits_zero(cmd): 2069 if subp.which(cmd[0]) is None: 2070 return False 2071 try: 2072 subp.subp(cmd) 2073 except subp.ProcessExecutionError: 2074 return False 2075 return True 2076 2077 2078def _is_container_systemd(): 2079 return _cmd_exits_zero(["systemd-detect-virt", "--quiet", "--container"]) 2080 2081 2082def _is_container_upstart(): 2083 return _cmd_exits_zero(["running-in-container"]) 2084 2085 2086def _is_container_old_lxc(): 2087 return _cmd_exits_zero(["lxc-is-container"]) 2088 2089 2090def _is_container_freebsd(): 2091 if not is_FreeBSD(): 2092 return False 2093 cmd = ["sysctl", "-qn", "security.jail.jailed"] 2094 if subp.which(cmd[0]) is None: 2095 return False 2096 out, _ = subp.subp(cmd) 2097 return out.strip() == "1" 2098 2099 2100@lru_cache() 2101def is_container(): 2102 """ 2103 Checks to see if this code running in a container of some sort 2104 """ 2105 checks = ( 2106 _is_container_systemd, 2107 _is_container_freebsd, 2108 _is_container_upstart, 2109 _is_container_old_lxc) 2110 2111 for helper in checks: 2112 if helper(): 2113 return True 2114 2115 # this code is largely from the logic in 2116 # ubuntu's /etc/init/container-detect.conf 2117 try: 2118 # Detect old-style libvirt 2119 # Detect OpenVZ containers 2120 pid1env = get_proc_env(1) 2121 if "container" in pid1env: 2122 return True 2123 if "LIBVIRT_LXC_UUID" in pid1env: 2124 return True 2125 except (IOError, OSError): 2126 pass 2127 2128 # Detect OpenVZ containers 2129 if os.path.isdir("/proc/vz") and not os.path.isdir("/proc/bc"): 2130 return True 2131 2132 try: 2133 # Detect Vserver containers 2134 lines = load_file("/proc/self/status").splitlines() 2135 for line in lines: 2136 if line.startswith("VxID:"): 2137 (_key, val) = line.strip().split(":", 1) 2138 if val != "0": 2139 return True 2140 except (IOError, OSError): 2141 pass 2142 2143 return False 2144 2145 2146def is_lxd(): 2147 """Check to see if we are running in a lxd container.""" 2148 return os.path.exists('/dev/lxd/sock') 2149 2150 2151def get_proc_env(pid, encoding='utf-8', errors='replace'): 2152 """ 2153 Return the environment in a dict that a given process id was started with. 2154 2155 @param encoding: if true, then decoding will be done with 2156 .decode(encoding, errors) and text will be returned. 2157 if false then binary will be returned. 2158 @param errors: only used if encoding is true.""" 2159 fn = os.path.join("/proc", str(pid), "environ") 2160 2161 try: 2162 contents = load_file(fn, decode=False) 2163 except (IOError, OSError): 2164 return {} 2165 2166 env = {} 2167 null, equal = (b"\x00", b"=") 2168 if encoding: 2169 null, equal = ("\x00", "=") 2170 contents = contents.decode(encoding, errors) 2171 2172 for tok in contents.split(null): 2173 if not tok: 2174 continue 2175 (name, val) = tok.split(equal, 1) 2176 if name: 2177 env[name] = val 2178 return env 2179 2180 2181def keyval_str_to_dict(kvstring): 2182 ret = {} 2183 for tok in kvstring.split(): 2184 try: 2185 (key, val) = tok.split("=", 1) 2186 except ValueError: 2187 key = tok 2188 val = True 2189 ret[key] = val 2190 return ret 2191 2192 2193def is_partition(device): 2194 if device.startswith("/dev/"): 2195 device = device[5:] 2196 2197 return os.path.isfile("/sys/class/block/%s/partition" % device) 2198 2199 2200def expand_package_list(version_fmt, pkgs): 2201 # we will accept tuples, lists of tuples, or just plain lists 2202 if not isinstance(pkgs, list): 2203 pkgs = [pkgs] 2204 2205 pkglist = [] 2206 for pkg in pkgs: 2207 if isinstance(pkg, str): 2208 pkglist.append(pkg) 2209 continue 2210 2211 if isinstance(pkg, (tuple, list)): 2212 if len(pkg) < 1 or len(pkg) > 2: 2213 raise RuntimeError("Invalid package & version tuple.") 2214 2215 if len(pkg) == 2 and pkg[1]: 2216 pkglist.append(version_fmt % tuple(pkg)) 2217 continue 2218 2219 pkglist.append(pkg[0]) 2220 2221 else: 2222 raise RuntimeError("Invalid package type.") 2223 2224 return pkglist 2225 2226 2227def parse_mount_info(path, mountinfo_lines, log=LOG, get_mnt_opts=False): 2228 """Return the mount information for PATH given the lines from 2229 /proc/$$/mountinfo.""" 2230 2231 path_elements = [e for e in path.split('/') if e] 2232 devpth = None 2233 fs_type = None 2234 match_mount_point = None 2235 match_mount_point_elements = None 2236 for i, line in enumerate(mountinfo_lines): 2237 parts = line.split() 2238 2239 # Completely fail if there is anything in any line that is 2240 # unexpected, as continuing to parse past a bad line could 2241 # cause an incorrect result to be returned, so it's better 2242 # return nothing than an incorrect result. 2243 2244 # The minimum number of elements in a valid line is 10. 2245 if len(parts) < 10: 2246 log.debug("Line %d has two few columns (%d): %s", 2247 i + 1, len(parts), line) 2248 return None 2249 2250 mount_point = parts[4] 2251 mount_point_elements = [e for e in mount_point.split('/') if e] 2252 2253 # Ignore mounts deeper than the path in question. 2254 if len(mount_point_elements) > len(path_elements): 2255 continue 2256 2257 # Ignore mounts where the common path is not the same. 2258 x = min(len(mount_point_elements), len(path_elements)) 2259 if mount_point_elements[0:x] != path_elements[0:x]: 2260 continue 2261 2262 # Ignore mount points higher than an already seen mount 2263 # point. 2264 if (match_mount_point_elements is not None and 2265 len(match_mount_point_elements) > len(mount_point_elements)): 2266 continue 2267 2268 # Find the '-' which terminates a list of optional columns to 2269 # find the filesystem type and the path to the device. See 2270 # man 5 proc for the format of this file. 2271 try: 2272 i = parts.index('-') 2273 except ValueError: 2274 log.debug("Did not find column named '-' in line %d: %s", 2275 i + 1, line) 2276 return None 2277 2278 # Get the path to the device. 2279 try: 2280 fs_type = parts[i + 1] 2281 devpth = parts[i + 2] 2282 except IndexError: 2283 log.debug("Too few columns after '-' column in line %d: %s", 2284 i + 1, line) 2285 return None 2286 2287 match_mount_point = mount_point 2288 match_mount_point_elements = mount_point_elements 2289 mount_options = parts[5] 2290 2291 if get_mnt_opts: 2292 if devpth and fs_type and match_mount_point and mount_options: 2293 return (devpth, fs_type, match_mount_point, mount_options) 2294 else: 2295 if devpth and fs_type and match_mount_point: 2296 return (devpth, fs_type, match_mount_point) 2297 2298 return None 2299 2300 2301def parse_mtab(path): 2302 """On older kernels there's no /proc/$$/mountinfo, so use mtab.""" 2303 for line in load_file("/etc/mtab").splitlines(): 2304 devpth, mount_point, fs_type = line.split()[:3] 2305 if mount_point == path: 2306 return devpth, fs_type, mount_point 2307 return None 2308 2309 2310def find_freebsd_part(fs): 2311 splitted = fs.split('/') 2312 if len(splitted) == 3: 2313 return splitted[2] 2314 elif splitted[2] in ['label', 'gpt', 'ufs']: 2315 target_label = fs[5:] 2316 (part, _err) = subp.subp(['glabel', 'status', '-s']) 2317 for labels in part.split("\n"): 2318 items = labels.split() 2319 if len(items) > 0 and items[0] == target_label: 2320 part = items[2] 2321 break 2322 return str(part) 2323 else: 2324 LOG.warning("Unexpected input in find_freebsd_part: %s", fs) 2325 2326 2327def find_dragonflybsd_part(fs): 2328 splitted = fs.split('/') 2329 if len(splitted) == 3 and splitted[1] == 'dev': 2330 return splitted[2] 2331 else: 2332 LOG.warning("Unexpected input in find_dragonflybsd_part: %s", fs) 2333 2334 2335def get_path_dev_freebsd(path, mnt_list): 2336 path_found = None 2337 for line in mnt_list.split("\n"): 2338 items = line.split() 2339 if (len(items) > 2 and os.path.exists(items[1] + path)): 2340 path_found = line 2341 break 2342 return path_found 2343 2344 2345def get_mount_info_freebsd(path): 2346 (result, err) = subp.subp(['mount', '-p', path], rcs=[0, 1]) 2347 if len(err): 2348 # find a path if the input is not a mounting point 2349 (mnt_list, err) = subp.subp(['mount', '-p']) 2350 path_found = get_path_dev_freebsd(path, mnt_list) 2351 if (path_found is None): 2352 return None 2353 result = path_found 2354 ret = result.split() 2355 label_part = find_freebsd_part(ret[0]) 2356 return "/dev/" + label_part, ret[2], ret[1] 2357 2358 2359def get_device_info_from_zpool(zpool): 2360 # zpool has 10 second timeout waiting for /dev/zfs LP: #1760173 2361 if not os.path.exists('/dev/zfs'): 2362 LOG.debug('Cannot get zpool info, no /dev/zfs') 2363 return None 2364 try: 2365 (zpoolstatus, err) = subp.subp(['zpool', 'status', zpool]) 2366 except subp.ProcessExecutionError as err: 2367 LOG.warning("Unable to get zpool status of %s: %s", zpool, err) 2368 return None 2369 if len(err): 2370 return None 2371 r = r'.*(ONLINE).*' 2372 for line in zpoolstatus.split("\n"): 2373 if re.search(r, line) and zpool not in line and "state" not in line: 2374 disk = line.split()[0] 2375 LOG.debug('found zpool "%s" on disk %s', zpool, disk) 2376 return disk 2377 2378 2379def parse_mount(path): 2380 (mountoutput, _err) = subp.subp(['mount']) 2381 mount_locs = mountoutput.splitlines() 2382 # there are 2 types of mount outputs we have to parse therefore 2383 # the regex is a bit complex. to better understand this regex see: 2384 # https://regex101.com/r/2F6c1k/1 2385 # https://regex101.com/r/T2en7a/1 2386 regex = (r'^(/dev/[\S]+|.*zroot\S*?) on (/[\S]*) ' 2387 r'(?=(?:type)[\s]+([\S]+)|\(([^,]*))') 2388 if is_DragonFlyBSD(): 2389 regex = (r'^(/dev/[\S]+|\S*?) on (/[\S]*) ' 2390 r'(?=(?:type)[\s]+([\S]+)|\(([^,]*))') 2391 for line in mount_locs: 2392 m = re.search(regex, line) 2393 if not m: 2394 continue 2395 devpth = m.group(1) 2396 mount_point = m.group(2) 2397 # above regex will either fill the fs_type in group(3) 2398 # or group(4) depending on the format we have. 2399 fs_type = m.group(3) 2400 if fs_type is None: 2401 fs_type = m.group(4) 2402 LOG.debug('found line in mount -> devpth: %s, mount_point: %s, ' 2403 'fs_type: %s', devpth, mount_point, fs_type) 2404 # check whether the dev refers to a label on FreeBSD 2405 # for example, if dev is '/dev/label/rootfs', we should 2406 # continue finding the real device like '/dev/da0'. 2407 # this is only valid for non zfs file systems as a zpool 2408 # can have gpt labels as disk. 2409 devm = re.search('^(/dev/.+)p([0-9])$', devpth) 2410 if not devm and is_FreeBSD() and fs_type != 'zfs': 2411 return get_mount_info_freebsd(path) 2412 elif mount_point == path: 2413 return devpth, fs_type, mount_point 2414 return None 2415 2416 2417def get_mount_info(path, log=LOG, get_mnt_opts=False): 2418 # Use /proc/$$/mountinfo to find the device where path is mounted. 2419 # This is done because with a btrfs filesystem using os.stat(path) 2420 # does not return the ID of the device. 2421 # 2422 # Here, / has a device of 18 (decimal). 2423 # 2424 # $ stat / 2425 # File: '/' 2426 # Size: 234 Blocks: 0 IO Block: 4096 directory 2427 # Device: 12h/18d Inode: 256 Links: 1 2428 # Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root) 2429 # Access: 2013-01-13 07:31:04.358011255 +0000 2430 # Modify: 2013-01-13 18:48:25.930011255 +0000 2431 # Change: 2013-01-13 18:48:25.930011255 +0000 2432 # Birth: - 2433 # 2434 # Find where / is mounted: 2435 # 2436 # $ mount | grep ' / ' 2437 # /dev/vda1 on / type btrfs (rw,subvol=@,compress=lzo) 2438 # 2439 # And the device ID for /dev/vda1 is not 18: 2440 # 2441 # $ ls -l /dev/vda1 2442 # brw-rw---- 1 root disk 253, 1 Jan 13 08:29 /dev/vda1 2443 # 2444 # So use /proc/$$/mountinfo to find the device underlying the 2445 # input path. 2446 mountinfo_path = '/proc/%s/mountinfo' % os.getpid() 2447 if os.path.exists(mountinfo_path): 2448 lines = load_file(mountinfo_path).splitlines() 2449 return parse_mount_info(path, lines, log, get_mnt_opts) 2450 elif os.path.exists("/etc/mtab"): 2451 return parse_mtab(path) 2452 else: 2453 return parse_mount(path) 2454 2455 2456def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): 2457 if args is None: 2458 args = [] 2459 if kwargs is None: 2460 kwargs = {} 2461 2462 start = time.time() 2463 2464 ustart = None 2465 if get_uptime: 2466 try: 2467 ustart = float(uptime()) 2468 except ValueError: 2469 pass 2470 2471 try: 2472 ret = func(*args, **kwargs) 2473 finally: 2474 delta = time.time() - start 2475 udelta = None 2476 if ustart is not None: 2477 try: 2478 udelta = float(uptime()) - ustart 2479 except ValueError: 2480 pass 2481 2482 tmsg = " took %0.3f seconds" % delta 2483 if get_uptime: 2484 if isinstance(udelta, (float)): 2485 tmsg += " (%0.2f)" % udelta 2486 else: 2487 tmsg += " (N/A)" 2488 try: 2489 logfunc(msg + tmsg) 2490 except Exception: 2491 pass 2492 return ret 2493 2494 2495def expand_dotted_devname(dotted): 2496 toks = dotted.rsplit(".", 1) 2497 if len(toks) > 1: 2498 return toks 2499 else: 2500 return (dotted, None) 2501 2502 2503def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): 2504 # return a dictionary populated with keys in 'required' and 'optional' 2505 # by reading files in prefix + delim + entry 2506 if required is None: 2507 required = [] 2508 if optional is None: 2509 optional = [] 2510 2511 missing = [] 2512 ret = {} 2513 for f in required + optional: 2514 try: 2515 ret[f] = load_file(base + delim + f, quiet=False, decode=False) 2516 except IOError as e: 2517 if e.errno != ENOENT: 2518 raise 2519 if f in required: 2520 missing.append(f) 2521 2522 if len(missing): 2523 raise ValueError( 2524 'Missing required files: {files}'.format(files=','.join(missing))) 2525 2526 return ret 2527 2528 2529def read_meminfo(meminfo="/proc/meminfo", raw=False): 2530 # read a /proc/meminfo style file and return 2531 # a dict with 'total', 'free', and 'available' 2532 mpliers = {'kB': 2 ** 10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} 2533 kmap = {'MemTotal:': 'total', 'MemFree:': 'free', 2534 'MemAvailable:': 'available'} 2535 ret = {} 2536 for line in load_file(meminfo).splitlines(): 2537 try: 2538 key, value, unit = line.split() 2539 except ValueError: 2540 key, value = line.split() 2541 unit = 'B' 2542 if raw: 2543 ret[key] = int(value) * mpliers[unit] 2544 elif key in kmap: 2545 ret[kmap[key]] = int(value) * mpliers[unit] 2546 2547 return ret 2548 2549 2550def human2bytes(size): 2551 """Convert human string or integer to size in bytes 2552 10M => 10485760 2553 .5G => 536870912 2554 """ 2555 size_in = size 2556 if size.endswith("B"): 2557 size = size[:-1] 2558 2559 mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} 2560 2561 num = size 2562 mplier = 'B' 2563 for m in mpliers: 2564 if size.endswith(m): 2565 mplier = m 2566 num = size[0:-len(m)] 2567 2568 try: 2569 num = float(num) 2570 except ValueError as e: 2571 raise ValueError("'%s' is not valid input." % size_in) from e 2572 2573 if num < 0: 2574 raise ValueError("'%s': cannot be negative" % size_in) 2575 2576 return int(num * mpliers[mplier]) 2577 2578 2579def is_x86(uname_arch=None): 2580 """Return True if platform is x86-based""" 2581 if uname_arch is None: 2582 uname_arch = os.uname()[4] 2583 x86_arch_match = ( 2584 uname_arch == 'x86_64' or 2585 (uname_arch[0] == 'i' and uname_arch[2:] == '86')) 2586 return x86_arch_match 2587 2588 2589def message_from_string(string): 2590 if sys.version_info[:2] < (2, 7): 2591 return email.message_from_file(io.StringIO(string)) 2592 return email.message_from_string(string) 2593 2594 2595def get_installed_packages(target=None): 2596 (out, _) = subp.subp(['dpkg-query', '--list'], target=target, capture=True) 2597 2598 pkgs_inst = set() 2599 for line in out.splitlines(): 2600 try: 2601 (state, pkg, _) = line.split(None, 2) 2602 except ValueError: 2603 continue 2604 if state.startswith("hi") or state.startswith("ii"): 2605 pkgs_inst.add(re.sub(":.*", "", pkg)) 2606 2607 return pkgs_inst 2608 2609 2610def system_is_snappy(): 2611 # channel.ini is configparser loadable. 2612 # snappy will move to using /etc/system-image/config.d/*.ini 2613 # this is certainly not a perfect test, but good enough for now. 2614 orpath = "/etc/os-release" 2615 try: 2616 orinfo = load_shell_content(load_file(orpath, quiet=True)) 2617 if orinfo.get('ID', '').lower() == "ubuntu-core": 2618 return True 2619 except ValueError as e: 2620 LOG.warning("Unexpected error loading '%s': %s", orpath, e) 2621 2622 cmdline = get_cmdline() 2623 if 'snap_core=' in cmdline: 2624 return True 2625 2626 content = load_file("/etc/system-image/channel.ini", quiet=True) 2627 if 'ubuntu-core' in content.lower(): 2628 return True 2629 if os.path.isdir("/etc/system-image/config.d/"): 2630 return True 2631 return False 2632 2633 2634def indent(text, prefix): 2635 """replacement for indent from textwrap that is not available in 2.7.""" 2636 lines = [] 2637 for line in text.splitlines(True): 2638 lines.append(prefix + line) 2639 return ''.join(lines) 2640 2641 2642def rootdev_from_cmdline(cmdline): 2643 found = None 2644 for tok in cmdline.split(): 2645 if tok.startswith("root="): 2646 found = tok[5:] 2647 break 2648 if found is None: 2649 return None 2650 2651 if found.startswith("/dev/"): 2652 return found 2653 if found.startswith("LABEL="): 2654 return "/dev/disk/by-label/" + found[len("LABEL="):] 2655 if found.startswith("UUID="): 2656 return "/dev/disk/by-uuid/" + found[len("UUID="):].lower() 2657 if found.startswith("PARTUUID="): 2658 disks_path = ("/dev/disk/by-partuuid/" + 2659 found[len("PARTUUID="):].lower()) 2660 if os.path.exists(disks_path): 2661 return disks_path 2662 results = find_devs_with(found) 2663 if results: 2664 return results[0] 2665 # we know this doesn't exist, but for consistency return the path as 2666 # it /would/ exist 2667 return disks_path 2668 2669 return "/dev/" + found 2670 2671 2672def load_shell_content(content, add_empty=False, empty_val=None): 2673 """Given shell like syntax (key=value\nkey2=value2\n) in content 2674 return the data in dictionary form. If 'add_empty' is True 2675 then add entries in to the returned dictionary for 'VAR=' 2676 variables. Set their value to empty_val.""" 2677 2678 def _shlex_split(blob): 2679 return shlex.split(blob, comments=True) 2680 2681 data = {} 2682 for line in _shlex_split(content): 2683 key, value = line.split("=", 1) 2684 if not value: 2685 value = empty_val 2686 if add_empty or value: 2687 data[key] = value 2688 2689 return data 2690 2691 2692def wait_for_files(flist, maxwait, naplen=.5, log_pre=""): 2693 need = set(flist) 2694 waited = 0 2695 while True: 2696 need -= set([f for f in need if os.path.exists(f)]) 2697 if len(need) == 0: 2698 LOG.debug("%sAll files appeared after %s seconds: %s", 2699 log_pre, waited, flist) 2700 return [] 2701 if waited == 0: 2702 LOG.debug("%sWaiting up to %s seconds for the following files: %s", 2703 log_pre, maxwait, flist) 2704 if waited + naplen > maxwait: 2705 break 2706 time.sleep(naplen) 2707 waited += naplen 2708 2709 LOG.debug("%sStill missing files after %s seconds: %s", 2710 log_pre, maxwait, need) 2711 return need 2712 2713 2714def mount_is_read_write(mount_point): 2715 """Check whether the given mount point is mounted rw""" 2716 result = get_mount_info(mount_point, get_mnt_opts=True) 2717 mount_opts = result[-1].split(',') 2718 return mount_opts[0] == 'rw' 2719 2720 2721def udevadm_settle(exists=None, timeout=None): 2722 """Invoke udevadm settle with optional exists and timeout parameters""" 2723 settle_cmd = ["udevadm", "settle"] 2724 if exists: 2725 # skip the settle if the requested path already exists 2726 if os.path.exists(exists): 2727 return 2728 settle_cmd.extend(['--exit-if-exists=%s' % exists]) 2729 if timeout: 2730 settle_cmd.extend(['--timeout=%s' % timeout]) 2731 2732 return subp.subp(settle_cmd) 2733 2734 2735def get_proc_ppid(pid): 2736 """ 2737 Return the parent pid of a process. 2738 """ 2739 ppid = 0 2740 try: 2741 contents = load_file("/proc/%s/stat" % pid, quiet=True) 2742 except IOError as e: 2743 LOG.warning('Failed to load /proc/%s/stat. %s', pid, e) 2744 if contents: 2745 parts = contents.split(" ", 4) 2746 # man proc says 2747 # ppid %d (4) The PID of the parent. 2748 ppid = int(parts[3]) 2749 return ppid 2750 2751# vi: ts=4 expandtab 2752