1# Copyright (C) 2012 Canonical Ltd. 2# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. 3# Copyright (C) 2012 Yahoo! Inc. 4# 5# Author: Scott Moser <scott.moser@canonical.com> 6# Author: Juerg Haefliger <juerg.haefliger@hp.com> 7# Author: Joshua Harlow <harlowja@yahoo-inc.com> 8# 9# This file is part of cloud-init. See LICENSE file for license information. 10 11from time import time 12 13import contextlib 14import os 15from configparser import NoSectionError, NoOptionError, RawConfigParser 16from io import StringIO 17 18from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, 19 CFG_ENV_NAME) 20 21from cloudinit import log as logging 22from cloudinit import type_utils 23from cloudinit import persistence 24from cloudinit import util 25 26LOG = logging.getLogger(__name__) 27 28 29class LockFailure(Exception): 30 pass 31 32 33class DummyLock(object): 34 pass 35 36 37class DummySemaphores(object): 38 def __init__(self): 39 pass 40 41 @contextlib.contextmanager 42 def lock(self, _name, _freq, _clear_on_fail=False): 43 yield DummyLock() 44 45 def has_run(self, _name, _freq): 46 return False 47 48 def clear(self, _name, _freq): 49 return True 50 51 def clear_all(self): 52 pass 53 54 55class FileLock(object): 56 def __init__(self, fn): 57 self.fn = fn 58 59 def __str__(self): 60 return "<%s using file %r>" % (type_utils.obj_name(self), self.fn) 61 62 63def canon_sem_name(name): 64 return name.replace("-", "_") 65 66 67class FileSemaphores(object): 68 def __init__(self, sem_path): 69 self.sem_path = sem_path 70 71 @contextlib.contextmanager 72 def lock(self, name, freq, clear_on_fail=False): 73 name = canon_sem_name(name) 74 try: 75 yield self._acquire(name, freq) 76 except Exception: 77 if clear_on_fail: 78 self.clear(name, freq) 79 raise 80 81 def clear(self, name, freq): 82 name = canon_sem_name(name) 83 sem_file = self._get_path(name, freq) 84 try: 85 util.del_file(sem_file) 86 except (IOError, OSError): 87 util.logexc(LOG, "Failed deleting semaphore %s", sem_file) 88 return False 89 return True 90 91 def clear_all(self): 92 try: 93 util.del_dir(self.sem_path) 94 except (IOError, OSError): 95 util.logexc(LOG, "Failed deleting semaphore directory %s", 96 self.sem_path) 97 98 def _acquire(self, name, freq): 99 # Check again if its been already gotten 100 if self.has_run(name, freq): 101 return None 102 # This is a race condition since nothing atomic is happening 103 # here, but this should be ok due to the nature of when 104 # and where cloud-init runs... (file writing is not a lock...) 105 sem_file = self._get_path(name, freq) 106 contents = "%s: %s\n" % (os.getpid(), time()) 107 try: 108 util.write_file(sem_file, contents) 109 except (IOError, OSError): 110 util.logexc(LOG, "Failed writing semaphore file %s", sem_file) 111 return None 112 return FileLock(sem_file) 113 114 def has_run(self, name, freq): 115 if not freq or freq == PER_ALWAYS: 116 return False 117 118 cname = canon_sem_name(name) 119 sem_file = self._get_path(cname, freq) 120 # This isn't really a good atomic check 121 # but it suffices for where and when cloudinit runs 122 if os.path.exists(sem_file): 123 return True 124 125 # this case could happen if the migrator module hadn't run yet 126 # but the item had run before we did canon_sem_name. 127 if cname != name and os.path.exists(self._get_path(name, freq)): 128 LOG.warning("%s has run without canonicalized name [%s].\n" 129 "likely the migrator has not yet run. " 130 "It will run next boot.\n" 131 "run manually with: cloud-init single --name=migrator", 132 name, cname) 133 return True 134 135 return False 136 137 def _get_path(self, name, freq): 138 sem_path = self.sem_path 139 if not freq or freq == PER_INSTANCE: 140 return os.path.join(sem_path, name) 141 else: 142 return os.path.join(sem_path, "%s.%s" % (name, freq)) 143 144 145class Runners(object): 146 def __init__(self, paths): 147 self.paths = paths 148 self.sems = {} 149 150 def _get_sem(self, freq): 151 if freq == PER_ALWAYS or not freq: 152 return None 153 sem_path = None 154 if freq == PER_INSTANCE: 155 # This may not exist, 156 # so thats why we still check for none 157 # below if say the paths object 158 # doesn't have a datasource that can 159 # provide this instance path... 160 sem_path = self.paths.get_ipath("sem") 161 elif freq == PER_ONCE: 162 sem_path = self.paths.get_cpath("sem") 163 if not sem_path: 164 return None 165 if sem_path not in self.sems: 166 self.sems[sem_path] = FileSemaphores(sem_path) 167 return self.sems[sem_path] 168 169 def run(self, name, functor, args, freq=None, clear_on_fail=False): 170 sem = self._get_sem(freq) 171 if not sem: 172 sem = DummySemaphores() 173 if not args: 174 args = [] 175 if sem.has_run(name, freq): 176 LOG.debug("%s already ran (freq=%s)", name, freq) 177 return (False, None) 178 with sem.lock(name, freq, clear_on_fail) as lk: 179 if not lk: 180 raise LockFailure("Failed to acquire lock for %s" % name) 181 else: 182 LOG.debug("Running %s using lock (%s)", name, lk) 183 if isinstance(args, (dict)): 184 results = functor(**args) 185 else: 186 results = functor(*args) 187 return (True, results) 188 189 190class ConfigMerger(object): 191 def __init__(self, paths=None, datasource=None, 192 additional_fns=None, base_cfg=None, 193 include_vendor=True): 194 self._paths = paths 195 self._ds = datasource 196 self._fns = additional_fns 197 self._base_cfg = base_cfg 198 self._include_vendor = include_vendor 199 # Created on first use 200 self._cfg = None 201 202 def _get_datasource_configs(self): 203 d_cfgs = [] 204 if self._ds: 205 try: 206 ds_cfg = self._ds.get_config_obj() 207 if ds_cfg and isinstance(ds_cfg, (dict)): 208 d_cfgs.append(ds_cfg) 209 except Exception: 210 util.logexc(LOG, "Failed loading of datasource config object " 211 "from %s", self._ds) 212 return d_cfgs 213 214 def _get_env_configs(self): 215 e_cfgs = [] 216 if CFG_ENV_NAME in os.environ: 217 e_fn = os.environ[CFG_ENV_NAME] 218 try: 219 e_cfgs.append(util.read_conf(e_fn)) 220 except Exception: 221 util.logexc(LOG, 'Failed loading of env. config from %s', 222 e_fn) 223 return e_cfgs 224 225 def _get_instance_configs(self): 226 i_cfgs = [] 227 # If cloud-config was written, pick it up as 228 # a configuration file to use when running... 229 if not self._paths: 230 return i_cfgs 231 232 cc_paths = ['cloud_config'] 233 if self._include_vendor: 234 # the order is important here: we want vendor2 235 # (dynamic vendor data from OpenStack) 236 # to override vendor (static data from OpenStack) 237 cc_paths.append('vendor2_cloud_config') 238 cc_paths.append('vendor_cloud_config') 239 240 for cc_p in cc_paths: 241 cc_fn = self._paths.get_ipath_cur(cc_p) 242 if cc_fn and os.path.isfile(cc_fn): 243 try: 244 i_cfgs.append(util.read_conf(cc_fn)) 245 except PermissionError: 246 LOG.debug( 247 'Skipped loading cloud-config from %s due to' 248 ' non-root.', cc_fn) 249 except Exception: 250 util.logexc(LOG, 'Failed loading of cloud-config from %s', 251 cc_fn) 252 return i_cfgs 253 254 def _read_cfg(self): 255 # Input config files override 256 # env config files which 257 # override instance configs 258 # which override datasource 259 # configs which override 260 # base configuration 261 cfgs = [] 262 if self._fns: 263 for c_fn in self._fns: 264 try: 265 cfgs.append(util.read_conf(c_fn)) 266 except Exception: 267 util.logexc(LOG, "Failed loading of configuration from %s", 268 c_fn) 269 270 cfgs.extend(self._get_env_configs()) 271 cfgs.extend(self._get_instance_configs()) 272 cfgs.extend(self._get_datasource_configs()) 273 if self._base_cfg: 274 cfgs.append(self._base_cfg) 275 return util.mergemanydict(cfgs) 276 277 @property 278 def cfg(self): 279 # None check to avoid empty case causing re-reading 280 if self._cfg is None: 281 self._cfg = self._read_cfg() 282 return self._cfg 283 284 285class ContentHandlers(object): 286 287 def __init__(self): 288 self.registered = {} 289 self.initialized = [] 290 291 def __contains__(self, item): 292 return self.is_registered(item) 293 294 def __getitem__(self, key): 295 return self._get_handler(key) 296 297 def is_registered(self, content_type): 298 return content_type in self.registered 299 300 def register(self, mod, initialized=False, overwrite=True): 301 types = set() 302 for t in mod.list_types(): 303 if overwrite: 304 types.add(t) 305 else: 306 if not self.is_registered(t): 307 types.add(t) 308 for t in types: 309 self.registered[t] = mod 310 if initialized and mod not in self.initialized: 311 self.initialized.append(mod) 312 return types 313 314 def _get_handler(self, content_type): 315 return self.registered[content_type] 316 317 def items(self): 318 return list(self.registered.items()) 319 320 321class Paths(persistence.CloudInitPickleMixin): 322 _ci_pkl_version = 1 323 324 def __init__(self, path_cfgs, ds=None): 325 self.cfgs = path_cfgs 326 # Populate all the initial paths 327 self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') 328 self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') 329 self.instance_link = os.path.join(self.cloud_dir, 'instance') 330 self.boot_finished = os.path.join(self.instance_link, "boot-finished") 331 self.upstart_conf_d = path_cfgs.get('upstart_dir') 332 self.seed_dir = os.path.join(self.cloud_dir, 'seed') 333 # This one isn't joined, since it should just be read-only 334 template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/') 335 self.template_tpl = os.path.join(template_dir, '%s.tmpl') 336 self.lookups = { 337 "handlers": "handlers", 338 "scripts": "scripts", 339 "vendor_scripts": "scripts/vendor", 340 "sem": "sem", 341 "boothooks": "boothooks", 342 "userdata_raw": "user-data.txt", 343 "userdata": "user-data.txt.i", 344 "obj_pkl": "obj.pkl", 345 "cloud_config": "cloud-config.txt", 346 "vendor_cloud_config": "vendor-cloud-config.txt", 347 "vendor2_cloud_config": "vendor2-cloud-config.txt", 348 "data": "data", 349 "vendordata_raw": "vendor-data.txt", 350 "vendordata2_raw": "vendor-data2.txt", 351 "vendordata": "vendor-data.txt.i", 352 "vendordata2": "vendor-data2.txt.i", 353 "instance_id": ".instance-id", 354 "manual_clean_marker": "manual-clean", 355 "warnings": "warnings", 356 } 357 # Set when a datasource becomes active 358 self.datasource = ds 359 360 def _unpickle(self, ci_pkl_version: int) -> None: 361 """Perform deserialization fixes for Paths.""" 362 if not hasattr(self, "run_dir"): 363 # On older versions of cloud-init the Paths class do not 364 # have the run_dir attribute. This is problematic because 365 # when loading the pickle object on newer versions of cloud-init 366 # we will rely on this attribute. To fix that, we are now 367 # manually adding that attribute here. 368 self.run_dir = Paths( 369 path_cfgs=self.cfgs, 370 ds=self.datasource).run_dir 371 372 # get_ipath_cur: get the current instance path for an item 373 def get_ipath_cur(self, name=None): 374 return self._get_path(self.instance_link, name) 375 376 # get_cpath : get the "clouddir" (/var/lib/cloud/<name>) 377 # for a name in dirmap 378 def get_cpath(self, name=None): 379 return self._get_path(self.cloud_dir, name) 380 381 # _get_ipath : get the instance path for a name in pathmap 382 # (/var/lib/cloud/instances/<instance>/<name>) 383 def _get_ipath(self, name=None): 384 if not self.datasource: 385 return None 386 iid = self.datasource.get_instance_id() 387 if iid is None: 388 return None 389 path_safe_iid = str(iid).replace(os.sep, '_') 390 ipath = os.path.join(self.cloud_dir, 'instances', path_safe_iid) 391 add_on = self.lookups.get(name) 392 if add_on: 393 ipath = os.path.join(ipath, add_on) 394 return ipath 395 396 # get_ipath : get the instance path for a name in pathmap 397 # (/var/lib/cloud/instances/<instance>/<name>) 398 # returns None + warns if no active datasource.... 399 def get_ipath(self, name=None): 400 ipath = self._get_ipath(name) 401 if not ipath: 402 LOG.warning(("No per instance data available, " 403 "is there an datasource/iid set?")) 404 return None 405 else: 406 return ipath 407 408 def _get_path(self, base, name=None): 409 if name is None: 410 return base 411 return os.path.join(base, self.lookups[name]) 412 413 def get_runpath(self, name=None): 414 return self._get_path(self.run_dir, name) 415 416 417# This config parser will not throw when sections don't exist 418# and you are setting values on those sections which is useful 419# when writing to new options that may not have corresponding 420# sections. Also it can default other values when doing gets 421# so that if those sections/options do not exist you will 422# get a default instead of an error. Another useful case where 423# you can avoid catching exceptions that you typically don't 424# care about... 425 426class DefaultingConfigParser(RawConfigParser): 427 DEF_INT = 0 428 DEF_FLOAT = 0.0 429 DEF_BOOLEAN = False 430 DEF_BASE = None 431 432 def get(self, section, option): 433 value = self.DEF_BASE 434 try: 435 value = RawConfigParser.get(self, section, option) 436 except NoSectionError: 437 pass 438 except NoOptionError: 439 pass 440 return value 441 442 def set(self, section, option, value=None): 443 if not self.has_section(section) and section.lower() != 'default': 444 self.add_section(section) 445 RawConfigParser.set(self, section, option, value) 446 447 def remove_option(self, section, option): 448 if self.has_option(section, option): 449 RawConfigParser.remove_option(self, section, option) 450 451 def getboolean(self, section, option): 452 if not self.has_option(section, option): 453 return self.DEF_BOOLEAN 454 return RawConfigParser.getboolean(self, section, option) 455 456 def getfloat(self, section, option): 457 if not self.has_option(section, option): 458 return self.DEF_FLOAT 459 return RawConfigParser.getfloat(self, section, option) 460 461 def getint(self, section, option): 462 if not self.has_option(section, option): 463 return self.DEF_INT 464 return RawConfigParser.getint(self, section, option) 465 466 def stringify(self, header=None): 467 contents = '' 468 outputstream = StringIO() 469 self.write(outputstream) 470 outputstream.flush() 471 contents = outputstream.getvalue() 472 if header: 473 contents = '\n'.join([header, contents, '']) 474 return contents 475 476# vi: ts=4 expandtab 477