1# Copyright 2012 OpenStack Foundation 2# All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may 5# not use this file except in compliance with the License. You may obtain 6# a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations 14# under the License. 15 16import collections 17from oslo_serialization import jsonutils 18import six 19from six.moves.urllib import error 20from six.moves.urllib import parse 21from six.moves.urllib import request 22 23from heatclient._i18n import _ 24from heatclient.common import environment_format 25from heatclient.common import template_format 26from heatclient.common import utils 27from heatclient import exc 28 29 30def process_template_path(template_path, object_request=None, 31 existing=False, fetch_child=True): 32 """Read template from template path. 33 34 Attempt to read template first as a file or url. If that is unsuccessful, 35 try again assuming path is to a template object. 36 37 :param template_path: local or uri path to template 38 :param object_request: custom object request function used to get template 39 if local or uri path fails 40 :param existing: if the current stack's template should be used 41 :param fetch_child: Whether to fetch the child templates 42 :returns: get_file dict and template contents 43 :raises: error.URLError 44 """ 45 try: 46 return get_template_contents(template_file=template_path, 47 existing=existing, 48 fetch_child=fetch_child) 49 except error.URLError as template_file_exc: 50 try: 51 return get_template_contents(template_object=template_path, 52 object_request=object_request, 53 existing=existing, 54 fetch_child=fetch_child) 55 except exc.HTTPNotFound: 56 # The initial exception gives the user better failure context. 57 raise template_file_exc 58 59 60def get_template_contents(template_file=None, template_url=None, 61 template_object=None, object_request=None, 62 files=None, existing=False, 63 fetch_child=True): 64 65 is_object = False 66 # Transform a bare file path to a file:// URL. 67 if template_file: 68 template_url = utils.normalise_file_path_to_url(template_file) 69 70 if template_url: 71 tpl = request.urlopen(template_url).read() 72 73 elif template_object: 74 is_object = True 75 template_url = template_object 76 tpl = object_request and object_request('GET', 77 template_object) 78 elif existing: 79 return {}, None 80 else: 81 raise exc.CommandError(_('Need to specify exactly one of ' 82 '[%(arg1)s, %(arg2)s or %(arg3)s]' 83 ' or %(arg4)s') % 84 { 85 'arg1': '--template-file', 86 'arg2': '--template-url', 87 'arg3': '--template-object', 88 'arg4': '--existing'}) 89 90 if not tpl: 91 raise exc.CommandError(_('Could not fetch template from %s') 92 % template_url) 93 94 try: 95 if isinstance(tpl, six.binary_type): 96 tpl = tpl.decode('utf-8') 97 template = template_format.parse(tpl) 98 except ValueError as e: 99 raise exc.CommandError(_('Error parsing template %(url)s %(error)s') % 100 {'url': template_url, 'error': e}) 101 if files is None: 102 files = {} 103 104 if fetch_child: 105 tmpl_base_url = utils.base_url_for_url(template_url) 106 resolve_template_get_files(template, files, tmpl_base_url, is_object, 107 object_request) 108 return files, template 109 110 111def resolve_template_get_files(template, files, template_base_url, 112 is_object=False, object_request=None): 113 114 def ignore_if(key, value): 115 if key != 'get_file' and key != 'type': 116 return True 117 if not isinstance(value, six.string_types): 118 return True 119 if (key == 'type' and 120 not value.endswith(('.yaml', '.template'))): 121 return True 122 return False 123 124 def recurse_if(value): 125 return isinstance(value, (dict, list)) 126 127 get_file_contents(template, files, template_base_url, 128 ignore_if, recurse_if, is_object, object_request) 129 130 131def is_template(file_content): 132 try: 133 if isinstance(file_content, six.binary_type): 134 file_content = file_content.decode('utf-8') 135 template_format.parse(file_content) 136 except (ValueError, TypeError): 137 return False 138 return True 139 140 141def get_file_contents(from_data, files, base_url=None, 142 ignore_if=None, recurse_if=None, 143 is_object=False, object_request=None): 144 145 if recurse_if and recurse_if(from_data): 146 if isinstance(from_data, dict): 147 recurse_data = six.itervalues(from_data) 148 else: 149 recurse_data = from_data 150 for value in recurse_data: 151 get_file_contents(value, files, base_url, ignore_if, recurse_if, 152 is_object, object_request) 153 154 if isinstance(from_data, dict): 155 for key, value in from_data.items(): 156 if ignore_if and ignore_if(key, value): 157 continue 158 159 if base_url and not base_url.endswith('/'): 160 base_url = base_url + '/' 161 162 str_url = parse.urljoin(base_url, value) 163 if str_url not in files: 164 if is_object and object_request: 165 file_content = object_request('GET', str_url) 166 else: 167 file_content = utils.read_url_content(str_url) 168 if is_template(file_content): 169 if is_object: 170 template = get_template_contents( 171 template_object=str_url, files=files, 172 object_request=object_request)[1] 173 else: 174 template = get_template_contents( 175 template_url=str_url, files=files)[1] 176 file_content = jsonutils.dumps(template) 177 files[str_url] = file_content 178 # replace the data value with the normalised absolute URL 179 from_data[key] = str_url 180 181 182def read_url_content(url): 183 '''DEPRECATED! Use 'utils.read_url_content' instead.''' 184 return utils.read_url_content(url) 185 186 187def base_url_for_url(url): 188 '''DEPRECATED! Use 'utils.base_url_for_url' instead.''' 189 return utils.base_url_for_url(url) 190 191 192def normalise_file_path_to_url(path): 193 '''DEPRECATED! Use 'utils.normalise_file_path_to_url' instead.''' 194 return utils.normalise_file_path_to_url(path) 195 196 197def deep_update(old, new): 198 '''Merge nested dictionaries.''' 199 200 # Prevents an error if in a previous iteration 201 # old[k] = None but v[k] = {...}, 202 if old is None: 203 old = {} 204 205 for k, v in new.items(): 206 if isinstance(v, collections.Mapping): 207 r = deep_update(old.get(k, {}), v) 208 old[k] = r 209 elif v is None and isinstance(old.get(k), collections.Mapping): 210 # Don't override empty data, to work around yaml syntax issue 211 pass 212 else: 213 old[k] = new[k] 214 return old 215 216 217def process_multiple_environments_and_files(env_paths=None, template=None, 218 template_url=None, 219 env_path_is_object=None, 220 object_request=None, 221 env_list_tracker=None, 222 fetch_env_files=True): 223 """Reads one or more environment files. 224 225 Reads in each specified environment file and returns a dictionary 226 of the filenames->contents (suitable for the files dict) 227 and the consolidated environment (after having applied the correct 228 overrides based on order). 229 230 If a list is provided in the env_list_tracker parameter, the behavior 231 is altered to take advantage of server-side environment resolution. 232 Specifically, this means: 233 234 * Populating env_list_tracker with an ordered list of environment file 235 URLs to be passed to the server 236 * Including the contents of each environment file in the returned 237 files dict, keyed by one of the URLs in env_list_tracker 238 239 :param env_paths: list of paths to the environment files to load; if 240 None, empty results will be returned 241 :type env_paths: list or None 242 :param template: unused; only included for API compatibility 243 :param template_url: unused; only included for API compatibility 244 :param env_list_tracker: if specified, environment filenames will be 245 stored within 246 :type env_list_tracker: list or None 247 :return: tuple of files dict and a dict of the consolidated environment 248 :rtype: tuple 249 :param fetch_env_files: fetch env_files or leave it to server 250 """ 251 merged_files = {} 252 merged_env = {} 253 254 # If we're keeping a list of environment files separately, include the 255 # contents of the files in the files dict 256 include_env_in_files = env_list_tracker is not None 257 258 if env_paths: 259 for env_path in env_paths: 260 if fetch_env_files: 261 files, env = process_environment_and_files( 262 env_path=env_path, 263 template=template, 264 template_url=template_url, 265 env_path_is_object=env_path_is_object, 266 object_request=object_request, 267 include_env_in_files=include_env_in_files) 268 269 # 'files' looks like: 270 # {"filename1": contents, "filename2": contents} 271 # so a simple update is enough for merging 272 merged_files.update(files) 273 274 # 'env' can be a deeply nested dictionary, so a simple 275 # update is not enough 276 merged_env = deep_update(merged_env, env) 277 env_url = utils.normalise_file_path_to_url(env_path) 278 else: 279 env_url = env_path 280 281 if env_list_tracker is not None: 282 env_list_tracker.append(env_url) 283 284 return merged_files, merged_env 285 286 287def process_environment_and_files(env_path=None, 288 template=None, 289 template_url=None, 290 env_path_is_object=None, 291 object_request=None, 292 include_env_in_files=False): 293 """Loads a single environment file. 294 295 Returns an entry suitable for the files dict which maps the environment 296 filename to its contents. 297 298 :param env_path: full path to the file to load 299 :type env_path: str or None 300 :param include_env_in_files: if specified, the raw environment file itself 301 will be included in the returned files dict 302 :type include_env_in_files: bool 303 :return: tuple of files dict and the loaded environment as a dict 304 :rtype: (dict, dict) 305 """ 306 files = {} 307 env = {} 308 309 is_object = env_path_is_object and env_path_is_object(env_path) 310 311 if is_object: 312 raw_env = object_request and object_request('GET', env_path) 313 env = environment_format.parse(raw_env) 314 env_base_url = utils.base_url_for_url(env_path) 315 316 resolve_environment_urls( 317 env.get('resource_registry'), 318 files, 319 env_base_url, is_object=True, object_request=object_request) 320 321 elif env_path: 322 env_url = utils.normalise_file_path_to_url(env_path) 323 env_base_url = utils.base_url_for_url(env_url) 324 raw_env = request.urlopen(env_url).read() 325 326 env = environment_format.parse(raw_env) 327 328 resolve_environment_urls( 329 env.get('resource_registry'), 330 files, 331 env_base_url) 332 333 if include_env_in_files: 334 files[env_url] = jsonutils.dumps(env) 335 336 return files, env 337 338 339def resolve_environment_urls(resource_registry, files, env_base_url, 340 is_object=False, object_request=None): 341 """Handles any resource URLs specified in an environment. 342 343 :param resource_registry: mapping of type name to template filename 344 :type resource_registry: dict 345 :param files: dict to store loaded file contents into 346 :type files: dict 347 :param env_base_url: base URL to look in when loading files 348 :type env_base_url: str or None 349 """ 350 if resource_registry is None: 351 return 352 353 rr = resource_registry 354 base_url = rr.get('base_url', env_base_url) 355 356 def ignore_if(key, value): 357 if key == 'base_url': 358 return True 359 if isinstance(value, dict): 360 return True 361 if '::' in value: 362 # Built in providers like: "X::Compute::Server" 363 # don't need downloading. 364 return True 365 if key in ['hooks', 'restricted_actions']: 366 return True 367 368 get_file_contents(rr, files, base_url, ignore_if, 369 is_object=is_object, object_request=object_request) 370 371 for res_name, res_dict in rr.get('resources', {}).items(): 372 res_base_url = res_dict.get('base_url', base_url) 373 get_file_contents( 374 res_dict, files, res_base_url, ignore_if, 375 is_object=is_object, object_request=object_request) 376 377 378def hooks_to_env(env, arg_hooks, hook): 379 """Add hooks from args to environment's resource_registry section. 380 381 Hooks are either "resource_name" (if it's a top-level resource) or 382 "nested_stack/resource_name" (if the resource is in a nested stack). 383 384 The environment expects each hook to be associated with the resource 385 within `resource_registry/resources` using the `hooks: pre-create` format. 386 """ 387 if 'resource_registry' not in env: 388 env['resource_registry'] = {} 389 if 'resources' not in env['resource_registry']: 390 env['resource_registry']['resources'] = {} 391 for hook_declaration in arg_hooks: 392 hook_path = [r for r in hook_declaration.split('/') if r] 393 resources = env['resource_registry']['resources'] 394 for nested_stack in hook_path: 395 if nested_stack not in resources: 396 resources[nested_stack] = {} 397 resources = resources[nested_stack] 398 else: 399 resources['hooks'] = hook 400