1# Licensed under the Apache License, Version 2.0 (the "License"); you may 2# not use this file except in compliance with the License. You may obtain 3# a copy of the License at 4# 5# http://www.apache.org/licenses/LICENSE-2.0 6# 7# Unless required by applicable law or agreed to in writing, software 8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10# License for the specific language governing permissions and limitations 11# under the License. 12 13import logging 14import sys 15import textwrap 16import warnings 17import yaml 18 19from oslo_config import cfg 20from oslo_serialization import jsonutils 21import stevedore 22 23from oslo_policy import policy 24 25LOG = logging.getLogger(__name__) 26 27GENERATOR_OPTS = [ 28 cfg.StrOpt('output-file', 29 help='Path of the file to write to. Defaults to stdout.'), 30] 31 32RULE_OPTS = [ 33 cfg.MultiStrOpt('namespace', 34 help='Option namespace(s) under "oslo.policy.policies" in ' 35 'which to query for options.'), 36 cfg.StrOpt('format', 37 deprecated_for_removal=True, 38 deprecated_since='Victoria', 39 deprecated_reason=""" 40``policy_file`` support for JSON formatted file is deprecated. 41So these tools also deprecate the support of generating or 42upgrading policy file in JSON format. 43""", 44 help='Desired format for the output.', 45 default='yaml', 46 choices=['json', 'yaml']), 47] 48 49ENFORCER_OPTS = [ 50 cfg.StrOpt('namespace', 51 help='Option namespace under "oslo.policy.enforcer" in ' 52 'which to look for a policy.Enforcer.'), 53] 54 55UPGRADE_OPTS = [ 56 cfg.StrOpt('policy', 57 required=True, 58 help='Path to the policy file which need to be updated.') 59] 60 61CONVERT_OPTS = [ 62 cfg.MultiStrOpt('namespace', 63 required=True, 64 help='Option namespace(s) under "oslo.policy.policies" in ' 65 'which to query for options.'), 66 cfg.StrOpt('policy-file', 67 required=True, 68 help='Path to the policy file which need to be converted to ' 69 'yaml format.') 70] 71 72 73def get_policies_dict(namespaces): 74 """Find the options available via the given namespaces. 75 76 :param namespaces: a list of namespaces registered under 77 'oslo.policy.policies' 78 :returns: a dict of {namespace1: [rule_default_1, rule_default_2], 79 namespace2: [rule_default_3]...} 80 """ 81 mgr = stevedore.named.NamedExtensionManager( 82 'oslo.policy.policies', 83 names=namespaces, 84 on_load_failure_callback=on_load_failure_callback, 85 invoke_on_load=True) 86 opts = {ep.name: ep.obj for ep in mgr} 87 88 return opts 89 90 91def _get_enforcer(namespace): 92 """Find a policy.Enforcer via an entry point with the given namespace. 93 94 :param namespace: a namespace under oslo.policy.enforcer where the desired 95 enforcer object can be found. 96 :returns: a policy.Enforcer object 97 """ 98 mgr = stevedore.named.NamedExtensionManager( 99 'oslo.policy.enforcer', 100 names=[namespace], 101 on_load_failure_callback=on_load_failure_callback, 102 invoke_on_load=True) 103 if namespace not in mgr: 104 raise KeyError('Namespace "%s" not found.' % namespace) 105 enforcer = mgr[namespace].obj 106 107 return enforcer 108 109 110def _format_help_text(description): 111 """Format a comment for a policy based on the description provided. 112 113 :param description: A string with helpful text. 114 :returns: A line wrapped comment, or blank comment if description is None 115 """ 116 if not description: 117 return '#' 118 119 formatted_lines = [] 120 paragraph = [] 121 122 def _wrap_paragraph(lines): 123 return textwrap.wrap(' '.join(lines), 70, initial_indent='# ', 124 subsequent_indent='# ') 125 126 for line in description.strip().splitlines(): 127 if not line.strip(): 128 # empty line -> line break, so dump anything we have 129 formatted_lines.extend(_wrap_paragraph(paragraph)) 130 formatted_lines.append('#') 131 paragraph = [] 132 elif len(line) == len(line.lstrip()): 133 # no leading whitespace = paragraph, which should be wrapped 134 paragraph.append(line.rstrip()) 135 else: 136 # leading whitespace - literal block, which should not be wrapping 137 if paragraph: 138 # ...however, literal blocks need a new line before them to 139 # delineate things 140 # TODO(stephenfin): Raise an exception here and stop doing 141 # anything else in oslo.policy 2.0 142 warnings.warn( 143 'Invalid policy description: literal blocks must be ' 144 'preceded by a new line. This will raise an exception in ' 145 'a future version of oslo.policy:\n%s' % description, 146 FutureWarning) 147 formatted_lines.extend(_wrap_paragraph(paragraph)) 148 formatted_lines.append('#') 149 paragraph = [] 150 151 formatted_lines.append('# %s' % line.rstrip()) 152 153 if paragraph: 154 # dump anything we might still have in the buffer 155 formatted_lines.extend(_wrap_paragraph(paragraph)) 156 157 return '\n'.join(formatted_lines) 158 159 160def _format_rule_default_yaml(default, include_help=True, comment_rule=True, 161 add_deprecated_rules=True): 162 """Create a yaml node from policy.RuleDefault or policy.DocumentedRuleDefault. 163 164 :param default: A policy.RuleDefault or policy.DocumentedRuleDefault object 165 :param comment_rule: By default rules will be commented out in generated 166 yaml format text. If you want to keep few or all rules 167 uncommented then pass this arg as False. 168 :param add_deprecated_rules: Whether to add the deprecated rules in format 169 text. 170 :returns: A string containing a yaml representation of the RuleDefault 171 """ 172 text = ('"%(name)s": "%(check_str)s"\n' % 173 {'name': default.name, 174 'check_str': default.check_str}) 175 176 if include_help: 177 op = "" 178 if hasattr(default, 'operations'): 179 for operation in default.operations: 180 if operation['method'] and operation['path']: 181 op += ('# %(method)s %(path)s\n' % 182 {'method': operation['method'], 183 'path': operation['path']}) 184 intended_scope = "" 185 if getattr(default, 'scope_types', None) is not None: 186 intended_scope = ( 187 '# Intended scope(s): ' + ', '.join(default.scope_types) + '\n' 188 ) 189 comment = '#' if comment_rule else '' 190 text = ('%(op)s%(scope)s%(comment)s%(text)s\n' % 191 {'op': op, 192 'scope': intended_scope, 193 'comment': comment, 194 'text': text}) 195 if default.description: 196 text = _format_help_text(default.description) + '\n' + text 197 198 if add_deprecated_rules and default.deprecated_for_removal: 199 text = ( 200 '# DEPRECATED\n# "%(name)s" has been deprecated since ' 201 '%(since)s.\n%(reason)s\n%(text)s' 202 ) % {'name': default.name, 203 'since': default.deprecated_since, 204 'reason': _format_help_text(default.deprecated_reason), 205 'text': text} 206 elif add_deprecated_rules and default.deprecated_rule: 207 deprecated_reason = ( 208 default.deprecated_rule.deprecated_reason or 209 default.deprecated_reason 210 ) 211 deprecated_since = ( 212 default.deprecated_rule.deprecated_since or 213 default.deprecated_since 214 ) 215 216 # This issues a deprecation warning but aliases the old policy name 217 # with the new policy name for compatibility. 218 deprecated_text = ( 219 '"%(old_name)s":"%(old_check_str)s" has been deprecated ' 220 'since %(since)s in favor of "%(name)s":"%(check_str)s".' 221 ) % { 222 'old_name': default.deprecated_rule.name, 223 'old_check_str': default.deprecated_rule.check_str, 224 'since': deprecated_since, 225 'name': default.name, 226 'check_str': default.check_str, 227 } 228 text = '%(text)s# DEPRECATED\n%(deprecated_text)s\n%(reason)s\n' % { 229 'text': text, 230 'reason': _format_help_text(deprecated_reason), 231 'deprecated_text': _format_help_text(deprecated_text) 232 } 233 234 if default.name != default.deprecated_rule.name: 235 text += ('"%(old_name)s": "rule:%(name)s"\n' % 236 {'old_name': default.deprecated_rule.name, 237 'name': default.name}) 238 text += '\n' 239 240 return text 241 242 243def _format_rule_default_json(default): 244 """Create a json node from policy.RuleDefault or policy.DocumentedRuleDefault. 245 246 :param default: A policy.RuleDefault or policy.DocumentedRuleDefault object 247 :returns: A string containing a json representation of the RuleDefault 248 """ 249 return ('"%(name)s": "%(check_str)s"' % 250 {'name': default.name, 251 'check_str': default.check_str}) 252 253 254def _sort_and_format_by_section(policies, output_format='yaml', 255 include_help=True): 256 """Generate a list of policy section texts 257 258 The text for a section will be created and returned one at a time. The 259 sections are sorted first to provide for consistent output. 260 261 Text is created in yaml format. This is done manually because PyYaml 262 does not facilitate outputing comments. 263 264 :param policies: A dict of {section1: [rule_default_1, rule_default_2], 265 section2: [rule_default_3]} 266 :param output_format: The format of the file to output to. 267 """ 268 for section in sorted(policies.keys()): 269 rule_defaults = policies[section] 270 for rule_default in rule_defaults: 271 if output_format == 'yaml': 272 yield _format_rule_default_yaml(rule_default, 273 include_help=include_help) 274 elif output_format == 'json': 275 LOG.warning(policy.WARN_JSON) 276 yield _format_rule_default_json(rule_default) 277 278 279def _generate_sample(namespaces, output_file=None, output_format='yaml', 280 include_help=True): 281 """Generate a sample policy file. 282 283 List all of the policies available via the namespace specified in the 284 given configuration and write them to the specified output file. 285 286 :param namespaces: a list of namespaces registered under 287 'oslo.policy.policies'. Stevedore will look here for 288 policy options. 289 :param output_file: The path of a file to output to. stdout used if None. 290 :param output_format: The format of the file to output to. 291 :param include_help: True, generates a sample-policy file with help text 292 along with rules in which everything is commented out. 293 False, generates a sample-policy file with only rules. 294 """ 295 policies = get_policies_dict(namespaces) 296 297 output_file = (open(output_file, 'w') if output_file 298 else sys.stdout) 299 300 sections_text = [] 301 for section in _sort_and_format_by_section(policies, output_format, 302 include_help=include_help): 303 sections_text.append(section) 304 305 if output_format == 'yaml': 306 output_file.writelines(sections_text) 307 elif output_format == 'json': 308 LOG.warning(policy.WARN_JSON) 309 output_file.writelines(( 310 '{\n ', 311 ',\n '.join(sections_text), 312 '\n}\n')) 313 314 if output_file != sys.stdout: 315 output_file.close() 316 317 318def _generate_policy(namespace, output_file=None): 319 """Generate a policy file showing what will be used. 320 321 This takes all registered policies and merges them with what's defined in 322 a policy file and outputs the result. That result is the effective policy 323 that will be honored by policy checks. 324 325 :param output_file: The path of a file to output to. stdout used if None. 326 """ 327 enforcer = _get_enforcer(namespace) 328 # Ensure that files have been parsed 329 enforcer.load_rules() 330 331 file_rules = [policy.RuleDefault(name, default.check_str) 332 for name, default in enforcer.file_rules.items()] 333 registered_rules = [policy.RuleDefault(name, default.check_str) 334 for name, default in enforcer.registered_rules.items() 335 if name not in enforcer.file_rules] 336 policies = {'rules': file_rules + registered_rules} 337 338 output_file = (open(output_file, 'w') if output_file 339 else sys.stdout) 340 341 for section in _sort_and_format_by_section(policies, include_help=False): 342 output_file.write(section) 343 344 if output_file != sys.stdout: 345 output_file.close() 346 347 348def _list_redundant(namespace): 349 """Generate a list of configured policies which match defaults. 350 351 This checks all policies loaded from policy files and checks to see if they 352 match registered policies. If so then it is redundant to have them defined 353 in a policy file and operators should consider removing them. 354 """ 355 enforcer = _get_enforcer(namespace) 356 # NOTE(bnemec): We don't want to see policy deprecation warnings in the 357 # output of this tool. They tend to overwhelm the output that the user 358 # actually cares about, and checking for deprecations isn't the purpose of 359 # this tool. 360 enforcer.suppress_deprecation_warnings = True 361 # Ensure that files have been parsed 362 enforcer.load_rules() 363 364 for name, file_rule in enforcer.file_rules.items(): 365 reg_rule = enforcer.registered_rules.get(name) 366 if reg_rule: 367 if file_rule == reg_rule: 368 print(reg_rule) 369 370 371def _validate_policy(namespace): 372 """Perform basic sanity checks on a policy file 373 374 Checks for the following errors in the configured policy file: 375 376 * A missing policy file 377 * Rules which have invalid syntax 378 * Rules which reference non-existent other rules 379 * Rules which form a cyclical reference with another rule 380 * Rules which do not exist in the specified namespace 381 382 :param namespace: The name under which the oslo.policy enforcer is 383 registered. 384 :returns: 0 if all policies validated correctly, 1 if not. 385 """ 386 return_code = 0 387 enforcer = _get_enforcer(namespace) 388 # NOTE(bnemec): We don't want to see policy deprecation warnings in the 389 # output of this tool. They tend to overwhelm the output that the user 390 # actually cares about. If we check for deprecated rules in this tool, 391 # we need to do it another way. 392 enforcer.suppress_deprecation_warnings = True 393 # Disable logging from the parser code. We'll be printing any errors we 394 # find below. 395 logging.disable(logging.ERROR) 396 # Ensure that files have been parsed 397 enforcer.load_rules() 398 399 if enforcer._informed_no_policy_file: 400 print('Configured policy file "%s" not found' % enforcer.policy_file) 401 # If the policy file is completely missing then the rest of our checks 402 # don't make sense. 403 return 1 404 405 # Re-enable logging so we get messages for things like cyclical references 406 logging.disable(logging.NOTSET) 407 result = enforcer.check_rules() 408 if not result: 409 print('Invalid rules found') 410 return_code = 1 411 412 # TODO(bnemec): Allow this to handle policy_dir 413 with open(cfg.CONF.oslo_policy.policy_file) as f: 414 unparsed_policies = yaml.safe_load(f.read()) 415 for name, file_rule in enforcer.file_rules.items(): 416 reg_rule = enforcer.registered_rules.get(name) 417 if reg_rule is None: 418 print('Unknown rule found in policy file:', name) 419 return_code = 1 420 # If a rule has invalid syntax it will be forced to '!'. If the literal 421 # rule from the policy file isn't '!' then this means there was an 422 # error parsing it. 423 if str(enforcer.rules[name]) == '!' and unparsed_policies[name] != '!': 424 print('Failed to parse rule:', unparsed_policies[name]) 425 return_code = 1 426 return return_code 427 428 429def _convert_policy_json_to_yaml(namespace, policy_file, output_file=None): 430 with open(policy_file, 'r') as rule_data: 431 file_policies = jsonutils.loads(rule_data.read()) 432 433 yaml_format_rules = [] 434 default_policies = get_policies_dict(namespace) 435 for section in sorted(default_policies): 436 default_rules = default_policies[section] 437 for default_rule in default_rules: 438 if default_rule.name not in file_policies: 439 continue 440 file_rule_check_str = file_policies.pop(default_rule.name) 441 # Some rules might be still RuleDefault object so let's prepare 442 # empty 'operations' list and rule name as description for 443 # those. 444 operations = [{ 445 'method': '', 446 'path': '' 447 }] 448 if hasattr(default_rule, 'operations'): 449 operations = default_rule.operations 450 # Converting JSON file rules to DocumentedRuleDefault rules so 451 # that we can convert the JSON file to YAML including 452 # descriptions which is what 'oslopolicy-sample-generator' 453 # tool does. 454 file_rule = policy.DocumentedRuleDefault( 455 default_rule.name, 456 file_rule_check_str, 457 default_rule.description or default_rule.name, 458 operations, 459 default_rule.deprecated_rule, 460 default_rule.deprecated_for_removal, 461 default_rule.deprecated_reason, 462 default_rule.deprecated_since, 463 scope_types=default_rule.scope_types) 464 if file_rule == default_rule: 465 rule_text = _format_rule_default_yaml( 466 file_rule, add_deprecated_rules=False) 467 else: 468 # NOTE(gmann): If json file rule is not same as default 469 # means rule is overridden then do not comment out it in 470 # yaml file. 471 rule_text = _format_rule_default_yaml( 472 file_rule, comment_rule=False, 473 add_deprecated_rules=False) 474 yaml_format_rules.append(rule_text) 475 476 extra_rules_text = ("# WARNING: Below rules are either deprecated rules\n" 477 "# or extra rules in policy file, it is strongly\n" 478 "# recommended to switch to new rules.\n") 479 # NOTE(gmann): If policy json file still using the deprecated rules which 480 # will not be present in default rules list. Or it can be case of any 481 # extra rule (old rule which is now removed) present in json file. 482 # so let's keep these as it is (not commented out) to avoid breaking 483 # existing deployment. 484 if file_policies: 485 yaml_format_rules.append(extra_rules_text) 486 for file_rule, check_str in file_policies.items(): 487 rule_text = ('"%(name)s": "%(check_str)s"\n' % 488 {'name': file_rule, 489 'check_str': check_str}) 490 yaml_format_rules.append(rule_text) 491 492 if output_file: 493 with open(output_file, 'w') as fh: 494 fh.writelines(yaml_format_rules) 495 else: 496 sys.stdout.writelines(yaml_format_rules) 497 498 499def on_load_failure_callback(*args, **kwargs): 500 raise 501 502 503def _check_for_namespace_opt(conf): 504 # NOTE(bnemec): This opt is required, but due to lp#1849518 we need to 505 # make it optional while our consumers migrate to the new method of 506 # parsing cli args. Making the arg itself optional and explicitly checking 507 # for it in the tools will allow us to migrate projects without breaking 508 # anything. Once everyone has migrated, we can make the arg required again 509 # and remove this check. 510 if conf.namespace is None: 511 raise cfg.RequiredOptError('namespace', 'DEFAULT') 512 513 514def generate_sample(args=None, conf=None): 515 logging.basicConfig(level=logging.WARN) 516 # Allow the caller to pass in a local conf object for unit testing 517 if conf is None: 518 conf = cfg.CONF 519 conf.register_cli_opts(GENERATOR_OPTS + RULE_OPTS) 520 conf.register_opts(GENERATOR_OPTS + RULE_OPTS) 521 conf(args) 522 _check_for_namespace_opt(conf) 523 _generate_sample(conf.namespace, conf.output_file, conf.format) 524 525 526def generate_policy(args=None): 527 logging.basicConfig(level=logging.WARN) 528 conf = cfg.CONF 529 conf.register_cli_opts(GENERATOR_OPTS + ENFORCER_OPTS) 530 conf.register_opts(GENERATOR_OPTS + ENFORCER_OPTS) 531 conf(args) 532 _check_for_namespace_opt(conf) 533 _generate_policy(conf.namespace, conf.output_file) 534 535 536def _upgrade_policies(policies, default_policies): 537 old_policies_keys = list(policies.keys()) 538 for section in sorted(default_policies.keys()): 539 rule_defaults = default_policies[section] 540 for rule_default in rule_defaults: 541 if (rule_default.deprecated_rule and 542 rule_default.deprecated_rule.name in old_policies_keys): 543 policies[rule_default.name] = policies.pop( 544 rule_default.deprecated_rule.name) 545 LOG.info('The name of policy %(old_name)s has been upgraded to' 546 '%(new_name)', 547 {'old_name': rule_default.deprecated_rule.name, 548 'new_name': rule_default.name}) 549 550 551def upgrade_policy(args=None, conf=None): 552 logging.basicConfig(level=logging.WARN) 553 # Allow the caller to pass in a local conf object for unit testing 554 if conf is None: 555 conf = cfg.CONF 556 conf.register_cli_opts(GENERATOR_OPTS + RULE_OPTS + UPGRADE_OPTS) 557 conf.register_opts(GENERATOR_OPTS + RULE_OPTS + UPGRADE_OPTS) 558 conf(args) 559 _check_for_namespace_opt(conf) 560 with open(conf.policy, 'r') as input_data: 561 policies = policy.parse_file_contents(input_data.read()) 562 default_policies = get_policies_dict(conf.namespace) 563 564 _upgrade_policies(policies, default_policies) 565 566 if conf.output_file: 567 with open(conf.output_file, 'w') as fh: 568 if conf.format == 'yaml': 569 yaml.safe_dump(policies, fh, default_flow_style=False) 570 elif conf.format == 'json': 571 LOG.warning(policy.WARN_JSON) 572 jsonutils.dump(policies, fh, indent=4) 573 else: 574 if conf.format == 'yaml': 575 sys.stdout.write(yaml.safe_dump(policies, 576 default_flow_style=False)) 577 elif conf.format == 'json': 578 LOG.warning(policy.WARN_JSON) 579 sys.stdout.write(jsonutils.dumps(policies, indent=4)) 580 581 582def list_redundant(args=None): 583 logging.basicConfig(level=logging.WARN) 584 conf = cfg.CONF 585 conf.register_cli_opts(ENFORCER_OPTS) 586 conf.register_opts(ENFORCER_OPTS) 587 conf(args) 588 _check_for_namespace_opt(conf) 589 _list_redundant(conf.namespace) 590 591 592def validate_policy(args=None): 593 logging.basicConfig(level=logging.WARN) 594 conf = cfg.CONF 595 conf.register_cli_opts(ENFORCER_OPTS) 596 conf.register_opts(ENFORCER_OPTS) 597 conf(args) 598 sys.exit(_validate_policy(conf.namespace)) 599 600 601def convert_policy_json_to_yaml(args=None, conf=None): 602 logging.basicConfig(level=logging.WARN) 603 # Allow the caller to pass in a local conf object for unit testing 604 if conf is None: 605 conf = cfg.CONF 606 conf.register_cli_opts(GENERATOR_OPTS + CONVERT_OPTS) 607 conf.register_opts(GENERATOR_OPTS + CONVERT_OPTS) 608 conf(args) 609 _check_for_namespace_opt(conf) 610 _convert_policy_json_to_yaml(conf.namespace, conf.policy_file, 611 conf.output_file) 612