1from __future__ import absolute_import 2from __future__ import unicode_literals 3 4import json 5import logging 6import os 7import re 8import sys 9 10import six 11from docker.utils.ports import split_port 12from jsonschema import Draft4Validator 13from jsonschema import FormatChecker 14from jsonschema import RefResolver 15from jsonschema import ValidationError 16 17from ..const import COMPOSEFILE_V1 as V1 18from ..const import NANOCPUS_SCALE 19from .errors import ConfigurationError 20from .errors import VERSION_EXPLANATION 21from .sort_services import get_service_name_from_network_mode 22 23 24log = logging.getLogger(__name__) 25 26 27DOCKER_CONFIG_HINTS = { 28 'cpu_share': 'cpu_shares', 29 'add_host': 'extra_hosts', 30 'hosts': 'extra_hosts', 31 'extra_host': 'extra_hosts', 32 'device': 'devices', 33 'link': 'links', 34 'memory_swap': 'memswap_limit', 35 'port': 'ports', 36 'privilege': 'privileged', 37 'priviliged': 'privileged', 38 'privilige': 'privileged', 39 'volume': 'volumes', 40 'workdir': 'working_dir', 41} 42 43 44VALID_NAME_CHARS = r'[a-zA-Z0-9\._\-]' 45VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' 46 47VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' 48VALID_IPV4_ADDR = r"({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) 49VALID_REGEX_IPV4_CIDR = r"^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) 50 51VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' 52VALID_REGEX_IPV6_CIDR = "".join(r""" 53^ 54( 55 (({IPV6_SEG}:){{7}}{IPV6_SEG})| 56 (({IPV6_SEG}:){{1,7}}:)| 57 (({IPV6_SEG}:){{1,6}}(:{IPV6_SEG}){{1,1}})| 58 (({IPV6_SEG}:){{1,5}}(:{IPV6_SEG}){{1,2}})| 59 (({IPV6_SEG}:){{1,4}}(:{IPV6_SEG}){{1,3}})| 60 (({IPV6_SEG}:){{1,3}}(:{IPV6_SEG}){{1,4}})| 61 (({IPV6_SEG}:){{1,2}}(:{IPV6_SEG}){{1,5}})| 62 (({IPV6_SEG}:){{1,1}}(:{IPV6_SEG}){{1,6}})| 63 (:((:{IPV6_SEG}){{1,7}}|:))| 64 (fe80:(:{IPV6_SEG}){{0,4}}%[0-9a-zA-Z]{{1,}})| 65 (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| 66 (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) 67) 68/(\d|[1-9]\d|1[0-1]\d|12[0-8]) 69$ 70""".format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) 71 72 73@FormatChecker.cls_checks(format="ports", raises=ValidationError) 74def format_ports(instance): 75 try: 76 split_port(instance) 77 except ValueError as e: 78 raise ValidationError(six.text_type(e)) 79 return True 80 81 82@FormatChecker.cls_checks(format="expose", raises=ValidationError) 83def format_expose(instance): 84 if isinstance(instance, six.string_types): 85 if not re.match(VALID_EXPOSE_FORMAT, instance): 86 raise ValidationError( 87 "should be of the format 'PORT[/PROTOCOL]'") 88 89 return True 90 91 92@FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) 93def format_subnet_ip_address(instance): 94 if isinstance(instance, six.string_types): 95 if not re.match(VALID_REGEX_IPV4_CIDR, instance) and \ 96 not re.match(VALID_REGEX_IPV6_CIDR, instance): 97 raise ValidationError("should use the CIDR format") 98 99 return True 100 101 102def match_named_volumes(service_dict, project_volumes): 103 service_volumes = service_dict.get('volumes', []) 104 for volume_spec in service_volumes: 105 if volume_spec.is_named_volume and volume_spec.external not in project_volumes: 106 raise ConfigurationError( 107 'Named volume "{0}" is used in service "{1}" but no' 108 ' declaration was found in the volumes section.'.format( 109 volume_spec.repr(), service_dict.get('name') 110 ) 111 ) 112 113 114def python_type_to_yaml_type(type_): 115 type_name = type(type_).__name__ 116 return { 117 'dict': 'mapping', 118 'list': 'array', 119 'int': 'number', 120 'float': 'number', 121 'bool': 'boolean', 122 'unicode': 'string', 123 'str': 'string', 124 'bytes': 'string', 125 }.get(type_name, type_name) 126 127 128def validate_config_section(filename, config, section): 129 """Validate the structure of a configuration section. This must be done 130 before interpolation so it's separate from schema validation. 131 """ 132 if not isinstance(config, dict): 133 raise ConfigurationError( 134 "In file '{filename}', {section} must be a mapping, not " 135 "{type}.".format( 136 filename=filename, 137 section=section, 138 type=anglicize_json_type(python_type_to_yaml_type(config)))) 139 140 for key, value in config.items(): 141 if not isinstance(key, six.string_types): 142 raise ConfigurationError( 143 "In file '{filename}', the {section} name {name} must be a " 144 "quoted string, i.e. '{name}'.".format( 145 filename=filename, 146 section=section, 147 name=key)) 148 149 if not isinstance(value, (dict, type(None))): 150 raise ConfigurationError( 151 "In file '{filename}', {section} '{name}' must be a mapping not " 152 "{type}.".format( 153 filename=filename, 154 section=section, 155 name=key, 156 type=anglicize_json_type(python_type_to_yaml_type(value)))) 157 158 159def validate_top_level_object(config_file): 160 if not isinstance(config_file.config, dict): 161 raise ConfigurationError( 162 "Top level object in '{}' needs to be an object not '{}'.".format( 163 config_file.filename, 164 type(config_file.config))) 165 166 167def validate_ulimits(service_config): 168 ulimit_config = service_config.config.get('ulimits', {}) 169 for limit_name, soft_hard_values in six.iteritems(ulimit_config): 170 if isinstance(soft_hard_values, dict): 171 if not soft_hard_values['soft'] <= soft_hard_values['hard']: 172 raise ConfigurationError( 173 "Service '{s.name}' has invalid ulimit '{ulimit}'. " 174 "'soft' value can not be greater than 'hard' value ".format( 175 s=service_config, 176 ulimit=ulimit_config)) 177 178 179def validate_extends_file_path(service_name, extends_options, filename): 180 """ 181 The service to be extended must either be defined in the config key 'file', 182 or within 'filename'. 183 """ 184 error_prefix = "Invalid 'extends' configuration for %s:" % service_name 185 186 if 'file' not in extends_options and filename is None: 187 raise ConfigurationError( 188 "%s you need to specify a 'file', e.g. 'file: something.yml'" % error_prefix 189 ) 190 191 192def validate_network_mode(service_config, service_names): 193 network_mode = service_config.config.get('network_mode') 194 if not network_mode: 195 return 196 197 if 'networks' in service_config.config: 198 raise ConfigurationError("'network_mode' and 'networks' cannot be combined") 199 200 dependency = get_service_name_from_network_mode(network_mode) 201 if not dependency: 202 return 203 204 if dependency not in service_names: 205 raise ConfigurationError( 206 "Service '{s.name}' uses the network stack of service '{dep}' which " 207 "is undefined.".format(s=service_config, dep=dependency)) 208 209 210def validate_pid_mode(service_config, service_names): 211 pid_mode = service_config.config.get('pid') 212 if not pid_mode: 213 return 214 215 dependency = get_service_name_from_network_mode(pid_mode) 216 if not dependency: 217 return 218 if dependency not in service_names: 219 raise ConfigurationError( 220 "Service '{s.name}' uses the PID namespace of service '{dep}' which " 221 "is undefined.".format(s=service_config, dep=dependency) 222 ) 223 224 225def validate_links(service_config, service_names): 226 for link in service_config.config.get('links', []): 227 if link.split(':')[0] not in service_names: 228 raise ConfigurationError( 229 "Service '{s.name}' has a link to service '{link}' which is " 230 "undefined.".format(s=service_config, link=link)) 231 232 233def validate_depends_on(service_config, service_names): 234 deps = service_config.config.get('depends_on', {}) 235 for dependency in deps.keys(): 236 if dependency not in service_names: 237 raise ConfigurationError( 238 "Service '{s.name}' depends on service '{dep}' which is " 239 "undefined.".format(s=service_config, dep=dependency) 240 ) 241 242 243def validate_credential_spec(service_config): 244 credential_spec = service_config.config.get('credential_spec') 245 if not credential_spec: 246 return 247 248 if 'registry' not in credential_spec and 'file' not in credential_spec: 249 raise ConfigurationError( 250 "Service '{s.name}' is missing 'credential_spec.file' or " 251 "credential_spec.registry'".format(s=service_config) 252 ) 253 254 255def get_unsupported_config_msg(path, error_key): 256 msg = "Unsupported config option for {}: '{}'".format(path_string(path), error_key) 257 if error_key in DOCKER_CONFIG_HINTS: 258 msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) 259 return msg 260 261 262def anglicize_json_type(json_type): 263 if json_type.startswith(('a', 'e', 'i', 'o', 'u')): 264 return 'an ' + json_type 265 return 'a ' + json_type 266 267 268def is_service_dict_schema(schema_id): 269 return schema_id in ('config_schema_v1.json', '#/properties/services') 270 271 272def handle_error_for_schema_with_id(error, path): 273 schema_id = error.schema['id'] 274 275 if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': 276 return "Invalid service name '{}' - only {} characters are allowed".format( 277 # The service_name is one of the keys in the json object 278 [i for i in list(error.instance) if not i or any(filter( 279 lambda c: not re.match(VALID_NAME_CHARS, c), i 280 ))][0], 281 VALID_NAME_CHARS 282 ) 283 284 if error.validator == 'additionalProperties': 285 if schema_id == '#/definitions/service': 286 invalid_config_key = parse_key_from_error_msg(error) 287 return get_unsupported_config_msg(path, invalid_config_key) 288 289 if schema_id.startswith('config_schema_v'): 290 invalid_config_key = parse_key_from_error_msg(error) 291 return ('Invalid top-level property "{key}". Valid top-level ' 292 'sections for this Compose file are: {properties}, and ' 293 'extensions starting with "x-".\n\n{explanation}').format( 294 key=invalid_config_key, 295 properties=', '.join(error.schema['properties'].keys()), 296 explanation=VERSION_EXPLANATION 297 ) 298 299 if not error.path: 300 return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) 301 302 303def handle_generic_error(error, path): 304 msg_format = None 305 error_msg = error.message 306 307 if error.validator == 'oneOf': 308 msg_format = "{path} {msg}" 309 config_key, error_msg = _parse_oneof_validator(error) 310 if config_key: 311 path.append(config_key) 312 313 elif error.validator == 'type': 314 msg_format = "{path} contains an invalid type, it should be {msg}" 315 error_msg = _parse_valid_types_from_validator(error.validator_value) 316 317 elif error.validator == 'required': 318 error_msg = ", ".join(error.validator_value) 319 msg_format = "{path} is invalid, {msg} is required." 320 321 elif error.validator == 'dependencies': 322 config_key = list(error.validator_value.keys())[0] 323 required_keys = ",".join(error.validator_value[config_key]) 324 325 msg_format = "{path} is invalid: {msg}" 326 path.append(config_key) 327 error_msg = "when defining '{}' you must set '{}' as well".format( 328 config_key, 329 required_keys) 330 331 elif error.cause: 332 error_msg = six.text_type(error.cause) 333 msg_format = "{path} is invalid: {msg}" 334 335 elif error.path: 336 msg_format = "{path} value {msg}" 337 338 if msg_format: 339 return msg_format.format(path=path_string(path), msg=error_msg) 340 341 return error.message 342 343 344def parse_key_from_error_msg(error): 345 try: 346 return error.message.split("'")[1] 347 except IndexError: 348 return error.message.split('(')[1].split(' ')[0].strip("'") 349 350 351def path_string(path): 352 return ".".join(c for c in path if isinstance(c, six.string_types)) 353 354 355def _parse_valid_types_from_validator(validator): 356 """A validator value can be either an array of valid types or a string of 357 a valid type. Parse the valid types and prefix with the correct article. 358 """ 359 if not isinstance(validator, list): 360 return anglicize_json_type(validator) 361 362 if len(validator) == 1: 363 return anglicize_json_type(validator[0]) 364 365 return "{}, or {}".format( 366 ", ".join([anglicize_json_type(validator[0])] + validator[1:-1]), 367 anglicize_json_type(validator[-1])) 368 369 370def _parse_oneof_validator(error): 371 """oneOf has multiple schemas, so we need to reason about which schema, sub 372 schema or constraint the validation is failing on. 373 Inspecting the context value of a ValidationError gives us information about 374 which sub schema failed and which kind of error it is. 375 """ 376 types = [] 377 for context in error.context: 378 if context.validator == 'oneOf': 379 _, error_msg = _parse_oneof_validator(context) 380 return path_string(context.path), error_msg 381 382 if context.validator == 'required': 383 return (None, context.message) 384 385 if context.validator == 'additionalProperties': 386 invalid_config_key = parse_key_from_error_msg(context) 387 return (None, "contains unsupported option: '{}'".format(invalid_config_key)) 388 389 if context.validator == 'uniqueItems': 390 return ( 391 path_string(context.path) if context.path else None, 392 "contains non-unique items, please remove duplicates from {}".format( 393 context.instance), 394 ) 395 396 if context.path: 397 return ( 398 path_string(context.path), 399 "contains {}, which is an invalid type, it should be {}".format( 400 json.dumps(context.instance), 401 _parse_valid_types_from_validator(context.validator_value)), 402 ) 403 404 if context.validator == 'type': 405 types.append(context.validator_value) 406 407 valid_types = _parse_valid_types_from_validator(types) 408 return (None, "contains an invalid type, it should be {}".format(valid_types)) 409 410 411def process_service_constraint_errors(error, service_name, version): 412 if version == V1: 413 if 'image' in error.instance and 'build' in error.instance: 414 return ( 415 "Service {} has both an image and build path specified. " 416 "A service can either be built to image or use an existing " 417 "image, not both.".format(service_name)) 418 419 if 'image' in error.instance and 'dockerfile' in error.instance: 420 return ( 421 "Service {} has both an image and alternate Dockerfile. " 422 "A service can either be built to image or use an existing " 423 "image, not both.".format(service_name)) 424 425 if 'image' not in error.instance and 'build' not in error.instance: 426 return ( 427 "Service {} has neither an image nor a build context specified. " 428 "At least one must be provided.".format(service_name)) 429 430 431def process_config_schema_errors(error): 432 path = list(error.path) 433 434 if 'id' in error.schema: 435 error_msg = handle_error_for_schema_with_id(error, path) 436 if error_msg: 437 return error_msg 438 439 return handle_generic_error(error, path) 440 441 442def validate_against_config_schema(config_file): 443 schema = load_jsonschema(config_file) 444 format_checker = FormatChecker(["ports", "expose", "subnet_ip_address"]) 445 validator = Draft4Validator( 446 schema, 447 resolver=RefResolver(get_resolver_path(), schema), 448 format_checker=format_checker) 449 handle_errors( 450 validator.iter_errors(config_file.config), 451 process_config_schema_errors, 452 config_file.filename) 453 454 455def validate_service_constraints(config, service_name, config_file): 456 def handler(errors): 457 return process_service_constraint_errors( 458 errors, service_name, config_file.version) 459 460 schema = load_jsonschema(config_file) 461 validator = Draft4Validator(schema['definitions']['constraints']['service']) 462 handle_errors(validator.iter_errors(config), handler, None) 463 464 465def validate_cpu(service_config): 466 cpus = service_config.config.get('cpus') 467 if not cpus: 468 return 469 nano_cpus = cpus * NANOCPUS_SCALE 470 if isinstance(nano_cpus, float) and not nano_cpus.is_integer(): 471 raise ConfigurationError( 472 "cpus must have nine or less digits after decimal point") 473 474 475def get_schema_path(): 476 return os.path.dirname(os.path.abspath(__file__)) 477 478 479def load_jsonschema(config_file): 480 filename = os.path.join( 481 get_schema_path(), 482 "config_schema_v{0}.json".format(config_file.version)) 483 484 if not os.path.exists(filename): 485 raise ConfigurationError( 486 'Version in "{}" is unsupported. {}' 487 .format(config_file.filename, VERSION_EXPLANATION)) 488 489 with open(filename, "r") as fh: 490 return json.load(fh) 491 492 493def get_resolver_path(): 494 schema_path = get_schema_path() 495 if sys.platform == "win32": 496 scheme = "///" 497 # TODO: why is this necessary? 498 schema_path = schema_path.replace('\\', '/') 499 else: 500 scheme = "//" 501 return "file:{}{}/".format(scheme, schema_path) 502 503 504def handle_errors(errors, format_error_func, filename): 505 """jsonschema returns an error tree full of information to explain what has 506 gone wrong. Process each error and pull out relevant information and re-write 507 helpful error messages that are relevant. 508 """ 509 errors = list(sorted(errors, key=str)) 510 if not errors: 511 return 512 513 error_msg = '\n'.join(format_error_func(error) for error in errors) 514 raise ConfigurationError( 515 "The Compose file{file_msg} is invalid because:\n{error_msg}".format( 516 file_msg=" '{}'".format(filename) if filename else "", 517 error_msg=error_msg)) 518 519 520def validate_healthcheck(service_config): 521 healthcheck = service_config.config.get('healthcheck', {}) 522 523 if 'test' in healthcheck and isinstance(healthcheck['test'], list): 524 if len(healthcheck['test']) == 0: 525 raise ConfigurationError( 526 'Service "{}" defines an invalid healthcheck: ' 527 '"test" is an empty list' 528 .format(service_config.name)) 529 530 # when disable is true config.py::process_healthcheck adds "test: ['NONE']" to service_config 531 elif healthcheck['test'][0] == 'NONE' and len(healthcheck) > 1: 532 raise ConfigurationError( 533 'Service "{}" defines an invalid healthcheck: ' 534 '"disable: true" cannot be combined with other options' 535 .format(service_config.name)) 536 537 elif healthcheck['test'][0] not in ('NONE', 'CMD', 'CMD-SHELL'): 538 raise ConfigurationError( 539 'Service "{}" defines an invalid healthcheck: ' 540 'when "test" is a list the first item must be either NONE, CMD or CMD-SHELL' 541 .format(service_config.name)) 542