1# Copyright 2007 Google LLC. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""AppInfo tools. 16 17This library allows you to work with AppInfo records in memory, as well as store 18and load from configuration files. 19""" 20 21 22# WARNING: This file is externally viewable by our users. All comments from 23# this file will be stripped. The docstrings will NOT. Do not put sensitive 24# information in docstrings. If you must communicate internal information in 25# this source file, please place them in comments only. 26 27# Parts of the code in this file are duplicated in 28# //java/com/google/apphosting/admin/legacy/... 29# This is part of an ongoing effort to replace the deployment API. 30# Until we can delete this code, please check to see if your changes need 31# to be reflected in the java code. For questions, talk to clouser@ or 32 33from __future__ import absolute_import 34from __future__ import division 35from __future__ import print_function 36import logging 37import os 38import re 39import string 40import sys 41import wsgiref.util 42 43# pylint: disable=g-import-not-at-top 44if os.environ.get('APPENGINE_RUNTIME') == 'python27': 45 from google.appengine.api import validation 46 from google.appengine.api import yaml_builder 47 from google.appengine.api import yaml_listener 48 from google.appengine.api import yaml_object 49else: 50 # This case covers both Python 2.5 and unittests, which are 2.5 only. 51 from googlecloudsdk.third_party.appengine.api import validation 52 from googlecloudsdk.third_party.appengine.api import yaml_builder 53 from googlecloudsdk.third_party.appengine.api import yaml_listener 54 from googlecloudsdk.third_party.appengine.api import yaml_object 55 56from googlecloudsdk.third_party.appengine.api import appinfo_errors 57from googlecloudsdk.third_party.appengine.api import backendinfo 58from googlecloudsdk.third_party.appengine._internal import six_subset 59 60 61# pylint: enable=g-import-not-at-top 62 63# Regular expression for matching URL, file, URL root regular expressions. 64# `url_root` is identical to url except it additionally imposes not ending with 65# *. 66# TODO(user): `url_root` should generally allow a URL but not a regex or 67# glob. 68_URL_REGEX = r'(?!\^)/.*|\..*|(\(.).*(?!\$).' 69_FILES_REGEX = r'.+' 70_URL_ROOT_REGEX = r'/.*' 71 72# Regular expression for matching cache expiration deltas. 73_DELTA_REGEX = r'([0-9]+)([DdHhMm]|[sS]?)' 74_EXPIRATION_REGEX = r'\s*(%s)(\s+%s)*\s*' % (_DELTA_REGEX, _DELTA_REGEX) 75_START_PATH = '/_ah/start' 76 77_NON_WHITE_SPACE_REGEX = r'^\S+$' 78 79# Regular expression for matching service names. 80# TODO(user): this may need altering so as to not leak unreleased service names 81# TODO(user): Re-add sms to list of services. 82_ALLOWED_SERVICES = ['mail', 'mail_bounce', 'xmpp_message', 'xmpp_subscribe', 83 'xmpp_presence', 'xmpp_error', 'channel_presence', 'rest', 84 'warmup'] 85_SERVICE_RE_STRING = '(' + '|'.join(_ALLOWED_SERVICES) + ')' 86 87# Regular expression for matching page names. 88_PAGE_NAME_REGEX = r'^.+$' 89 90# Constants for interpreting expiration deltas. 91_EXPIRATION_CONVERSIONS = { 92 'd': 60 * 60 * 24, 93 'h': 60 * 60, 94 'm': 60, 95 's': 1, 96} 97 98# Constant values from `apphosting/base/constants.h` 99# TODO(user): Maybe a python constants file. 100APP_ID_MAX_LEN = 100 101MODULE_ID_MAX_LEN = 63 102# See b/5485871 for why this is 100 and not 63. 103# NOTE(user): See b/5485871 for why this is different from the 104# `apphosting/base/constants.h` value. 105MODULE_VERSION_ID_MAX_LEN = 63 106MAX_URL_MAPS = 100 107 108# The character separating the partition from the domain. 109PARTITION_SEPARATOR = '~' 110 111# The character separating the domain from the display-app-id. 112DOMAIN_SEPARATOR = ':' 113 114# The character separating major and minor versions. 115VERSION_SEPARATOR = '.' 116 117# The character separating module from module version. 118MODULE_SEPARATOR = ':' 119 120# The name of the default module 121DEFAULT_MODULE = 'default' 122 123# Regular expression for ID types. Defined in apphosting/base/id_util.cc. 124PARTITION_RE_STRING_WITHOUT_SEPARATOR = (r'[a-z\d\-]{1,%d}' % APP_ID_MAX_LEN) 125PARTITION_RE_STRING = (r'%s\%s' % 126 (PARTITION_RE_STRING_WITHOUT_SEPARATOR, 127 PARTITION_SEPARATOR)) 128DOMAIN_RE_STRING_WITHOUT_SEPARATOR = (r'(?!\-)[a-z\d\-\.]{1,%d}' % 129 APP_ID_MAX_LEN) 130DOMAIN_RE_STRING = (r'%s%s' % 131 (DOMAIN_RE_STRING_WITHOUT_SEPARATOR, DOMAIN_SEPARATOR)) 132DISPLAY_APP_ID_RE_STRING = r'(?!-)[a-z\d\-]{0,%d}[a-z\d]' % (APP_ID_MAX_LEN - 1) 133APPLICATION_RE_STRING = (r'(?:%s)?(?:%s)?%s' % 134 (PARTITION_RE_STRING, 135 DOMAIN_RE_STRING, 136 DISPLAY_APP_ID_RE_STRING)) 137 138# NOTE(user,user): These regexes have been copied to multiple other 139# locations in google.apphosting so we don't have to pull this file into 140# python_lib for other modules to work in production. 141# Other known locations as of 2016-08-15: 142# - java/com/google/apphosting/admin/legacy/LegacyAppInfo.java 143# - apphosting/client/app_config_old.cc 144# - apphosting/api/app_config/app_config_server2.cc 145MODULE_ID_RE_STRING = r'^(?!-)[a-z\d\-]{0,%d}[a-z\d]$' % (MODULE_ID_MAX_LEN - 1) 146MODULE_VERSION_ID_RE_STRING = (r'^(?!-)[a-z\d\-]{0,%d}[a-z\d]$' % 147 (MODULE_VERSION_ID_MAX_LEN - 1)) 148 149_IDLE_INSTANCES_REGEX = r'^([\d]+|automatic)$' 150# Note that this regex will not allow zero-prefixed numbers, e.g. 0001. 151_INSTANCES_REGEX = r'^[1-9][\d]*$' 152 153_CONCURRENT_REQUESTS_REGEX = r'^([1-9]\d*)$' 154 155# This enforces that we will only accept a single decimal point of accuracy at 156# the granularity of seconds and no decimal point with a granularity of 157# milliseconds. 158_PENDING_LATENCY_REGEX = r'^(\d+((\.\d{1,3})?s|ms)|automatic)$' 159 160_IDLE_TIMEOUT_REGEX = r'^[\d]+(s|m)$' 161 162GCE_RESOURCE_PATH_REGEX = r'^[a-z\d-]+(/[a-z\d-]+)*$' 163 164GCE_RESOURCE_NAME_REGEX = r'^[a-z]([a-z\d-]{0,61}[a-z\d])?$' 165 166VPC_ACCESS_CONNECTOR_NAME_REGEX = r'^[a-z\d-]+(/.+)*$' 167 168ALTERNATE_HOSTNAME_SEPARATOR = '-dot-' 169 170# Note(user): This must match api/app_config.py 171BUILTIN_NAME_PREFIX = 'ah-builtin' 172 173# Here we expect either normal runtimes (such as 'nodejs' or 'java') or 174# pinned runtime builders, which take the form of the path to a cloudbuild.yaml 175# manifest file in GCS (written as gs://bucket/path/to/build.yaml). 176RUNTIME_RE_STRING = r'((gs://[a-z0-9\-\._/]+)|([a-z][a-z0-9\-\.]{0,29}))' 177 178API_VERSION_RE_STRING = r'[\w.]{1,32}' 179ENV_RE_STRING = r'(1|2|standard|flex|flexible)' 180 181SOURCE_LANGUAGE_RE_STRING = r'[\w.\-]{1,32}' 182 183HANDLER_STATIC_FILES = 'static_files' 184HANDLER_STATIC_DIR = 'static_dir' 185HANDLER_SCRIPT = 'script' 186HANDLER_API_ENDPOINT = 'api_endpoint' 187 188LOGIN_OPTIONAL = 'optional' 189LOGIN_REQUIRED = 'required' 190LOGIN_ADMIN = 'admin' 191 192AUTH_FAIL_ACTION_REDIRECT = 'redirect' 193AUTH_FAIL_ACTION_UNAUTHORIZED = 'unauthorized' 194 195DATASTORE_ID_POLICY_LEGACY = 'legacy' 196DATASTORE_ID_POLICY_DEFAULT = 'default' 197 198SECURE_HTTP = 'never' 199SECURE_HTTPS = 'always' 200SECURE_HTTP_OR_HTTPS = 'optional' 201# Used for missing values; see http://b/issue?id=2073962. 202SECURE_DEFAULT = 'default' 203 204REQUIRE_MATCHING_FILE = 'require_matching_file' 205 206DEFAULT_SKIP_FILES = (r'^(.*/)?(' 207 r'(#.*#)|' 208 r'(.*~)|' 209 r'(.*\.py[co])|' 210 r'(.*/RCS/.*)|' 211 r'(\..*)|' 212 r')$') 213# Expression meaning to skip no files, which is the default for AppInclude. 214SKIP_NO_FILES = r'(?!)' 215 216DEFAULT_NOBUILD_FILES = (r'^$') 217 218# Attributes for `URLMap` 219LOGIN = 'login' 220AUTH_FAIL_ACTION = 'auth_fail_action' 221SECURE = 'secure' 222URL = 'url' 223POSITION = 'position' 224POSITION_HEAD = 'head' 225POSITION_TAIL = 'tail' 226STATIC_FILES = 'static_files' 227UPLOAD = 'upload' 228STATIC_DIR = 'static_dir' 229MIME_TYPE = 'mime_type' 230SCRIPT = 'script' 231EXPIRATION = 'expiration' 232API_ENDPOINT = 'api_endpoint' 233HTTP_HEADERS = 'http_headers' 234APPLICATION_READABLE = 'application_readable' 235REDIRECT_HTTP_RESPONSE_CODE = 'redirect_http_response_code' 236 237# Attributes for `AppInfoExternal` 238APPLICATION = 'application' 239PROJECT = 'project' # An alias for 'application' 240MODULE = 'module' 241SERVICE = 'service' 242AUTOMATIC_SCALING = 'automatic_scaling' 243MANUAL_SCALING = 'manual_scaling' 244BASIC_SCALING = 'basic_scaling' 245VM = 'vm' 246VM_SETTINGS = 'vm_settings' 247ZONES = 'zones' 248BETA_SETTINGS = 'beta_settings' 249VM_HEALTH_CHECK = 'vm_health_check' 250HEALTH_CHECK = 'health_check' 251RESOURCES = 'resources' 252LIVENESS_CHECK = 'liveness_check' 253READINESS_CHECK = 'readiness_check' 254NETWORK = 'network' 255VPC_ACCESS_CONNECTOR = 'vpc_access_connector' 256VERSION = 'version' 257MAJOR_VERSION = 'major_version' 258MINOR_VERSION = 'minor_version' 259RUNTIME = 'runtime' 260RUNTIME_CHANNEL = 'runtime_channel' 261API_VERSION = 'api_version' 262MAIN = 'main' 263ENDPOINTS_API_SERVICE = 'endpoints_api_service' 264ENV = 'env' 265ENTRYPOINT = 'entrypoint' 266RUNTIME_CONFIG = 'runtime_config' 267SOURCE_LANGUAGE = 'source_language' 268BUILTINS = 'builtins' 269INCLUDES = 'includes' 270HANDLERS = 'handlers' 271LIBRARIES = 'libraries' 272DEFAULT_EXPIRATION = 'default_expiration' 273SKIP_FILES = 'skip_files' 274NOBUILD_FILES = 'nobuild_files' 275SERVICES = 'inbound_services' 276DERIVED_FILE_TYPE = 'derived_file_type' 277JAVA_PRECOMPILED = 'java_precompiled' 278PYTHON_PRECOMPILED = 'python_precompiled' 279ADMIN_CONSOLE = 'admin_console' 280ERROR_HANDLERS = 'error_handlers' 281BACKENDS = 'backends' 282THREADSAFE = 'threadsafe' 283DATASTORE_AUTO_ID_POLICY = 'auto_id_policy' 284API_CONFIG = 'api_config' 285CODE_LOCK = 'code_lock' 286ENV_VARIABLES = 'env_variables' 287STANDARD_WEBSOCKET = 'standard_websocket' 288APP_ENGINE_APIS = 'app_engine_apis' 289 290SOURCE_REPO_RE_STRING = r'^[a-z][a-z0-9\-\+\.]*:[^#]*$' 291SOURCE_REVISION_RE_STRING = r'^[0-9a-fA-F]+$' 292 293# Maximum size of all source references (in bytes) for a deployment. 294SOURCE_REFERENCES_MAX_SIZE = 2048 295 296INSTANCE_CLASS = 'instance_class' 297 298# Attributes for Standard App Engine (only) AutomaticScaling. 299MINIMUM_PENDING_LATENCY = 'min_pending_latency' 300MAXIMUM_PENDING_LATENCY = 'max_pending_latency' 301MINIMUM_IDLE_INSTANCES = 'min_idle_instances' 302MAXIMUM_IDLE_INSTANCES = 'max_idle_instances' 303MAXIMUM_CONCURRENT_REQUEST = 'max_concurrent_requests' 304 305# Attributes for Managed VMs (only) AutomaticScaling. These are very 306# different than Standard App Engine because scaling settings are 307# mapped to Cloud Autoscaler (as opposed to the clone scheduler). See 308MIN_NUM_INSTANCES = 'min_num_instances' 309MAX_NUM_INSTANCES = 'max_num_instances' 310COOL_DOWN_PERIOD_SEC = 'cool_down_period_sec' 311CPU_UTILIZATION = 'cpu_utilization' 312CPU_UTILIZATION_UTILIZATION = 'target_utilization' 313CPU_UTILIZATION_AGGREGATION_WINDOW_LENGTH_SEC = 'aggregation_window_length_sec' 314# Managed VMs Richer Autoscaling. These (MVMs only) scaling settings 315# are supported for both vm:true and env:2|flex, but are not yet 316# publicly documented. 317TARGET_NETWORK_SENT_BYTES_PER_SEC = 'target_network_sent_bytes_per_sec' 318TARGET_NETWORK_SENT_PACKETS_PER_SEC = 'target_network_sent_packets_per_sec' 319TARGET_NETWORK_RECEIVED_BYTES_PER_SEC = 'target_network_received_bytes_per_sec' 320TARGET_NETWORK_RECEIVED_PACKETS_PER_SEC = ( 321 'target_network_received_packets_per_sec') 322TARGET_DISK_WRITE_BYTES_PER_SEC = 'target_disk_write_bytes_per_sec' 323TARGET_DISK_WRITE_OPS_PER_SEC = 'target_disk_write_ops_per_sec' 324TARGET_DISK_READ_BYTES_PER_SEC = 'target_disk_read_bytes_per_sec' 325TARGET_DISK_READ_OPS_PER_SEC = 'target_disk_read_ops_per_sec' 326TARGET_REQUEST_COUNT_PER_SEC = 'target_request_count_per_sec' 327TARGET_CONCURRENT_REQUESTS = 'target_concurrent_requests' 328# Custom Metric autoscaling. These are supported for Flex only. 329CUSTOM_METRICS = 'custom_metrics' 330METRIC_NAME = 'metric_name' 331TARGET_TYPE = 'target_type' 332TARGET_TYPE_REGEX = r'^(GAUGE|DELTA_PER_SECOND|DELTA_PER_MINUTE)$' 333CUSTOM_METRIC_UTILIZATION = 'target_utilization' 334SINGLE_INSTANCE_ASSIGNMENT = 'single_instance_assignment' 335FILTER = 'filter' 336 337 338# Attributes for ManualScaling 339INSTANCES = 'instances' 340 341# Attributes for BasicScaling 342MAX_INSTANCES = 'max_instances' 343IDLE_TIMEOUT = 'idle_timeout' 344 345# Attributes for AdminConsole 346PAGES = 'pages' 347NAME = 'name' 348 349# Attributes for EndpointsApiService 350ENDPOINTS_NAME = 'name' 351CONFIG_ID = 'config_id' 352ROLLOUT_STRATEGY = 'rollout_strategy' 353ROLLOUT_STRATEGY_FIXED = 'fixed' 354ROLLOUT_STRATEGY_MANAGED = 'managed' 355TRACE_SAMPLING = 'trace_sampling' 356 357# Attributes for ErrorHandlers 358ERROR_CODE = 'error_code' 359FILE = 'file' 360_ERROR_CODE_REGEX = r'(default|over_quota|dos_api_denial|timeout)' 361 362# Attributes for BuiltinHandler 363ON = 'on' 364ON_ALIASES = ['yes', 'y', 'True', 't', '1', 'true'] 365OFF = 'off' 366OFF_ALIASES = ['no', 'n', 'False', 'f', '0', 'false'] 367 368# Attributes for `VmHealthCheck`. Please refer to message `VmHealthCheck` in 369# `request_path` and `port` are not configurable yet. 370ENABLE_HEALTH_CHECK = 'enable_health_check' 371CHECK_INTERVAL_SEC = 'check_interval_sec' 372TIMEOUT_SEC = 'timeout_sec' 373APP_START_TIMEOUT_SEC = 'app_start_timeout_sec' 374UNHEALTHY_THRESHOLD = 'unhealthy_threshold' 375HEALTHY_THRESHOLD = 'healthy_threshold' 376FAILURE_THRESHOLD = 'failure_threshold' 377SUCCESS_THRESHOLD = 'success_threshold' 378RESTART_THRESHOLD = 'restart_threshold' 379INITIAL_DELAY_SEC = 'initial_delay_sec' 380HOST = 'host' 381PATH = 'path' 382 383# Attributes for Resources. 384CPU = 'cpu' 385MEMORY_GB = 'memory_gb' 386DISK_SIZE_GB = 'disk_size_gb' 387 388# Attributes for Resources:Volumes. 389VOLUMES = 'volumes' 390VOLUME_NAME = 'name' 391VOLUME_TYPE = 'volume_type' 392SIZE_GB = 'size_gb' 393 394# Attributes for Network. 395FORWARDED_PORTS = 'forwarded_ports' 396INSTANCE_TAG = 'instance_tag' 397NETWORK_NAME = 'name' 398SUBNETWORK_NAME = 'subnetwork_name' 399SESSION_AFFINITY = 'session_affinity' 400 401# Attributes for Scheduler Settings 402STANDARD_MIN_INSTANCES = 'min_instances' 403STANDARD_MAX_INSTANCES = 'max_instances' 404STANDARD_TARGET_CPU_UTILIZATION = 'target_cpu_utilization' 405STANDARD_TARGET_THROUGHPUT_UTILIZATION = 'target_throughput_utilization' 406 407# Attributes for `VpcAccessConnector`. 408VPC_ACCESS_CONNECTOR_NAME = 'name' 409 410 411class _VersionedLibrary(object): 412 """A versioned library supported by App Engine.""" 413 414 def __init__(self, 415 name, 416 url, 417 description, 418 supported_versions, 419 latest_version, 420 default_version=None, 421 deprecated_versions=None, 422 experimental_versions=None, 423 hidden_versions=None): 424 """Initializer for `_VersionedLibrary`. 425 426 Args: 427 name: The name of the library; for example, `django`. 428 url: The URL for the library's project page; for example, 429 `http://www.djangoproject.com/`. 430 description: A short description of the library; for example, 431 `A framework...`. 432 supported_versions: A list of supported version names, ordered by release 433 date; for example, `["v1", "v2", "v3"]`. 434 latest_version: The version of the library that will be used when you 435 specify `latest.` The rule of thumb is that this value should be the 436 newest version that is neither deprecated nor experimental; however 437 this value might be an experimental version if all of the supported 438 versions are either deprecated or experimental. 439 default_version: The version of the library that is enabled by default 440 in the Python 2.7 runtime, or `None` if the library is not available 441 by default; for example, `v1`. 442 deprecated_versions: A list of the versions of the library that have been 443 deprecated; for example, `["v1", "v2"]`. Order by release version. 444 experimental_versions: A list of the versions of the library that are 445 currently experimental; for example, `["v1"]`. Order by release 446 version. 447 hidden_versions: A list of versions that will not show up in public 448 documentation for release purposes. If, as a result, the library 449 has no publicly documented versions, the entire library won't show 450 up in the docs. Order by release version. 451 """ 452 self.name = name 453 self.url = url 454 self.description = description 455 self.supported_versions = supported_versions 456 self.latest_version = latest_version 457 self.default_version = default_version 458 self.deprecated_versions = deprecated_versions or [] 459 self.experimental_versions = experimental_versions or [] 460 self.hidden_versions = hidden_versions or [] 461 462 @property 463 def hidden(self): 464 """Determines if the entire library should be hidden from public docs. 465 466 Returns: 467 True if there is every supported version is hidden. 468 """ 469 return sorted(self.supported_versions) == sorted(self.hidden_versions) 470 471 @property 472 def non_deprecated_versions(self): 473 """Retrieves the versions of the library that are not deprecated. 474 475 Returns: 476 A list of the versions of the library that are not deprecated. 477 """ 478 return [version for version in self.supported_versions 479 if version not in self.deprecated_versions] 480 481 482_SUPPORTED_LIBRARIES = [ 483 _VersionedLibrary( 484 'clearsilver', 485 'http://www.clearsilver.net/', 486 'A fast, powerful, and language-neutral HTML template system.', 487 ['0.10.5'], 488 latest_version='0.10.5', 489 hidden_versions=['0.10.5'], 490 ), 491 _VersionedLibrary( 492 'click', 493 'http://click.pocoo.org/', 494 'A command line library for Python.', 495 ['6.6'], 496 latest_version='6.6', 497 hidden_versions=['6.6'], 498 ), 499 _VersionedLibrary( 500 'django', 501 'http://www.djangoproject.com/', 502 'A full-featured web application framework for Python.', 503 ['1.2', '1.3', '1.4', '1.5', '1.9', '1.11'], 504 latest_version='1.4', 505 deprecated_versions=['1.2', '1.3', '1.5', '1.9'], 506 # TODO(b/78247136) Deprecate 1.4 and update latest_version to 1.11 507 ), 508 _VersionedLibrary( 509 'enum', 510 'https://pypi.python.org/pypi/enum34', 511 'A backport of the enum module introduced in python 3.4', 512 ['0.9.23'], 513 latest_version='0.9.23', 514 ), 515 _VersionedLibrary( 516 'endpoints', 517 'https://cloud.google.com/appengine/docs/standard/python/endpoints/', 518 'Libraries for building APIs in an App Engine application.', 519 ['1.0'], 520 latest_version='1.0', 521 ), 522 _VersionedLibrary( 523 'flask', 524 'http://flask.pocoo.org/', 525 'Flask is a microframework for Python based on Werkzeug, Jinja 2 ' 526 'and good intentions.', 527 ['0.12'], 528 latest_version='0.12', 529 ), 530 _VersionedLibrary( 531 'futures', 532 'https://docs.python.org/3/library/concurrent.futures.html', 533 'Backport of Python 3.2 Futures.', 534 ['3.0.5'], 535 latest_version='3.0.5', 536 ), 537 _VersionedLibrary( 538 'grpcio', 539 'http://www.grpc.io/', 540 'A high performance general RPC framework', 541 # Note: For documentation this is overridden to display 1.1.0dev0 542 ['1.0.0'], 543 latest_version='1.0.0', 544 experimental_versions=['1.0.0'], 545 ), 546 _VersionedLibrary( 547 'itsdangerous', 548 'http://pythonhosted.org/itsdangerous/', 549 'HMAC and SHA1 signing for Python.', 550 ['0.24'], 551 latest_version='0.24', 552 hidden_versions=['0.24'], 553 ), 554 _VersionedLibrary( 555 'jinja2', 556 'http://jinja.pocoo.org/docs/', 557 'A modern and designer friendly templating language for Python.', 558 ['2.6'], 559 latest_version='2.6', 560 ), 561 _VersionedLibrary( 562 'lxml', 563 'http://lxml.de/', 564 'A Pythonic binding for the C libraries libxml2 and libxslt.', 565 ['2.3', '2.3.5', '3.7.3'], 566 latest_version='3.7.3', 567 deprecated_versions=['2.3', '2.3.5'], 568 ), 569 _VersionedLibrary( 570 'markupsafe', 571 'http://pypi.python.org/pypi/MarkupSafe', 572 'A XML/HTML/XHTML markup safe string for Python.', 573 ['0.15', '0.23'], 574 latest_version='0.15', 575 ), 576 _VersionedLibrary( 577 'matplotlib', 578 'http://matplotlib.org/', 579 'A 2D plotting library which produces publication-quality figures.', 580 ['1.2.0'], 581 latest_version='1.2.0', 582 ), 583 _VersionedLibrary( 584 'MySQLdb', 585 'http://mysql-python.sourceforge.net/', 586 'A Python DB API v2.0 compatible interface to MySQL.', 587 ['1.2.4b4', '1.2.4', '1.2.5'], 588 latest_version='1.2.5', 589 deprecated_versions=['1.2.4b4', '1.2.4'], 590 ), 591 _VersionedLibrary( 592 'mysqlclient', 593 'http://mysql-python.sourceforge.net/', 594 'A Python DB API v2.0 compatible interface to MySQL.', 595 ['1.4.4'], 596 latest_version='1.4.4', 597 ), 598 _VersionedLibrary( 599 'numpy', 600 'http://numpy.scipy.org/', 601 'A general-purpose library for array-processing.', 602 ['1.6.1'], 603 latest_version='1.6.1', 604 ), 605 _VersionedLibrary( 606 'PIL', 607 'http://www.pythonware.com/library/pil/handbook/', 608 'A library for creating and transforming images.', 609 ['1.1.7'], 610 latest_version='1.1.7', 611 ), 612 _VersionedLibrary( 613 'protorpc', 614 'https://github.com/google/protorpc', 615 'A framework for implementing HTTP-based remote procedure call (RPC) ' 616 'services.', 617 ['1.0'], 618 latest_version='1.0', 619 default_version='1.0', 620 ), 621 _VersionedLibrary( 622 'pytz', 623 'https://pypi.python.org/pypi/pytz?', 624 'A library for cross-platform timezone calculations', 625 ['2016.4', '2017.2', '2017.3'], 626 latest_version='2017.3', 627 default_version='2017.3', 628 deprecated_versions=['2016.4', '2017.2'], 629 ), 630 _VersionedLibrary( 631 'crcmod', 632 'http://crcmod.sourceforge.net/', 633 'A library for generating Cyclic Redundancy Checks (CRC).', 634 ['1.7'], 635 latest_version='1.7', 636 ), 637 _VersionedLibrary( 638 'protobuf', 639 'https://developers.google.com/protocol-buffers/', 640 'A library for serializing structured data', 641 ['3.0.0'], 642 latest_version='3.0.0', 643 experimental_versions=['3.0.0'], 644 ), 645 _VersionedLibrary( 646 'psycopg2', 647 'http://initd.org/psycopg/', 648 'A Python DB API v2.0 compatible interface to PostgreSQL.', 649 ['2.8.3'], 650 latest_version='2.8.3', 651 ), 652 _VersionedLibrary( 653 'PyAMF', 654 'https://pypi.python.org/pypi/PyAMF', 655 'A library that provides (AMF) Action Message Format functionality.', 656 ['0.6.1', '0.7.2'], 657 latest_version='0.6.1', 658 experimental_versions=['0.7.2'], 659 ), 660 _VersionedLibrary( 661 'pycrypto', 662 'https://www.dlitz.net/software/pycrypto/', 663 'A library of cryptography functions such as random number generation.', 664 ['2.3', '2.6', '2.6.1'], 665 latest_version='2.6', 666 deprecated_versions=['2.3'], 667 # TODO(b/78247136) Deprecate 2.6 and update latest_version to 2.6.1 668 ), 669 _VersionedLibrary( 670 'setuptools', 671 'http://pypi.python.org/pypi/setuptools', 672 'A library that provides package and module discovery capabilities.', 673 ['0.6c11', '36.6.0'], 674 latest_version='36.6.0', 675 deprecated_versions=['0.6c11'], 676 ), 677 _VersionedLibrary( 678 'six', 679 'https://pypi.python.org/pypi/six', 680 'Abstract differences between py2.x and py3', 681 ['1.9.0', '1.12.0'], 682 latest_version='1.12.0', 683 default_version='1.12.0', 684 ), 685 _VersionedLibrary( 686 'ssl', 687 'http://docs.python.org/dev/library/ssl.html', 688 'The SSL socket wrapper built-in module.', 689 ['2.7', '2.7.11', '2.7.16', '2.7.current'], 690 latest_version='2.7.11', 691 deprecated_versions=['2.7', '2.7.16'] 692 ), 693 _VersionedLibrary( 694 'ujson', 695 'https://pypi.python.org/pypi/ujson', 696 'UltraJSON is an ultra fast JSON encoder and decoder written in pure C', 697 ['1.35'], 698 latest_version='1.35', 699 ), 700 _VersionedLibrary( 701 'webapp2', 702 'http://webapp-improved.appspot.com/', 703 'A lightweight Python web framework.', 704 ['2.3', '2.5.1', '2.5.2'], 705 latest_version='2.5.2', 706 # Keep default version at 2.3 because apps in production depend on it. 707 default_version='2.3', 708 deprecated_versions=['2.5.1'] 709 ), 710 _VersionedLibrary( 711 'webob', 712 'http://www.webob.org/', 713 'A library that provides wrappers around the WSGI request environment.', 714 ['1.1.1', '1.2.3'], 715 latest_version='1.2.3', 716 # Keep default version at 1.1.1 because apps in production depend on it. 717 default_version='1.1.1', 718 ), 719 _VersionedLibrary( 720 'werkzeug', 721 'http://www.werkzeug.pocoo.org/', 722 'A WSGI utility library.', 723 ['0.11.10'], 724 latest_version='0.11.10', 725 default_version='0.11.10', 726 ), 727 _VersionedLibrary( 728 'yaml', 729 'http://www.yaml.org/', 730 'A library for YAML serialization and deserialization.', 731 ['3.10'], 732 latest_version='3.10', 733 default_version='3.10' 734 ), 735 ] 736 737_NAME_TO_SUPPORTED_LIBRARY = dict((library.name, library) 738 for library in _SUPPORTED_LIBRARIES) 739 740# A mapping from third-party name/version to a list of that library's 741# dependencies. 742REQUIRED_LIBRARIES = { 743 ('django', '1.11'): [('pytz', '2017.2')], 744 ('flask', '0.12'): [('click', '6.6'), ('itsdangerous', '0.24'), 745 ('jinja2', '2.6'), ('werkzeug', '0.11.10')], 746 ('jinja2', '2.6'): [('markupsafe', '0.15'), ('setuptools', '0.6c11')], 747 ('jinja2', 'latest'): [('markupsafe', 'latest'), ('setuptools', 'latest')], 748 ('matplotlib', '1.2.0'): [('numpy', '1.6.1')], 749 ('matplotlib', 'latest'): [('numpy', 'latest')], 750 ('protobuf', '3.0.0'): [('six', 'latest')], 751 ('protobuf', 'latest'): [('six', 'latest')], 752 ('grpcio', '1.0.0'): [('protobuf', '3.0.0'), ('enum', '0.9.23'), 753 ('futures', '3.0.5'), ('six', 'latest'), 754 ('setuptools', '36.6.0')], 755 ('grpcio', 'latest'): [('protobuf', 'latest'), ('enum', 'latest'), 756 ('futures', 'latest'), ('six', 'latest'), 757 ('setuptools', 'latest')] 758} 759 760_USE_VERSION_FORMAT = ('use one of: "%s"') 761 762 763# See RFC 2616 section 2.2. 764_HTTP_SEPARATOR_CHARS = frozenset('()<>@,;:\\"/[]?={} \t') 765_HTTP_TOKEN_CHARS = frozenset(string.printable[:-5]) - _HTTP_SEPARATOR_CHARS 766_HTTP_TOKEN_RE = re.compile('[%s]+$' % re.escape(''.join(_HTTP_TOKEN_CHARS))) 767 768# Source: http://www.cs.tut.fi/~jkorpela/http.html 769_HTTP_REQUEST_HEADERS = frozenset([ 770 'accept', 771 'accept-charset', 772 'accept-encoding', 773 'accept-language', 774 'authorization', 775 'expect', 776 'from', 777 'host', 778 'if-match', 779 'if-modified-since', 780 'if-none-match', 781 'if-range', 782 'if-unmodified-since', 783 'max-forwards', 784 'proxy-authorization', 785 'range', 786 'referer', 787 'te', 788 'user-agent', 789]) 790 791# The minimum cookie length (i.e. number of bytes) that HTTP clients should 792# support, per RFCs 2109 and 2965. 793_MAX_COOKIE_LENGTH = 4096 794 795# trailing NULL character, which is why this is not 2048. 796_MAX_URL_LENGTH = 2047 797 798# We allow certain headers to be larger than the normal limit of 8192 bytes. 799_MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS = 10240 800 801_CANNED_RUNTIMES = ('contrib-dart', 'dart', 'go', 'php', 'php55', 'php72', 802 'python', 'python27', 'python-compat', 'java', 'java7', 803 'java8', 'vm', 'custom', 'nodejs', 'ruby', 'go111', 804 'go112') 805_all_runtimes = _CANNED_RUNTIMES 806 807 808def GetAllRuntimes(): 809 """Returns the list of all valid runtimes. 810 811 This list can include third-party runtimes as well as canned runtimes. 812 813 Returns: 814 Tuple of strings. 815 """ 816 return _all_runtimes 817 818 819def EnsureAsciiBytes(s, err): 820 """Ensure s contains only ASCII-safe characters; return it as bytes-type. 821 822 Arguments: 823 s: the string or bytes to check 824 err: the error to raise if not good. 825 Raises: 826 err if it's not ASCII-safe. 827 Returns: 828 s as a byte string 829 """ 830 try: 831 return s.encode('ascii') 832 except UnicodeEncodeError: 833 raise err 834 except UnicodeDecodeError: 835 # Python 2 hilariously raises UnicodeDecodeError on trying to 836 # ascii-_en_code a byte string invalidly. 837 raise err 838 except AttributeError: 839 try: 840 return s.decode('ascii').encode('ascii') 841 except UnicodeDecodeError: 842 raise err 843 844 845class HandlerBase(validation.Validated): 846 """Base class for URLMap and ApiConfigHandler.""" 847 ATTRIBUTES = { 848 # Common fields. 849 URL: validation.Optional(_URL_REGEX), 850 LOGIN: validation.Options(LOGIN_OPTIONAL, 851 LOGIN_REQUIRED, 852 LOGIN_ADMIN, 853 default=LOGIN_OPTIONAL), 854 855 AUTH_FAIL_ACTION: validation.Options(AUTH_FAIL_ACTION_REDIRECT, 856 AUTH_FAIL_ACTION_UNAUTHORIZED, 857 default=AUTH_FAIL_ACTION_REDIRECT), 858 859 SECURE: validation.Options(SECURE_HTTP, 860 SECURE_HTTPS, 861 SECURE_HTTP_OR_HTTPS, 862 SECURE_DEFAULT, 863 default=SECURE_DEFAULT), 864 865 # Python/CGI fields. 866 HANDLER_SCRIPT: validation.Optional(_FILES_REGEX) 867 } 868 869 870class HttpHeadersDict(validation.ValidatedDict): 871 """A dict that limits keys and values to what `http_headers` allows. 872 873 `http_headers` is an static handler key; it applies to handlers with 874 `static_dir` or `static_files` keys. The following code is an example of how 875 `http_headers` is used:: 876 877 handlers: 878 - url: /static 879 static_dir: static 880 http_headers: 881 X-Foo-Header: foo value 882 X-Bar-Header: bar value 883 884 """ 885 886 DISALLOWED_HEADERS = frozenset([ 887 # TODO(user): I don't think there's any reason to disallow users 888 # from setting Content-Encoding, but other parts of the system prevent 889 # this; therefore, we disallow it here. See the following discussion: 890 'content-encoding', 891 'content-length', 892 'date', 893 'server' 894 ]) 895 896 MAX_HEADER_LENGTH = 500 897 MAX_HEADER_VALUE_LENGTHS = { 898 'content-security-policy': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, 899 'x-content-security-policy': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, 900 'x-webkit-csp': _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, 901 'content-security-policy-report-only': 902 _MAX_HEADER_SIZE_FOR_EXEMPTED_HEADERS, 903 'set-cookie': _MAX_COOKIE_LENGTH, 904 'set-cookie2': _MAX_COOKIE_LENGTH, 905 'location': _MAX_URL_LENGTH} 906 MAX_LEN = 500 907 908 class KeyValidator(validation.Validator): 909 """Ensures that keys in `HttpHeadersDict` are valid. 910 911 `HttpHeadersDict` contains a list of headers. An instance is used as 912 `HttpHeadersDict`'s `KEY_VALIDATOR`. 913 """ 914 915 def Validate(self, name, unused_key=None): 916 """Returns an argument, or raises an exception if the argument is invalid. 917 918 HTTP header names are defined by `RFC 2616, section 4.2`_. 919 920 Args: 921 name: HTTP header field value. 922 unused_key: Unused. 923 924 Returns: 925 name argument, unchanged. 926 927 Raises: 928 appinfo_errors.InvalidHttpHeaderName: An argument cannot be used as an 929 HTTP header name. 930 931 .. _RFC 2616, section 4.2: 932 https://www.ietf.org/rfc/rfc2616.txt 933 """ 934 original_name = name 935 936 # Make sure only ASCII data is used. 937 if isinstance(name, six_subset.string_types): 938 name = EnsureAsciiBytes(name, appinfo_errors.InvalidHttpHeaderName( 939 'HTTP header values must not contain non-ASCII data')) 940 941 # HTTP headers are case-insensitive. 942 name = name.lower().decode('ascii') 943 944 if not _HTTP_TOKEN_RE.match(name): 945 raise appinfo_errors.InvalidHttpHeaderName( 946 'An HTTP header must be a non-empty RFC 2616 token.') 947 948 # Request headers shouldn't be used in responses. 949 if name in _HTTP_REQUEST_HEADERS: 950 raise appinfo_errors.InvalidHttpHeaderName( 951 '%r can only be used in HTTP requests, not responses.' 952 % original_name) 953 954 # Make sure that none of the reserved prefixes is used. 955 if name.startswith('x-appengine'): 956 raise appinfo_errors.InvalidHttpHeaderName( 957 'HTTP header names that begin with X-Appengine are reserved.') 958 959 if wsgiref.util.is_hop_by_hop(name): 960 raise appinfo_errors.InvalidHttpHeaderName( 961 'Only use end-to-end headers may be used. See RFC 2616 section' 962 ' 13.5.1.') 963 964 if name in HttpHeadersDict.DISALLOWED_HEADERS: 965 raise appinfo_errors.InvalidHttpHeaderName( 966 '%s is a disallowed header.' % name) 967 968 return original_name 969 970 class ValueValidator(validation.Validator): 971 """Ensures that values in `HttpHeadersDict` are valid. 972 973 An instance is used as `HttpHeadersDict`'s `VALUE_VALIDATOR`. 974 """ 975 976 def Validate(self, value, key=None): 977 """Returns a value, or raises an exception if the value is invalid. 978 979 According to `RFC 2616 section 4.2`_ header field values must consist "of 980 either *TEXT or combinations of token, separators, and quoted-string":: 981 982 TEXT = <any OCTET except CTLs, but including LWS> 983 984 Args: 985 value: HTTP header field value. 986 key: HTTP header field name. 987 988 Returns: 989 A value argument. 990 991 Raises: 992 appinfo_errors.InvalidHttpHeaderValue: An argument cannot be used as an 993 HTTP header value. 994 995 .. _RFC 2616, section 4.2: 996 https://www.ietf.org/rfc/rfc2616.txt 997 """ 998 # Make sure only ASCII data is used. 999 error = appinfo_errors.InvalidHttpHeaderValue( 1000 'HTTP header values must not contain non-ASCII data') 1001 if isinstance(value, six_subset.string_types): 1002 b_value = EnsureAsciiBytes(value, error) 1003 else: 1004 b_value = EnsureAsciiBytes(('%s' % value), error) 1005 1006 # HTTP headers are case-insensitive. 1007 key = key.lower() 1008 1009 # TODO(user): This is the same check that appserver performs, but it 1010 # could be stronger. e.g. `"foo` should not be considered valid, because 1011 # HTTP does not allow unclosed double quote marks in header values, per 1012 # RFC 2616 section 4.2. 1013 printable = set(string.printable[:-5].encode('ascii')) 1014 if not all(b in printable for b in b_value): 1015 raise appinfo_errors.InvalidHttpHeaderValue( 1016 'HTTP header field values must consist of printable characters.') 1017 1018 HttpHeadersDict.ValueValidator.AssertHeaderNotTooLong(key, value) 1019 1020 return value 1021 1022 @staticmethod 1023 def AssertHeaderNotTooLong(name, value): 1024 header_length = len(('%s: %s\r\n' % (name, value)).encode('ascii')) 1025 1026 # The `>=` operator here is a little counter-intuitive. The reason for it 1027 # is that I'm trying to follow the 1028 # `HTTPProto::IsValidHeader` implementation. 1029 if header_length >= HttpHeadersDict.MAX_HEADER_LENGTH: 1030 # If execution reaches this point, it generally means the header is too 1031 # long, but there are a few exceptions, which are listed in the next 1032 # dict. 1033 try: 1034 max_len = HttpHeadersDict.MAX_HEADER_VALUE_LENGTHS[name] 1035 except KeyError: 1036 raise appinfo_errors.InvalidHttpHeaderValue( 1037 'HTTP header (name + value) is too long.') 1038 1039 # We are dealing with one of the exceptional headers with larger maximum 1040 # value lengths. 1041 if len(value) > max_len: 1042 insert = name, len(value), max_len 1043 raise appinfo_errors.InvalidHttpHeaderValue( 1044 '%r header value has length %d, which exceed the maximum allowed,' 1045 ' %d.' % insert) 1046 1047 KEY_VALIDATOR = KeyValidator() 1048 VALUE_VALIDATOR = ValueValidator() 1049 1050 def Get(self, header_name): 1051 """Gets a header value. 1052 1053 Args: 1054 header_name: HTTP header name to look for. 1055 1056 Returns: 1057 A header value that corresponds to `header_name`. If more than one such 1058 value is in `self`, one of the values is selected arbitrarily and 1059 returned. The selection is not deterministic. 1060 """ 1061 for name in self: 1062 if name.lower() == header_name.lower(): 1063 return self[name] 1064 1065 # TODO(user): Perhaps, this functionality should be part of 1066 # `validation.ValidatedDict`. 1067 def __setitem__(self, key, value): 1068 is_addition = self.Get(key) is None 1069 if is_addition and len(self) >= self.MAX_LEN: 1070 raise appinfo_errors.TooManyHttpHeaders( 1071 'Tried to add another header when the current set of HTTP headers' 1072 ' already has the maximum allowed number of headers, %d.' 1073 % HttpHeadersDict.MAX_LEN) 1074 super(HttpHeadersDict, self).__setitem__(key, value) 1075 1076 1077class URLMap(HandlerBase): 1078 r"""Maps from URLs to handlers. 1079 1080 This class acts similar to a union type. Its purpose is to describe a mapping 1081 between a set of URLs and their handlers. The handler type of a given instance 1082 is determined by which `handler-id` attribute is used. 1083 1084 Every mapping can have one and only one handler type. Attempting to use more 1085 than one `handler-id` attribute will cause an `UnknownHandlerType` to be 1086 raised during validation. Failure to provide any `handler-id` attributes will 1087 cause `MissingHandlerType` to be raised during validation. 1088 1089 The regular expression used by the `url` field will be used to match against 1090 the entire URL path and query string of the request; therefore, partial maps 1091 will not be matched. Specifying a `url`, such as `/admin`, is the same as 1092 matching against the regular expression `^/admin$`. Don't start your matching 1093 `url` with `^` or end them with `$`. These regular expressions won't be 1094 accepted and will raise `ValueError`. 1095 1096 Attributes: 1097 login: Specifies whether a user should be logged in to access a URL. 1098 The default value of this argument is `optional`. 1099 secure: Sets the restriction on the protocol that can be used to serve this 1100 URL or handler. This value can be set to `HTTP`, `HTTPS` or `either`. 1101 url: Specifies a regular expression that is used to fully match against the 1102 request URLs path. See the "Special cases" section of this document to 1103 learn more. 1104 static_files: Specifies the handler ID attribute that maps `url` to the 1105 appropriate file. You can specify regular expression backreferences to 1106 the string matched to `url`. 1107 upload: Specifies the regular expression that is used by the application 1108 configuration program to determine which files are uploaded as blobs. 1109 Because it is difficult to determine this information using just the 1110 `url` and `static_files` arguments, this attribute must be included. 1111 This attribute is required when you define a `static_files` mapping. A 1112 matching file name must fully match against the `upload` regular 1113 expression, similar to how `url` is matched against the request path. Do 1114 not begin the `upload` argument with the `^` character or end it with 1115 the `$` character. 1116 static_dir: Specifies the handler ID that maps the provided `url` to a 1117 sub-directory within the application directory. See "Special cases." 1118 mime_type: When used with `static_files` and `static_dir`, this argument 1119 specifies that the MIME type of the files that are served from those 1120 directories must be overridden with this value. 1121 script: Specifies the handler ID that maps URLs to a script handler within 1122 the application directory that will run using CGI. 1123 position: Used in `AppInclude` objects to specify whether a handler should 1124 be inserted at the beginning of the primary handler list or at the end. 1125 If `tail` is specified, the handler is inserted at the end; otherwise, 1126 the handler is inserted at the beginning. This behavior implies that 1127 `head` is the effective default. 1128 expiration: When used with static files and directories, this argument 1129 specifies the time delta to use for cache expiration. This argument 1130 should use the following format: `4d 5h 30m 15s`, where each letter 1131 signifies days, hours, minutes, and seconds, respectively. The `s` for 1132 "seconds" can be omitted. Only one amount must be specified, though 1133 combining multiple amounts is optional. The following list contains 1134 examples of values that are acceptable: `10`, `1d 6h`, `1h 30m`, 1135 `7d 7d 7d`, `5m 30`. 1136 api_endpoint: Specifies the handler ID that identifies an endpoint as an API 1137 endpoint. Calls that terminate here will be handled by the API serving 1138 framework. 1139 1140 Special cases: 1141 When defining a `static_dir` handler, do not use a regular expression in the 1142 `url` attribute. Both the `url` and `static_dir` attributes are 1143 automatically mapped to these equivalents:: 1144 1145 <url>/(.*) 1146 <static_dir>/\1 1147 1148 For example, this declaration...:: 1149 1150 url: /images 1151 static_dir: images_folder 1152 1153 ...is equivalent to this `static_files` declaration:: 1154 1155 url: /images/(.*) 1156 static_files: images_folder/\1 1157 upload: images_folder/(.*) 1158 1159 """ 1160 ATTRIBUTES = { 1161 # Static file fields. 1162 # File mappings are allowed to have regex back references. 1163 HANDLER_STATIC_FILES: validation.Optional(_FILES_REGEX), 1164 UPLOAD: validation.Optional(_FILES_REGEX), 1165 APPLICATION_READABLE: validation.Optional(bool), 1166 1167 # Static directory fields. 1168 HANDLER_STATIC_DIR: validation.Optional(_FILES_REGEX), 1169 1170 # Used in both static mappings. 1171 MIME_TYPE: validation.Optional(str), 1172 EXPIRATION: validation.Optional(_EXPIRATION_REGEX), 1173 REQUIRE_MATCHING_FILE: validation.Optional(bool), 1174 HTTP_HEADERS: validation.Optional(HttpHeadersDict), 1175 1176 # Python/CGI fields. 1177 POSITION: validation.Optional(validation.Options(POSITION_HEAD, 1178 POSITION_TAIL)), 1179 1180 HANDLER_API_ENDPOINT: validation.Optional(validation.Options( 1181 (ON, ON_ALIASES), 1182 (OFF, OFF_ALIASES))), 1183 1184 REDIRECT_HTTP_RESPONSE_CODE: validation.Optional(validation.Options( 1185 '301', '302', '303', '307')), 1186 } 1187 ATTRIBUTES.update(HandlerBase.ATTRIBUTES) 1188 1189 COMMON_FIELDS = set([ 1190 URL, LOGIN, AUTH_FAIL_ACTION, SECURE, REDIRECT_HTTP_RESPONSE_CODE]) 1191 1192 # The keys of this map are attributes which can be used to identify each 1193 # mapping type in addition to the handler identifying attribute itself. 1194 ALLOWED_FIELDS = { 1195 HANDLER_STATIC_FILES: (MIME_TYPE, UPLOAD, EXPIRATION, 1196 REQUIRE_MATCHING_FILE, HTTP_HEADERS, 1197 APPLICATION_READABLE), 1198 HANDLER_STATIC_DIR: (MIME_TYPE, EXPIRATION, REQUIRE_MATCHING_FILE, 1199 HTTP_HEADERS, APPLICATION_READABLE), 1200 HANDLER_SCRIPT: (POSITION), 1201 HANDLER_API_ENDPOINT: (POSITION, SCRIPT), 1202 } 1203 1204 def GetHandler(self): 1205 """Gets the handler for a mapping. 1206 1207 Returns: 1208 The value of the handler, as determined by the handler ID attribute. 1209 """ 1210 return getattr(self, self.GetHandlerType()) 1211 1212 def GetHandlerType(self): 1213 """Gets the handler type of a mapping. 1214 1215 Returns: 1216 The handler type as determined by which handler ID attribute is set. 1217 1218 Raises: 1219 UnknownHandlerType: If none of the handler ID attributes are set. 1220 UnexpectedHandlerAttribute: If an unexpected attribute is set for the 1221 discovered handler type. 1222 HandlerTypeMissingAttribute: If the handler is missing a required 1223 attribute for its handler type. 1224 MissingHandlerAttribute: If a URL handler is missing an attribute. 1225 """ 1226 # Special case for the `api_endpoint` handler as it may have a `script` 1227 # attribute as well. 1228 if getattr(self, HANDLER_API_ENDPOINT) is not None: 1229 # Matched id attribute, break out of loop. 1230 mapping_type = HANDLER_API_ENDPOINT 1231 else: 1232 for id_field in URLMap.ALLOWED_FIELDS: 1233 # Attributes always exist as defined by ATTRIBUTES. 1234 if getattr(self, id_field) is not None: 1235 # Matched id attribute, break out of loop. 1236 mapping_type = id_field 1237 break 1238 else: 1239 # If no mapping type is found raise exception. 1240 raise appinfo_errors.UnknownHandlerType( 1241 'Unknown url handler type.\n%s' % str(self)) 1242 1243 allowed_fields = URLMap.ALLOWED_FIELDS[mapping_type] 1244 1245 # Make sure that none of the set attributes on this handler 1246 # are not allowed for the discovered handler type. 1247 for attribute in self.ATTRIBUTES: 1248 if (getattr(self, attribute) is not None and 1249 not (attribute in allowed_fields or 1250 attribute in URLMap.COMMON_FIELDS or 1251 attribute == mapping_type)): 1252 raise appinfo_errors.UnexpectedHandlerAttribute( 1253 'Unexpected attribute "%s" for mapping type %s.' % 1254 (attribute, mapping_type)) 1255 1256 # Also check that static file map has 'upload'. 1257 # NOTE: Add REQUIRED_FIELDS along with ALLOWED_FIELDS if any more 1258 # exceptional cases arise. 1259 if mapping_type == HANDLER_STATIC_FILES and not self.upload: 1260 raise appinfo_errors.MissingHandlerAttribute( 1261 'Missing "%s" attribute for URL "%s".' % (UPLOAD, self.url)) 1262 1263 return mapping_type 1264 1265 def CheckInitialized(self): 1266 """Adds additional checking to make sure a handler has correct fields. 1267 1268 In addition to normal `ValidatedCheck`, this method calls `GetHandlerType`, 1269 which validates whether all of the handler fields are configured properly. 1270 1271 Raises: 1272 UnknownHandlerType: If none of the handler ID attributes are set. 1273 UnexpectedHandlerAttribute: If an unexpected attribute is set for the 1274 discovered handler type. 1275 HandlerTypeMissingAttribute: If the handler is missing a required 1276 attribute for its handler type. 1277 ContentTypeSpecifiedMultipleTimes: If `mime_type` is inconsistent with 1278 `http_headers`. 1279 """ 1280 super(URLMap, self).CheckInitialized() 1281 if self.GetHandlerType() in (STATIC_DIR, STATIC_FILES): 1282 # re how headers that affect caching interact per RFC 2616: 1283 # 1284 # Section 13.1.3 says that when there is "apparent conflict between 1285 # [Cache-Control] header values, the most restrictive interpretation is 1286 # applied". 1287 # 1288 # Section 14.21 says that Cache-Control: max-age overrides Expires 1289 # headers. 1290 # 1291 # Section 14.32 says that Pragma: no-cache has no meaning in responses; 1292 # therefore, we do not need to be concerned about that header here. 1293 self.AssertUniqueContentType() 1294 1295 def AssertUniqueContentType(self): 1296 """Makes sure that `self.http_headers` is consistent with `self.mime_type`. 1297 1298 This method assumes that `self` is a static handler, either 1299 `self.static_dir` or `self.static_files`. You cannot specify `None`. 1300 1301 Raises: 1302 appinfo_errors.ContentTypeSpecifiedMultipleTimes: If `self.http_headers` 1303 contains a `Content-Type` header, and `self.mime_type` is set. For 1304 example, the following configuration would be rejected:: 1305 1306 handlers: 1307 - url: /static 1308 static_dir: static 1309 mime_type: text/html 1310 http_headers: 1311 content-type: text/html 1312 1313 1314 As this example shows, a configuration will be rejected when 1315 `http_headers` and `mime_type` specify a content type, even when they 1316 specify the same content type. 1317 """ 1318 used_both_fields = self.mime_type and self.http_headers 1319 if not used_both_fields: 1320 return 1321 1322 content_type = self.http_headers.Get('Content-Type') 1323 if content_type is not None: 1324 raise appinfo_errors.ContentTypeSpecifiedMultipleTimes( 1325 'http_header specified a Content-Type header of %r in a handler that' 1326 ' also specified a mime_type of %r.' % (content_type, self.mime_type)) 1327 1328 def FixSecureDefaults(self): 1329 """Forces omitted `secure` handler fields to be set to 'secure: optional'. 1330 1331 The effect is that `handler.secure` is never equal to the nominal default. 1332 """ 1333 # See http://b/issue?id=2073962. 1334 if self.secure == SECURE_DEFAULT: 1335 self.secure = SECURE_HTTP_OR_HTTPS 1336 1337 def WarnReservedURLs(self): 1338 """Generates a warning for reserved URLs. 1339 1340 See the `version element documentation`_ to learn which URLs are reserved. 1341 1342 .. _`version element documentation`: 1343 https://cloud.google.com/appengine/docs/python/config/appref#syntax 1344 """ 1345 if self.url == '/form': 1346 logging.warning( 1347 'The URL path "/form" is reserved and will not be matched.') 1348 1349 def ErrorOnPositionForAppInfo(self): 1350 """Raises an error if position is specified outside of AppInclude objects. 1351 1352 Raises: 1353 PositionUsedInAppYamlHandler: If the `position` attribute is specified for 1354 an `app.yaml` file instead of an `include.yaml` file. 1355 """ 1356 if self.position: 1357 raise appinfo_errors.PositionUsedInAppYamlHandler( 1358 'The position attribute was specified for this handler, but this is ' 1359 'an app.yaml file. Position attribute is only valid for ' 1360 'include.yaml files.') 1361 1362 1363class AdminConsolePage(validation.Validated): 1364 """Class representing the admin console page in an `AdminConsole` object.""" 1365 ATTRIBUTES = { 1366 URL: _URL_REGEX, 1367 NAME: _PAGE_NAME_REGEX, 1368 } 1369 1370 1371class AdminConsole(validation.Validated): 1372 """Class representing an admin console directives in application info.""" 1373 ATTRIBUTES = { 1374 PAGES: validation.Optional(validation.Repeated(AdminConsolePage)), 1375 } 1376 1377 @classmethod 1378 def Merge(cls, adminconsole_one, adminconsole_two): 1379 """Returns the result of merging two `AdminConsole` objects.""" 1380 # Right now this method only needs to worry about the pages attribute of 1381 # `AdminConsole`. However, since this object is valid as part of an 1382 # `AppInclude` object, any objects added to `AdminConsole` in the future 1383 # must also be merged. Rather than burying the merge logic in the process 1384 # of merging two `AppInclude` objects, it is centralized here. If you modify 1385 # the `AdminConsole` object to support other objects, you must also modify 1386 # this method to support merging those additional objects. 1387 1388 if not adminconsole_one or not adminconsole_two: 1389 return adminconsole_one or adminconsole_two 1390 1391 if adminconsole_one.pages: 1392 if adminconsole_two.pages: 1393 adminconsole_one.pages.extend(adminconsole_two.pages) 1394 else: 1395 adminconsole_one.pages = adminconsole_two.pages 1396 1397 return adminconsole_one 1398 1399 1400class ErrorHandlers(validation.Validated): 1401 """Class representing error handler directives in application info.""" 1402 ATTRIBUTES = { 1403 ERROR_CODE: validation.Optional(_ERROR_CODE_REGEX), 1404 FILE: _FILES_REGEX, 1405 MIME_TYPE: validation.Optional(str), 1406 } 1407 1408 1409class BuiltinHandler(validation.Validated): 1410 """Class representing built-in handler directives in application info. 1411 1412 This class permits arbitrary keys, but their values must be described by the 1413 `validation.Options` object that is returned by `ATTRIBUTES`. 1414 """ 1415 1416 # `Validated` is a somewhat complicated class. It actually maintains two 1417 # dictionaries: the `ATTRIBUTES` dictionary and an internal `__dict__` object 1418 # that maintains key value pairs. 1419 # 1420 # The normal flow is that a key must exist in `ATTRIBUTES` in order to be able 1421 # to be inserted into `__dict__`. So that's why we force the 1422 # `ATTRIBUTES.__contains__` method to always return `True`; we want to accept 1423 # any attribute. Once the method returns `True`, then its value will be 1424 # fetched, which returns `ATTRIBUTES[key]`; that's why we override 1425 # `ATTRIBUTES.__getitem__` to return the validator for a `BuiltinHandler` 1426 # object. 1427 # 1428 # This is where it gets tricky. Once the validator object is returned, then 1429 # `__dict__[key]` is set to the validated object for that key. However, when 1430 # `CheckInitialized()` is called, it uses iteritems from `ATTRIBUTES` in order 1431 # to generate a list of keys to validate. This expects the `BuiltinHandler` 1432 # instance to contain every item in `ATTRIBUTES`, which contains every 1433 # built-in name seen so far by any `BuiltinHandler`. To work around this, 1434 # `__getattr__` always returns `None` for public attribute names. Note that 1435 # `__getattr__` is only called if `__dict__` does not contain the key. Thus, 1436 # the single built-in value set is validated. 1437 # 1438 # What's important to know is that in this implementation, only the keys in 1439 # `ATTRIBUTES` matter, and only the values in `__dict__` matter. The values in 1440 # `ATTRIBUTES` and the keys in `__dict__` are both ignored. The key in 1441 # `__dict__` is only used for the `__getattr__` function, but to find out what 1442 # keys are available, only `ATTRIBUTES` is ever read. 1443 1444 class DynamicAttributes(dict): 1445 """Provides a dictionary object that will always claim to have a key. 1446 1447 This dictionary returns a fixed value for any `get` operation. The fixed 1448 value that you pass in as a constructor parameter should be a 1449 `validation.Validated` object. 1450 """ 1451 1452 def __init__(self, return_value, **parameters): 1453 self.__return_value = return_value 1454 dict.__init__(self, parameters) 1455 1456 def __contains__(self, _): 1457 return True 1458 1459 def __getitem__(self, _): 1460 return self.__return_value 1461 1462 ATTRIBUTES = DynamicAttributes( 1463 validation.Optional(validation.Options((ON, ON_ALIASES), 1464 (OFF, OFF_ALIASES)))) 1465 1466 def __init__(self, **attributes): 1467 """Ensures all BuiltinHandler objects at least use the `default` attribute. 1468 1469 Args: 1470 **attributes: The attributes that you want to use. 1471 """ 1472 self.builtin_name = '' 1473 super(BuiltinHandler, self).__init__(**attributes) 1474 1475 def __setattr__(self, key, value): 1476 """Allows `ATTRIBUTES.iteritems()` to return set of items that have values. 1477 1478 Whenever `validate` calls `iteritems()`, it is always called on 1479 `ATTRIBUTES`, not on `__dict__`, so this override is important to ensure 1480 that functions such as `ToYAML()` return the correct set of keys. 1481 1482 Args: 1483 key: The key for the `iteritem` that you want to set. 1484 value: The value for the `iteritem` that you want to set. 1485 1486 Raises: 1487 MultipleBuiltinsSpecified: If more than one built-in is defined in a list 1488 element. 1489 """ 1490 if key == 'builtin_name': 1491 object.__setattr__(self, key, value) 1492 elif not self.builtin_name: 1493 self.ATTRIBUTES[key] = '' 1494 self.builtin_name = key 1495 super(BuiltinHandler, self).__setattr__(key, value) 1496 else: 1497 # Only the name of a built-in handler is currently allowed as an attribute 1498 # so the object can only be set once. If later attributes are desired of 1499 # a different form, this clause should be used to catch whenever more than 1500 # one object does not match a predefined attribute name. 1501 raise appinfo_errors.MultipleBuiltinsSpecified( 1502 'More than one builtin defined in list element. Each new builtin ' 1503 'should be prefixed by "-".') 1504 1505 def __getattr__(self, key): 1506 if key.startswith('_'): 1507 # `__getattr__` is only called for attributes that don't exist in the 1508 # instance dictionary. 1509 raise AttributeError 1510 return None 1511 1512 def GetUnnormalized(self, key): 1513 try: 1514 return super(BuiltinHandler, self).GetUnnormalized(key) 1515 except AttributeError: 1516 return getattr(self, key) 1517 1518 def ToDict(self): 1519 """Converts a `BuiltinHander` object to a dictionary. 1520 1521 Returns: 1522 A dictionary in `{builtin_handler_name: on/off}` form 1523 """ 1524 return {self.builtin_name: getattr(self, self.builtin_name)} 1525 1526 @classmethod 1527 def IsDefined(cls, builtins_list, builtin_name): 1528 """Finds if a builtin is defined in a given list of builtin handler objects. 1529 1530 Args: 1531 builtins_list: A list of `BuiltinHandler` objects, typically 1532 `yaml.builtins`. 1533 builtin_name: The name of the built-in that you want to determine whether 1534 it is defined. 1535 1536 Returns: 1537 `True` if `builtin_name` is defined by a member of `builtins_list`; all 1538 other results return `False`. 1539 """ 1540 for b in builtins_list: 1541 if b.builtin_name == builtin_name: 1542 return True 1543 return False 1544 1545 @classmethod 1546 def ListToTuples(cls, builtins_list): 1547 """Converts a list of `BuiltinHandler` objects. 1548 1549 Args: 1550 builtins_list: A list of `BuildinHandler` objects to convert to tuples. 1551 1552 Returns: 1553 A list of `(name, status)` that is derived from the `BuiltinHandler` 1554 objects. 1555 """ 1556 return [(b.builtin_name, getattr(b, b.builtin_name)) for b in builtins_list] 1557 1558 @classmethod 1559 def Validate(cls, builtins_list, runtime=None): 1560 """Verifies that all `BuiltinHandler` objects are valid and not repeated. 1561 1562 Args: 1563 builtins_list: A list of `BuiltinHandler` objects to validate. 1564 runtime: If you specify this argument, warnings are generated for 1565 built-ins that have been deprecated in the given runtime. 1566 1567 Raises: 1568 InvalidBuiltinFormat: If the name of a `BuiltinHandler` object cannot be 1569 determined. 1570 DuplicateBuiltinsSpecified: If a `BuiltinHandler` name is used more than 1571 once in the list. 1572 """ 1573 seen = set() 1574 for b in builtins_list: 1575 if not b.builtin_name: 1576 raise appinfo_errors.InvalidBuiltinFormat( 1577 'Name of builtin for list object %s could not be determined.' 1578 % b) 1579 if b.builtin_name in seen: 1580 raise appinfo_errors.DuplicateBuiltinsSpecified( 1581 'Builtin %s was specified more than once in one yaml file.' 1582 % b.builtin_name) 1583 1584 # This checking must be done here rather than in `apphosting/ext/builtins` 1585 # because `apphosting/ext/builtins` cannot differentiate between between 1586 # built-ins specified in `app.yaml` versus ones added in a built-in 1587 # include. There is a hole here where warnings are not generated for 1588 # deprecated built-ins that appear in user-created include files. 1589 if b.builtin_name == 'datastore_admin' and runtime == 'python': 1590 logging.warning( 1591 'The datastore_admin builtin is deprecated. You can find ' 1592 'information on how to enable it through the Administrative ' 1593 'Console here: ' 1594 'http://developers.google.com/appengine/docs/adminconsole/' 1595 'datastoreadmin.html') 1596 elif b.builtin_name == 'mapreduce' and runtime == 'python': 1597 logging.warning( 1598 'The mapreduce builtin is deprecated. You can find more ' 1599 'information on how to configure and use it here: ' 1600 'http://developers.google.com/appengine/docs/python/dataprocessing/' 1601 'overview.html') 1602 1603 seen.add(b.builtin_name) 1604 1605 1606class ApiConfigHandler(HandlerBase): 1607 """Class representing `api_config` handler directives in application info.""" 1608 ATTRIBUTES = HandlerBase.ATTRIBUTES 1609 ATTRIBUTES.update({ 1610 # Make `URL` and `SCRIPT` required for `api_config` stanza 1611 URL: validation.Regex(_URL_REGEX), 1612 HANDLER_SCRIPT: validation.Regex(_FILES_REGEX) 1613 }) 1614 1615 1616class Library(validation.Validated): 1617 """Class representing the configuration of a single library.""" 1618 1619 ATTRIBUTES = {'name': validation.Type(str), 1620 'version': validation.Type(str)} 1621 1622 def CheckInitialized(self): 1623 """Determines if the library configuration is not valid. 1624 1625 Raises: 1626 appinfo_errors.InvalidLibraryName: If the specified library is not 1627 supported. 1628 appinfo_errors.InvalidLibraryVersion: If the specified library version is 1629 not supported. 1630 """ 1631 super(Library, self).CheckInitialized() 1632 if self.name not in _NAME_TO_SUPPORTED_LIBRARY: 1633 raise appinfo_errors.InvalidLibraryName( 1634 'the library "%s" is not supported' % self.name) 1635 supported_library = _NAME_TO_SUPPORTED_LIBRARY[self.name] 1636 if self.version == 'latest': 1637 self.version = supported_library.latest_version 1638 elif self.version not in supported_library.supported_versions: 1639 raise appinfo_errors.InvalidLibraryVersion( 1640 ('%s version "%s" is not supported, ' + _USE_VERSION_FORMAT) % ( 1641 self.name, 1642 self.version, 1643 '", "'.join(supported_library.non_deprecated_versions))) 1644 elif self.version in supported_library.deprecated_versions: 1645 use_vers = '", "'.join(supported_library.non_deprecated_versions) 1646 logging.warning( 1647 '%s version "%s" is deprecated, ' + _USE_VERSION_FORMAT, 1648 self.name, 1649 self.version, 1650 use_vers) 1651 1652 1653class CpuUtilization(validation.Validated): 1654 """Class representing the configuration of VM CPU utilization.""" 1655 1656 ATTRIBUTES = { 1657 CPU_UTILIZATION_UTILIZATION: validation.Optional( 1658 validation.Range(1e-6, 1.0, float)), 1659 CPU_UTILIZATION_AGGREGATION_WINDOW_LENGTH_SEC: validation.Optional( 1660 validation.Range(1, sys.maxsize)), 1661 } 1662 1663 1664class CustomMetric(validation.Validated): 1665 """Class representing CustomMetrics in AppInfoExternal.""" 1666 1667 ATTRIBUTES = { 1668 METRIC_NAME: validation.Regex(_NON_WHITE_SPACE_REGEX), 1669 TARGET_TYPE: validation.Regex(TARGET_TYPE_REGEX), 1670 CUSTOM_METRIC_UTILIZATION: validation.Optional(validation.TYPE_FLOAT), 1671 SINGLE_INSTANCE_ASSIGNMENT: validation.Optional(validation.TYPE_FLOAT), 1672 FILTER: validation.Optional(validation.TYPE_STR), 1673 } 1674 1675 def CheckInitialized(self): 1676 """Determines if the CustomMetric is not valid. 1677 1678 Raises: 1679 appinfo_errors.TooManyAutoscalingUtilizationTargetsError: If too many 1680 scaling targets are set. 1681 appinfo_errors.NotEnoughAutoscalingUtilizationTargetsError: If no scaling 1682 targets are set. 1683 """ 1684 super(CustomMetric, self).CheckInitialized() 1685 if bool(self.target_utilization) and bool(self.single_instance_assignment): 1686 raise appinfo_errors.TooManyAutoscalingUtilizationTargetsError( 1687 ("There may be only one of '%s' or '%s'." % CUSTOM_METRIC_UTILIZATION, 1688 SINGLE_INSTANCE_ASSIGNMENT)) 1689 elif not (bool(self.target_utilization) or 1690 bool(self.single_instance_assignment)): 1691 raise appinfo_errors.NotEnoughAutoscalingUtilizationTargetsError( 1692 ("There must be one of '%s' or '%s'." % CUSTOM_METRIC_UTILIZATION, 1693 SINGLE_INSTANCE_ASSIGNMENT)) 1694 1695 1696class EndpointsApiService(validation.Validated): 1697 """Class representing EndpointsApiService in AppInfoExternal.""" 1698 ATTRIBUTES = { 1699 ENDPOINTS_NAME: 1700 validation.Regex(_NON_WHITE_SPACE_REGEX), 1701 ROLLOUT_STRATEGY: 1702 validation.Optional( 1703 validation.Options(ROLLOUT_STRATEGY_FIXED, 1704 ROLLOUT_STRATEGY_MANAGED)), 1705 CONFIG_ID: 1706 validation.Optional(_NON_WHITE_SPACE_REGEX), 1707 TRACE_SAMPLING: 1708 validation.Optional(validation.TYPE_BOOL), 1709 } 1710 1711 def CheckInitialized(self): 1712 """Determines if the Endpoints API Service is not valid. 1713 1714 Raises: 1715 appinfo_errors.MissingEndpointsConfigId: If the config id is missing when 1716 the rollout strategy is unspecified or set to "fixed". 1717 appinfo_errors.UnexpectedEndpointsConfigId: If the config id is set when 1718 the rollout strategy is "managed". 1719 """ 1720 super(EndpointsApiService, self).CheckInitialized() 1721 if (self.rollout_strategy != ROLLOUT_STRATEGY_MANAGED and 1722 self.config_id is None): 1723 raise appinfo_errors.MissingEndpointsConfigId( 1724 'config_id must be specified when rollout_strategy is unspecified or' 1725 ' set to "fixed"') 1726 elif (self.rollout_strategy == ROLLOUT_STRATEGY_MANAGED and 1727 self.config_id is not None): 1728 raise appinfo_errors.UnexpectedEndpointsConfigId( 1729 'config_id is forbidden when rollout_strategy is set to "managed"') 1730 1731 1732class AutomaticScaling(validation.Validated): 1733 """Class representing automatic scaling settings in AppInfoExternal.""" 1734 ATTRIBUTES = { 1735 MINIMUM_IDLE_INSTANCES: 1736 validation.Optional(_IDLE_INSTANCES_REGEX), 1737 MAXIMUM_IDLE_INSTANCES: 1738 validation.Optional(_IDLE_INSTANCES_REGEX), 1739 MINIMUM_PENDING_LATENCY: 1740 validation.Optional(_PENDING_LATENCY_REGEX), 1741 MAXIMUM_PENDING_LATENCY: 1742 validation.Optional(_PENDING_LATENCY_REGEX), 1743 MAXIMUM_CONCURRENT_REQUEST: 1744 validation.Optional(_CONCURRENT_REQUESTS_REGEX), 1745 # Attributes for VM-based AutomaticScaling. 1746 MIN_NUM_INSTANCES: 1747 validation.Optional(validation.Range(1, sys.maxsize)), 1748 MAX_NUM_INSTANCES: 1749 validation.Optional(validation.Range(1, sys.maxsize)), 1750 COOL_DOWN_PERIOD_SEC: 1751 validation.Optional(validation.Range(60, sys.maxsize, int)), 1752 CPU_UTILIZATION: 1753 validation.Optional(CpuUtilization), 1754 STANDARD_MAX_INSTANCES: 1755 validation.Optional(validation.TYPE_INT), 1756 STANDARD_MIN_INSTANCES: 1757 validation.Optional(validation.TYPE_INT), 1758 STANDARD_TARGET_CPU_UTILIZATION: 1759 validation.Optional(validation.TYPE_FLOAT), 1760 STANDARD_TARGET_THROUGHPUT_UTILIZATION: 1761 validation.Optional(validation.TYPE_FLOAT), 1762 TARGET_NETWORK_SENT_BYTES_PER_SEC: 1763 validation.Optional(validation.Range(1, sys.maxsize)), 1764 TARGET_NETWORK_SENT_PACKETS_PER_SEC: 1765 validation.Optional(validation.Range(1, sys.maxsize)), 1766 TARGET_NETWORK_RECEIVED_BYTES_PER_SEC: 1767 validation.Optional(validation.Range(1, sys.maxsize)), 1768 TARGET_NETWORK_RECEIVED_PACKETS_PER_SEC: 1769 validation.Optional(validation.Range(1, sys.maxsize)), 1770 TARGET_DISK_WRITE_BYTES_PER_SEC: 1771 validation.Optional(validation.Range(1, sys.maxsize)), 1772 TARGET_DISK_WRITE_OPS_PER_SEC: 1773 validation.Optional(validation.Range(1, sys.maxsize)), 1774 TARGET_DISK_READ_BYTES_PER_SEC: 1775 validation.Optional(validation.Range(1, sys.maxsize)), 1776 TARGET_DISK_READ_OPS_PER_SEC: 1777 validation.Optional(validation.Range(1, sys.maxsize)), 1778 TARGET_REQUEST_COUNT_PER_SEC: 1779 validation.Optional(validation.Range(1, sys.maxsize)), 1780 TARGET_CONCURRENT_REQUESTS: 1781 validation.Optional(validation.Range(1, sys.maxsize)), 1782 CUSTOM_METRICS: validation.Optional(validation.Repeated(CustomMetric)), 1783 } 1784 1785 1786class ManualScaling(validation.Validated): 1787 """Class representing manual scaling settings in AppInfoExternal.""" 1788 ATTRIBUTES = { 1789 INSTANCES: validation.Regex(_INSTANCES_REGEX), 1790 } 1791 1792 1793class BasicScaling(validation.Validated): 1794 """Class representing basic scaling settings in AppInfoExternal.""" 1795 ATTRIBUTES = { 1796 MAX_INSTANCES: validation.Regex(_INSTANCES_REGEX), 1797 IDLE_TIMEOUT: validation.Optional(_IDLE_TIMEOUT_REGEX), 1798 } 1799 1800 1801class RuntimeConfig(validation.ValidatedDict): 1802 """Class for "vanilla" runtime configuration. 1803 1804 Fields used vary by runtime, so validation is delegated to the per-runtime 1805 build processes. 1806 1807 These are intended to be used during Dockerfile generation, not after VM boot. 1808 """ 1809 1810 KEY_VALIDATOR = validation.Regex('[a-zA-Z_][a-zA-Z0-9_]*') 1811 VALUE_VALIDATOR = str 1812 1813 1814class VmSettings(validation.ValidatedDict): 1815 """Class for VM settings. 1816 1817 The settings are not further validated here. The settings are validated on 1818 the server side. 1819 """ 1820 1821 KEY_VALIDATOR = validation.Regex('[a-zA-Z_][a-zA-Z0-9_]*') 1822 VALUE_VALIDATOR = str 1823 1824 @classmethod 1825 def Merge(cls, vm_settings_one, vm_settings_two): 1826 """Merges two `VmSettings` instances. 1827 1828 If a variable is specified by both instances, the value from 1829 `vm_settings_one` is used. 1830 1831 Args: 1832 vm_settings_one: The first `VmSettings` instance, or `None`. 1833 vm_settings_two: The second `VmSettings` instance, or `None`. 1834 1835 Returns: 1836 The merged `VmSettings` instance, or `None` if both input instances are 1837 `None` or empty. 1838 """ 1839 # Note that `VmSettings.copy()` results in a dict. 1840 result_vm_settings = (vm_settings_two or {}).copy() 1841 # TODO(user): Apply merge logic when feature is fully defined. 1842 # For now, we will merge the two dict and `vm_settings_one` will win 1843 # if key collides. 1844 result_vm_settings.update(vm_settings_one or {}) 1845 return VmSettings(**result_vm_settings) if result_vm_settings else None 1846 1847 1848class BetaSettings(VmSettings): 1849 """Class for Beta (internal or unreleased) settings. 1850 1851 This class is meant to replace `VmSettings` eventually. 1852 1853 Note: 1854 All new beta settings must be registered in `shared_constants.py`. 1855 1856 These settings are not validated further here. The settings are validated on 1857 the server side. 1858 """ 1859 1860 @classmethod 1861 def Merge(cls, beta_settings_one, beta_settings_two): 1862 """Merges two `BetaSettings` instances. 1863 1864 Args: 1865 beta_settings_one: The first `BetaSettings` instance, or `None`. 1866 beta_settings_two: The second `BetaSettings` instance, or `None`. 1867 1868 Returns: 1869 The merged `BetaSettings` instance, or `None` if both input instances are 1870 `None` or empty. 1871 """ 1872 merged = VmSettings.Merge(beta_settings_one, beta_settings_two) 1873 return BetaSettings(**merged.ToDict()) if merged else None 1874 1875 1876class EnvironmentVariables(validation.ValidatedDict): 1877 """Class representing a mapping of environment variable key/value pairs.""" 1878 1879 KEY_VALIDATOR = validation.Regex('[a-zA-Z_][a-zA-Z0-9_]*') 1880 VALUE_VALIDATOR = str 1881 1882 @classmethod 1883 def Merge(cls, env_variables_one, env_variables_two): 1884 """Merges two `EnvironmentVariables` instances. 1885 1886 If a variable is specified by both instances, the value from 1887 `env_variables_two` is used. 1888 1889 Args: 1890 env_variables_one: The first `EnvironmentVariables` instance or `None`. 1891 env_variables_two: The second `EnvironmentVariables` instance or `None`. 1892 1893 Returns: 1894 The merged `EnvironmentVariables` instance, or `None` if both input 1895 instances are `None` or empty. 1896 """ 1897 # Note that `EnvironmentVariables.copy()` results in a dict. 1898 result_env_variables = (env_variables_one or {}).copy() 1899 result_env_variables.update(env_variables_two or {}) 1900 return (EnvironmentVariables(**result_env_variables) 1901 if result_env_variables else None) 1902 1903 1904def ValidateSourceReference(ref): 1905 """Determines if a source reference is valid. 1906 1907 Args: 1908 ref: A source reference in the following format: 1909 `[repository_uri#]revision`. 1910 1911 Raises: 1912 ValidationError: If the reference is malformed. 1913 """ 1914 repo_revision = ref.split('#', 1) 1915 revision_id = repo_revision[-1] 1916 if not re.match(SOURCE_REVISION_RE_STRING, revision_id): 1917 raise validation.ValidationError('Bad revision identifier: %s' % 1918 revision_id) 1919 1920 if len(repo_revision) == 2: 1921 uri = repo_revision[0] 1922 if not re.match(SOURCE_REPO_RE_STRING, uri): 1923 raise validation.ValidationError('Bad repository URI: %s' % uri) 1924 1925 1926def ValidateCombinedSourceReferencesString(source_refs): 1927 """Determines if `source_refs` contains a valid list of source references. 1928 1929 Args: 1930 source_refs: A multi-line string containing one source reference per line. 1931 1932 Raises: 1933 ValidationError: If the reference is malformed. 1934 """ 1935 if len(source_refs) > SOURCE_REFERENCES_MAX_SIZE: 1936 raise validation.ValidationError( 1937 'Total source reference(s) size exceeds the limit: %d > %d' % ( 1938 len(source_refs), SOURCE_REFERENCES_MAX_SIZE)) 1939 1940 for ref in source_refs.splitlines(): 1941 ValidateSourceReference(ref.strip()) 1942 1943 1944class HealthCheck(validation.Validated): 1945 """Class representing the health check configuration.""" 1946 ATTRIBUTES = { 1947 ENABLE_HEALTH_CHECK: validation.Optional(validation.TYPE_BOOL), 1948 CHECK_INTERVAL_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1949 TIMEOUT_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1950 UNHEALTHY_THRESHOLD: validation.Optional( 1951 validation.Range(0, sys.maxsize)), 1952 HEALTHY_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1953 RESTART_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1954 HOST: validation.Optional(validation.TYPE_STR)} 1955 1956 1957class LivenessCheck(validation.Validated): 1958 """Class representing the liveness check configuration.""" 1959 ATTRIBUTES = { 1960 CHECK_INTERVAL_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1961 TIMEOUT_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1962 FAILURE_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1963 SUCCESS_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1964 INITIAL_DELAY_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1965 PATH: validation.Optional(validation.TYPE_STR), 1966 HOST: validation.Optional(validation.TYPE_STR)} 1967 1968 1969class ReadinessCheck(validation.Validated): 1970 """Class representing the readiness check configuration.""" 1971 ATTRIBUTES = { 1972 CHECK_INTERVAL_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1973 TIMEOUT_SEC: validation.Optional(validation.Range(0, sys.maxsize)), 1974 APP_START_TIMEOUT_SEC: validation.Optional( 1975 validation.Range(0, sys.maxsize)), 1976 FAILURE_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1977 SUCCESS_THRESHOLD: validation.Optional(validation.Range(0, sys.maxsize)), 1978 PATH: validation.Optional(validation.TYPE_STR), 1979 HOST: validation.Optional(validation.TYPE_STR)} 1980 1981 1982class VmHealthCheck(HealthCheck): 1983 """Class representing the configuration of the VM health check. 1984 1985 Note: 1986 This class is deprecated and will be removed in a future release. Use 1987 `HealthCheck` instead. 1988 """ 1989 pass 1990 1991 1992class Volume(validation.Validated): 1993 """Class representing the configuration of a volume.""" 1994 1995 ATTRIBUTES = { 1996 VOLUME_NAME: validation.TYPE_STR, 1997 SIZE_GB: validation.TYPE_FLOAT, 1998 VOLUME_TYPE: validation.TYPE_STR, 1999 } 2000 2001 2002class Resources(validation.Validated): 2003 """Class representing the configuration of VM resources.""" 2004 2005 ATTRIBUTES = { 2006 CPU: validation.Optional(validation.TYPE_FLOAT), 2007 MEMORY_GB: validation.Optional(validation.TYPE_FLOAT), 2008 DISK_SIZE_GB: validation.Optional(validation.TYPE_INT), 2009 VOLUMES: validation.Optional(validation.Repeated(Volume)) 2010 } 2011 2012 2013class Network(validation.Validated): 2014 """Class representing the VM network configuration.""" 2015 2016 ATTRIBUTES = { 2017 # A list of port mappings in the form 'port' or 'external:internal'. 2018 FORWARDED_PORTS: validation.Optional(validation.Repeated(validation.Regex( 2019 '[0-9]+(:[0-9]+)?(/(udp|tcp))?'))), 2020 2021 INSTANCE_TAG: validation.Optional(validation.Regex( 2022 GCE_RESOURCE_NAME_REGEX)), 2023 2024 NETWORK_NAME: validation.Optional(validation.Regex( 2025 GCE_RESOURCE_PATH_REGEX)), 2026 2027 SUBNETWORK_NAME: validation.Optional(validation.Regex( 2028 GCE_RESOURCE_NAME_REGEX)), 2029 2030 SESSION_AFFINITY: 2031 validation.Optional(bool) 2032 } 2033 2034 2035class VpcAccessConnector(validation.Validated): 2036 """Class representing the VPC Access connector configuration.""" 2037 2038 ATTRIBUTES = { 2039 VPC_ACCESS_CONNECTOR_NAME: 2040 validation.Regex(VPC_ACCESS_CONNECTOR_NAME_REGEX), 2041 } 2042 2043 2044class AppInclude(validation.Validated): 2045 """Class representing the contents of an included `app.yaml` file. 2046 2047 This class is used for both `builtins` and `includes` directives. 2048 """ 2049 2050 # TODO(user): It probably makes sense to have a scheme where we do a 2051 # deep-copy of fields from `AppInfoExternal` when setting the `ATTRIBUTES` 2052 # here. Right now it's just copypasta. 2053 ATTRIBUTES = { 2054 BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)), 2055 INCLUDES: validation.Optional(validation.Type(list)), 2056 HANDLERS: validation.Optional(validation.Repeated(URLMap), default=[]), 2057 ADMIN_CONSOLE: validation.Optional(AdminConsole), 2058 MANUAL_SCALING: validation.Optional(ManualScaling), 2059 VM: validation.Optional(bool), 2060 VM_SETTINGS: validation.Optional(VmSettings), 2061 BETA_SETTINGS: validation.Optional(BetaSettings), 2062 ENV_VARIABLES: validation.Optional(EnvironmentVariables), 2063 SKIP_FILES: validation.RegexStr(default=SKIP_NO_FILES), 2064 # TODO(user): add `LIBRARIES` here when we have a good story for 2065 # handling contradictory library requests. 2066 } 2067 2068 @classmethod 2069 def MergeManualScaling(cls, appinclude_one, appinclude_two): 2070 """Takes the greater of `<manual_scaling.instances>` from the arguments. 2071 2072 `appinclude_one` is mutated to be the merged result in this process. 2073 2074 Also, this function must be updated if `ManualScaling` gets additional 2075 fields. 2076 2077 Args: 2078 appinclude_one: The first object to merge. The object must have a 2079 `manual_scaling` field that contains a `ManualScaling()`. 2080 appinclude_two: The second object to merge. The object must have a 2081 `manual_scaling` field that contains a `ManualScaling()`. 2082 Returns: 2083 An object that is the result of merging 2084 `appinclude_one.manual_scaling.instances` and 2085 `appinclude_two.manual_scaling.instances`; this is returned as a revised 2086 `appinclude_one` object after the mutations are complete. 2087 """ 2088 2089 def _Instances(appinclude): 2090 """Determines the number of `manual_scaling.instances` sets. 2091 2092 Args: 2093 appinclude: The include for which you want to determine the number of 2094 `manual_scaling.instances` sets. 2095 2096 Returns: 2097 The number of instances as an integer. If the value of 2098 `manual_scaling.instances` evaluates to False (e.g. 0 or None), then 2099 return 0. 2100 """ 2101 if appinclude.manual_scaling: 2102 if appinclude.manual_scaling.instances: 2103 return int(appinclude.manual_scaling.instances) 2104 return 0 2105 2106 # We only want to mutate a param if at least one of the given 2107 # arguments has manual_scaling.instances set. 2108 if _Instances(appinclude_one) or _Instances(appinclude_two): 2109 instances = max(_Instances(appinclude_one), _Instances(appinclude_two)) 2110 appinclude_one.manual_scaling = ManualScaling(instances=str(instances)) 2111 return appinclude_one 2112 2113 @classmethod 2114 def _CommonMergeOps(cls, one, two): 2115 """This function performs common merge operations. 2116 2117 Args: 2118 one: The first object that you want to merge. 2119 two: The second object that you want to merge. 2120 2121 Returns: 2122 An updated `one` object containing all merged data. 2123 """ 2124 # Merge `ManualScaling`. 2125 AppInclude.MergeManualScaling(one, two) 2126 2127 # Merge `AdminConsole` objects. 2128 one.admin_console = AdminConsole.Merge(one.admin_console, 2129 two.admin_console) 2130 2131 # Preserve the specific value of `one.vm` (`None` or `False`) when neither 2132 # are `True`. 2133 one.vm = two.vm or one.vm 2134 2135 # Merge `VmSettings` objects. 2136 one.vm_settings = VmSettings.Merge(one.vm_settings, 2137 two.vm_settings) 2138 2139 # Merge `BetaSettings` objects. 2140 if hasattr(one, 'beta_settings'): 2141 one.beta_settings = BetaSettings.Merge(one.beta_settings, 2142 two.beta_settings) 2143 2144 # Merge `EnvironmentVariables` objects. The values in `two.env_variables` 2145 # override the ones in `one.env_variables` in case of conflict. 2146 one.env_variables = EnvironmentVariables.Merge(one.env_variables, 2147 two.env_variables) 2148 2149 one.skip_files = cls.MergeSkipFiles(one.skip_files, two.skip_files) 2150 2151 return one 2152 2153 @classmethod 2154 def MergeAppYamlAppInclude(cls, appyaml, appinclude): 2155 """Merges an `app.yaml` file with referenced builtins/includes. 2156 2157 Args: 2158 appyaml: The `app.yaml` file that you want to update with `appinclude`. 2159 appinclude: The includes that you want to merge into `appyaml`. 2160 2161 Returns: 2162 An updated `app.yaml` file that includes the directives you specified in 2163 `appinclude`. 2164 """ 2165 # All merge operations should occur in this function or in functions 2166 # referenced from this one. That makes it much easier to understand what 2167 # goes wrong when included files are not merged correctly. 2168 2169 if not appinclude: 2170 return appyaml 2171 2172 # Merge handlers while paying attention to `position` attribute. 2173 if appinclude.handlers: 2174 tail = appyaml.handlers or [] 2175 appyaml.handlers = [] 2176 2177 for h in appinclude.handlers: 2178 if not h.position or h.position == 'head': 2179 appyaml.handlers.append(h) 2180 else: 2181 tail.append(h) 2182 # Get rid of the `position` attribute since we no longer need it, and is 2183 # technically invalid to include in the resulting merged `app.yaml` file 2184 # that will be sent when deploying the application. 2185 h.position = None 2186 2187 appyaml.handlers.extend(tail) 2188 2189 appyaml = cls._CommonMergeOps(appyaml, appinclude) 2190 appyaml.NormalizeVmSettings() 2191 return appyaml 2192 2193 @classmethod 2194 def MergeAppIncludes(cls, appinclude_one, appinclude_two): 2195 """Merges the non-referential state of the provided `AppInclude`. 2196 2197 That is, `builtins` and `includes` directives are not preserved, but any 2198 static objects are copied into an aggregate `AppInclude` object that 2199 preserves the directives of both provided `AppInclude` objects. 2200 2201 `appinclude_one` is updated to be the merged result in this process. 2202 2203 Args: 2204 appinclude_one: First `AppInclude` to merge. 2205 appinclude_two: Second `AppInclude` to merge. 2206 2207 Returns: 2208 `AppInclude` object that is the result of merging the static directives of 2209 `appinclude_one` and `appinclude_two`. An updated version of 2210 `appinclude_one` is returned. 2211 """ 2212 2213 # If one or both `appinclude` objects were `None`, return the object that 2214 # was not `None` or return `None`. 2215 if not appinclude_one or not appinclude_two: 2216 return appinclude_one or appinclude_two 2217 # Now, both `appincludes` are non-`None`. 2218 2219 # Merge handlers. 2220 if appinclude_one.handlers: 2221 if appinclude_two.handlers: 2222 appinclude_one.handlers.extend(appinclude_two.handlers) 2223 else: 2224 appinclude_one.handlers = appinclude_two.handlers 2225 2226 return cls._CommonMergeOps(appinclude_one, appinclude_two) 2227 2228 @staticmethod 2229 def MergeSkipFiles(skip_files_one, skip_files_two): 2230 """Merges two `skip_files` directives. 2231 2232 Args: 2233 skip_files_one: The first `skip_files` element that you want to merge. 2234 skip_files_two: The second `skip_files` element that you want to merge. 2235 2236 Returns: 2237 A list of regular expressions that are merged. 2238 """ 2239 if skip_files_one == SKIP_NO_FILES: 2240 return skip_files_two 2241 if skip_files_two == SKIP_NO_FILES: 2242 return skip_files_one 2243 return validation.RegexStr().Validate( 2244 [skip_files_one, skip_files_two], SKIP_FILES) 2245 # We exploit the handling of RegexStr where regex properties can be 2246 # specified as a list of regexes that are then joined with |. 2247 2248 2249class AppInfoExternal(validation.Validated): 2250 """Class representing users application info. 2251 2252 This class is passed to a `yaml_object` builder to provide the validation 2253 for the application information file format parser. 2254 2255 Attributes: 2256 application: Unique identifier for application. 2257 version: Application's major version. 2258 runtime: Runtime used by application. 2259 api_version: Which version of APIs to use. 2260 source_language: Optional specification of the source language. For example, 2261 you could specify `php-quercus` if this is a Java app that was generated 2262 from PHP source using Quercus. 2263 handlers: List of URL handlers. 2264 default_expiration: Default time delta to use for cache expiration for 2265 all static files, unless they have their own specific `expiration` set. 2266 See the documentation for the `URLMap.expiration` field for more 2267 information. 2268 skip_files: A regular expression object. Files that match this regular 2269 expression will not be uploaded by `appcfg.py`. For example:: 2270 skip_files: | 2271 .svn.*| 2272 #.*# 2273 nobuild_files: A regular expression object. Files that match this regular 2274 expression will not be built into the app. This directive is valid for 2275 Go only. 2276 api_config: URL root and script or servlet path for enhanced API serving. 2277 """ 2278 2279 ATTRIBUTES = { 2280 # Regular expressions for these attributes are defined in 2281 # //apphosting/base/id_util.cc. 2282 APPLICATION: validation.Optional(APPLICATION_RE_STRING), 2283 # An alias for `APPLICATION`. 2284 PROJECT: validation.Optional(APPLICATION_RE_STRING), 2285 SERVICE: validation.Preferred(MODULE, 2286 validation.Optional(MODULE_ID_RE_STRING)), 2287 MODULE: validation.Deprecated(SERVICE, 2288 validation.Optional(MODULE_ID_RE_STRING)), 2289 VERSION: validation.Optional(MODULE_VERSION_ID_RE_STRING), 2290 RUNTIME: validation.Optional(RUNTIME_RE_STRING), 2291 RUNTIME_CHANNEL: validation.Optional(validation.Type(str)), 2292 # A new `api_version` requires a release of the `dev_appserver`, so it 2293 # is ok to hardcode the version names here. 2294 API_VERSION: validation.Optional(API_VERSION_RE_STRING), 2295 MAIN: validation.Optional(_FILES_REGEX), 2296 # The App Engine environment to run this version in. (VM vs. non-VM, etc.) 2297 ENV: validation.Optional(ENV_RE_STRING), 2298 ENDPOINTS_API_SERVICE: validation.Optional(EndpointsApiService), 2299 # The SDK will use this for generated Dockerfiles 2300 # hasattr guard the new Exec() validator temporarily 2301 ENTRYPOINT: validation.Optional( 2302 validation.Exec() if hasattr( 2303 validation, 'Exec') else validation.Type(str)), 2304 RUNTIME_CONFIG: validation.Optional(RuntimeConfig), 2305 INSTANCE_CLASS: validation.Optional(validation.Type(str)), 2306 SOURCE_LANGUAGE: validation.Optional( 2307 validation.Regex(SOURCE_LANGUAGE_RE_STRING)), 2308 AUTOMATIC_SCALING: validation.Optional(AutomaticScaling), 2309 MANUAL_SCALING: validation.Optional(ManualScaling), 2310 BASIC_SCALING: validation.Optional(BasicScaling), 2311 VM: validation.Optional(bool), 2312 VM_SETTINGS: validation.Optional(VmSettings), # Deprecated 2313 BETA_SETTINGS: validation.Optional(BetaSettings), 2314 VM_HEALTH_CHECK: validation.Optional(VmHealthCheck), # Deprecated 2315 HEALTH_CHECK: validation.Optional(HealthCheck), 2316 RESOURCES: validation.Optional(Resources), 2317 LIVENESS_CHECK: validation.Optional(LivenessCheck), 2318 READINESS_CHECK: validation.Optional(ReadinessCheck), 2319 NETWORK: validation.Optional(Network), 2320 VPC_ACCESS_CONNECTOR: validation.Optional(VpcAccessConnector), 2321 ZONES: validation.Optional(validation.Repeated(validation.TYPE_STR)), 2322 BUILTINS: validation.Optional(validation.Repeated(BuiltinHandler)), 2323 INCLUDES: validation.Optional(validation.Type(list)), 2324 HANDLERS: validation.Optional(validation.Repeated(URLMap), default=[]), 2325 LIBRARIES: validation.Optional(validation.Repeated(Library)), 2326 # TODO(user): change to a regex when `validation.Repeated` supports it 2327 SERVICES: validation.Optional(validation.Repeated( 2328 validation.Regex(_SERVICE_RE_STRING))), 2329 DEFAULT_EXPIRATION: validation.Optional(_EXPIRATION_REGEX), 2330 SKIP_FILES: validation.RegexStr(default=DEFAULT_SKIP_FILES), 2331 NOBUILD_FILES: validation.RegexStr(default=DEFAULT_NOBUILD_FILES), 2332 DERIVED_FILE_TYPE: validation.Optional(validation.Repeated( 2333 validation.Options(JAVA_PRECOMPILED, PYTHON_PRECOMPILED))), 2334 ADMIN_CONSOLE: validation.Optional(AdminConsole), 2335 ERROR_HANDLERS: validation.Optional(validation.Repeated(ErrorHandlers)), 2336 BACKENDS: validation.Optional(validation.Repeated( 2337 backendinfo.BackendEntry)), 2338 THREADSAFE: validation.Optional(bool), 2339 DATASTORE_AUTO_ID_POLICY: validation.Optional( 2340 validation.Options(DATASTORE_ID_POLICY_LEGACY, 2341 DATASTORE_ID_POLICY_DEFAULT)), 2342 API_CONFIG: validation.Optional(ApiConfigHandler), 2343 CODE_LOCK: validation.Optional(bool), 2344 ENV_VARIABLES: validation.Optional(EnvironmentVariables), 2345 STANDARD_WEBSOCKET: validation.Optional(bool), 2346 APP_ENGINE_APIS: validation.Optional(bool), 2347 } 2348 2349 def CheckInitialized(self): 2350 """Performs non-regular expression-based validation. 2351 2352 The following are verified: 2353 - At least one URL mapping is provided in the URL mappers. 2354 - The number of URL mappers doesn't exceed `MAX_URL_MAPS`. 2355 - The major version does not contain the string `-dot-`. 2356 - If `api_endpoints` are defined, an `api_config` stanza must be 2357 defined. 2358 - If the `runtime` is `python27` and `threadsafe` is set, then no CGI 2359 handlers can be used. 2360 - The version name doesn't start with `BUILTIN_NAME_PREFIX`. 2361 - If `redirect_http_response_code` exists, it is in the list of valid 2362 300s. 2363 - Module and service aren't both set. Services were formerly known as 2364 modules. 2365 2366 Raises: 2367 DuplicateLibrary: If `library_name` is specified more than once. 2368 MissingURLMapping: If no `URLMap` object is present in the object. 2369 TooManyURLMappings: If there are too many `URLMap` entries. 2370 MissingApiConfig: If `api_endpoints` exists without an `api_config`. 2371 MissingThreadsafe: If `threadsafe` is not set but the runtime requires it. 2372 ThreadsafeWithCgiHandler: If the `runtime` is `python27`, `threadsafe` is 2373 set and CGI handlers are specified. 2374 TooManyScalingSettingsError: If more than one scaling settings block is 2375 present. 2376 RuntimeDoesNotSupportLibraries: If the libraries clause is used for a 2377 runtime that does not support it, such as `python25`. 2378 """ 2379 super(AppInfoExternal, self).CheckInitialized() 2380 if self.runtime is None and not self.IsVm(): 2381 raise appinfo_errors.MissingRuntimeError( 2382 'You must specify a "runtime" field for non-vm applications.') 2383 elif self.runtime is None: 2384 # Default optional to custom (we don't do that in attributes just so 2385 # we know that it's been defaulted) 2386 self.runtime = 'custom' 2387 if self.handlers and len(self.handlers) > MAX_URL_MAPS: 2388 raise appinfo_errors.TooManyURLMappings( 2389 'Found more than %d URLMap entries in application configuration' % 2390 MAX_URL_MAPS) 2391 2392 vm_runtime_python27 = ( 2393 self.runtime == 'vm' and 2394 (hasattr(self, 'vm_settings') and 2395 self.vm_settings and 2396 self.vm_settings.get('vm_runtime') == 'python27') or 2397 (hasattr(self, 'beta_settings') and 2398 self.beta_settings and 2399 self.beta_settings.get('vm_runtime') == 'python27')) 2400 2401 if (self.threadsafe is None and 2402 (self.runtime == 'python27' or vm_runtime_python27)): 2403 raise appinfo_errors.MissingThreadsafe( 2404 'threadsafe must be present and set to a true or false YAML value') 2405 2406 if self.auto_id_policy == DATASTORE_ID_POLICY_LEGACY: 2407 datastore_auto_ids_url = ('http://developers.google.com/' 2408 'appengine/docs/python/datastore/' 2409 'entities#Kinds_and_Identifiers') 2410 appcfg_auto_ids_url = ('http://developers.google.com/appengine/docs/' 2411 'python/config/appconfig#auto_id_policy') 2412 logging.warning( 2413 "You have set the datastore auto_id_policy to 'legacy'. It is " 2414 "recommended that you select 'default' instead.\n" 2415 "Legacy auto ids are deprecated. You can continue to allocate\n" 2416 "legacy ids manually using the allocate_ids() API functions.\n" 2417 "For more information see:\n" 2418 + datastore_auto_ids_url + '\n' + appcfg_auto_ids_url + '\n') 2419 2420 if (hasattr(self, 'beta_settings') and self.beta_settings 2421 and self.beta_settings.get('source_reference')): 2422 ValidateCombinedSourceReferencesString( 2423 self.beta_settings.get('source_reference')) 2424 2425 if self.libraries: 2426 if not (vm_runtime_python27 or self.runtime == 'python27'): 2427 raise appinfo_errors.RuntimeDoesNotSupportLibraries( 2428 'libraries entries are only supported by the "python27" runtime') 2429 2430 library_names = [library.name for library in self.libraries] 2431 for library_name in library_names: 2432 if library_names.count(library_name) > 1: 2433 raise appinfo_errors.DuplicateLibrary( 2434 'Duplicate library entry for %s' % library_name) 2435 2436 if self.version and self.version.find(ALTERNATE_HOSTNAME_SEPARATOR) != -1: 2437 raise validation.ValidationError( 2438 'Version "%s" cannot contain the string "%s"' % ( 2439 self.version, ALTERNATE_HOSTNAME_SEPARATOR)) 2440 if self.version and self.version.startswith(BUILTIN_NAME_PREFIX): 2441 raise validation.ValidationError( 2442 ('Version "%s" cannot start with "%s" because it is a ' 2443 'reserved version name prefix.') % (self.version, 2444 BUILTIN_NAME_PREFIX)) 2445 if self.handlers: 2446 api_endpoints = [handler.url for handler in self.handlers 2447 if handler.GetHandlerType() == HANDLER_API_ENDPOINT] 2448 if api_endpoints and not self.api_config: 2449 raise appinfo_errors.MissingApiConfig( 2450 'An api_endpoint handler was specified, but the required ' 2451 'api_config stanza was not configured.') 2452 if self.threadsafe and self.runtime == 'python27': 2453 # VMEngines can handle python25 handlers, so we don't include 2454 # vm_runtime_python27 in the if statement above. 2455 for handler in self.handlers: 2456 if (handler.script and (handler.script.endswith('.py') or 2457 '/' in handler.script)): 2458 raise appinfo_errors.ThreadsafeWithCgiHandler( 2459 'threadsafe cannot be enabled with CGI handler: %s' % 2460 handler.script) 2461 if sum([bool(self.automatic_scaling), 2462 bool(self.manual_scaling), 2463 bool(self.basic_scaling)]) > 1: 2464 raise appinfo_errors.TooManyScalingSettingsError( 2465 "There may be only one of 'automatic_scaling', 'manual_scaling', " 2466 "or 'basic_scaling'.") 2467 2468 def GetAllLibraries(self): 2469 """Returns a list of all `Library` instances active for this configuration. 2470 2471 Returns: 2472 The list of active `Library` instances for this configuration. This 2473 includes directly-specified libraries as well as any required 2474 dependencies. 2475 """ 2476 if not self.libraries: 2477 return [] 2478 2479 library_names = set(library.name for library in self.libraries) 2480 required_libraries = [] 2481 2482 for library in self.libraries: 2483 for required_name, required_version in REQUIRED_LIBRARIES.get( 2484 (library.name, library.version), []): 2485 if required_name not in library_names: 2486 required_libraries.append(Library(name=required_name, 2487 version=required_version)) 2488 2489 return [Library(**library.ToDict()) 2490 for library in self.libraries + required_libraries] 2491 2492 def GetNormalizedLibraries(self): 2493 """Returns a list of normalized `Library` instances for this configuration. 2494 2495 Returns: 2496 The list of active `Library` instances for this configuration. This 2497 includes directly-specified libraries, their required dependencies, and 2498 any libraries enabled by default. Any libraries with `latest` as their 2499 version will be replaced with the latest available version. 2500 """ 2501 libraries = self.GetAllLibraries() 2502 enabled_libraries = set(library.name for library in libraries) 2503 for library in _SUPPORTED_LIBRARIES: 2504 if library.default_version and library.name not in enabled_libraries: 2505 libraries.append(Library(name=library.name, 2506 version=library.default_version)) 2507 return libraries 2508 2509 def ApplyBackendSettings(self, backend_name): 2510 """Applies settings from the indicated backend to the `AppInfoExternal`. 2511 2512 Backend entries can contain directives that modify other parts of the 2513 `app.yaml` file, such as the `start` directive, which adds a handler for the 2514 start request. This method performs those modifications. 2515 2516 Args: 2517 backend_name: The name of a backend that is defined in the `backends` 2518 directive. 2519 2520 Raises: 2521 BackendNotFound: If the indicated backend was not listed in the 2522 `backends` directive. 2523 DuplicateBackend: If the backend is found more than once in the `backends` 2524 directive. 2525 """ 2526 if backend_name is None: 2527 return 2528 2529 if self.backends is None: 2530 raise appinfo_errors.BackendNotFound 2531 2532 self.version = backend_name 2533 2534 match = None 2535 for backend in self.backends: 2536 if backend.name != backend_name: 2537 continue 2538 if match: 2539 raise appinfo_errors.DuplicateBackend 2540 else: 2541 match = backend 2542 2543 if match is None: 2544 raise appinfo_errors.BackendNotFound 2545 2546 if match.start is None: 2547 return 2548 2549 start_handler = URLMap(url=_START_PATH, script=match.start) 2550 self.handlers.insert(0, start_handler) 2551 2552 def GetEffectiveRuntime(self): 2553 """Returns the app's runtime, resolving VMs to the underlying `vm_runtime`. 2554 2555 Returns: 2556 The effective runtime: The value of `beta/vm_settings.vm_runtime` if 2557 `runtime` is `vm`, or `runtime` otherwise. 2558 """ 2559 if (self.runtime == 'vm' and hasattr(self, 'vm_settings') 2560 and self.vm_settings is not None): 2561 return self.vm_settings.get('vm_runtime') 2562 if (self.runtime == 'vm' and hasattr(self, 'beta_settings') 2563 and self.beta_settings is not None): 2564 return self.beta_settings.get('vm_runtime') 2565 return self.runtime 2566 2567 def SetEffectiveRuntime(self, runtime): 2568 """Sets the runtime while respecting vm runtimes rules for runtime settings. 2569 2570 Args: 2571 runtime: The runtime to use. 2572 """ 2573 if self.IsVm(): 2574 if not self.vm_settings: 2575 self.vm_settings = VmSettings() 2576 2577 # Patch up vm runtime setting. Copy `runtime` to `vm_runtime` and set 2578 # runtime to the string `vm`. 2579 self.vm_settings['vm_runtime'] = runtime 2580 self.runtime = 'vm' 2581 else: 2582 self.runtime = runtime 2583 2584 def NormalizeVmSettings(self): 2585 """Normalizes VM settings.""" 2586 # NOTE(user): In the input files, `vm` is not a type of runtime, but 2587 # rather is specified as `vm: true|false`. In the code, `vm` is represented 2588 # as a value of `AppInfoExternal.runtime`. 2589 # NOTE(user): This hack is only being applied after the parsing of 2590 # `AppInfoExternal`. If the `vm` attribute can ever be specified in the 2591 # `AppInclude`, then this processing will need to be done there too. 2592 if self.IsVm(): 2593 if not self.vm_settings: 2594 self.vm_settings = VmSettings() 2595 2596 if 'vm_runtime' not in self.vm_settings: 2597 self.SetEffectiveRuntime(self.runtime) 2598 2599 # Copy fields that are automatically added by the SDK or this class 2600 # to `beta_settings`. 2601 if hasattr(self, 'beta_settings') and self.beta_settings: 2602 # Only copy if `beta_settings` already exists, because we have logic in 2603 # `appversion.py` to discard all of `vm_settings` if anything is in 2604 # `beta_settings`. So we won't create an empty one just to add these 2605 # fields. 2606 for field in ['vm_runtime', 2607 'has_docker_image', 2608 'image', 2609 'module_yaml_path']: 2610 if field not in self.beta_settings and field in self.vm_settings: 2611 self.beta_settings[field] = self.vm_settings[field] 2612 2613 # TODO(user): `env` replaces `vm`. Remove `vm` when field is removed. 2614 def IsVm(self): 2615 return (self.vm or 2616 self.env in ['2', 'flex', 'flexible']) 2617 2618 2619def ValidateHandlers(handlers, is_include_file=False): 2620 """Validates a list of handler (`URLMap`) objects. 2621 2622 Args: 2623 handlers: A list of a handler (`URLMap`) objects. 2624 is_include_file: If this argument is set to `True`, the handlers that are 2625 added as part of the `includes` directive are validated. 2626 """ 2627 if not handlers: 2628 return 2629 2630 for handler in handlers: 2631 handler.FixSecureDefaults() 2632 handler.WarnReservedURLs() 2633 if not is_include_file: 2634 handler.ErrorOnPositionForAppInfo() 2635 2636 2637def LoadSingleAppInfo(app_info): 2638 """Loads a single `AppInfo` object where one and only one is expected. 2639 2640 This method validates that the values in the `AppInfo` match the 2641 validators that are defined in this file, in particular, 2642 `AppInfoExternal.ATTRIBUTES`. 2643 2644 Args: 2645 app_info: A file-like object or string. If the argument is a string, the 2646 argument is parsed as a configuration file. If the argument is a 2647 file-like object, the data is read, then parsed. 2648 2649 Returns: 2650 An instance of `AppInfoExternal` as loaded from a YAML file. 2651 2652 Raises: 2653 ValueError: If a specified service is not valid. 2654 EmptyConfigurationFile: If there are no documents in YAML file. 2655 MultipleConfigurationFile: If more than one document exists in the YAML 2656 file. 2657 DuplicateBackend: If a backend is found more than once in the `backends` 2658 directive. 2659 yaml_errors.EventError: If the `app.yaml` file fails validation. 2660 appinfo_errors.MultipleProjectNames: If the `app.yaml` file has both an 2661 `application` directive and a `project` directive. 2662 """ 2663 builder = yaml_object.ObjectBuilder(AppInfoExternal) 2664 handler = yaml_builder.BuilderHandler(builder) 2665 listener = yaml_listener.EventListener(handler) 2666 listener.Parse(app_info) 2667 2668 app_infos = handler.GetResults() 2669 if len(app_infos) < 1: 2670 raise appinfo_errors.EmptyConfigurationFile() 2671 if len(app_infos) > 1: 2672 raise appinfo_errors.MultipleConfigurationFile() 2673 2674 appyaml = app_infos[0] 2675 ValidateHandlers(appyaml.handlers) 2676 if appyaml.builtins: 2677 BuiltinHandler.Validate(appyaml.builtins, appyaml.runtime) 2678 2679 # Allow `project: name` as an alias for `application: name`. If found, we 2680 # change the `project` field to `None`. (Deleting it would make a distinction 2681 # between loaded and constructed `AppInfoExternal` objects, since the latter 2682 # would still have the project field.) 2683 if appyaml.application and appyaml.project: 2684 raise appinfo_errors.MultipleProjectNames( 2685 'Specify one of "application: name" or "project: name"') 2686 elif appyaml.project: 2687 appyaml.application = appyaml.project 2688 appyaml.project = None 2689 2690 appyaml.NormalizeVmSettings() 2691 return appyaml 2692 2693 2694class AppInfoSummary(validation.Validated): 2695 """This class contains only basic summary information about an app. 2696 2697 This class is used to pass back information about the newly created app to 2698 users after a new version has been created. 2699 """ 2700 # NOTE(user): Before you consider adding anything to this YAML definition, 2701 # you must solve the issue that old SDK versions will try to parse this new 2702 # value with the old definition and fail. Basically we are stuck with this 2703 # definition for the time being. The parsing of the value is done in 2704 ATTRIBUTES = { 2705 APPLICATION: APPLICATION_RE_STRING, 2706 MAJOR_VERSION: MODULE_VERSION_ID_RE_STRING, 2707 MINOR_VERSION: validation.TYPE_LONG 2708 } 2709 2710 2711def LoadAppInclude(app_include): 2712 """Loads a single `AppInclude` object where one and only one is expected. 2713 2714 Args: 2715 app_include: A file-like object or string. The argument is set to a string, 2716 the argument is parsed as a configuration file. If the argument is set 2717 to a file-like object, the data is read and parsed. 2718 2719 Returns: 2720 An instance of `AppInclude` as loaded from a YAML file. 2721 2722 Raises: 2723 EmptyConfigurationFile: If there are no documents in the YAML file. 2724 MultipleConfigurationFile: If there is more than one document in the YAML 2725 file. 2726 """ 2727 builder = yaml_object.ObjectBuilder(AppInclude) 2728 handler = yaml_builder.BuilderHandler(builder) 2729 listener = yaml_listener.EventListener(handler) 2730 listener.Parse(app_include) 2731 2732 includes = handler.GetResults() 2733 if len(includes) < 1: 2734 raise appinfo_errors.EmptyConfigurationFile() 2735 if len(includes) > 1: 2736 raise appinfo_errors.MultipleConfigurationFile() 2737 2738 includeyaml = includes[0] 2739 if includeyaml.handlers: 2740 for handler in includeyaml.handlers: 2741 handler.FixSecureDefaults() 2742 handler.WarnReservedURLs() 2743 if includeyaml.builtins: 2744 BuiltinHandler.Validate(includeyaml.builtins) 2745 2746 return includeyaml 2747 2748 2749def ParseExpiration(expiration): 2750 """Parses an expiration delta string. 2751 2752 Args: 2753 expiration: String that matches `_DELTA_REGEX`. 2754 2755 Returns: 2756 Time delta in seconds. 2757 """ 2758 delta = 0 2759 for match in re.finditer(_DELTA_REGEX, expiration): 2760 amount = int(match.group(1)) 2761 units = _EXPIRATION_CONVERSIONS.get(match.group(2).lower(), 1) 2762 delta += amount * units 2763 return delta 2764 2765 2766##################################################################### 2767# These regexps must be the same as those in: 2768# - apphosting/api/app_config/request_validator.cc 2769# - java/com/google/appengine/tools/admin/AppVersionUpload.java 2770# - java/com/google/apphosting/admin/legacy/LegacyAppInfo.java 2771 2772# Forbid `.`, `..`, and leading `-`, `_ah/` or `/` 2773_file_path_negative_1_re = re.compile(r'\.\.|^\./|\.$|/\./|^-|^_ah/|^/') 2774 2775# Forbid `//` and trailing `/` 2776_file_path_negative_2_re = re.compile(r'//|/$') 2777 2778# Forbid any use of space other than in the middle of a directory 2779# or file name. Forbid line feeds and carriage returns. 2780_file_path_negative_3_re = re.compile(r'^ | $|/ | /|\r|\n') 2781 2782 2783# (erinjerison) Lint seems to think I'm specifying the word "character" as an 2784# argument. This isn't the case; it's part of a list to enable the list to 2785# build properly. Disabling it for now. 2786# pylint: disable=g-doc-args 2787def ValidFilename(filename): 2788 """Determines if a file name is valid. 2789 2790 Args: 2791 filename: The file name to validate. The file name must be a valid file 2792 name: 2793 - It must only contain letters, numbers, and the following special 2794 characters: `@`, `_`, `+`, `/` `$`, `.`, `-`, or '~'. 2795 - It must be less than 256 characters. 2796 - It must not contain `/./`, `/../`, or `//`. 2797 - It must not end in `/`. 2798 - All spaces must be in the middle of a directory or file name. 2799 2800 Returns: 2801 An error string if the file name is invalid. `''` is returned if the file 2802 name is valid. 2803 """ 2804 if not filename: 2805 return 'Filename cannot be empty' 2806 if len(filename) > 1024: 2807 return 'Filename cannot exceed 1024 characters: %s' % filename 2808 if _file_path_negative_1_re.search(filename) is not None: 2809 return ('Filename cannot contain "." or ".." ' 2810 'or start with "-" or "_ah/": %s' % 2811 filename) 2812 if _file_path_negative_2_re.search(filename) is not None: 2813 return 'Filename cannot have trailing / or contain //: %s' % filename 2814 if _file_path_negative_3_re.search(filename) is not None: 2815 return 'Any spaces must be in the middle of a filename: %s' % filename 2816 return '' 2817