1# Copyright 2015 Google Inc. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Support for externalized runtimes.""" 15 16from __future__ import absolute_import 17from __future__ import division 18from __future__ import print_function 19 20import json 21import logging 22import os 23import subprocess 24import sys 25import threading 26 27from . import comm 28import ruamel.yaml as yaml 29from six.moves import input 30 31# Try importing these modules from the cloud SDK first. 32try: 33 from googlecloudsdk.third_party.appengine.admin.tools.conversion import schema 34except ImportError: 35 from yaml_conversion import schema 36 37try: 38 from googlecloudsdk.third_party.py27 import py27_subprocess as subprocess 39except ImportError: 40 import subprocess 41 42 43WRITING_FILE_MESSAGE = 'Writing [{0}] to [{1}].' 44FILE_EXISTS_MESSAGE = 'Not writing [{0}], it already exists.' 45 46 47class Error(Exception): 48 """Base class for exceptions in this module.""" 49 50 51class PluginInvocationFailed(Error): 52 """Raised when a plugin invocation returns a non-zero result code.""" 53 54 55class InvalidRuntimeDefinition(Error): 56 """Raised when an inconsistency is found in the runtime definition.""" 57 pass 58 59 60class Params(object): 61 """Parameters passed to the the runtime module Fingerprint() methods. 62 63 Attributes: 64 appinfo: (apphosting.api.appinfo.AppInfoExternal or None) The parsed 65 app.yaml file for the module if it exists. 66 custom: (bool) True if the Configurator should generate a custom runtime. 67 runtime (str or None) Runtime (alias allowed) that should be enforced. 68 deploy: (bool) True if this is happening from deployment. 69 """ 70 71 def __init__(self, appinfo=None, custom=False, runtime=None, deploy=False): 72 self.appinfo = appinfo 73 self.custom = custom 74 self.runtime = runtime 75 self.deploy = deploy 76 77 def ToDict(self): 78 """Returns the object converted to a dictionary. 79 80 Returns: 81 ({str: object}) A dictionary that can be converted to json using 82 json.dump(). 83 """ 84 return {'appinfo': self.appinfo and self.appinfo.ToDict(), 85 'custom': self.custom, 86 'runtime': self.runtime, 87 'deploy': self.deploy} 88 89 90class Configurator(object): 91 """Base configurator class. 92 93 Configurators generate config files for specific classes of runtimes. They 94 are returned by the Fingerprint functions in the runtimes sub-package after 95 a successful match of the runtime's heuristics. 96 """ 97 98 def CollectData(self): 99 """Collect all information on this application. 100 101 This is called after the runtime type is detected and may gather 102 additional information from the source code and from the user. Whereas 103 performing user queries during detection is deprecated, user queries are 104 allowed in CollectData(). 105 106 The base class version of this does nothing. 107 """ 108 109 def Prebuild(self): 110 """Run additional build behavior before the application is deployed. 111 112 This is called after the runtime type has been detected and after any 113 additional data has been collected. 114 115 The base class version of this does nothing. 116 """ 117 118 def GenerateConfigs(self): 119 """Generate all configuration files for the module. 120 121 Generates config files in the current working directory. 122 123 Returns: 124 (callable()) Function that will delete all of the generated files. 125 """ 126 raise NotImplementedError() 127 128 129class ExecutionEnvironment(object): 130 """An interface for providing system functionality to a runtime definition. 131 132 Abstract interface containing methods for console IO and system 133 introspection. This exists to allow gcloud to inject special functionality. 134 """ 135 136 def GetPythonExecutable(self): 137 """Returns the full path of the python executable (str).""" 138 raise NotImplementedError() 139 140 def CanPrompt(self): 141 """Returns true """ 142 raise NotImplementedError() 143 144 def PromptResponse(self, message): 145 raise NotImplementedError() 146 147 148 def Print(self, message): 149 """Print a message to the console. 150 151 Args: 152 message: (str) 153 """ 154 raise NotImplementedError() 155 156 157class DefaultExecutionEnvironment(ExecutionEnvironment): 158 """Standard implementation of the ExecutionEnvironment.""" 159 160 def GetPythonExecutable(self): 161 return sys.executable 162 163 def CanPrompt(self): 164 return sys.stdin.isatty() 165 166 def PromptResponse(self, message): 167 sys.stdout.write(message) 168 sys.stdout.flush() 169 return input('> ') 170 171 def Print(self, message): 172 print(message) 173 174 175class ExternalRuntimeConfigurator(Configurator): 176 """Configurator for general externalized runtimes. 177 178 Attributes: 179 runtime: (ExternalizedRuntime) The runtime that produced this. 180 params: (Params) Runtime parameters. 181 data: ({str: object, ...} or None) Optional dictionary of runtime data 182 passed back through a runtime_parameters message. 183 generated_appinfo: ({str: object, ...} or None) Generated appinfo if any 184 is produced by the runtime. 185 path: (str) Path to the user's source directory. 186 """ 187 188 def __init__(self, runtime, params, data, generated_appinfo, path, env): 189 """Constructor. 190 191 Args: 192 runtime: (ExternalizedRuntime) The runtime that produced this. 193 params: (Params) Runtime parameters. 194 data: ({str: object, ...} or None) Optional dictionary of runtime data 195 passed back through a runtime_parameters message. 196 generated_appinfo: ({str: object, ...} or None) Optional dictionary 197 representing the contents of app.yaml if the runtime produces this. 198 path: (str) Path to the user's source directory. 199 env: (ExecutionEnvironment) 200 """ 201 self.runtime = runtime 202 self.params = params 203 self.data = data 204 if generated_appinfo: 205 206 # Add env: flex if we don't have an "env" field. 207 self.generated_appinfo = {} 208 if not 'env' in generated_appinfo: 209 self.generated_appinfo['env'] = 'flex' 210 211 # And then update with the values provided by the runtime def. 212 self.generated_appinfo.update(generated_appinfo) 213 else: 214 self.generated_appinfo = None 215 self.path = path 216 self.env = env 217 218 def MaybeWriteAppYaml(self): 219 """Generates the app.yaml file if it doesn't already exist.""" 220 221 if not self.generated_appinfo: 222 return 223 224 notify = logging.info if self.params.deploy else self.env.Print 225 # TODO(user): The config file need not be named app.yaml. We need to 226 # pass the appinfo file name in through params. and use it here. 227 filename = os.path.join(self.path, 'app.yaml') 228 229 # Don't generate app.yaml if we've already got it. We consider the 230 # presence of appinfo to be an indicator of the existence of app.yaml as 231 # well as the existence of the file itself because this helps with 232 # testability, as well as preventing us from writing the file if another 233 # config file is being used. 234 if self.params.appinfo or os.path.exists(filename): 235 notify(FILE_EXISTS_MESSAGE.format('app.yaml')) 236 return 237 238 notify(WRITING_FILE_MESSAGE.format('app.yaml', self.path)) 239 with open(filename, 'w') as f: 240 yaml.safe_dump(self.generated_appinfo, f, default_flow_style=False) 241 242 def SetGeneratedAppInfo(self, generated_appinfo): 243 """Sets the generated appinfo.""" 244 self.generated_appinfo = generated_appinfo 245 246 def CollectData(self): 247 self.runtime.CollectData(self) 248 249 def Prebuild(self): 250 self.runtime.Prebuild(self) 251 252 def GenerateConfigs(self): 253 self.MaybeWriteAppYaml() 254 255 # At this point, if we have don't have appinfo, but we do have generated 256 # appinfo, we want to use the generated appinfo and pass it to config 257 # generation. 258 if not self.params.appinfo and self.generated_appinfo: 259 self.params.appinfo = comm.dict_to_object(self.generated_appinfo) 260 return self.runtime.GenerateConfigs(self) 261 262 def GenerateConfigData(self): 263 self.MaybeWriteAppYaml() 264 265 # At this point, if we have don't have appinfo, but we do have generated 266 # appinfo, we want to use the generated appinfo and pass it to config 267 # generation. 268 if not self.params.appinfo and self.generated_appinfo: 269 self.params.appinfo = comm.dict_to_object(self.generated_appinfo) 270 return self.runtime.GenerateConfigData(self) 271 272 273def _NormalizePath(basedir, pathname): 274 """Get the absolute path from a unix-style relative path. 275 276 Args: 277 basedir: (str) Platform-specific encoding of the base directory. 278 pathname: (str) A unix-style (forward slash separated) path relative to 279 the runtime definition root directory. 280 281 Returns: 282 (str) An absolute path conforming to the conventions of the operating 283 system. Note: in order for this to work, 'pathname' must not contain 284 any characters with special meaning in any of the targeted operating 285 systems. Keep those names simple. 286 """ 287 components = pathname.split('/') 288 return os.path.join(basedir, *components) 289 290 291class GeneratedFile(object): 292 """Wraps the name and contents of a generated file.""" 293 294 def __init__(self, filename, contents): 295 """Constructor. 296 297 Args: 298 filename: (str) Unix style file path relative to the target source 299 directory. 300 contents: (str) File contents. 301 """ 302 self.filename = filename 303 self.contents = contents 304 305 def WriteTo(self, dest_dir, notify): 306 """Write the file to the destination directory. 307 308 Args: 309 dest_dir: (str) Destination directory. 310 notify: (callable(str)) Function to notify the user. 311 312 Returns: 313 (str or None) The full normalized path name of the destination file, 314 None if it wasn't generated because it already exists. 315 """ 316 path = _NormalizePath(dest_dir, self.filename) 317 if not os.path.exists(path): 318 notify(WRITING_FILE_MESSAGE.format(self.filename, dest_dir)) 319 with open(path, 'w') as f: 320 f.write(self.contents) 321 return path 322 else: 323 notify(FILE_EXISTS_MESSAGE.format(self.filename)) 324 325 return None 326 327 328class PluginResult(object): 329 330 def __init__(self): 331 self.exit_code = -1 332 self.runtime_data = None 333 self.generated_appinfo = None 334 self.docker_context = None 335 self.files = [] 336 337 338class _Collector(object): 339 """Manages a PluginResult in a thread-safe context.""" 340 341 def __init__(self): 342 self.result = PluginResult() 343 self.lock = threading.Lock() 344 345 346_LOG_FUNCS = { 347 'info': logging.info, 348 'error': logging.error, 349 'warn': logging.warn, 350 'debug': logging.debug 351} 352 353# A section consisting only of scripts. 354_EXEC_SECTION = schema.Message( 355 python=schema.Value(converter=str)) 356 357_RUNTIME_SCHEMA = schema.Message( 358 name=schema.Value(converter=str), 359 description=schema.Value(converter=str), 360 author=schema.Value(converter=str), 361 api_version=schema.Value(converter=str), 362 generate_configs=schema.Message( 363 python=schema.Value(converter=str), 364 files_to_copy=schema.RepeatedField(element=schema.Value(converter=str)), 365 ), 366 detect=_EXEC_SECTION, 367 collect_data=_EXEC_SECTION, 368 prebuild=_EXEC_SECTION, 369 postbuild=_EXEC_SECTION) 370 371_MISSING_FIELD_ERROR = 'Missing [{0}] field in [{1}] message' 372_NO_DEFAULT_ERROR = ('User input requested: [{0}] while running ' 373 'non-interactive with no default specified.') 374 375 376class ExternalizedRuntime(object): 377 """Encapsulates an externalized runtime.""" 378 379 def __init__(self, path, config, env): 380 """ 381 Args: 382 path: (str) Path to the root of the runtime definition. 383 config: ({str: object, ...}) The runtime definition configuration (from 384 runtime.yaml). 385 env: (ExecutionEnvironment) 386 """ 387 388 self.root = path 389 self.env = env 390 try: 391 # Do validation up front, after this we can assume all of the types are 392 # correct. 393 self.config = _RUNTIME_SCHEMA.ConvertValue(config) 394 except ValueError as ex: 395 raise InvalidRuntimeDefinition( 396 'Invalid runtime definition: {0}'.format(ex.message)) 397 398 @property 399 def name(self): 400 return self.config.get('name', 'unnamed') 401 402 @staticmethod 403 def Load(path, env): 404 """Loads the externalized runtime from the specified path. 405 406 Args: 407 path: (str) root directory of the runtime definition. Should 408 contain a "runtime.yaml" file. 409 410 Returns: 411 (ExternalizedRuntime) 412 """ 413 with open(os.path.join(path, 'runtime.yaml')) as f: 414 return ExternalizedRuntime(path, yaml.load(f), env) 415 416 def _ProcessPluginStderr(self, section_name, stderr): 417 """Process the standard error stream of a plugin. 418 419 Standard error output is just written to the log at "warning" priority and 420 otherwise ignored. 421 422 Args: 423 section_name: (str) Section name, to be attached to log messages. 424 stderr: (file) Process standard error stream. 425 """ 426 while True: 427 line = stderr.readline() 428 if not line: 429 break 430 logging.warn('%s: %s' % (section_name, line.rstrip())) 431 432 def _ProcessMessage(self, plugin_stdin, message, result, params, 433 runtime_data): 434 """Process a message received from the plugin. 435 436 Args: 437 plugin_stdin: (file) The standard input stream of the plugin process. 438 message: ({str: object, ...}) The message (this maps directly to the 439 message's json object). 440 result: (PluginResult) A result object in which to store data collected 441 from some types of message. 442 params: (Params) Parameters passed in through the 443 fingerprinter. 444 runtime_data: (object or None) Arbitrary runtime data obtained from the 445 "detect" plugin. This will be None if we are processing a message for 446 the detect plugin itself or if no runtime data was provided. 447 """ 448 449 def SendResponse(response): 450 json.dump(response, plugin_stdin) 451 plugin_stdin.write('\n') 452 plugin_stdin.flush() 453 454 msg_type = message.get('type') 455 if msg_type is None: 456 logging.error('Missing type in message: %0.80s' % str(message)) 457 elif msg_type in _LOG_FUNCS: 458 _LOG_FUNCS[msg_type](message.get('message')) 459 elif msg_type == 'runtime_parameters': 460 try: 461 result.runtime_data = message['runtime_data'] 462 except KeyError: 463 logging.error(_MISSING_FIELD_ERROR.format('runtime_data', msg_type)) 464 result.generated_appinfo = message.get('appinfo') 465 elif msg_type == 'gen_file': 466 try: 467 # TODO(user): deal with 'encoding' 468 filename = message['filename'] 469 contents = message['contents'] 470 result.files.append(GeneratedFile(filename, contents)) 471 except KeyError as ex: 472 logging.error(_MISSING_FIELD_ERROR.format(ex, msg_type)) 473 elif msg_type == 'get_config': 474 response = {'type': 'get_config_response', 475 'params': params.ToDict(), 476 'runtime_data': runtime_data} 477 SendResponse(response) 478 elif msg_type == 'query_user': 479 try: 480 prompt = message['prompt'] 481 except KeyError as ex: 482 logging.error(_MISSING_FIELD_ERROR.format('prompt', msg_type)) 483 return 484 default = message.get('default') 485 486 if self.env.CanPrompt(): 487 if default: 488 message = '{0} [{1}]: '.format(prompt, default) 489 else: 490 message = prompt + ':' 491 result = self.env.PromptResponse(message) 492 else: 493 # TODO(user): Support the "id" field once there is a way to pass 494 # these through. 495 if default is not None: 496 result = default 497 else: 498 result = '' 499 logging.error(_NO_DEFAULT_ERROR.format(prompt)) 500 501 SendResponse({'type': 'query_user_response', 'result': result}) 502 elif msg_type == 'set_docker_context': 503 try: 504 result.docker_context = message['path'] 505 except KeyError: 506 logging.error(_MISSING_FIELD_ERROR.format('path', msg_type)) 507 return 508 # TODO(user): implement remaining message types. 509 else: 510 logging.error('Unknown message type %s' % msg_type) 511 512 def _ProcessPluginPipes(self, section_name, proc, result, params, 513 runtime_data): 514 """Process the standard output and input streams of a plugin.""" 515 while True: 516 line = proc.stdout.readline() 517 if not line: 518 break 519 520 # Parse and process the message. 521 try: 522 message = json.loads(line) 523 self._ProcessMessage(proc.stdin, message, result, params, runtime_data) 524 except ValueError: 525 # Unstructured lines get logged as "info". 526 logging.info('%s: %s' % (section_name, line.rstrip())) 527 528 def RunPlugin(self, section_name, plugin_spec, params, args=None, 529 valid_exit_codes=(0,), 530 runtime_data=None): 531 """Run a plugin. 532 533 Args: 534 section_name: (str) Name of the config section that the plugin spec is 535 from. 536 plugin_spec: ({str: str, ...}) A dictionary mapping plugin locales to 537 script names 538 params: (Params or None) Parameters for the plugin. 539 args: ([str, ...] or None) Command line arguments for the plugin. 540 valid_exit_codes: (int, ...) Exit codes that will be accepted without 541 raising an exception. 542 runtime_data: ({str: object, ...}) A dictionary of runtime data passed 543 back from detect. 544 545 Returns: 546 (PluginResult) A bundle of the exit code and data produced by the plugin. 547 548 Raises: 549 PluginInvocationFailed: The plugin terminated with a non-zero exit code. 550 """ 551 # TODO(user): Support other script types. 552 if 'python' in plugin_spec: 553 normalized_path = _NormalizePath(self.root, plugin_spec['python']) 554 555 # We're sharing 'result' with the output collection thread, we can get 556 # away with this without locking because we pass it into the thread at 557 # creation and do not use it again until after we've joined the thread. 558 result = PluginResult() 559 p = subprocess.Popen([self.env.GetPythonExecutable(), normalized_path] + 560 (args if args else []), 561 stdout=subprocess.PIPE, 562 stdin=subprocess.PIPE, 563 stderr=subprocess.PIPE) 564 stderr_thread = threading.Thread(target=self._ProcessPluginStderr, 565 args=(section_name, p.stderr,)) 566 stderr_thread.start() 567 stdout_thread = threading.Thread(target=self._ProcessPluginPipes, 568 args=(section_name, p, result, 569 params, runtime_data)) 570 stdout_thread.start() 571 572 stderr_thread.join() 573 stdout_thread.join() 574 exit_code = p.wait() 575 result.exit_code = exit_code 576 if exit_code not in valid_exit_codes: 577 raise PluginInvocationFailed('Failed during execution of plugin %s ' 578 'for section %s of runtime %s. rc = %s' % 579 (normalized_path, section_name, 580 self.config.get('name', 'unknown'), 581 exit_code)) 582 return result 583 else: 584 logging.error('No usable plugin type found for %s' % section_name) 585 586 def Detect(self, path, params): 587 """Determine if 'path' contains an instance of the runtime type. 588 589 Checks to see if the 'path' directory looks like an instance of the 590 runtime type. 591 592 Args: 593 path: (str) The path name. 594 params: (Params) Parameters used by the framework. 595 596 Returns: 597 (Configurator) An object containing parameters inferred from source 598 inspection. 599 """ 600 detect = self.config.get('detect') 601 if detect: 602 result = self.RunPlugin('detect', detect, params, [path], (0, 1)) 603 if result.exit_code: 604 return None 605 else: 606 return ExternalRuntimeConfigurator(self, params, result.runtime_data, 607 result.generated_appinfo, 608 path, 609 self.env) 610 611 else: 612 return None 613 614 def CollectData(self, configurator): 615 """Do data collection on a detected runtime. 616 617 Args: 618 configurator: (ExternalRuntimeConfigurator) The configurator retuned by 619 Detect(). 620 621 Raises: 622 InvalidRuntimeDefinition: For a variety of problems with the runtime 623 definition. 624 """ 625 collect_data = self.config.get('collectData') 626 if collect_data: 627 result = self.RunPlugin('collect_data', collect_data, 628 configurator.params, 629 runtime_data=configurator.data) 630 if result.generated_appinfo: 631 configurator.SetGeneratedAppInfo(result.generated_appinfo) 632 633 def Prebuild(self, configurator): 634 """Perform any additional build behavior before the application is deployed. 635 636 Args: 637 configurator: (ExternalRuntimeConfigurator) The configurator returned by 638 Detect(). 639 """ 640 prebuild = self.config.get('prebuild') 641 if prebuild: 642 result = self.RunPlugin('prebuild', prebuild, configurator.params, 643 args=[configurator.path], runtime_data=configurator.data) 644 645 if result.docker_context: 646 configurator.path = result.docker_context 647 648 # The legacy runtimes use "Fingerprint" for this function, the externalized 649 # runtime code uses "Detect" to mirror the name in runtime.yaml, so alias it. 650 # b/25117700 651 Fingerprint = Detect 652 653 def GetAllConfigFiles(self, configurator): 654 """Generate list of GeneratedFile objects. 655 656 Args: 657 configurator: Configurator, the runtime configurator 658 659 Returns: 660 [GeneratedFile] a list of GeneratedFile objects. 661 662 Raises: 663 InvalidRuntimeDefinition: For a variety of problems with the runtime 664 definition. 665 """ 666 667 generate_configs = self.config.get('generateConfigs') 668 if generate_configs: 669 files_to_copy = generate_configs.get('filesToCopy') 670 if files_to_copy: 671 all_config_files = [] 672 673 # Make sure there's nothing else. 674 if len(generate_configs) != 1: 675 raise InvalidRuntimeDefinition('If "files_to_copy" is specified, ' 676 'it must be the only field in ' 677 'generate_configs.') 678 for filename in files_to_copy: 679 full_name = _NormalizePath(self.root, filename) 680 if not os.path.isfile(full_name): 681 raise InvalidRuntimeDefinition('File [%s] specified in ' 682 'files_to_copy, but is not in ' 683 'the runtime definition.' % 684 filename) 685 with open(full_name, 'r') as file_to_read: 686 file_contents = file_to_read.read() 687 all_config_files.append(GeneratedFile(filename, file_contents)) 688 return all_config_files 689 else: 690 result = self.RunPlugin('generate_configs', generate_configs, 691 configurator.params, 692 runtime_data=configurator.data) 693 return result.files 694 695 def GenerateConfigData(self, configurator): 696 """Do config generation on the runtime, return file objects. 697 698 Args: 699 configurator: (ExternalRuntimeConfigurator) The configurator retuned by 700 Detect(). 701 702 Returns: 703 [GeneratedFile] list of generated file objects. 704 """ 705 # Log or print status messages depending on whether we're in gen-config or 706 # deploy. 707 notify = logging.info if configurator.params.deploy else self.env.Print 708 all_config_files = self.GetAllConfigFiles(configurator) 709 if all_config_files is None: 710 return [] 711 for config_file in all_config_files: 712 if config_file.filename == 'app.yaml': 713 config_file.WriteTo(configurator.path, notify) 714 config_files = [] 715 for config_file in all_config_files: 716 if not os.path.exists(_NormalizePath(configurator.path, 717 config_file.filename)): 718 config_files.append(config_file) 719 return config_files 720 721 def GenerateConfigs(self, configurator): 722 """Do config generation on the runtime. 723 724 This should generally be called from the configurator's GenerateConfigs() 725 method. 726 727 Args: 728 configurator: (ExternalRuntimeConfigurator) The configurator retuned by 729 Detect(). 730 731 Returns: 732 (bool) True if files were generated, False if not 733 """ 734 # Log or print status messages depending on whether we're in gen-config or 735 # deploy. 736 notify = logging.info if configurator.params.deploy else self.env.Print 737 all_config_files = self.GetAllConfigFiles(configurator) 738 if all_config_files is None: 739 return 740 created = False 741 for gen_file in all_config_files: 742 if gen_file.WriteTo(configurator.path, notify) is not None: 743 created = True 744 if not created: 745 notify('All config files already exist, not generating anything.') 746 return created 747 748