1# Copyright 2015 Google Inc. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#    http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Support for externalized runtimes."""
15
16from __future__ import absolute_import
17from __future__ import division
18from __future__ import print_function
19
20import json
21import logging
22import os
23import subprocess
24import sys
25import threading
26
27from . import comm
28import ruamel.yaml as yaml
29from six.moves import input
30
31# Try importing these modules from the cloud SDK first.
32try:
33  from googlecloudsdk.third_party.appengine.admin.tools.conversion import schema
34except ImportError:
35  from yaml_conversion import schema
36
37try:
38  from googlecloudsdk.third_party.py27 import py27_subprocess as subprocess
39except ImportError:
40  import subprocess
41
42
43WRITING_FILE_MESSAGE = 'Writing [{0}] to [{1}].'
44FILE_EXISTS_MESSAGE = 'Not writing [{0}], it already exists.'
45
46
47class Error(Exception):
48  """Base class for exceptions in this module."""
49
50
51class PluginInvocationFailed(Error):
52  """Raised when a plugin invocation returns a non-zero result code."""
53
54
55class InvalidRuntimeDefinition(Error):
56  """Raised when an inconsistency is found in the runtime definition."""
57  pass
58
59
60class Params(object):
61  """Parameters passed to the the runtime module Fingerprint() methods.
62
63  Attributes:
64    appinfo: (apphosting.api.appinfo.AppInfoExternal or None) The parsed
65      app.yaml file for the module if it exists.
66    custom: (bool) True if the Configurator should generate a custom runtime.
67    runtime (str or None) Runtime (alias allowed) that should be enforced.
68    deploy: (bool) True if this is happening from deployment.
69  """
70
71  def __init__(self, appinfo=None, custom=False, runtime=None, deploy=False):
72    self.appinfo = appinfo
73    self.custom = custom
74    self.runtime = runtime
75    self.deploy = deploy
76
77  def ToDict(self):
78    """Returns the object converted to a dictionary.
79
80    Returns:
81      ({str: object}) A dictionary that can be converted to json using
82      json.dump().
83    """
84    return {'appinfo': self.appinfo and self.appinfo.ToDict(),
85            'custom': self.custom,
86            'runtime': self.runtime,
87            'deploy': self.deploy}
88
89
90class Configurator(object):
91  """Base configurator class.
92
93  Configurators generate config files for specific classes of runtimes.  They
94  are returned by the Fingerprint functions in the runtimes sub-package after
95  a successful match of the runtime's heuristics.
96  """
97
98  def CollectData(self):
99    """Collect all information on this application.
100
101    This is called after the runtime type is detected and may gather
102    additional information from the source code and from the user.  Whereas
103    performing user queries during detection is deprecated, user queries are
104    allowed in CollectData().
105
106    The base class version of this does nothing.
107    """
108
109  def Prebuild(self):
110    """Run additional build behavior before the application is deployed.
111
112    This is called after the runtime type has been detected and after any
113    additional data has been collected.
114
115    The base class version of this does nothing.
116    """
117
118  def GenerateConfigs(self):
119    """Generate all configuration files for the module.
120
121    Generates config files in the current working directory.
122
123    Returns:
124      (callable()) Function that will delete all of the generated files.
125    """
126    raise NotImplementedError()
127
128
129class ExecutionEnvironment(object):
130  """An interface for providing system functionality to a runtime definition.
131
132  Abstract interface containing methods for console IO and system
133  introspection.  This exists to allow gcloud to inject special functionality.
134  """
135
136  def GetPythonExecutable(self):
137    """Returns the full path of the python executable (str)."""
138    raise NotImplementedError()
139
140  def CanPrompt(self):
141    """Returns true """
142    raise NotImplementedError()
143
144  def PromptResponse(self, message):
145    raise NotImplementedError()
146
147
148  def Print(self, message):
149    """Print a message to the console.
150
151    Args:
152      message: (str)
153    """
154    raise NotImplementedError()
155
156
157class DefaultExecutionEnvironment(ExecutionEnvironment):
158  """Standard implementation of the ExecutionEnvironment."""
159
160  def GetPythonExecutable(self):
161    return sys.executable
162
163  def CanPrompt(self):
164    return sys.stdin.isatty()
165
166  def PromptResponse(self, message):
167    sys.stdout.write(message)
168    sys.stdout.flush()
169    return input('> ')
170
171  def Print(self, message):
172    print(message)
173
174
175class ExternalRuntimeConfigurator(Configurator):
176  """Configurator for general externalized runtimes.
177
178  Attributes:
179    runtime: (ExternalizedRuntime) The runtime that produced this.
180    params: (Params) Runtime parameters.
181    data: ({str: object, ...} or None) Optional dictionary of runtime data
182      passed back through a runtime_parameters message.
183    generated_appinfo: ({str: object, ...} or None) Generated appinfo if any
184      is produced by the runtime.
185    path: (str) Path to the user's source directory.
186  """
187
188  def __init__(self, runtime, params, data, generated_appinfo, path, env):
189    """Constructor.
190
191    Args:
192      runtime: (ExternalizedRuntime) The runtime that produced this.
193      params: (Params) Runtime parameters.
194      data: ({str: object, ...} or None) Optional dictionary of runtime data
195        passed back through a runtime_parameters message.
196      generated_appinfo: ({str: object, ...} or None) Optional dictionary
197        representing the contents of app.yaml if the runtime produces this.
198      path: (str) Path to the user's source directory.
199      env: (ExecutionEnvironment)
200    """
201    self.runtime = runtime
202    self.params = params
203    self.data = data
204    if generated_appinfo:
205
206      # Add env: flex if we don't have an "env" field.
207      self.generated_appinfo = {}
208      if not 'env' in generated_appinfo:
209        self.generated_appinfo['env'] = 'flex'
210
211      # And then update with the values provided by the runtime def.
212      self.generated_appinfo.update(generated_appinfo)
213    else:
214      self.generated_appinfo = None
215    self.path = path
216    self.env = env
217
218  def MaybeWriteAppYaml(self):
219    """Generates the app.yaml file if it doesn't already exist."""
220
221    if not self.generated_appinfo:
222      return
223
224    notify = logging.info if self.params.deploy else self.env.Print
225    # TODO(user): The config file need not be named app.yaml.  We need to
226    # pass the appinfo file name in through params. and use it here.
227    filename = os.path.join(self.path, 'app.yaml')
228
229    # Don't generate app.yaml if we've already got it.  We consider the
230    # presence of appinfo to be an indicator of the existence of app.yaml as
231    # well as the existence of the file itself because this helps with
232    # testability, as well as preventing us from writing the file if another
233    # config file is being used.
234    if self.params.appinfo or os.path.exists(filename):
235      notify(FILE_EXISTS_MESSAGE.format('app.yaml'))
236      return
237
238    notify(WRITING_FILE_MESSAGE.format('app.yaml', self.path))
239    with open(filename, 'w') as f:
240      yaml.safe_dump(self.generated_appinfo, f, default_flow_style=False)
241
242  def SetGeneratedAppInfo(self, generated_appinfo):
243    """Sets the generated appinfo."""
244    self.generated_appinfo = generated_appinfo
245
246  def CollectData(self):
247    self.runtime.CollectData(self)
248
249  def Prebuild(self):
250    self.runtime.Prebuild(self)
251
252  def GenerateConfigs(self):
253    self.MaybeWriteAppYaml()
254
255    # At this point, if we have don't have appinfo, but we do have generated
256    # appinfo, we want to use the generated appinfo and pass it to config
257    # generation.
258    if not self.params.appinfo and self.generated_appinfo:
259      self.params.appinfo = comm.dict_to_object(self.generated_appinfo)
260    return self.runtime.GenerateConfigs(self)
261
262  def GenerateConfigData(self):
263    self.MaybeWriteAppYaml()
264
265    # At this point, if we have don't have appinfo, but we do have generated
266    # appinfo, we want to use the generated appinfo and pass it to config
267    # generation.
268    if not self.params.appinfo and self.generated_appinfo:
269      self.params.appinfo = comm.dict_to_object(self.generated_appinfo)
270    return self.runtime.GenerateConfigData(self)
271
272
273def _NormalizePath(basedir, pathname):
274  """Get the absolute path from a unix-style relative path.
275
276  Args:
277    basedir: (str) Platform-specific encoding of the base directory.
278    pathname: (str) A unix-style (forward slash separated) path relative to
279      the runtime definition root directory.
280
281  Returns:
282    (str) An absolute path conforming to the conventions of the operating
283    system.  Note: in order for this to work, 'pathname' must not contain
284    any characters with special meaning in any of the targeted operating
285    systems.  Keep those names simple.
286  """
287  components = pathname.split('/')
288  return os.path.join(basedir, *components)
289
290
291class GeneratedFile(object):
292  """Wraps the name and contents of a generated file."""
293
294  def __init__(self, filename, contents):
295    """Constructor.
296
297    Args:
298      filename: (str) Unix style file path relative to the target source
299        directory.
300      contents: (str) File contents.
301    """
302    self.filename = filename
303    self.contents = contents
304
305  def WriteTo(self, dest_dir, notify):
306    """Write the file to the destination directory.
307
308    Args:
309      dest_dir: (str) Destination directory.
310      notify: (callable(str)) Function to notify the user.
311
312    Returns:
313      (str or None) The full normalized path name of the destination file,
314      None if it wasn't generated because it already exists.
315    """
316    path = _NormalizePath(dest_dir, self.filename)
317    if not os.path.exists(path):
318      notify(WRITING_FILE_MESSAGE.format(self.filename, dest_dir))
319      with open(path, 'w') as f:
320        f.write(self.contents)
321      return path
322    else:
323      notify(FILE_EXISTS_MESSAGE.format(self.filename))
324
325    return None
326
327
328class PluginResult(object):
329
330  def __init__(self):
331    self.exit_code = -1
332    self.runtime_data = None
333    self.generated_appinfo = None
334    self.docker_context = None
335    self.files = []
336
337
338class _Collector(object):
339  """Manages a PluginResult in a thread-safe context."""
340
341  def __init__(self):
342    self.result = PluginResult()
343    self.lock = threading.Lock()
344
345
346_LOG_FUNCS = {
347    'info': logging.info,
348    'error': logging.error,
349    'warn': logging.warn,
350    'debug': logging.debug
351}
352
353# A section consisting only of scripts.
354_EXEC_SECTION = schema.Message(
355    python=schema.Value(converter=str))
356
357_RUNTIME_SCHEMA = schema.Message(
358    name=schema.Value(converter=str),
359    description=schema.Value(converter=str),
360    author=schema.Value(converter=str),
361    api_version=schema.Value(converter=str),
362    generate_configs=schema.Message(
363        python=schema.Value(converter=str),
364        files_to_copy=schema.RepeatedField(element=schema.Value(converter=str)),
365        ),
366    detect=_EXEC_SECTION,
367    collect_data=_EXEC_SECTION,
368    prebuild=_EXEC_SECTION,
369    postbuild=_EXEC_SECTION)
370
371_MISSING_FIELD_ERROR = 'Missing [{0}] field in [{1}] message'
372_NO_DEFAULT_ERROR = ('User input requested: [{0}] while running '
373                     'non-interactive with no default specified.')
374
375
376class ExternalizedRuntime(object):
377  """Encapsulates an externalized runtime."""
378
379  def __init__(self, path, config, env):
380    """
381    Args:
382      path: (str) Path to the root of the runtime definition.
383      config: ({str: object, ...}) The runtime definition configuration (from
384        runtime.yaml).
385      env: (ExecutionEnvironment)
386    """
387
388    self.root = path
389    self.env = env
390    try:
391      # Do validation up front, after this we can assume all of the types are
392      # correct.
393      self.config = _RUNTIME_SCHEMA.ConvertValue(config)
394    except ValueError as ex:
395      raise InvalidRuntimeDefinition(
396          'Invalid runtime definition: {0}'.format(ex.message))
397
398  @property
399  def name(self):
400    return self.config.get('name', 'unnamed')
401
402  @staticmethod
403  def Load(path, env):
404    """Loads the externalized runtime from the specified path.
405
406    Args:
407      path: (str) root directory of the runtime definition.  Should
408        contain a "runtime.yaml" file.
409
410    Returns:
411      (ExternalizedRuntime)
412    """
413    with open(os.path.join(path, 'runtime.yaml')) as f:
414      return ExternalizedRuntime(path, yaml.load(f), env)
415
416  def _ProcessPluginStderr(self, section_name, stderr):
417    """Process the standard error stream of a plugin.
418
419    Standard error output is just written to the log at "warning" priority and
420    otherwise ignored.
421
422    Args:
423      section_name: (str) Section name, to be attached to log messages.
424      stderr: (file) Process standard error stream.
425    """
426    while True:
427      line = stderr.readline()
428      if not line:
429        break
430      logging.warn('%s: %s' % (section_name, line.rstrip()))
431
432  def _ProcessMessage(self, plugin_stdin, message, result, params,
433                      runtime_data):
434    """Process a message received from the plugin.
435
436    Args:
437      plugin_stdin: (file) The standard input stream of the plugin process.
438      message: ({str: object, ...}) The message (this maps directly to the
439        message's json object).
440      result: (PluginResult) A result object in which to store data collected
441        from some types of message.
442      params: (Params) Parameters passed in through the
443        fingerprinter.
444      runtime_data: (object or None) Arbitrary runtime data obtained from the
445        "detect" plugin.  This will be None if we are processing a message for
446        the detect plugin itself or if no runtime data was provided.
447    """
448
449    def SendResponse(response):
450      json.dump(response, plugin_stdin)
451      plugin_stdin.write('\n')
452      plugin_stdin.flush()
453
454    msg_type = message.get('type')
455    if msg_type is None:
456      logging.error('Missing type in message: %0.80s' % str(message))
457    elif msg_type in _LOG_FUNCS:
458      _LOG_FUNCS[msg_type](message.get('message'))
459    elif msg_type == 'runtime_parameters':
460      try:
461        result.runtime_data = message['runtime_data']
462      except KeyError:
463        logging.error(_MISSING_FIELD_ERROR.format('runtime_data', msg_type))
464      result.generated_appinfo = message.get('appinfo')
465    elif msg_type == 'gen_file':
466      try:
467        # TODO(user): deal with 'encoding'
468        filename = message['filename']
469        contents = message['contents']
470        result.files.append(GeneratedFile(filename, contents))
471      except KeyError as ex:
472        logging.error(_MISSING_FIELD_ERROR.format(ex, msg_type))
473    elif msg_type == 'get_config':
474      response = {'type': 'get_config_response',
475                  'params': params.ToDict(),
476                  'runtime_data': runtime_data}
477      SendResponse(response)
478    elif msg_type == 'query_user':
479      try:
480        prompt = message['prompt']
481      except KeyError as ex:
482        logging.error(_MISSING_FIELD_ERROR.format('prompt', msg_type))
483        return
484      default = message.get('default')
485
486      if self.env.CanPrompt():
487        if default:
488          message = '{0} [{1}]: '.format(prompt, default)
489        else:
490          message = prompt + ':'
491        result = self.env.PromptResponse(message)
492      else:
493        # TODO(user): Support the "id" field once there is a way to pass
494        # these through.
495        if default is not None:
496          result = default
497        else:
498          result = ''
499          logging.error(_NO_DEFAULT_ERROR.format(prompt))
500
501      SendResponse({'type': 'query_user_response', 'result': result})
502    elif msg_type == 'set_docker_context':
503      try:
504        result.docker_context = message['path']
505      except KeyError:
506        logging.error(_MISSING_FIELD_ERROR.format('path', msg_type))
507        return
508    # TODO(user): implement remaining message types.
509    else:
510      logging.error('Unknown message type %s' % msg_type)
511
512  def _ProcessPluginPipes(self, section_name, proc, result, params,
513                          runtime_data):
514    """Process the standard output and input streams of a plugin."""
515    while True:
516      line = proc.stdout.readline()
517      if not line:
518        break
519
520      # Parse and process the message.
521      try:
522        message = json.loads(line)
523        self._ProcessMessage(proc.stdin, message, result, params, runtime_data)
524      except ValueError:
525        # Unstructured lines get logged as "info".
526        logging.info('%s: %s' % (section_name, line.rstrip()))
527
528  def RunPlugin(self, section_name, plugin_spec, params, args=None,
529                valid_exit_codes=(0,),
530                runtime_data=None):
531    """Run a plugin.
532
533    Args:
534      section_name: (str) Name of the config section that the plugin spec is
535        from.
536      plugin_spec: ({str: str, ...}) A dictionary mapping plugin locales to
537        script names
538      params: (Params or None) Parameters for the plugin.
539      args: ([str, ...] or None) Command line arguments for the plugin.
540      valid_exit_codes: (int, ...) Exit codes that will be accepted without
541        raising an exception.
542      runtime_data: ({str: object, ...}) A dictionary of runtime data passed
543        back from detect.
544
545    Returns:
546      (PluginResult) A bundle of the exit code and data produced by the plugin.
547
548    Raises:
549      PluginInvocationFailed: The plugin terminated with a non-zero exit code.
550    """
551    # TODO(user): Support other script types.
552    if 'python' in plugin_spec:
553      normalized_path = _NormalizePath(self.root, plugin_spec['python'])
554
555      # We're sharing 'result' with the output collection thread, we can get
556      # away with this without locking because we pass it into the thread at
557      # creation and do not use it again until after we've joined the thread.
558      result = PluginResult()
559      p = subprocess.Popen([self.env.GetPythonExecutable(), normalized_path] +
560                           (args if args else []),
561                           stdout=subprocess.PIPE,
562                           stdin=subprocess.PIPE,
563                           stderr=subprocess.PIPE)
564      stderr_thread = threading.Thread(target=self._ProcessPluginStderr,
565                                       args=(section_name, p.stderr,))
566      stderr_thread.start()
567      stdout_thread = threading.Thread(target=self._ProcessPluginPipes,
568                                       args=(section_name, p, result,
569                                             params, runtime_data))
570      stdout_thread.start()
571
572      stderr_thread.join()
573      stdout_thread.join()
574      exit_code = p.wait()
575      result.exit_code = exit_code
576      if exit_code not in valid_exit_codes:
577        raise PluginInvocationFailed('Failed during execution of plugin %s '
578                                     'for section %s of runtime %s. rc = %s' %
579                                     (normalized_path, section_name,
580                                      self.config.get('name', 'unknown'),
581                                      exit_code))
582      return result
583    else:
584      logging.error('No usable plugin type found for %s' % section_name)
585
586  def Detect(self, path, params):
587    """Determine if 'path' contains an instance of the runtime type.
588
589    Checks to see if the 'path' directory looks like an instance of the
590    runtime type.
591
592    Args:
593      path: (str) The path name.
594      params: (Params) Parameters used by the framework.
595
596    Returns:
597      (Configurator) An object containing parameters inferred from source
598        inspection.
599    """
600    detect = self.config.get('detect')
601    if detect:
602      result = self.RunPlugin('detect', detect, params, [path], (0, 1))
603      if result.exit_code:
604        return None
605      else:
606        return ExternalRuntimeConfigurator(self, params, result.runtime_data,
607                                           result.generated_appinfo,
608                                           path,
609                                           self.env)
610
611    else:
612      return None
613
614  def CollectData(self, configurator):
615    """Do data collection on a detected runtime.
616
617    Args:
618      configurator: (ExternalRuntimeConfigurator) The configurator retuned by
619        Detect().
620
621    Raises:
622      InvalidRuntimeDefinition: For a variety of problems with the runtime
623        definition.
624    """
625    collect_data = self.config.get('collectData')
626    if collect_data:
627      result = self.RunPlugin('collect_data', collect_data,
628                              configurator.params,
629                              runtime_data=configurator.data)
630      if result.generated_appinfo:
631        configurator.SetGeneratedAppInfo(result.generated_appinfo)
632
633  def Prebuild(self, configurator):
634    """Perform any additional build behavior before the application is deployed.
635
636    Args:
637      configurator: (ExternalRuntimeConfigurator) The configurator returned by
638      Detect().
639    """
640    prebuild = self.config.get('prebuild')
641    if prebuild:
642      result = self.RunPlugin('prebuild', prebuild, configurator.params,
643          args=[configurator.path], runtime_data=configurator.data)
644
645      if result.docker_context:
646        configurator.path = result.docker_context
647
648  # The legacy runtimes use "Fingerprint" for this function, the externalized
649  # runtime code uses "Detect" to mirror the name in runtime.yaml, so alias it.
650  # b/25117700
651  Fingerprint = Detect
652
653  def GetAllConfigFiles(self, configurator):
654    """Generate list of GeneratedFile objects.
655
656    Args:
657      configurator: Configurator, the runtime configurator
658
659    Returns:
660      [GeneratedFile] a list of GeneratedFile objects.
661
662    Raises:
663      InvalidRuntimeDefinition: For a variety of problems with the runtime
664        definition.
665    """
666
667    generate_configs = self.config.get('generateConfigs')
668    if generate_configs:
669      files_to_copy = generate_configs.get('filesToCopy')
670      if files_to_copy:
671        all_config_files = []
672
673        # Make sure there's nothing else.
674        if len(generate_configs) != 1:
675          raise InvalidRuntimeDefinition('If "files_to_copy" is specified, '
676                                         'it must be the only field in '
677                                         'generate_configs.')
678        for filename in files_to_copy:
679          full_name = _NormalizePath(self.root, filename)
680          if not os.path.isfile(full_name):
681            raise InvalidRuntimeDefinition('File [%s] specified in '
682                                           'files_to_copy, but is not in '
683                                           'the runtime definition.' %
684                                           filename)
685          with open(full_name, 'r') as file_to_read:
686            file_contents = file_to_read.read()
687          all_config_files.append(GeneratedFile(filename, file_contents))
688        return all_config_files
689      else:
690        result = self.RunPlugin('generate_configs', generate_configs,
691                                configurator.params,
692                                runtime_data=configurator.data)
693        return result.files
694
695  def GenerateConfigData(self, configurator):
696    """Do config generation on the runtime, return file objects.
697
698    Args:
699      configurator: (ExternalRuntimeConfigurator) The configurator retuned by
700        Detect().
701
702    Returns:
703      [GeneratedFile] list of generated file objects.
704    """
705    # Log or print status messages depending on whether we're in gen-config or
706    # deploy.
707    notify = logging.info if configurator.params.deploy else self.env.Print
708    all_config_files = self.GetAllConfigFiles(configurator)
709    if all_config_files is None:
710      return []
711    for config_file in all_config_files:
712      if config_file.filename == 'app.yaml':
713        config_file.WriteTo(configurator.path, notify)
714    config_files = []
715    for config_file in all_config_files:
716      if not os.path.exists(_NormalizePath(configurator.path,
717                                           config_file.filename)):
718        config_files.append(config_file)
719    return config_files
720
721  def GenerateConfigs(self, configurator):
722    """Do config generation on the runtime.
723
724    This should generally be called from the configurator's GenerateConfigs()
725    method.
726
727    Args:
728      configurator: (ExternalRuntimeConfigurator) The configurator retuned by
729        Detect().
730
731    Returns:
732      (bool) True if files were generated, False if not
733    """
734    # Log or print status messages depending on whether we're in gen-config or
735    # deploy.
736    notify = logging.info if configurator.params.deploy else self.env.Print
737    all_config_files = self.GetAllConfigFiles(configurator)
738    if all_config_files is None:
739      return
740    created = False
741    for gen_file in all_config_files:
742      if gen_file.WriteTo(configurator.path, notify) is not None:
743        created = True
744    if not created:
745      notify('All config files already exist, not generating anything.')
746    return created
747
748