1# -*- coding: utf-8 -*- #
2# Copyright 2015 Google LLC. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#    http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Utility functions for gcloud app."""
17
18from __future__ import absolute_import
19from __future__ import division
20from __future__ import unicode_literals
21
22import datetime
23import os
24import posixpath
25import sys
26import time
27from googlecloudsdk.core import config
28from googlecloudsdk.core import exceptions
29from googlecloudsdk.core import log
30from googlecloudsdk.core.util import platforms
31from googlecloudsdk.third_party.appengine.api import client_deployinfo
32import six
33from six.moves import urllib
34
35
36class Error(exceptions.Error):
37  """Exceptions for the appcfg module."""
38
39
40class NoFieldsSpecifiedError(Error):
41  """The user specified no fields to a command which requires at least one."""
42
43
44class NoCloudSDKError(Error):
45  """The module was unable to find Cloud SDK."""
46
47  def __init__(self):
48    super(NoCloudSDKError, self).__init__(
49        'Unable to find a Cloud SDK installation.')
50
51
52class NoAppengineSDKError(Error):
53  """The module was unable to find the appengine SDK."""
54
55
56class TimeoutError(Error):
57  """An exception for when a retry with wait operation times out."""
58
59  def __init__(self):
60    super(TimeoutError, self).__init__(
61        'Timed out waiting for the operation to complete.')
62
63
64class RPCError(Error):
65  """For when an error occurs when making an RPC call."""
66
67  def __init__(self, url_error, body=''):
68    super(RPCError, self).__init__(
69        'Server responded with code [{code}]:\n  {reason}.\n  {body}'
70        .format(code=url_error.code,
71                reason=getattr(url_error, 'reason', '(unknown)'),
72                body=body))
73    self.url_error = url_error
74
75
76def GetCloudSDKRoot():
77  """Gets the directory of the root of the Cloud SDK, error if it doesn't exist.
78
79  Raises:
80    NoCloudSDKError: If there is no SDK root.
81
82  Returns:
83    str, The path to the root of the Cloud SDK.
84  """
85  sdk_root = config.Paths().sdk_root
86  if not sdk_root:
87    raise NoCloudSDKError()
88  log.debug('Found Cloud SDK root: %s', sdk_root)
89  return sdk_root
90
91
92def GetAppEngineSDKRoot():
93  """Gets the directory of the GAE SDK directory in the SDK.
94
95  Raises:
96    NoCloudSDKError: If there is no SDK root.
97    NoAppengineSDKError: If the GAE SDK cannot be found.
98
99  Returns:
100    str, The path to the root of the GAE SDK within the Cloud SDK.
101  """
102  sdk_root = GetCloudSDKRoot()
103  gae_sdk_dir = os.path.join(sdk_root, 'platform', 'google_appengine')
104  if not os.path.isdir(gae_sdk_dir):
105    raise NoAppengineSDKError()
106  log.debug('Found App Engine SDK root: %s', gae_sdk_dir)
107
108  return gae_sdk_dir
109
110
111def GenerateVersionId(datetime_getter=datetime.datetime.now):
112  """Generates a version id based off the current time.
113
114  Args:
115    datetime_getter: A function that returns a datetime.datetime instance.
116
117  Returns:
118    A version string based.
119  """
120  return datetime_getter().isoformat().lower().replace('-', '').replace(
121      ':', '')[:15]
122
123
124def ConvertToPosixPath(path):
125  """Converts a native-OS path to /-separated: os.path.join('a', 'b')->'a/b'."""
126  return posixpath.join(*path.split(os.path.sep))
127
128
129def ShouldSkip(skip_files, path):
130  """Returns whether the given path should be skipped by the skip_files field.
131
132  A user can specify a `skip_files` field in their .yaml file, which is a list
133  of regular expressions matching files that should be skipped. By this point in
134  the code, it's been turned into one mega-regex that matches any file to skip.
135
136  Args:
137    skip_files: A regular expression object for files/directories to skip.
138    path: str, the path to the file/directory which might be skipped (relative
139      to the application root)
140
141  Returns:
142    bool, whether the file/dir should be skipped.
143  """
144  # On Windows, os.path.join uses the path separator '\' instead of '/'.
145  # However, the skip_files regular expression always uses '/'.
146  # To handle this, we'll replace '\' characters with '/' characters.
147  path = ConvertToPosixPath(path)
148  return skip_files.match(path)
149
150
151def FileIterator(base, skip_files):
152  """Walks a directory tree, returning all the files. Follows symlinks.
153
154  Args:
155    base: The base path to search for files under.
156    skip_files: A regular expression object for files/directories to skip.
157
158  Yields:
159    Paths of files found, relative to base.
160  """
161  dirs = ['']
162
163  while dirs:
164    current_dir = dirs.pop()
165    entries = set(os.listdir(os.path.join(base, current_dir)))
166    for entry in sorted(entries):
167      name = os.path.join(current_dir, entry)
168      fullname = os.path.join(base, name)
169
170      if os.path.isfile(fullname):
171        if ShouldSkip(skip_files, name):
172          log.info('Ignoring file [%s]: File matches ignore regex.', name)
173        else:
174          yield name
175      elif os.path.isdir(fullname):
176        if ShouldSkip(skip_files, name):
177          log.info('Ignoring directory [%s]: Directory matches ignore regex.',
178                   name)
179        else:
180          dirs.append(name)
181
182
183def RetryWithBackoff(func, retry_notify_func,
184                     initial_delay=1, backoff_factor=2,
185                     max_delay=60, max_tries=20, raise_on_timeout=True):
186  """Calls a function multiple times, backing off more and more each time.
187
188  Args:
189    func: f() -> (bool, value), A function that performs some operation that
190      should be retried a number of times upon failure. If the first tuple
191      element is True, we'll immediately return (True, value). If False, we'll
192      delay a bit and try again, unless we've hit the 'max_tries' limit, in
193      which case we'll return (False, value).
194    retry_notify_func: f(value, delay) -> None, This function will be called
195      immediately before the next retry delay.  'value' is the value returned
196      by the last call to 'func'.  'delay' is the retry delay, in seconds
197    initial_delay: int, Initial delay after first try, in seconds.
198    backoff_factor: int, Delay will be multiplied by this factor after each
199      try.
200    max_delay: int, Maximum delay, in seconds.
201    max_tries: int, Maximum number of tries (the first one counts).
202    raise_on_timeout: bool, True to raise an exception if the operation times
203      out instead of returning False.
204
205  Returns:
206    What the last call to 'func' returned, which is of the form (done, value).
207    If 'done' is True, you know 'func' returned True before we ran out of
208    retries.  If 'done' is False, you know 'func' kept returning False and we
209    ran out of retries.
210
211  Raises:
212    TimeoutError: If raise_on_timeout is True and max_tries is exhausted.
213  """
214  delay = initial_delay
215  try_count = max_tries
216  value = None
217
218  while True:
219    try_count -= 1
220    done, value = func()
221    if done:
222      return True, value
223    if try_count <= 0:
224      if raise_on_timeout:
225        raise TimeoutError()
226      return False, value
227    retry_notify_func(value, delay)
228    time.sleep(delay)
229    delay = min(delay * backoff_factor, max_delay)
230
231
232def RetryNoBackoff(callable_func, retry_notify_func, delay=5, max_tries=200):
233  """Calls a function multiple times, with the same delay each time.
234
235  Args:
236    callable_func: A function that performs some operation that should be
237      retried a number of times upon failure.  Signature: () -> (done, value)
238      If 'done' is True, we'll immediately return (True, value)
239      If 'done' is False, we'll delay a bit and try again, unless we've
240      hit the 'max_tries' limit, in which case we'll return (False, value).
241    retry_notify_func: This function will be called immediately before the
242      next retry delay.  Signature: (value, delay) -> None
243      'value' is the value returned by the last call to 'callable_func'
244      'delay' is the retry delay, in seconds
245    delay: Delay between tries, in seconds.
246    max_tries: Maximum number of tries (the first one counts).
247
248  Returns:
249    What the last call to 'callable_func' returned, which is of the form
250    (done, value).  If 'done' is True, you know 'callable_func' returned True
251    before we ran out of retries.  If 'done' is False, you know 'callable_func'
252    kept returning False and we ran out of retries.
253
254  Raises:
255    Whatever the function raises--an exception will immediately stop retries.
256  """
257  # A backoff_factor of 1 means the delay won't grow.
258  return RetryWithBackoff(callable_func, retry_notify_func, delay, 1, delay,
259                          max_tries)
260
261
262def GetSourceName():
263  """Gets the name of this source version."""
264  return 'Google-appcfg-{0}'.format(config.CLOUD_SDK_VERSION)
265
266
267def GetUserAgent():
268  """Determines the value of the 'User-agent' header to use for HTTP requests.
269
270  Returns:
271    String containing the 'user-agent' header value.
272  """
273  product_tokens = []
274
275  # SDK version
276  product_tokens.append(config.CLOUDSDK_USER_AGENT)
277
278  # Platform
279  product_tokens.append(platforms.Platform.Current().UserAgentFragment())
280
281  # Python version
282  python_version = '.'.join(six.text_type(i) for i in sys.version_info)
283  product_tokens.append('Python/%s' % python_version)
284
285  return ' '.join(product_tokens)
286
287
288class ClientDeployLoggingContext(object):
289  """Context for sending and recording server rpc requests.
290
291  Attributes:
292    rpcserver: The AbstractRpcServer to use for the upload.
293    requests: A list of client_deployinfo.Request objects to include
294      with the client deploy log.
295    time_func: Function to get the current time in milliseconds.
296    request_params: A dictionary with params to append to requests
297  """
298
299  def __init__(self,
300               rpcserver,
301               request_params,
302               usage_reporting,
303               time_func=time.time):
304    """Creates a new AppVersionUpload.
305
306    Args:
307      rpcserver: The RPC server to use. Should be an instance of HttpRpcServer
308        or TestRpcServer.
309      request_params: A dictionary with params to append to requests
310      usage_reporting: Whether to actually upload data.
311      time_func: Function to return the current time in millisecods
312        (default time.time).
313    """
314    self.rpcserver = rpcserver
315    self.request_params = request_params
316    self.usage_reporting = usage_reporting
317    self.time_func = time_func
318    self.requests = []
319
320  def Send(self, url, payload='', **kwargs):
321    """Sends a request to the server, with common params."""
322    start_time_usec = self.GetCurrentTimeUsec()
323    request_size_bytes = len(payload)
324    try:
325      log.debug('Send: {0}, params={1}'.format(url, self.request_params))
326
327      kwargs.update(self.request_params)
328      result = self.rpcserver.Send(url, payload=payload, **kwargs)
329      self._RegisterReqestForLogging(url, 200, start_time_usec,
330                                     request_size_bytes)
331      return result
332    except RPCError as err:
333      self._RegisterReqestForLogging(url, err.url_error.code, start_time_usec,
334                                     request_size_bytes)
335      raise
336
337  def GetCurrentTimeUsec(self):
338    """Returns the current time in microseconds."""
339    return int(round(self.time_func() * 1000 * 1000))
340
341  def _RegisterReqestForLogging(self, path, response_code, start_time_usec,
342                                request_size_bytes):
343    """Registers a request for client deploy logging purposes."""
344    end_time_usec = self.GetCurrentTimeUsec()
345    self.requests.append(client_deployinfo.Request(
346        path=path,
347        response_code=response_code,
348        start_time_usec=start_time_usec,
349        end_time_usec=end_time_usec,
350        request_size_bytes=request_size_bytes))
351
352  def LogClientDeploy(self, runtime, start_time_usec, success):
353    """Logs a client deployment attempt.
354
355    Args:
356      runtime: The runtime for the app being deployed.
357      start_time_usec: The start time of the deployment in micro seconds.
358      success: True if the deployment succeeded otherwise False.
359    """
360    if not self.usage_reporting:
361      log.info('Skipping usage reporting.')
362      return
363    end_time_usec = self.GetCurrentTimeUsec()
364    try:
365      info = client_deployinfo.ClientDeployInfoExternal(
366          runtime=runtime,
367          start_time_usec=start_time_usec,
368          end_time_usec=end_time_usec,
369          requests=self.requests,
370          success=success,
371          sdk_version=config.CLOUD_SDK_VERSION)
372      self.Send('/api/logclientdeploy', info.ToYAML())
373    except BaseException as e:  # pylint: disable=broad-except
374      log.debug('Exception logging deploy info continuing - {0}'.format(e))
375
376
377class RPCServer(object):
378  """This wraps the underlying RPC server so we can make a nice error message.
379
380  This will go away once we switch to just using our own http object.
381  """
382
383  def __init__(self, original_server):
384    """Construct a new rpc server.
385
386    Args:
387      original_server: The server to wrap.
388    """
389    self._server = original_server
390
391  def Send(self, *args, **kwargs):
392    try:
393      response = self._server.Send(*args, **kwargs)
394      log.debug('Got response: %s', response)
395      return response
396    except urllib.error.HTTPError as e:
397      # This is the message body, if included in e
398      if hasattr(e, 'read'):
399        body = e.read()
400      else:
401        body = ''
402      exceptions.reraise(RPCError(e, body=body))
403