1# -*- coding: utf-8 -*- # 2# Copyright 2015 Google LLC. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Utility functions for gcloud app.""" 17 18from __future__ import absolute_import 19from __future__ import division 20from __future__ import unicode_literals 21 22import datetime 23import os 24import posixpath 25import sys 26import time 27from googlecloudsdk.core import config 28from googlecloudsdk.core import exceptions 29from googlecloudsdk.core import log 30from googlecloudsdk.core.util import platforms 31from googlecloudsdk.third_party.appengine.api import client_deployinfo 32import six 33from six.moves import urllib 34 35 36class Error(exceptions.Error): 37 """Exceptions for the appcfg module.""" 38 39 40class NoFieldsSpecifiedError(Error): 41 """The user specified no fields to a command which requires at least one.""" 42 43 44class NoCloudSDKError(Error): 45 """The module was unable to find Cloud SDK.""" 46 47 def __init__(self): 48 super(NoCloudSDKError, self).__init__( 49 'Unable to find a Cloud SDK installation.') 50 51 52class NoAppengineSDKError(Error): 53 """The module was unable to find the appengine SDK.""" 54 55 56class TimeoutError(Error): 57 """An exception for when a retry with wait operation times out.""" 58 59 def __init__(self): 60 super(TimeoutError, self).__init__( 61 'Timed out waiting for the operation to complete.') 62 63 64class RPCError(Error): 65 """For when an error occurs when making an RPC call.""" 66 67 def __init__(self, url_error, body=''): 68 super(RPCError, self).__init__( 69 'Server responded with code [{code}]:\n {reason}.\n {body}' 70 .format(code=url_error.code, 71 reason=getattr(url_error, 'reason', '(unknown)'), 72 body=body)) 73 self.url_error = url_error 74 75 76def GetCloudSDKRoot(): 77 """Gets the directory of the root of the Cloud SDK, error if it doesn't exist. 78 79 Raises: 80 NoCloudSDKError: If there is no SDK root. 81 82 Returns: 83 str, The path to the root of the Cloud SDK. 84 """ 85 sdk_root = config.Paths().sdk_root 86 if not sdk_root: 87 raise NoCloudSDKError() 88 log.debug('Found Cloud SDK root: %s', sdk_root) 89 return sdk_root 90 91 92def GetAppEngineSDKRoot(): 93 """Gets the directory of the GAE SDK directory in the SDK. 94 95 Raises: 96 NoCloudSDKError: If there is no SDK root. 97 NoAppengineSDKError: If the GAE SDK cannot be found. 98 99 Returns: 100 str, The path to the root of the GAE SDK within the Cloud SDK. 101 """ 102 sdk_root = GetCloudSDKRoot() 103 gae_sdk_dir = os.path.join(sdk_root, 'platform', 'google_appengine') 104 if not os.path.isdir(gae_sdk_dir): 105 raise NoAppengineSDKError() 106 log.debug('Found App Engine SDK root: %s', gae_sdk_dir) 107 108 return gae_sdk_dir 109 110 111def GenerateVersionId(datetime_getter=datetime.datetime.now): 112 """Generates a version id based off the current time. 113 114 Args: 115 datetime_getter: A function that returns a datetime.datetime instance. 116 117 Returns: 118 A version string based. 119 """ 120 return datetime_getter().isoformat().lower().replace('-', '').replace( 121 ':', '')[:15] 122 123 124def ConvertToPosixPath(path): 125 """Converts a native-OS path to /-separated: os.path.join('a', 'b')->'a/b'.""" 126 return posixpath.join(*path.split(os.path.sep)) 127 128 129def ShouldSkip(skip_files, path): 130 """Returns whether the given path should be skipped by the skip_files field. 131 132 A user can specify a `skip_files` field in their .yaml file, which is a list 133 of regular expressions matching files that should be skipped. By this point in 134 the code, it's been turned into one mega-regex that matches any file to skip. 135 136 Args: 137 skip_files: A regular expression object for files/directories to skip. 138 path: str, the path to the file/directory which might be skipped (relative 139 to the application root) 140 141 Returns: 142 bool, whether the file/dir should be skipped. 143 """ 144 # On Windows, os.path.join uses the path separator '\' instead of '/'. 145 # However, the skip_files regular expression always uses '/'. 146 # To handle this, we'll replace '\' characters with '/' characters. 147 path = ConvertToPosixPath(path) 148 return skip_files.match(path) 149 150 151def FileIterator(base, skip_files): 152 """Walks a directory tree, returning all the files. Follows symlinks. 153 154 Args: 155 base: The base path to search for files under. 156 skip_files: A regular expression object for files/directories to skip. 157 158 Yields: 159 Paths of files found, relative to base. 160 """ 161 dirs = [''] 162 163 while dirs: 164 current_dir = dirs.pop() 165 entries = set(os.listdir(os.path.join(base, current_dir))) 166 for entry in sorted(entries): 167 name = os.path.join(current_dir, entry) 168 fullname = os.path.join(base, name) 169 170 if os.path.isfile(fullname): 171 if ShouldSkip(skip_files, name): 172 log.info('Ignoring file [%s]: File matches ignore regex.', name) 173 else: 174 yield name 175 elif os.path.isdir(fullname): 176 if ShouldSkip(skip_files, name): 177 log.info('Ignoring directory [%s]: Directory matches ignore regex.', 178 name) 179 else: 180 dirs.append(name) 181 182 183def RetryWithBackoff(func, retry_notify_func, 184 initial_delay=1, backoff_factor=2, 185 max_delay=60, max_tries=20, raise_on_timeout=True): 186 """Calls a function multiple times, backing off more and more each time. 187 188 Args: 189 func: f() -> (bool, value), A function that performs some operation that 190 should be retried a number of times upon failure. If the first tuple 191 element is True, we'll immediately return (True, value). If False, we'll 192 delay a bit and try again, unless we've hit the 'max_tries' limit, in 193 which case we'll return (False, value). 194 retry_notify_func: f(value, delay) -> None, This function will be called 195 immediately before the next retry delay. 'value' is the value returned 196 by the last call to 'func'. 'delay' is the retry delay, in seconds 197 initial_delay: int, Initial delay after first try, in seconds. 198 backoff_factor: int, Delay will be multiplied by this factor after each 199 try. 200 max_delay: int, Maximum delay, in seconds. 201 max_tries: int, Maximum number of tries (the first one counts). 202 raise_on_timeout: bool, True to raise an exception if the operation times 203 out instead of returning False. 204 205 Returns: 206 What the last call to 'func' returned, which is of the form (done, value). 207 If 'done' is True, you know 'func' returned True before we ran out of 208 retries. If 'done' is False, you know 'func' kept returning False and we 209 ran out of retries. 210 211 Raises: 212 TimeoutError: If raise_on_timeout is True and max_tries is exhausted. 213 """ 214 delay = initial_delay 215 try_count = max_tries 216 value = None 217 218 while True: 219 try_count -= 1 220 done, value = func() 221 if done: 222 return True, value 223 if try_count <= 0: 224 if raise_on_timeout: 225 raise TimeoutError() 226 return False, value 227 retry_notify_func(value, delay) 228 time.sleep(delay) 229 delay = min(delay * backoff_factor, max_delay) 230 231 232def RetryNoBackoff(callable_func, retry_notify_func, delay=5, max_tries=200): 233 """Calls a function multiple times, with the same delay each time. 234 235 Args: 236 callable_func: A function that performs some operation that should be 237 retried a number of times upon failure. Signature: () -> (done, value) 238 If 'done' is True, we'll immediately return (True, value) 239 If 'done' is False, we'll delay a bit and try again, unless we've 240 hit the 'max_tries' limit, in which case we'll return (False, value). 241 retry_notify_func: This function will be called immediately before the 242 next retry delay. Signature: (value, delay) -> None 243 'value' is the value returned by the last call to 'callable_func' 244 'delay' is the retry delay, in seconds 245 delay: Delay between tries, in seconds. 246 max_tries: Maximum number of tries (the first one counts). 247 248 Returns: 249 What the last call to 'callable_func' returned, which is of the form 250 (done, value). If 'done' is True, you know 'callable_func' returned True 251 before we ran out of retries. If 'done' is False, you know 'callable_func' 252 kept returning False and we ran out of retries. 253 254 Raises: 255 Whatever the function raises--an exception will immediately stop retries. 256 """ 257 # A backoff_factor of 1 means the delay won't grow. 258 return RetryWithBackoff(callable_func, retry_notify_func, delay, 1, delay, 259 max_tries) 260 261 262def GetSourceName(): 263 """Gets the name of this source version.""" 264 return 'Google-appcfg-{0}'.format(config.CLOUD_SDK_VERSION) 265 266 267def GetUserAgent(): 268 """Determines the value of the 'User-agent' header to use for HTTP requests. 269 270 Returns: 271 String containing the 'user-agent' header value. 272 """ 273 product_tokens = [] 274 275 # SDK version 276 product_tokens.append(config.CLOUDSDK_USER_AGENT) 277 278 # Platform 279 product_tokens.append(platforms.Platform.Current().UserAgentFragment()) 280 281 # Python version 282 python_version = '.'.join(six.text_type(i) for i in sys.version_info) 283 product_tokens.append('Python/%s' % python_version) 284 285 return ' '.join(product_tokens) 286 287 288class ClientDeployLoggingContext(object): 289 """Context for sending and recording server rpc requests. 290 291 Attributes: 292 rpcserver: The AbstractRpcServer to use for the upload. 293 requests: A list of client_deployinfo.Request objects to include 294 with the client deploy log. 295 time_func: Function to get the current time in milliseconds. 296 request_params: A dictionary with params to append to requests 297 """ 298 299 def __init__(self, 300 rpcserver, 301 request_params, 302 usage_reporting, 303 time_func=time.time): 304 """Creates a new AppVersionUpload. 305 306 Args: 307 rpcserver: The RPC server to use. Should be an instance of HttpRpcServer 308 or TestRpcServer. 309 request_params: A dictionary with params to append to requests 310 usage_reporting: Whether to actually upload data. 311 time_func: Function to return the current time in millisecods 312 (default time.time). 313 """ 314 self.rpcserver = rpcserver 315 self.request_params = request_params 316 self.usage_reporting = usage_reporting 317 self.time_func = time_func 318 self.requests = [] 319 320 def Send(self, url, payload='', **kwargs): 321 """Sends a request to the server, with common params.""" 322 start_time_usec = self.GetCurrentTimeUsec() 323 request_size_bytes = len(payload) 324 try: 325 log.debug('Send: {0}, params={1}'.format(url, self.request_params)) 326 327 kwargs.update(self.request_params) 328 result = self.rpcserver.Send(url, payload=payload, **kwargs) 329 self._RegisterReqestForLogging(url, 200, start_time_usec, 330 request_size_bytes) 331 return result 332 except RPCError as err: 333 self._RegisterReqestForLogging(url, err.url_error.code, start_time_usec, 334 request_size_bytes) 335 raise 336 337 def GetCurrentTimeUsec(self): 338 """Returns the current time in microseconds.""" 339 return int(round(self.time_func() * 1000 * 1000)) 340 341 def _RegisterReqestForLogging(self, path, response_code, start_time_usec, 342 request_size_bytes): 343 """Registers a request for client deploy logging purposes.""" 344 end_time_usec = self.GetCurrentTimeUsec() 345 self.requests.append(client_deployinfo.Request( 346 path=path, 347 response_code=response_code, 348 start_time_usec=start_time_usec, 349 end_time_usec=end_time_usec, 350 request_size_bytes=request_size_bytes)) 351 352 def LogClientDeploy(self, runtime, start_time_usec, success): 353 """Logs a client deployment attempt. 354 355 Args: 356 runtime: The runtime for the app being deployed. 357 start_time_usec: The start time of the deployment in micro seconds. 358 success: True if the deployment succeeded otherwise False. 359 """ 360 if not self.usage_reporting: 361 log.info('Skipping usage reporting.') 362 return 363 end_time_usec = self.GetCurrentTimeUsec() 364 try: 365 info = client_deployinfo.ClientDeployInfoExternal( 366 runtime=runtime, 367 start_time_usec=start_time_usec, 368 end_time_usec=end_time_usec, 369 requests=self.requests, 370 success=success, 371 sdk_version=config.CLOUD_SDK_VERSION) 372 self.Send('/api/logclientdeploy', info.ToYAML()) 373 except BaseException as e: # pylint: disable=broad-except 374 log.debug('Exception logging deploy info continuing - {0}'.format(e)) 375 376 377class RPCServer(object): 378 """This wraps the underlying RPC server so we can make a nice error message. 379 380 This will go away once we switch to just using our own http object. 381 """ 382 383 def __init__(self, original_server): 384 """Construct a new rpc server. 385 386 Args: 387 original_server: The server to wrap. 388 """ 389 self._server = original_server 390 391 def Send(self, *args, **kwargs): 392 try: 393 response = self._server.Send(*args, **kwargs) 394 log.debug('Got response: %s', response) 395 return response 396 except urllib.error.HTTPError as e: 397 # This is the message body, if included in e 398 if hasattr(e, 'read'): 399 body = e.read() 400 else: 401 body = '' 402 exceptions.reraise(RPCError(e, body=body)) 403