1# -*- coding: utf-8 -*- 2# Copyright 2017 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""JSON gsutil Cloud API implementation for Google Cloud Storage.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import json 23import logging 24import traceback 25 26from apitools.base.py import exceptions as apitools_exceptions 27from boto import config 28from gslib.cloud_api import AccessDeniedException 29from gslib.cloud_api import BadRequestException 30from gslib.cloud_api import NotFoundException 31from gslib.cloud_api import PreconditionException 32from gslib.cloud_api import ServiceException 33from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache 34from gslib.no_op_credentials import NoOpCredentials 35from gslib.third_party.kms_apitools import cloudkms_v1_client as apitools_client 36from gslib.third_party.kms_apitools import cloudkms_v1_messages as apitools_messages 37from gslib.utils import system_util 38from gslib.utils.boto_util import GetCertsFile 39from gslib.utils.boto_util import GetMaxRetryDelay 40from gslib.utils.boto_util import GetNewHttp 41from gslib.utils.boto_util import GetNumRetries 42 43TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) 44 45if system_util.InvokedViaCloudSdk(): 46 _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 47 'Insufficient OAuth2 scope to perform this operation. ' 48 'Please re-run `gcloud auth login`') 49else: 50 _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 51 'Insufficient OAuth2 scope to perform this operation. ' 52 'Please re-run `gsutil config`') 53 54 55class KmsApi(object): 56 """Wraps calls to the Cloud KMS v1 interface via apitools.""" 57 58 def __init__(self, logger=None, credentials=None, debug=0): 59 """Performs necessary setup for interacting with Google Cloud KMS. 60 61 Args: 62 logger: logging.logger for outputting log messages. 63 credentials: Credentials to be used for interacting with Cloud KMS 64 debug: Debug level for the API implementation (0..3). 65 """ 66 super(KmsApi, self).__init__() 67 self.logger = logger 68 69 self.certs_file = GetCertsFile() 70 self.http = GetNewHttp() 71 self.http_base = 'https://' 72 self.host_base = config.get('Credentials', 'gs_kms_host', 73 'cloudkms.googleapis.com') 74 gs_kms_port = config.get('Credentials', 'gs_kms_port', None) 75 self.host_port = (':' + gs_kms_port) if gs_kms_port else '' 76 self.url_base = (self.http_base + self.host_base + self.host_port) 77 78 SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) 79 80 log_request = (debug >= 3) 81 log_response = (debug >= 3) 82 83 self.api_client = apitools_client.CloudkmsV1(url=self.url_base, 84 http=self.http, 85 log_request=log_request, 86 log_response=log_response, 87 credentials=self.credentials) 88 89 self.num_retries = GetNumRetries() 90 self.api_client.num_retries = self.num_retries 91 92 self.max_retry_wait = GetMaxRetryDelay() 93 self.api_client.max_retry_wait = self.max_retry_wait 94 95 if isinstance(self.credentials, NoOpCredentials): 96 # This API key is not secret and is used to identify gsutil during 97 # anonymous requests. 98 self.api_client.AddGlobalParam( 99 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') 100 101 def GetKeyIamPolicy(self, key_name): 102 request = (apitools_messages. 103 CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest( 104 resource=key_name)) 105 try: 106 return (self.api_client.projects_locations_keyRings_cryptoKeys. 107 GetIamPolicy(request)) 108 except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: 109 self._TranslateExceptionAndRaise(e, key_name=key_name) 110 111 def SetKeyIamPolicy(self, key_name, policy): 112 policy_request = apitools_messages.SetIamPolicyRequest(policy=policy) 113 request = (apitools_messages. 114 CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest( 115 resource=key_name, setIamPolicyRequest=policy_request)) 116 try: 117 return (self.api_client.projects_locations_keyRings_cryptoKeys. 118 SetIamPolicy(request)) 119 except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: 120 self._TranslateExceptionAndRaise(e, key_name=key_name) 121 122 def CreateKeyRing(self, project, keyring_name, location='global'): 123 """Attempts to create the specified keyRing. 124 125 Args: 126 project: (str) The project id in which to create the keyRing and key. 127 keyring_name: (str) The name of the keyRing, e.g. my-keyring. Note 128 that this must be unique within the location. 129 location: (str) The location in which to create the keyRing. Defaults to 130 'global'. 131 132 Returns: 133 (str) The fully-qualified name of the keyRing, e.g.: 134 projects/my-project/locations/global/keyRings/my-keyring 135 136 Raises: 137 Translated CloudApi exception if we were unable to create the keyRing. 138 Note that in the event of a 409 status code (resource already exists) when 139 attempting creation, we continue and treat this as a success. 140 """ 141 keyring_msg = apitools_messages.KeyRing( 142 name='projects/%s/locations/%s/keyRings/%s' % 143 (project, location, keyring_name)) 144 keyring_create_request = ( 145 apitools_messages.CloudkmsProjectsLocationsKeyRingsCreateRequest( 146 keyRing=keyring_msg, 147 keyRingId=keyring_name, 148 parent='projects/%s/locations/%s' % (project, location))) 149 try: 150 self.api_client.projects_locations_keyRings.Create(keyring_create_request) 151 except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: 152 if e.status_code != 409: 153 raise 154 return 'projects/%s/locations/%s/keyRings/%s' % (project, location, 155 keyring_name) 156 157 def CreateCryptoKey(self, keyring_fqn, key_name): 158 """Attempts to create the specified cryptoKey. 159 160 Args: 161 keyring_fqn: (str) The fully-qualified name of the keyRing, e.g. 162 projects/my-project/locations/global/keyRings/my-keyring. 163 key_name: (str) The name of the desired key, e.g. my-key. Note that 164 this must be unique within the keyRing. 165 166 Returns: 167 (str) The fully-qualified name of the cryptoKey, e.g.: 168 projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key 169 170 Raises: 171 Translated CloudApi exception if we were unable to create the cryptoKey. 172 Note that in the event of a 409 status code (resource already exists) when 173 attempting creation, we continue and treat this as a success. 174 """ 175 cryptokey_msg = apitools_messages.CryptoKey(purpose=( 176 apitools_messages.CryptoKey.PurposeValueValuesEnum.ENCRYPT_DECRYPT)) 177 cryptokey_create_request = ( 178 apitools_messages. 179 CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest( 180 cryptoKey=cryptokey_msg, cryptoKeyId=key_name, parent=keyring_fqn)) 181 try: 182 self.api_client.projects_locations_keyRings_cryptoKeys.Create( 183 cryptokey_create_request) 184 except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: 185 if e.status_code != 409: 186 raise 187 return '%s/cryptoKeys/%s' % (keyring_fqn.rstrip('/'), key_name) 188 189 def _TranslateExceptionAndRaise(self, e, key_name=None): 190 """Translates an HTTP exception and raises the translated or original value. 191 192 Args: 193 e: Any Exception. 194 key_name: Optional key name in request that caused the exception. 195 196 Raises: 197 Translated CloudApi exception, or the original exception if it was not 198 translatable. 199 """ 200 if self.logger.isEnabledFor(logging.DEBUG): 201 self.logger.debug('TranslateExceptionAndRaise: %s', 202 traceback.format_exc()) 203 translated_exception = self._TranslateApitoolsException(e, 204 key_name=key_name) 205 if translated_exception: 206 raise translated_exception 207 else: 208 raise 209 210 def _GetMessageFromHttpError(self, http_error): 211 if isinstance(http_error, apitools_exceptions.HttpError): 212 if getattr(http_error, 'content', None): 213 try: 214 json_obj = json.loads(http_error.content) 215 if 'error' in json_obj and 'message' in json_obj['error']: 216 return json_obj['error']['message'] 217 except Exception: # pylint: disable=broad-except 218 # If we couldn't decode anything, just leave the message as None. 219 pass 220 221 def _GetAcceptableScopesFromHttpError(self, http_error): 222 try: 223 www_authenticate = http_error.response['www-authenticate'] 224 # In the event of a scope error, the www-authenticate field of the HTTP 225 # response should contain text of the form 226 # 227 # 'Bearer realm="https://oauth2.googleapis.com/", 228 # error=insufficient_scope, 229 # scope="${space separated list of acceptable scopes}"' 230 # 231 # Here we use a quick string search to find the scope list, just looking 232 # for a substring with the form 'scope="${scopes}"'. 233 scope_idx = www_authenticate.find('scope="') 234 if scope_idx >= 0: 235 scopes = www_authenticate[scope_idx:].split('"')[1] 236 return 'Acceptable scopes: %s' % scopes 237 except Exception: # pylint: disable=broad-except 238 # Return None if we have any trouble parsing out the acceptable scopes. 239 pass 240 241 def _TranslateApitoolsException(self, e, key_name=None): 242 """Translates apitools exceptions into their gsutil equivalents. 243 244 Args: 245 e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. 246 key_name: Optional key name in request that caused the exception. 247 248 Returns: 249 CloudStorageApiServiceException for translatable exceptions, None 250 otherwise. 251 """ 252 253 if isinstance(e, apitools_exceptions.HttpError): 254 message = self._GetMessageFromHttpError(e) 255 if e.status_code == 400: 256 # It is possible that the Project ID is incorrect. Unfortunately the 257 # JSON API does not give us much information about what part of the 258 # request was bad. 259 return BadRequestException(message or 'Bad Request', 260 status=e.status_code) 261 elif e.status_code == 401: 262 if 'Login Required' in str(e): 263 return AccessDeniedException(message or 264 'Access denied: login required.', 265 status=e.status_code) 266 elif 'insufficient_scope' in str(e): 267 # If the service includes insufficient scope error detail in the 268 # response body, this check can be removed. 269 return AccessDeniedException( 270 _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, 271 status=e.status_code, 272 body=self._GetAcceptableScopesFromHttpError(e)) 273 elif e.status_code == 403: 274 if 'The account for the specified project has been disabled' in str(e): 275 return AccessDeniedException(message or 'Account disabled.', 276 status=e.status_code) 277 elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): 278 return AccessDeniedException(message or 279 'Access denied: quota exceeded. ' 280 'Is your project ID valid?', 281 status=e.status_code) 282 elif 'User Rate Limit Exceeded' in str(e): 283 return AccessDeniedException( 284 'Rate limit exceeded. Please retry this ' 285 'request later.', 286 status=e.status_code) 287 elif 'Access Not Configured' in str(e): 288 return AccessDeniedException( 289 'Access Not Configured. Please go to the Google Cloud Platform ' 290 'Console (https://cloud.google.com/console#/project) for your ' 291 'project, select APIs & services, and enable the Google Cloud ' 292 'KMS API.', 293 status=e.status_code) 294 elif 'insufficient_scope' in str(e): 295 # If the service includes insufficient scope error detail in the 296 # response body, this check can be removed. 297 return AccessDeniedException( 298 _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, 299 status=e.status_code, 300 body=self._GetAcceptableScopesFromHttpError(e)) 301 else: 302 return AccessDeniedException(message or e.message or key_name, 303 status=e.status_code) 304 elif e.status_code == 404: 305 return NotFoundException(message or e.message, status=e.status_code) 306 307 elif e.status_code == 409 and key_name: 308 return ServiceException('The key %s already exists.' % key_name, 309 status=e.status_code) 310 elif e.status_code == 412: 311 return PreconditionException(message, status=e.status_code) 312 return ServiceException(message, status=e.status_code) 313