1# -*- coding: utf-8 -*- # 2# Copyright 2019 Google LLC. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Module for making API requests.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import unicode_literals 20 21import copy 22import json 23 24from googlecloudsdk.api_lib.compute import batch_helper 25from googlecloudsdk.api_lib.compute import utils 26from googlecloudsdk.api_lib.compute import waiters 27from googlecloudsdk.core import log 28import six 29from six.moves import zip # pylint: disable=redefined-builtin 30 31 32def _RequestsAreListRequests(requests): 33 """Checks if all requests are of list requests.""" 34 list_requests = [ 35 method in ( 36 'List', 'AggregatedList', 'ListInstances', 'ListManagedInstances' 37 ) for _, method, _ in requests 38 ] 39 if all(list_requests): 40 return True 41 elif not any(list_requests): 42 return False 43 else: 44 raise ValueError( 45 'All requests must be either list requests or non-list requests.') 46 47 48def _HandleJsonList(response, service, method, errors): 49 """Extracts data from one *List response page as JSON and stores in dicts. 50 51 Args: 52 response: str, The *List response in JSON 53 service: The service which responded to *List request 54 method: str, Method used to list resources. One of 'List' or 55 'AggregatedList'. 56 errors: list, Errors from response will be appended to this list. 57 58 Returns: 59 Pair of: 60 - List of items returned in response as dicts 61 - Next page token (if present, otherwise None). 62 """ 63 items = [] 64 65 response = json.loads(response) 66 67 # If the request is a list call, then yield the items directly. 68 if method in ('List', 'ListInstances'): 69 items = response.get('items', []) 70 elif method == 'ListManagedInstances': 71 items = response.get('managedInstances', []) 72 73 # If the request is an aggregatedList call, then do all the 74 # magic necessary to get the actual resources because the 75 # aggregatedList responses are very complicated data 76 # structures... 77 elif method == 'AggregatedList': 78 items_field_name = service.GetMethodConfig( 79 'AggregatedList').relative_path.split('/')[-1] 80 for scope_result in six.itervalues(response['items']): 81 # If the given scope is unreachable, record the warning 82 # message in the errors list. 83 warning = scope_result.get('warning', None) 84 if warning and warning['code'] == 'UNREACHABLE': 85 errors.append((None, warning['message'])) 86 87 items.extend(scope_result.get(items_field_name, [])) 88 89 return items, response.get('nextPageToken', None) 90 91 92def _HandleMessageList(response, service, method, errors): 93 """Extracts data from one *List response page as Message object.""" 94 items = [] 95 96 # If the request is a list call, then yield the items directly. 97 if method in ('List', 'ListInstances'): 98 items = response.items 99 elif method == 'ListManagedInstances': 100 items = response.managedInstances 101 # If the request is an aggregatedList call, then do all the 102 # magic necessary to get the actual resources because the 103 # aggregatedList responses are very complicated data 104 # structures... 105 else: 106 items_field_name = service.GetMethodConfig( 107 'AggregatedList').relative_path.split('/')[-1] 108 for scope_result in response.items.additionalProperties: 109 # If the given scope is unreachable, record the warning 110 # message in the errors list. 111 warning = scope_result.value.warning 112 if warning and warning.code == warning.CodeValueValuesEnum.UNREACHABLE: 113 errors.append((None, warning.message)) 114 115 items.extend(getattr(scope_result.value, items_field_name)) 116 117 return items, response.nextPageToken 118 119 120def _ListCore(requests, http, batch_url, errors, response_handler): 121 """Makes a series of list and/or aggregatedList batch requests. 122 123 Args: 124 requests: A list of requests to make. Each element must be a 3-element tuple 125 where the first element is the service, the second element is the method 126 ('List' or 'AggregatedList'), and the third element is a protocol buffer 127 representing either a list or aggregatedList request. 128 http: An httplib2.Http-like object. 129 batch_url: The handler for making batch requests. 130 errors: A list for capturing errors. If any response contains an error, it 131 is added to this list. 132 response_handler: The function to extract information responses. 133 134 Yields: 135 Resources encapsulated in format chosen by response_handler as they are 136 received from the server. 137 """ 138 while requests: 139 responses, request_errors = batch_helper.MakeRequests( 140 requests=requests, http=http, batch_url=batch_url) 141 errors.extend(request_errors) 142 143 new_requests = [] 144 145 for i, response in enumerate(responses): 146 if not response: 147 continue 148 149 service, method, request_protobuf = requests[i] 150 151 items, next_page_token = response_handler(response, service, method, 152 errors) 153 for item in items: 154 yield item 155 156 if next_page_token: 157 new_request_protobuf = copy.deepcopy(request_protobuf) 158 new_request_protobuf.pageToken = next_page_token 159 new_requests.append((service, method, new_request_protobuf)) 160 161 requests = new_requests 162 163 164def _List(requests, http, batch_url, errors): 165 """Makes a series of list and/or aggregatedList batch requests. 166 167 Args: 168 requests: A list of requests to make. Each element must be a 3-element tuple 169 where the first element is the service, the second element is the method 170 ('List' or 'AggregatedList'), and the third element is a protocol buffer 171 representing either a list or aggregatedList request. 172 http: An httplib2.Http-like object. 173 batch_url: The handler for making batch requests. 174 errors: A list for capturing errors. If any response contains an error, it 175 is added to this list. 176 177 Returns: 178 Resources encapsulated as protocol buffers as they are received 179 from the server. 180 """ 181 return _ListCore(requests, http, batch_url, errors, _HandleMessageList) 182 183 184def ListJson(requests, http, batch_url, errors): 185 """Makes a series of list and/or aggregatedList batch requests. 186 187 This function does all of: 188 - Sends batch of List/AggragatedList requests 189 - Extracts items from responses 190 - Handles pagination 191 192 All requests must be sent to the same client - Compute. 193 194 Args: 195 requests: A list of requests to make. Each element must be a 3-element tuple 196 where the first element is the service, the second element is the method 197 ('List' or 'AggregatedList'), and the third element is a protocol buffer 198 representing either a list or aggregatedList request. 199 http: An httplib2.Http-like object. 200 batch_url: The handler for making batch requests. 201 errors: A list for capturing errors. If any response contains an error, it 202 is added to this list. 203 204 Yields: 205 Resources in dicts as they are received from the server. 206 """ 207 # This is compute-specific helper. It is assumed at this point that all 208 # requests are being sent to the same client (for example Compute). 209 with requests[0][0].client.JsonResponseModel(): 210 for item in _ListCore(requests, http, batch_url, errors, _HandleJsonList): 211 yield item 212 213 214def MakeRequests(requests, 215 http, 216 batch_url, 217 errors, 218 progress_tracker=None, 219 no_followup=False, 220 always_return_operation=False, 221 followup_overrides=None, 222 log_result=True, 223 timeout=None): 224 """Makes one or more requests to the API. 225 226 Each request can be either a synchronous API call or an asynchronous 227 one. For synchronous calls (e.g., get and list), the result from the 228 server is yielded immediately. For asynchronous calls (e.g., calls 229 that return operations like insert), this function waits until the 230 operation reaches the DONE state and fetches the corresponding 231 object and yields that object (nothing is yielded for deletions). 232 233 Currently, a heterogenous set of synchronous calls can be made 234 (e.g., get request to fetch a disk and instance), however, the 235 asynchronous requests must be homogenous (e.g., they must all be the 236 same verb on the same collection). In the future, heterogenous 237 asynchronous requests will be supported. For now, it is up to the 238 client to ensure that the asynchronous requests are 239 homogenous. Synchronous and asynchronous requests can be mixed. 240 241 Args: 242 requests: A list of requests to make. Each element must be a 3-element tuple 243 where the first element is the service, the second element is the string 244 name of the method on the service, and the last element is a protocol 245 buffer representing the request. 246 http: An httplib2.Http-like object. 247 batch_url: The handler for making batch requests. 248 errors: A list for capturing errors. If any response contains an error, it 249 is added to this list. 250 progress_tracker: progress tracker to be ticked while waiting for operations 251 to finish. 252 no_followup: If True, do not followup operation with a GET request. 253 always_return_operation: If True, return operation object even if operation 254 fails. 255 followup_overrides: A list of new resource names to GET once the operation 256 finishes. Generally used in renaming calls. 257 log_result: Whether the Operation Waiter should print the result in past 258 tense of each request. 259 timeout: The maximum amount of time, in seconds, to wait for the 260 operations to reach the DONE state. 261 262 Yields: 263 A response for each request. For deletion requests, no corresponding 264 responses are returned. 265 """ 266 if _RequestsAreListRequests(requests): 267 for item in _List( 268 requests=requests, http=http, batch_url=batch_url, errors=errors): 269 yield item 270 return 271 responses, new_errors = batch_helper.MakeRequests( 272 requests=requests, http=http, batch_url=batch_url) 273 errors.extend(new_errors) 274 275 operation_service = None 276 resource_service = None 277 278 # Collects all operation objects in a list so they can be waited on 279 # and yields all non-operation objects since non-operation responses 280 # cannot be waited on. 281 operations_data = [] 282 283 if not followup_overrides: 284 followup_overrides = [None for _ in requests] 285 for request, response, followup_override in zip(requests, responses, 286 followup_overrides): 287 if response is None: 288 continue 289 290 service, _, request_body = request 291 if (isinstance(response, service.client.MESSAGES_MODULE.Operation) and 292 service.__class__.__name__ not in ( 293 'GlobalOperationsService', 'RegionOperationsService', 294 'ZoneOperationsService', 'GlobalOrganizationOperationsService', 295 'GlobalAccountsOperationsService')): 296 297 resource_service = service 298 project = None 299 if hasattr(request_body, 'project'): 300 project = request_body.project 301 if response.zone: 302 operation_service = service.client.zoneOperations 303 elif response.region: 304 operation_service = service.client.regionOperations 305 else: 306 operation_service = service.client.globalOperations 307 else: 308 operation_service = service.client.globalOrganizationOperations 309 310 operations_data.append( 311 waiters.OperationData( 312 response, 313 operation_service, 314 resource_service, 315 project=project, 316 no_followup=no_followup, 317 followup_override=followup_override, 318 always_return_operation=always_return_operation)) 319 320 else: 321 yield response 322 323 if operations_data: 324 warnings = [] 325 for response in waiters.WaitForOperations( 326 operations_data=operations_data, 327 http=http, 328 batch_url=batch_url, 329 warnings=warnings, 330 progress_tracker=progress_tracker, 331 errors=errors, 332 log_result=log_result, 333 timeout=timeout): 334 yield response 335 336 if warnings: 337 log.warning( 338 utils.ConstructList('Some requests generated warnings:', warnings)) 339