1# -*- coding: utf-8 -*- #
2# Copyright 2019 Google LLC. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#    http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Module for making API requests."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import unicode_literals
20
21import copy
22import json
23
24from googlecloudsdk.api_lib.compute import batch_helper
25from googlecloudsdk.api_lib.compute import utils
26from googlecloudsdk.api_lib.compute import waiters
27from googlecloudsdk.core import log
28import six
29from six.moves import zip  # pylint: disable=redefined-builtin
30
31
32def _RequestsAreListRequests(requests):
33  """Checks if all requests are of list requests."""
34  list_requests = [
35      method in (
36          'List', 'AggregatedList', 'ListInstances', 'ListManagedInstances'
37          ) for _, method, _ in requests
38  ]
39  if all(list_requests):
40    return True
41  elif not any(list_requests):
42    return False
43  else:
44    raise ValueError(
45        'All requests must be either list requests or non-list requests.')
46
47
48def _HandleJsonList(response, service, method, errors):
49  """Extracts data from one *List response page as JSON and stores in dicts.
50
51  Args:
52    response: str, The *List response in JSON
53    service: The service which responded to *List request
54    method: str, Method used to list resources. One of 'List' or
55      'AggregatedList'.
56    errors: list, Errors from response will be appended to  this list.
57
58  Returns:
59    Pair of:
60    - List of items returned in response as dicts
61    - Next page token (if present, otherwise None).
62  """
63  items = []
64
65  response = json.loads(response)
66
67  # If the request is a list call, then yield the items directly.
68  if method in ('List', 'ListInstances'):
69    items = response.get('items', [])
70  elif method == 'ListManagedInstances':
71    items = response.get('managedInstances', [])
72
73  # If the request is an aggregatedList call, then do all the
74  # magic necessary to get the actual resources because the
75  # aggregatedList responses are very complicated data
76  # structures...
77  elif method == 'AggregatedList':
78    items_field_name = service.GetMethodConfig(
79        'AggregatedList').relative_path.split('/')[-1]
80    for scope_result in six.itervalues(response['items']):
81      # If the given scope is unreachable, record the warning
82      # message in the errors list.
83      warning = scope_result.get('warning', None)
84      if warning and warning['code'] == 'UNREACHABLE':
85        errors.append((None, warning['message']))
86
87      items.extend(scope_result.get(items_field_name, []))
88
89  return items, response.get('nextPageToken', None)
90
91
92def _HandleMessageList(response, service, method, errors):
93  """Extracts data from one *List response page as Message object."""
94  items = []
95
96  # If the request is a list call, then yield the items directly.
97  if method in ('List', 'ListInstances'):
98    items = response.items
99  elif method == 'ListManagedInstances':
100    items = response.managedInstances
101  # If the request is an aggregatedList call, then do all the
102  # magic necessary to get the actual resources because the
103  # aggregatedList responses are very complicated data
104  # structures...
105  else:
106    items_field_name = service.GetMethodConfig(
107        'AggregatedList').relative_path.split('/')[-1]
108    for scope_result in response.items.additionalProperties:
109      # If the given scope is unreachable, record the warning
110      # message in the errors list.
111      warning = scope_result.value.warning
112      if warning and warning.code == warning.CodeValueValuesEnum.UNREACHABLE:
113        errors.append((None, warning.message))
114
115      items.extend(getattr(scope_result.value, items_field_name))
116
117  return items, response.nextPageToken
118
119
120def _ListCore(requests, http, batch_url, errors, response_handler):
121  """Makes a series of list and/or aggregatedList batch requests.
122
123  Args:
124    requests: A list of requests to make. Each element must be a 3-element tuple
125      where the first element is the service, the second element is the method
126      ('List' or 'AggregatedList'), and the third element is a protocol buffer
127      representing either a list or aggregatedList request.
128    http: An httplib2.Http-like object.
129    batch_url: The handler for making batch requests.
130    errors: A list for capturing errors. If any response contains an error, it
131      is added to this list.
132    response_handler: The function to extract information responses.
133
134  Yields:
135    Resources encapsulated in format chosen by response_handler as they are
136      received from the server.
137  """
138  while requests:
139    responses, request_errors = batch_helper.MakeRequests(
140        requests=requests, http=http, batch_url=batch_url)
141    errors.extend(request_errors)
142
143    new_requests = []
144
145    for i, response in enumerate(responses):
146      if not response:
147        continue
148
149      service, method, request_protobuf = requests[i]
150
151      items, next_page_token = response_handler(response, service, method,
152                                                errors)
153      for item in items:
154        yield item
155
156      if next_page_token:
157        new_request_protobuf = copy.deepcopy(request_protobuf)
158        new_request_protobuf.pageToken = next_page_token
159        new_requests.append((service, method, new_request_protobuf))
160
161    requests = new_requests
162
163
164def _List(requests, http, batch_url, errors):
165  """Makes a series of list and/or aggregatedList batch requests.
166
167  Args:
168    requests: A list of requests to make. Each element must be a 3-element tuple
169      where the first element is the service, the second element is the method
170      ('List' or 'AggregatedList'), and the third element is a protocol buffer
171      representing either a list or aggregatedList request.
172    http: An httplib2.Http-like object.
173    batch_url: The handler for making batch requests.
174    errors: A list for capturing errors. If any response contains an error, it
175      is added to this list.
176
177  Returns:
178    Resources encapsulated as protocol buffers as they are received
179      from the server.
180  """
181  return _ListCore(requests, http, batch_url, errors, _HandleMessageList)
182
183
184def ListJson(requests, http, batch_url, errors):
185  """Makes a series of list and/or aggregatedList batch requests.
186
187  This function does all of:
188  - Sends batch of List/AggragatedList requests
189  - Extracts items from responses
190  - Handles pagination
191
192  All requests must be sent to the same client - Compute.
193
194  Args:
195    requests: A list of requests to make. Each element must be a 3-element tuple
196      where the first element is the service, the second element is the method
197      ('List' or 'AggregatedList'), and the third element is a protocol buffer
198      representing either a list or aggregatedList request.
199    http: An httplib2.Http-like object.
200    batch_url: The handler for making batch requests.
201    errors: A list for capturing errors. If any response contains an error, it
202      is added to this list.
203
204  Yields:
205    Resources in dicts as they are received from the server.
206  """
207  # This is compute-specific helper. It is assumed at this point that all
208  # requests are being sent to the same client (for example Compute).
209  with requests[0][0].client.JsonResponseModel():
210    for item in _ListCore(requests, http, batch_url, errors, _HandleJsonList):
211      yield item
212
213
214def MakeRequests(requests,
215                 http,
216                 batch_url,
217                 errors,
218                 progress_tracker=None,
219                 no_followup=False,
220                 always_return_operation=False,
221                 followup_overrides=None,
222                 log_result=True,
223                 timeout=None):
224  """Makes one or more requests to the API.
225
226  Each request can be either a synchronous API call or an asynchronous
227  one. For synchronous calls (e.g., get and list), the result from the
228  server is yielded immediately. For asynchronous calls (e.g., calls
229  that return operations like insert), this function waits until the
230  operation reaches the DONE state and fetches the corresponding
231  object and yields that object (nothing is yielded for deletions).
232
233  Currently, a heterogenous set of synchronous calls can be made
234  (e.g., get request to fetch a disk and instance), however, the
235  asynchronous requests must be homogenous (e.g., they must all be the
236  same verb on the same collection). In the future, heterogenous
237  asynchronous requests will be supported. For now, it is up to the
238  client to ensure that the asynchronous requests are
239  homogenous. Synchronous and asynchronous requests can be mixed.
240
241  Args:
242    requests: A list of requests to make. Each element must be a 3-element tuple
243      where the first element is the service, the second element is the string
244      name of the method on the service, and the last element is a protocol
245      buffer representing the request.
246    http: An httplib2.Http-like object.
247    batch_url: The handler for making batch requests.
248    errors: A list for capturing errors. If any response contains an error, it
249      is added to this list.
250    progress_tracker: progress tracker to be ticked while waiting for operations
251      to finish.
252    no_followup: If True, do not followup operation with a GET request.
253    always_return_operation: If True, return operation object even if operation
254      fails.
255    followup_overrides: A list of new resource names to GET once the operation
256      finishes. Generally used in renaming calls.
257    log_result: Whether the Operation Waiter should print the result in past
258      tense of each request.
259    timeout: The maximum amount of time, in seconds, to wait for the
260      operations to reach the DONE state.
261
262  Yields:
263    A response for each request. For deletion requests, no corresponding
264    responses are returned.
265  """
266  if _RequestsAreListRequests(requests):
267    for item in _List(
268        requests=requests, http=http, batch_url=batch_url, errors=errors):
269      yield item
270    return
271  responses, new_errors = batch_helper.MakeRequests(
272      requests=requests, http=http, batch_url=batch_url)
273  errors.extend(new_errors)
274
275  operation_service = None
276  resource_service = None
277
278  # Collects all operation objects in a list so they can be waited on
279  # and yields all non-operation objects since non-operation responses
280  # cannot be waited on.
281  operations_data = []
282
283  if not followup_overrides:
284    followup_overrides = [None for _ in requests]
285  for request, response, followup_override in zip(requests, responses,
286                                                  followup_overrides):
287    if response is None:
288      continue
289
290    service, _, request_body = request
291    if (isinstance(response, service.client.MESSAGES_MODULE.Operation) and
292        service.__class__.__name__ not in (
293            'GlobalOperationsService', 'RegionOperationsService',
294            'ZoneOperationsService', 'GlobalOrganizationOperationsService',
295            'GlobalAccountsOperationsService')):
296
297      resource_service = service
298      project = None
299      if hasattr(request_body, 'project'):
300        project = request_body.project
301        if response.zone:
302          operation_service = service.client.zoneOperations
303        elif response.region:
304          operation_service = service.client.regionOperations
305        else:
306          operation_service = service.client.globalOperations
307      else:
308        operation_service = service.client.globalOrganizationOperations
309
310      operations_data.append(
311          waiters.OperationData(
312              response,
313              operation_service,
314              resource_service,
315              project=project,
316              no_followup=no_followup,
317              followup_override=followup_override,
318              always_return_operation=always_return_operation))
319
320    else:
321      yield response
322
323  if operations_data:
324    warnings = []
325    for response in waiters.WaitForOperations(
326        operations_data=operations_data,
327        http=http,
328        batch_url=batch_url,
329        warnings=warnings,
330        progress_tracker=progress_tracker,
331        errors=errors,
332        log_result=log_result,
333        timeout=timeout):
334      yield response
335
336    if warnings:
337      log.warning(
338          utils.ConstructList('Some requests generated warnings:', warnings))
339