1# Copyright 2014 Google Inc. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Client for discovery based APIs. 16 17A client library for Google's discovery based APIs. 18""" 19from __future__ import absolute_import 20import six 21from six.moves import zip 22 23__author__ = 'jcgregorio@google.com (Joe Gregorio)' 24__all__ = [ 25 'build', 26 'build_from_document', 27 'fix_method_name', 28 'key2param', 29 ] 30 31from six import BytesIO 32from six.moves import http_client 33from six.moves.urllib.parse import urlencode, urlparse, urljoin, \ 34 urlunparse, parse_qsl 35 36# Standard library imports 37import copy 38try: 39 from email.generator import BytesGenerator 40except ImportError: 41 from email.generator import Generator as BytesGenerator 42from email.mime.multipart import MIMEMultipart 43from email.mime.nonmultipart import MIMENonMultipart 44import json 45import keyword 46import logging 47import mimetypes 48import os 49import re 50 51# Third-party imports 52import httplib2 53import uritemplate 54 55# Local imports 56from googleapiclient import _auth 57from googleapiclient import mimeparse 58from googleapiclient.errors import HttpError 59from googleapiclient.errors import InvalidJsonError 60from googleapiclient.errors import MediaUploadSizeError 61from googleapiclient.errors import UnacceptableMimeTypeError 62from googleapiclient.errors import UnknownApiNameOrVersion 63from googleapiclient.errors import UnknownFileType 64from googleapiclient.http import build_http 65from googleapiclient.http import BatchHttpRequest 66from googleapiclient.http import HttpMock 67from googleapiclient.http import HttpMockSequence 68from googleapiclient.http import HttpRequest 69from googleapiclient.http import MediaFileUpload 70from googleapiclient.http import MediaUpload 71from googleapiclient.model import JsonModel 72from googleapiclient.model import MediaModel 73from googleapiclient.model import RawModel 74from googleapiclient.schema import Schemas 75 76from googleapiclient._helpers import _add_query_parameter 77from googleapiclient._helpers import positional 78 79 80# The client library requires a version of httplib2 that supports RETRIES. 81httplib2.RETRIES = 1 82 83logger = logging.getLogger(__name__) 84 85URITEMPLATE = re.compile('{[^}]*}') 86VARNAME = re.compile('[a-zA-Z0-9_-]+') 87DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/' 88 '{api}/{apiVersion}/rest') 89V1_DISCOVERY_URI = DISCOVERY_URI 90V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?' 91 'version={apiVersion}') 92DEFAULT_METHOD_DOC = 'A description of how to use this function' 93HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH']) 94 95_MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} 96BODY_PARAMETER_DEFAULT_VALUE = { 97 'description': 'The request body.', 98 'type': 'object', 99 'required': True, 100} 101MEDIA_BODY_PARAMETER_DEFAULT_VALUE = { 102 'description': ('The filename of the media request body, or an instance ' 103 'of a MediaUpload object.'), 104 'type': 'string', 105 'required': False, 106} 107MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = { 108 'description': ('The MIME type of the media request body, or an instance ' 109 'of a MediaUpload object.'), 110 'type': 'string', 111 'required': False, 112} 113_PAGE_TOKEN_NAMES = ('pageToken', 'nextPageToken') 114 115# Parameters accepted by the stack, but not visible via discovery. 116# TODO(dhermes): Remove 'userip' in 'v2'. 117STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict']) 118STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'} 119 120# Library-specific reserved words beyond Python keywords. 121RESERVED_WORDS = frozenset(['body']) 122 123# patch _write_lines to avoid munging '\r' into '\n' 124# ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 ) 125class _BytesGenerator(BytesGenerator): 126 _write_lines = BytesGenerator.write 127 128def fix_method_name(name): 129 """Fix method names to avoid reserved word conflicts. 130 131 Args: 132 name: string, method name. 133 134 Returns: 135 The name with an '_' appended if the name is a reserved word. 136 """ 137 if keyword.iskeyword(name) or name in RESERVED_WORDS: 138 return name + '_' 139 else: 140 return name 141 142 143def key2param(key): 144 """Converts key names into parameter names. 145 146 For example, converting "max-results" -> "max_results" 147 148 Args: 149 key: string, the method key name. 150 151 Returns: 152 A safe method name based on the key name. 153 """ 154 result = [] 155 key = list(key) 156 if not key[0].isalpha(): 157 result.append('x') 158 for c in key: 159 if c.isalnum(): 160 result.append(c) 161 else: 162 result.append('_') 163 164 return ''.join(result) 165 166 167@positional(2) 168def build(serviceName, 169 version, 170 http=None, 171 discoveryServiceUrl=DISCOVERY_URI, 172 developerKey=None, 173 model=None, 174 requestBuilder=HttpRequest, 175 credentials=None, 176 cache_discovery=True, 177 cache=None): 178 """Construct a Resource for interacting with an API. 179 180 Construct a Resource object for interacting with an API. The serviceName and 181 version are the names from the Discovery service. 182 183 Args: 184 serviceName: string, name of the service. 185 version: string, the version of the service. 186 http: httplib2.Http, An instance of httplib2.Http or something that acts 187 like it that HTTP requests will be made through. 188 discoveryServiceUrl: string, a URI Template that points to the location of 189 the discovery service. It should have two parameters {api} and 190 {apiVersion} that when filled in produce an absolute URI to the discovery 191 document for that service. 192 developerKey: string, key obtained from 193 https://code.google.com/apis/console. 194 model: googleapiclient.Model, converts to and from the wire format. 195 requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP 196 request. 197 credentials: oauth2client.Credentials or 198 google.auth.credentials.Credentials, credentials to be used for 199 authentication. 200 cache_discovery: Boolean, whether or not to cache the discovery doc. 201 cache: googleapiclient.discovery_cache.base.CacheBase, an optional 202 cache object for the discovery documents. 203 204 Returns: 205 A Resource object with methods for interacting with the service. 206 """ 207 params = { 208 'api': serviceName, 209 'apiVersion': version 210 } 211 212 if http is None: 213 discovery_http = build_http() 214 else: 215 discovery_http = http 216 217 for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,): 218 requested_url = uritemplate.expand(discovery_url, params) 219 220 try: 221 content = _retrieve_discovery_doc( 222 requested_url, discovery_http, cache_discovery, cache, developerKey) 223 return build_from_document(content, base=discovery_url, http=http, 224 developerKey=developerKey, model=model, requestBuilder=requestBuilder, 225 credentials=credentials) 226 except HttpError as e: 227 if e.resp.status == http_client.NOT_FOUND: 228 continue 229 else: 230 raise e 231 232 raise UnknownApiNameOrVersion( 233 "name: %s version: %s" % (serviceName, version)) 234 235 236def _retrieve_discovery_doc(url, http, cache_discovery, cache=None, 237 developerKey=None): 238 """Retrieves the discovery_doc from cache or the internet. 239 240 Args: 241 url: string, the URL of the discovery document. 242 http: httplib2.Http, An instance of httplib2.Http or something that acts 243 like it through which HTTP requests will be made. 244 cache_discovery: Boolean, whether or not to cache the discovery doc. 245 cache: googleapiclient.discovery_cache.base.Cache, an optional cache 246 object for the discovery documents. 247 248 Returns: 249 A unicode string representation of the discovery document. 250 """ 251 if cache_discovery: 252 from . import discovery_cache 253 from .discovery_cache import base 254 if cache is None: 255 cache = discovery_cache.autodetect() 256 if cache: 257 content = cache.get(url) 258 if content: 259 return content 260 261 actual_url = url 262 # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment 263 # variable that contains the network address of the client sending the 264 # request. If it exists then add that to the request for the discovery 265 # document to avoid exceeding the quota on discovery requests. 266 if 'REMOTE_ADDR' in os.environ: 267 actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR']) 268 if developerKey: 269 actual_url = _add_query_parameter(url, 'key', developerKey) 270 logger.info('URL being requested: GET %s', actual_url) 271 272 resp, content = http.request(actual_url) 273 274 if resp.status >= 400: 275 raise HttpError(resp, content, uri=actual_url) 276 277 try: 278 content = content.decode('utf-8') 279 except AttributeError: 280 pass 281 282 try: 283 service = json.loads(content) 284 except ValueError as e: 285 logger.error('Failed to parse as JSON: ' + content) 286 raise InvalidJsonError() 287 if cache_discovery and cache: 288 cache.set(url, content) 289 return content 290 291 292@positional(1) 293def build_from_document( 294 service, 295 base=None, 296 future=None, 297 http=None, 298 developerKey=None, 299 model=None, 300 requestBuilder=HttpRequest, 301 credentials=None): 302 """Create a Resource for interacting with an API. 303 304 Same as `build()`, but constructs the Resource object from a discovery 305 document that is it given, as opposed to retrieving one over HTTP. 306 307 Args: 308 service: string or object, the JSON discovery document describing the API. 309 The value passed in may either be the JSON string or the deserialized 310 JSON. 311 base: string, base URI for all HTTP requests, usually the discovery URI. 312 This parameter is no longer used as rootUrl and servicePath are included 313 within the discovery document. (deprecated) 314 future: string, discovery document with future capabilities (deprecated). 315 http: httplib2.Http, An instance of httplib2.Http or something that acts 316 like it that HTTP requests will be made through. 317 developerKey: string, Key for controlling API usage, generated 318 from the API Console. 319 model: Model class instance that serializes and de-serializes requests and 320 responses. 321 requestBuilder: Takes an http request and packages it up to be executed. 322 credentials: oauth2client.Credentials or 323 google.auth.credentials.Credentials, credentials to be used for 324 authentication. 325 326 Returns: 327 A Resource object with methods for interacting with the service. 328 """ 329 330 if http is not None and credentials is not None: 331 raise ValueError('Arguments http and credentials are mutually exclusive.') 332 333 if isinstance(service, six.string_types): 334 service = json.loads(service) 335 336 if 'rootUrl' not in service and (isinstance(http, (HttpMock, 337 HttpMockSequence))): 338 logger.error("You are using HttpMock or HttpMockSequence without" + 339 "having the service discovery doc in cache. Try calling " + 340 "build() without mocking once first to populate the " + 341 "cache.") 342 raise InvalidJsonError() 343 344 base = urljoin(service['rootUrl'], service['servicePath']) 345 schema = Schemas(service) 346 347 # If the http client is not specified, then we must construct an http client 348 # to make requests. If the service has scopes, then we also need to setup 349 # authentication. 350 if http is None: 351 # Does the service require scopes? 352 scopes = list( 353 service.get('auth', {}).get('oauth2', {}).get('scopes', {}).keys()) 354 355 # If so, then the we need to setup authentication if no developerKey is 356 # specified. 357 if scopes and not developerKey: 358 # If the user didn't pass in credentials, attempt to acquire application 359 # default credentials. 360 if credentials is None: 361 credentials = _auth.default_credentials() 362 363 # The credentials need to be scoped. 364 credentials = _auth.with_scopes(credentials, scopes) 365 366 # If credentials are provided, create an authorized http instance; 367 # otherwise, skip authentication. 368 if credentials: 369 http = _auth.authorized_http(credentials) 370 371 # If the service doesn't require scopes then there is no need for 372 # authentication. 373 else: 374 http = build_http() 375 376 if model is None: 377 features = service.get('features', []) 378 model = JsonModel('dataWrapper' in features) 379 380 return Resource(http=http, baseUrl=base, model=model, 381 developerKey=developerKey, requestBuilder=requestBuilder, 382 resourceDesc=service, rootDesc=service, schema=schema) 383 384 385def _cast(value, schema_type): 386 """Convert value to a string based on JSON Schema type. 387 388 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on 389 JSON Schema. 390 391 Args: 392 value: any, the value to convert 393 schema_type: string, the type that value should be interpreted as 394 395 Returns: 396 A string representation of 'value' based on the schema_type. 397 """ 398 if schema_type == 'string': 399 if type(value) == type('') or type(value) == type(u''): 400 return value 401 else: 402 return str(value) 403 elif schema_type == 'integer': 404 return str(int(value)) 405 elif schema_type == 'number': 406 return str(float(value)) 407 elif schema_type == 'boolean': 408 return str(bool(value)).lower() 409 else: 410 if type(value) == type('') or type(value) == type(u''): 411 return value 412 else: 413 return str(value) 414 415 416def _media_size_to_long(maxSize): 417 """Convert a string media size, such as 10GB or 3TB into an integer. 418 419 Args: 420 maxSize: string, size as a string, such as 2MB or 7GB. 421 422 Returns: 423 The size as an integer value. 424 """ 425 if len(maxSize) < 2: 426 return 0 427 units = maxSize[-2:].upper() 428 bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units) 429 if bit_shift is not None: 430 return int(maxSize[:-2]) << bit_shift 431 else: 432 return int(maxSize) 433 434 435def _media_path_url_from_info(root_desc, path_url): 436 """Creates an absolute media path URL. 437 438 Constructed using the API root URI and service path from the discovery 439 document and the relative path for the API method. 440 441 Args: 442 root_desc: Dictionary; the entire original deserialized discovery document. 443 path_url: String; the relative URL for the API method. Relative to the API 444 root, which is specified in the discovery document. 445 446 Returns: 447 String; the absolute URI for media upload for the API method. 448 """ 449 return '%(root)supload/%(service_path)s%(path)s' % { 450 'root': root_desc['rootUrl'], 451 'service_path': root_desc['servicePath'], 452 'path': path_url, 453 } 454 455 456def _fix_up_parameters(method_desc, root_desc, http_method, schema): 457 """Updates parameters of an API method with values specific to this library. 458 459 Specifically, adds whatever global parameters are specified by the API to the 460 parameters for the individual method. Also adds parameters which don't 461 appear in the discovery document, but are available to all discovery based 462 APIs (these are listed in STACK_QUERY_PARAMETERS). 463 464 SIDE EFFECTS: This updates the parameters dictionary object in the method 465 description. 466 467 Args: 468 method_desc: Dictionary with metadata describing an API method. Value comes 469 from the dictionary of methods stored in the 'methods' key in the 470 deserialized discovery document. 471 root_desc: Dictionary; the entire original deserialized discovery document. 472 http_method: String; the HTTP method used to call the API method described 473 in method_desc. 474 schema: Object, mapping of schema names to schema descriptions. 475 476 Returns: 477 The updated Dictionary stored in the 'parameters' key of the method 478 description dictionary. 479 """ 480 parameters = method_desc.setdefault('parameters', {}) 481 482 # Add in the parameters common to all methods. 483 for name, description in six.iteritems(root_desc.get('parameters', {})): 484 parameters[name] = description 485 486 # Add in undocumented query parameters. 487 for name in STACK_QUERY_PARAMETERS: 488 parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy() 489 490 # Add 'body' (our own reserved word) to parameters if the method supports 491 # a request payload. 492 if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc: 493 body = BODY_PARAMETER_DEFAULT_VALUE.copy() 494 body.update(method_desc['request']) 495 # Make body optional for requests with no parameters. 496 if not _methodProperties(method_desc, schema, 'request'): 497 body['required'] = False 498 parameters['body'] = body 499 500 return parameters 501 502 503def _fix_up_media_upload(method_desc, root_desc, path_url, parameters): 504 """Adds 'media_body' and 'media_mime_type' parameters if supported by method. 505 506 SIDE EFFECTS: If the method supports media upload and has a required body, 507 sets body to be optional (required=False) instead. Also, if there is a 508 'mediaUpload' in the method description, adds 'media_upload' key to 509 parameters. 510 511 Args: 512 method_desc: Dictionary with metadata describing an API method. Value comes 513 from the dictionary of methods stored in the 'methods' key in the 514 deserialized discovery document. 515 root_desc: Dictionary; the entire original deserialized discovery document. 516 path_url: String; the relative URL for the API method. Relative to the API 517 root, which is specified in the discovery document. 518 parameters: A dictionary describing method parameters for method described 519 in method_desc. 520 521 Returns: 522 Triple (accept, max_size, media_path_url) where: 523 - accept is a list of strings representing what content types are 524 accepted for media upload. Defaults to empty list if not in the 525 discovery document. 526 - max_size is a long representing the max size in bytes allowed for a 527 media upload. Defaults to 0L if not in the discovery document. 528 - media_path_url is a String; the absolute URI for media upload for the 529 API method. Constructed using the API root URI and service path from 530 the discovery document and the relative path for the API method. If 531 media upload is not supported, this is None. 532 """ 533 media_upload = method_desc.get('mediaUpload', {}) 534 accept = media_upload.get('accept', []) 535 max_size = _media_size_to_long(media_upload.get('maxSize', '')) 536 media_path_url = None 537 538 if media_upload: 539 media_path_url = _media_path_url_from_info(root_desc, path_url) 540 parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy() 541 parameters['media_mime_type'] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy() 542 if 'body' in parameters: 543 parameters['body']['required'] = False 544 545 return accept, max_size, media_path_url 546 547 548def _fix_up_method_description(method_desc, root_desc, schema): 549 """Updates a method description in a discovery document. 550 551 SIDE EFFECTS: Changes the parameters dictionary in the method description with 552 extra parameters which are used locally. 553 554 Args: 555 method_desc: Dictionary with metadata describing an API method. Value comes 556 from the dictionary of methods stored in the 'methods' key in the 557 deserialized discovery document. 558 root_desc: Dictionary; the entire original deserialized discovery document. 559 schema: Object, mapping of schema names to schema descriptions. 560 561 Returns: 562 Tuple (path_url, http_method, method_id, accept, max_size, media_path_url) 563 where: 564 - path_url is a String; the relative URL for the API method. Relative to 565 the API root, which is specified in the discovery document. 566 - http_method is a String; the HTTP method used to call the API method 567 described in the method description. 568 - method_id is a String; the name of the RPC method associated with the 569 API method, and is in the method description in the 'id' key. 570 - accept is a list of strings representing what content types are 571 accepted for media upload. Defaults to empty list if not in the 572 discovery document. 573 - max_size is a long representing the max size in bytes allowed for a 574 media upload. Defaults to 0L if not in the discovery document. 575 - media_path_url is a String; the absolute URI for media upload for the 576 API method. Constructed using the API root URI and service path from 577 the discovery document and the relative path for the API method. If 578 media upload is not supported, this is None. 579 """ 580 path_url = method_desc['path'] 581 http_method = method_desc['httpMethod'] 582 method_id = method_desc['id'] 583 584 parameters = _fix_up_parameters(method_desc, root_desc, http_method, schema) 585 # Order is important. `_fix_up_media_upload` needs `method_desc` to have a 586 # 'parameters' key and needs to know if there is a 'body' parameter because it 587 # also sets a 'media_body' parameter. 588 accept, max_size, media_path_url = _fix_up_media_upload( 589 method_desc, root_desc, path_url, parameters) 590 591 return path_url, http_method, method_id, accept, max_size, media_path_url 592 593 594def _urljoin(base, url): 595 """Custom urljoin replacement supporting : before / in url.""" 596 # In general, it's unsafe to simply join base and url. However, for 597 # the case of discovery documents, we know: 598 # * base will never contain params, query, or fragment 599 # * url will never contain a scheme or net_loc. 600 # In general, this means we can safely join on /; we just need to 601 # ensure we end up with precisely one / joining base and url. The 602 # exception here is the case of media uploads, where url will be an 603 # absolute url. 604 if url.startswith('http://') or url.startswith('https://'): 605 return urljoin(base, url) 606 new_base = base if base.endswith('/') else base + '/' 607 new_url = url[1:] if url.startswith('/') else url 608 return new_base + new_url 609 610 611# TODO(dhermes): Convert this class to ResourceMethod and make it callable 612class ResourceMethodParameters(object): 613 """Represents the parameters associated with a method. 614 615 Attributes: 616 argmap: Map from method parameter name (string) to query parameter name 617 (string). 618 required_params: List of required parameters (represented by parameter 619 name as string). 620 repeated_params: List of repeated parameters (represented by parameter 621 name as string). 622 pattern_params: Map from method parameter name (string) to regular 623 expression (as a string). If the pattern is set for a parameter, the 624 value for that parameter must match the regular expression. 625 query_params: List of parameters (represented by parameter name as string) 626 that will be used in the query string. 627 path_params: Set of parameters (represented by parameter name as string) 628 that will be used in the base URL path. 629 param_types: Map from method parameter name (string) to parameter type. Type 630 can be any valid JSON schema type; valid values are 'any', 'array', 631 'boolean', 'integer', 'number', 'object', or 'string'. Reference: 632 http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1 633 enum_params: Map from method parameter name (string) to list of strings, 634 where each list of strings is the list of acceptable enum values. 635 """ 636 637 def __init__(self, method_desc): 638 """Constructor for ResourceMethodParameters. 639 640 Sets default values and defers to set_parameters to populate. 641 642 Args: 643 method_desc: Dictionary with metadata describing an API method. Value 644 comes from the dictionary of methods stored in the 'methods' key in 645 the deserialized discovery document. 646 """ 647 self.argmap = {} 648 self.required_params = [] 649 self.repeated_params = [] 650 self.pattern_params = {} 651 self.query_params = [] 652 # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE 653 # parsing is gotten rid of. 654 self.path_params = set() 655 self.param_types = {} 656 self.enum_params = {} 657 658 self.set_parameters(method_desc) 659 660 def set_parameters(self, method_desc): 661 """Populates maps and lists based on method description. 662 663 Iterates through each parameter for the method and parses the values from 664 the parameter dictionary. 665 666 Args: 667 method_desc: Dictionary with metadata describing an API method. Value 668 comes from the dictionary of methods stored in the 'methods' key in 669 the deserialized discovery document. 670 """ 671 for arg, desc in six.iteritems(method_desc.get('parameters', {})): 672 param = key2param(arg) 673 self.argmap[param] = arg 674 675 if desc.get('pattern'): 676 self.pattern_params[param] = desc['pattern'] 677 if desc.get('enum'): 678 self.enum_params[param] = desc['enum'] 679 if desc.get('required'): 680 self.required_params.append(param) 681 if desc.get('repeated'): 682 self.repeated_params.append(param) 683 if desc.get('location') == 'query': 684 self.query_params.append(param) 685 if desc.get('location') == 'path': 686 self.path_params.add(param) 687 self.param_types[param] = desc.get('type', 'string') 688 689 # TODO(dhermes): Determine if this is still necessary. Discovery based APIs 690 # should have all path parameters already marked with 691 # 'location: path'. 692 for match in URITEMPLATE.finditer(method_desc['path']): 693 for namematch in VARNAME.finditer(match.group(0)): 694 name = key2param(namematch.group(0)) 695 self.path_params.add(name) 696 if name in self.query_params: 697 self.query_params.remove(name) 698 699 700def createMethod(methodName, methodDesc, rootDesc, schema): 701 """Creates a method for attaching to a Resource. 702 703 Args: 704 methodName: string, name of the method to use. 705 methodDesc: object, fragment of deserialized discovery document that 706 describes the method. 707 rootDesc: object, the entire deserialized discovery document. 708 schema: object, mapping of schema names to schema descriptions. 709 """ 710 methodName = fix_method_name(methodName) 711 (pathUrl, httpMethod, methodId, accept, 712 maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc, schema) 713 714 parameters = ResourceMethodParameters(methodDesc) 715 716 def method(self, **kwargs): 717 # Don't bother with doc string, it will be over-written by createMethod. 718 719 for name in six.iterkeys(kwargs): 720 if name not in parameters.argmap: 721 raise TypeError('Got an unexpected keyword argument "%s"' % name) 722 723 # Remove args that have a value of None. 724 keys = list(kwargs.keys()) 725 for name in keys: 726 if kwargs[name] is None: 727 del kwargs[name] 728 729 for name in parameters.required_params: 730 if name not in kwargs: 731 # temporary workaround for non-paging methods incorrectly requiring 732 # page token parameter (cf. drive.changes.watch vs. drive.changes.list) 733 if name not in _PAGE_TOKEN_NAMES or _findPageTokenName( 734 _methodProperties(methodDesc, schema, 'response')): 735 raise TypeError('Missing required parameter "%s"' % name) 736 737 for name, regex in six.iteritems(parameters.pattern_params): 738 if name in kwargs: 739 if isinstance(kwargs[name], six.string_types): 740 pvalues = [kwargs[name]] 741 else: 742 pvalues = kwargs[name] 743 for pvalue in pvalues: 744 if re.match(regex, pvalue) is None: 745 raise TypeError( 746 'Parameter "%s" value "%s" does not match the pattern "%s"' % 747 (name, pvalue, regex)) 748 749 for name, enums in six.iteritems(parameters.enum_params): 750 if name in kwargs: 751 # We need to handle the case of a repeated enum 752 # name differently, since we want to handle both 753 # arg='value' and arg=['value1', 'value2'] 754 if (name in parameters.repeated_params and 755 not isinstance(kwargs[name], six.string_types)): 756 values = kwargs[name] 757 else: 758 values = [kwargs[name]] 759 for value in values: 760 if value not in enums: 761 raise TypeError( 762 'Parameter "%s" value "%s" is not an allowed value in "%s"' % 763 (name, value, str(enums))) 764 765 actual_query_params = {} 766 actual_path_params = {} 767 for key, value in six.iteritems(kwargs): 768 to_type = parameters.param_types.get(key, 'string') 769 # For repeated parameters we cast each member of the list. 770 if key in parameters.repeated_params and type(value) == type([]): 771 cast_value = [_cast(x, to_type) for x in value] 772 else: 773 cast_value = _cast(value, to_type) 774 if key in parameters.query_params: 775 actual_query_params[parameters.argmap[key]] = cast_value 776 if key in parameters.path_params: 777 actual_path_params[parameters.argmap[key]] = cast_value 778 body_value = kwargs.get('body', None) 779 media_filename = kwargs.get('media_body', None) 780 media_mime_type = kwargs.get('media_mime_type', None) 781 782 if self._developerKey: 783 actual_query_params['key'] = self._developerKey 784 785 model = self._model 786 if methodName.endswith('_media'): 787 model = MediaModel() 788 elif 'response' not in methodDesc: 789 model = RawModel() 790 791 headers = {} 792 headers, params, query, body = model.request(headers, 793 actual_path_params, actual_query_params, body_value) 794 795 expanded_url = uritemplate.expand(pathUrl, params) 796 url = _urljoin(self._baseUrl, expanded_url + query) 797 798 resumable = None 799 multipart_boundary = '' 800 801 if media_filename: 802 # Ensure we end up with a valid MediaUpload object. 803 if isinstance(media_filename, six.string_types): 804 if media_mime_type is None: 805 logger.warning( 806 'media_mime_type argument not specified: trying to auto-detect for %s', 807 media_filename) 808 media_mime_type, _ = mimetypes.guess_type(media_filename) 809 if media_mime_type is None: 810 raise UnknownFileType(media_filename) 811 if not mimeparse.best_match([media_mime_type], ','.join(accept)): 812 raise UnacceptableMimeTypeError(media_mime_type) 813 media_upload = MediaFileUpload(media_filename, 814 mimetype=media_mime_type) 815 elif isinstance(media_filename, MediaUpload): 816 media_upload = media_filename 817 else: 818 raise TypeError('media_filename must be str or MediaUpload.') 819 820 # Check the maxSize 821 if media_upload.size() is not None and media_upload.size() > maxSize > 0: 822 raise MediaUploadSizeError("Media larger than: %s" % maxSize) 823 824 # Use the media path uri for media uploads 825 expanded_url = uritemplate.expand(mediaPathUrl, params) 826 url = _urljoin(self._baseUrl, expanded_url + query) 827 if media_upload.resumable(): 828 url = _add_query_parameter(url, 'uploadType', 'resumable') 829 830 if media_upload.resumable(): 831 # This is all we need to do for resumable, if the body exists it gets 832 # sent in the first request, otherwise an empty body is sent. 833 resumable = media_upload 834 else: 835 # A non-resumable upload 836 if body is None: 837 # This is a simple media upload 838 headers['content-type'] = media_upload.mimetype() 839 body = media_upload.getbytes(0, media_upload.size()) 840 url = _add_query_parameter(url, 'uploadType', 'media') 841 else: 842 # This is a multipart/related upload. 843 msgRoot = MIMEMultipart('related') 844 # msgRoot should not write out it's own headers 845 setattr(msgRoot, '_write_headers', lambda self: None) 846 847 # attach the body as one part 848 msg = MIMENonMultipart(*headers['content-type'].split('/')) 849 msg.set_payload(body) 850 msgRoot.attach(msg) 851 852 # attach the media as the second part 853 msg = MIMENonMultipart(*media_upload.mimetype().split('/')) 854 msg['Content-Transfer-Encoding'] = 'binary' 855 856 payload = media_upload.getbytes(0, media_upload.size()) 857 msg.set_payload(payload) 858 msgRoot.attach(msg) 859 # encode the body: note that we can't use `as_string`, because 860 # it plays games with `From ` lines. 861 fp = BytesIO() 862 g = _BytesGenerator(fp, mangle_from_=False) 863 g.flatten(msgRoot, unixfrom=False) 864 body = fp.getvalue() 865 866 multipart_boundary = msgRoot.get_boundary() 867 headers['content-type'] = ('multipart/related; ' 868 'boundary="%s"') % multipart_boundary 869 url = _add_query_parameter(url, 'uploadType', 'multipart') 870 871 logger.info('URL being requested: %s %s' % (httpMethod,url)) 872 return self._requestBuilder(self._http, 873 model.response, 874 url, 875 method=httpMethod, 876 body=body, 877 headers=headers, 878 methodId=methodId, 879 resumable=resumable) 880 881 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n'] 882 if len(parameters.argmap) > 0: 883 docs.append('Args:\n') 884 885 # Skip undocumented params and params common to all methods. 886 skip_parameters = list(rootDesc.get('parameters', {}).keys()) 887 skip_parameters.extend(STACK_QUERY_PARAMETERS) 888 889 all_args = list(parameters.argmap.keys()) 890 args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])] 891 892 # Move body to the front of the line. 893 if 'body' in all_args: 894 args_ordered.append('body') 895 896 for name in all_args: 897 if name not in args_ordered: 898 args_ordered.append(name) 899 900 for arg in args_ordered: 901 if arg in skip_parameters: 902 continue 903 904 repeated = '' 905 if arg in parameters.repeated_params: 906 repeated = ' (repeated)' 907 required = '' 908 if arg in parameters.required_params: 909 required = ' (required)' 910 paramdesc = methodDesc['parameters'][parameters.argmap[arg]] 911 paramdoc = paramdesc.get('description', 'A parameter') 912 if '$ref' in paramdesc: 913 docs.append( 914 (' %s: object, %s%s%s\n The object takes the' 915 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated, 916 schema.prettyPrintByName(paramdesc['$ref']))) 917 else: 918 paramtype = paramdesc.get('type', 'string') 919 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required, 920 repeated)) 921 enum = paramdesc.get('enum', []) 922 enumDesc = paramdesc.get('enumDescriptions', []) 923 if enum and enumDesc: 924 docs.append(' Allowed values\n') 925 for (name, desc) in zip(enum, enumDesc): 926 docs.append(' %s - %s\n' % (name, desc)) 927 if 'response' in methodDesc: 928 if methodName.endswith('_media'): 929 docs.append('\nReturns:\n The media object as a string.\n\n ') 930 else: 931 docs.append('\nReturns:\n An object of the form:\n\n ') 932 docs.append(schema.prettyPrintSchema(methodDesc['response'])) 933 934 setattr(method, '__doc__', ''.join(docs)) 935 return (methodName, method) 936 937 938def createNextMethod(methodName, 939 pageTokenName='pageToken', 940 nextPageTokenName='nextPageToken', 941 isPageTokenParameter=True): 942 """Creates any _next methods for attaching to a Resource. 943 944 The _next methods allow for easy iteration through list() responses. 945 946 Args: 947 methodName: string, name of the method to use. 948 pageTokenName: string, name of request page token field. 949 nextPageTokenName: string, name of response page token field. 950 isPageTokenParameter: Boolean, True if request page token is a query 951 parameter, False if request page token is a field of the request body. 952 """ 953 methodName = fix_method_name(methodName) 954 955 def methodNext(self, previous_request, previous_response): 956 """Retrieves the next page of results. 957 958Args: 959 previous_request: The request for the previous page. (required) 960 previous_response: The response from the request for the previous page. (required) 961 962Returns: 963 A request object that you can call 'execute()' on to request the next 964 page. Returns None if there are no more items in the collection. 965 """ 966 # Retrieve nextPageToken from previous_response 967 # Use as pageToken in previous_request to create new request. 968 969 nextPageToken = previous_response.get(nextPageTokenName, None) 970 if not nextPageToken: 971 return None 972 973 request = copy.copy(previous_request) 974 975 if isPageTokenParameter: 976 # Replace pageToken value in URI 977 request.uri = _add_query_parameter( 978 request.uri, pageTokenName, nextPageToken) 979 logger.info('Next page request URL: %s %s' % (methodName, request.uri)) 980 else: 981 # Replace pageToken value in request body 982 model = self._model 983 body = model.deserialize(request.body) 984 body[pageTokenName] = nextPageToken 985 request.body = model.serialize(body) 986 logger.info('Next page request body: %s %s' % (methodName, body)) 987 988 return request 989 990 return (methodName, methodNext) 991 992 993class Resource(object): 994 """A class for interacting with a resource.""" 995 996 def __init__(self, http, baseUrl, model, requestBuilder, developerKey, 997 resourceDesc, rootDesc, schema): 998 """Build a Resource from the API description. 999 1000 Args: 1001 http: httplib2.Http, Object to make http requests with. 1002 baseUrl: string, base URL for the API. All requests are relative to this 1003 URI. 1004 model: googleapiclient.Model, converts to and from the wire format. 1005 requestBuilder: class or callable that instantiates an 1006 googleapiclient.HttpRequest object. 1007 developerKey: string, key obtained from 1008 https://code.google.com/apis/console 1009 resourceDesc: object, section of deserialized discovery document that 1010 describes a resource. Note that the top level discovery document 1011 is considered a resource. 1012 rootDesc: object, the entire deserialized discovery document. 1013 schema: object, mapping of schema names to schema descriptions. 1014 """ 1015 self._dynamic_attrs = [] 1016 1017 self._http = http 1018 self._baseUrl = baseUrl 1019 self._model = model 1020 self._developerKey = developerKey 1021 self._requestBuilder = requestBuilder 1022 self._resourceDesc = resourceDesc 1023 self._rootDesc = rootDesc 1024 self._schema = schema 1025 1026 self._set_service_methods() 1027 1028 def _set_dynamic_attr(self, attr_name, value): 1029 """Sets an instance attribute and tracks it in a list of dynamic attributes. 1030 1031 Args: 1032 attr_name: string; The name of the attribute to be set 1033 value: The value being set on the object and tracked in the dynamic cache. 1034 """ 1035 self._dynamic_attrs.append(attr_name) 1036 self.__dict__[attr_name] = value 1037 1038 def __getstate__(self): 1039 """Trim the state down to something that can be pickled. 1040 1041 Uses the fact that the instance variable _dynamic_attrs holds attrs that 1042 will be wiped and restored on pickle serialization. 1043 """ 1044 state_dict = copy.copy(self.__dict__) 1045 for dynamic_attr in self._dynamic_attrs: 1046 del state_dict[dynamic_attr] 1047 del state_dict['_dynamic_attrs'] 1048 return state_dict 1049 1050 def __setstate__(self, state): 1051 """Reconstitute the state of the object from being pickled. 1052 1053 Uses the fact that the instance variable _dynamic_attrs holds attrs that 1054 will be wiped and restored on pickle serialization. 1055 """ 1056 self.__dict__.update(state) 1057 self._dynamic_attrs = [] 1058 self._set_service_methods() 1059 1060 def _set_service_methods(self): 1061 self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema) 1062 self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema) 1063 self._add_next_methods(self._resourceDesc, self._schema) 1064 1065 def _add_basic_methods(self, resourceDesc, rootDesc, schema): 1066 # If this is the root Resource, add a new_batch_http_request() method. 1067 if resourceDesc == rootDesc: 1068 batch_uri = '%s%s' % ( 1069 rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch')) 1070 def new_batch_http_request(callback=None): 1071 """Create a BatchHttpRequest object based on the discovery document. 1072 1073 Args: 1074 callback: callable, A callback to be called for each response, of the 1075 form callback(id, response, exception). The first parameter is the 1076 request id, and the second is the deserialized response object. The 1077 third is an apiclient.errors.HttpError exception object if an HTTP 1078 error occurred while processing the request, or None if no error 1079 occurred. 1080 1081 Returns: 1082 A BatchHttpRequest object based on the discovery document. 1083 """ 1084 return BatchHttpRequest(callback=callback, batch_uri=batch_uri) 1085 self._set_dynamic_attr('new_batch_http_request', new_batch_http_request) 1086 1087 # Add basic methods to Resource 1088 if 'methods' in resourceDesc: 1089 for methodName, methodDesc in six.iteritems(resourceDesc['methods']): 1090 fixedMethodName, method = createMethod( 1091 methodName, methodDesc, rootDesc, schema) 1092 self._set_dynamic_attr(fixedMethodName, 1093 method.__get__(self, self.__class__)) 1094 # Add in _media methods. The functionality of the attached method will 1095 # change when it sees that the method name ends in _media. 1096 if methodDesc.get('supportsMediaDownload', False): 1097 fixedMethodName, method = createMethod( 1098 methodName + '_media', methodDesc, rootDesc, schema) 1099 self._set_dynamic_attr(fixedMethodName, 1100 method.__get__(self, self.__class__)) 1101 1102 def _add_nested_resources(self, resourceDesc, rootDesc, schema): 1103 # Add in nested resources 1104 if 'resources' in resourceDesc: 1105 1106 def createResourceMethod(methodName, methodDesc): 1107 """Create a method on the Resource to access a nested Resource. 1108 1109 Args: 1110 methodName: string, name of the method to use. 1111 methodDesc: object, fragment of deserialized discovery document that 1112 describes the method. 1113 """ 1114 methodName = fix_method_name(methodName) 1115 1116 def methodResource(self): 1117 return Resource(http=self._http, baseUrl=self._baseUrl, 1118 model=self._model, developerKey=self._developerKey, 1119 requestBuilder=self._requestBuilder, 1120 resourceDesc=methodDesc, rootDesc=rootDesc, 1121 schema=schema) 1122 1123 setattr(methodResource, '__doc__', 'A collection resource.') 1124 setattr(methodResource, '__is_resource__', True) 1125 1126 return (methodName, methodResource) 1127 1128 for methodName, methodDesc in six.iteritems(resourceDesc['resources']): 1129 fixedMethodName, method = createResourceMethod(methodName, methodDesc) 1130 self._set_dynamic_attr(fixedMethodName, 1131 method.__get__(self, self.__class__)) 1132 1133 def _add_next_methods(self, resourceDesc, schema): 1134 # Add _next() methods if and only if one of the names 'pageToken' or 1135 # 'nextPageToken' occurs among the fields of both the method's response 1136 # type either the method's request (query parameters) or request body. 1137 if 'methods' not in resourceDesc: 1138 return 1139 for methodName, methodDesc in six.iteritems(resourceDesc['methods']): 1140 nextPageTokenName = _findPageTokenName( 1141 _methodProperties(methodDesc, schema, 'response')) 1142 if not nextPageTokenName: 1143 continue 1144 isPageTokenParameter = True 1145 pageTokenName = _findPageTokenName(methodDesc.get('parameters', {})) 1146 if not pageTokenName: 1147 isPageTokenParameter = False 1148 pageTokenName = _findPageTokenName( 1149 _methodProperties(methodDesc, schema, 'request')) 1150 if not pageTokenName: 1151 continue 1152 fixedMethodName, method = createNextMethod( 1153 methodName + '_next', pageTokenName, nextPageTokenName, 1154 isPageTokenParameter) 1155 self._set_dynamic_attr(fixedMethodName, 1156 method.__get__(self, self.__class__)) 1157 1158 1159def _findPageTokenName(fields): 1160 """Search field names for one like a page token. 1161 1162 Args: 1163 fields: container of string, names of fields. 1164 1165 Returns: 1166 First name that is either 'pageToken' or 'nextPageToken' if one exists, 1167 otherwise None. 1168 """ 1169 return next((tokenName for tokenName in _PAGE_TOKEN_NAMES 1170 if tokenName in fields), None) 1171 1172def _methodProperties(methodDesc, schema, name): 1173 """Get properties of a field in a method description. 1174 1175 Args: 1176 methodDesc: object, fragment of deserialized discovery document that 1177 describes the method. 1178 schema: object, mapping of schema names to schema descriptions. 1179 name: string, name of top-level field in method description. 1180 1181 Returns: 1182 Object representing fragment of deserialized discovery document 1183 corresponding to 'properties' field of object corresponding to named field 1184 in method description, if it exists, otherwise empty dict. 1185 """ 1186 desc = methodDesc.get(name, {}) 1187 if '$ref' in desc: 1188 desc = schema.get(desc['$ref'], {}) 1189 return desc.get('properties', {}) 1190