1# Copyright 2015 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Client for interacting with the Google Cloud Storage API.""" 16 17import base64 18import binascii 19import collections 20import datetime 21import functools 22import json 23import warnings 24import google.api_core.client_options 25 26from google.auth.credentials import AnonymousCredentials 27 28from google import resumable_media 29 30from google.api_core import page_iterator 31from google.cloud._helpers import _LocalStack, _NOW 32from google.cloud.client import ClientWithProject 33from google.cloud.exceptions import NotFound 34from google.cloud.storage._helpers import _get_storage_host 35from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST 36from google.cloud.storage._helpers import _bucket_bound_hostname_url 37from google.cloud.storage._helpers import _add_etag_match_headers 38from google.cloud.storage._http import Connection 39from google.cloud.storage._signing import ( 40 get_expiration_seconds_v4, 41 get_v4_now_dtstamps, 42 ensure_signed_credentials, 43 _sign_message, 44) 45from google.cloud.storage.batch import Batch 46from google.cloud.storage.bucket import Bucket, _item_to_blob, _blobs_page_start 47from google.cloud.storage.blob import ( 48 Blob, 49 _get_encryption_headers, 50 _raise_from_invalid_response, 51) 52from google.cloud.storage.hmac_key import HMACKeyMetadata 53from google.cloud.storage.acl import BucketACL 54from google.cloud.storage.acl import DefaultObjectACL 55from google.cloud.storage.constants import _DEFAULT_TIMEOUT 56from google.cloud.storage.retry import DEFAULT_RETRY 57from google.cloud.storage.retry import ConditionalRetryPolicy 58 59 60_marker = object() 61 62 63class Client(ClientWithProject): 64 """Client to bundle configuration needed for API requests. 65 66 :type project: str or None 67 :param project: the project which the client acts on behalf of. Will be 68 passed when creating a topic. If not passed, 69 falls back to the default inferred from the environment. 70 71 :type credentials: :class:`~google.auth.credentials.Credentials` 72 :param credentials: (Optional) The OAuth2 Credentials to use for this 73 client. If not passed (and if no ``_http`` object is 74 passed), falls back to the default inferred from the 75 environment. 76 77 :type _http: :class:`~requests.Session` 78 :param _http: (Optional) HTTP object to make requests. Can be any object 79 that defines ``request()`` with the same interface as 80 :meth:`requests.Session.request`. If not passed, an 81 ``_http`` object is created that is bound to the 82 ``credentials`` for the current object. 83 This parameter should be considered private, and could 84 change in the future. 85 86 :type client_info: :class:`~google.api_core.client_info.ClientInfo` 87 :param client_info: 88 The client info used to send a user-agent string along with API 89 requests. If ``None``, then default info will be used. Generally, 90 you only need to set this if you're developing your own library 91 or partner tool. 92 93 :type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict` 94 :param client_options: (Optional) Client options used to set user options on the client. 95 API Endpoint should be set through client_options. 96 """ 97 98 SCOPE = ( 99 "https://www.googleapis.com/auth/devstorage.full_control", 100 "https://www.googleapis.com/auth/devstorage.read_only", 101 "https://www.googleapis.com/auth/devstorage.read_write", 102 ) 103 """The scopes required for authenticating as a Cloud Storage consumer.""" 104 105 def __init__( 106 self, 107 project=_marker, 108 credentials=None, 109 _http=None, 110 client_info=None, 111 client_options=None, 112 ): 113 self._base_connection = None 114 115 if project is None: 116 no_project = True 117 project = "<none>" 118 else: 119 no_project = False 120 121 if project is _marker: 122 project = None 123 124 super(Client, self).__init__( 125 project=project, 126 credentials=credentials, 127 client_options=client_options, 128 _http=_http, 129 ) 130 131 kw_args = {"client_info": client_info} 132 133 # `api_endpoint` should be only set by the user via `client_options`, 134 # or if the _get_storage_host() returns a non-default value. 135 # `api_endpoint` plays an important role for mTLS, if it is not set, 136 # then mTLS logic will be applied to decide which endpoint will be used. 137 storage_host = _get_storage_host() 138 kw_args["api_endpoint"] = ( 139 storage_host if storage_host != _DEFAULT_STORAGE_HOST else None 140 ) 141 142 if client_options: 143 if type(client_options) == dict: 144 client_options = google.api_core.client_options.from_dict( 145 client_options 146 ) 147 if client_options.api_endpoint: 148 api_endpoint = client_options.api_endpoint 149 kw_args["api_endpoint"] = api_endpoint 150 151 if no_project: 152 self.project = None 153 154 self._connection = Connection(self, **kw_args) 155 self._batch_stack = _LocalStack() 156 157 @classmethod 158 def create_anonymous_client(cls): 159 """Factory: return client with anonymous credentials. 160 161 .. note:: 162 163 Such a client has only limited access to "public" buckets: 164 listing their contents and downloading their blobs. 165 166 :rtype: :class:`google.cloud.storage.client.Client` 167 :returns: Instance w/ anonymous credentials and no project. 168 """ 169 client = cls(project="<none>", credentials=AnonymousCredentials()) 170 client.project = None 171 return client 172 173 @property 174 def _connection(self): 175 """Get connection or batch on the client. 176 177 :rtype: :class:`google.cloud.storage._http.Connection` 178 :returns: The connection set on the client, or the batch 179 if one is set. 180 """ 181 if self.current_batch is not None: 182 return self.current_batch 183 else: 184 return self._base_connection 185 186 @_connection.setter 187 def _connection(self, value): 188 """Set connection on the client. 189 190 Intended to be used by constructor (since the base class calls) 191 self._connection = connection 192 Will raise if the connection is set more than once. 193 194 :type value: :class:`google.cloud.storage._http.Connection` 195 :param value: The connection set on the client. 196 197 :raises: :class:`ValueError` if connection has already been set. 198 """ 199 if self._base_connection is not None: 200 raise ValueError("Connection already set on client") 201 self._base_connection = value 202 203 def _push_batch(self, batch): 204 """Push a batch onto our stack. 205 206 "Protected", intended for use by batch context mgrs. 207 208 :type batch: :class:`google.cloud.storage.batch.Batch` 209 :param batch: newly-active batch 210 """ 211 self._batch_stack.push(batch) 212 213 def _pop_batch(self): 214 """Pop a batch from our stack. 215 216 "Protected", intended for use by batch context mgrs. 217 218 :raises: IndexError if the stack is empty. 219 :rtype: :class:`google.cloud.storage.batch.Batch` 220 :returns: the top-most batch/transaction, after removing it. 221 """ 222 return self._batch_stack.pop() 223 224 @property 225 def current_batch(self): 226 """Currently-active batch. 227 228 :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if 229 no batch is active). 230 :returns: The batch at the top of the batch stack. 231 """ 232 return self._batch_stack.top 233 234 def get_service_account_email( 235 self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY 236 ): 237 """Get the email address of the project's GCS service account 238 239 :type project: str 240 :param project: 241 (Optional) Project ID to use for retreiving GCS service account 242 email address. Defaults to the client's project. 243 :type timeout: float or tuple 244 :param timeout: 245 (Optional) The amount of time, in seconds, to wait 246 for the server response. See: :ref:`configuring_timeouts` 247 248 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 249 :param retry: 250 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 251 252 :rtype: str 253 :returns: service account email address 254 """ 255 if project is None: 256 project = self.project 257 258 path = "/projects/%s/serviceAccount" % (project,) 259 api_response = self._get_resource(path, timeout=timeout, retry=retry) 260 return api_response["email_address"] 261 262 def bucket(self, bucket_name, user_project=None): 263 """Factory constructor for bucket object. 264 265 .. note:: 266 This will not make an HTTP request; it simply instantiates 267 a bucket object owned by this client. 268 269 :type bucket_name: str 270 :param bucket_name: The name of the bucket to be instantiated. 271 272 :type user_project: str 273 :param user_project: (Optional) The project ID to be billed for API 274 requests made via the bucket. 275 276 :rtype: :class:`google.cloud.storage.bucket.Bucket` 277 :returns: The bucket object created. 278 """ 279 return Bucket(client=self, name=bucket_name, user_project=user_project) 280 281 def batch(self): 282 """Factory constructor for batch object. 283 284 .. note:: 285 This will not make an HTTP request; it simply instantiates 286 a batch object owned by this client. 287 288 :rtype: :class:`google.cloud.storage.batch.Batch` 289 :returns: The batch object created. 290 """ 291 return Batch(client=self) 292 293 def _get_resource( 294 self, 295 path, 296 query_params=None, 297 headers=None, 298 timeout=_DEFAULT_TIMEOUT, 299 retry=DEFAULT_RETRY, 300 _target_object=None, 301 ): 302 """Helper for bucket / blob methods making API 'GET' calls. 303 304 Args: 305 path str: 306 The path of the resource to fetch. 307 308 query_params Optional[dict]: 309 HTTP query parameters to be passed 310 311 headers Optional[dict]: 312 HTTP headers to be passed 313 314 timeout (Optional[Union[float, Tuple[float, float]]]): 315 The amount of time, in seconds, to wait for the server response. 316 317 Can also be passed as a tuple (connect_timeout, read_timeout). 318 See :meth:`requests.Session.request` documentation for details. 319 320 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 321 How to retry the RPC. A None value will disable retries. 322 A google.api_core.retry.Retry value will enable retries, and the object will 323 define retriable response codes and errors and configure backoff and timeout options. 324 325 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 326 activates it only if certain conditions are met. This class exists to provide safe defaults 327 for RPC calls that are not technically safe to retry normally (due to potential data 328 duplication or other side-effects) but become safe to retry if a condition such as 329 if_metageneration_match is set. 330 331 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 332 information on retry types and how to configure them. 333 334 _target_object (Union[ \ 335 :class:`~google.cloud.storage.bucket.Bucket`, \ 336 :class:`~google.cloud.storage.bucket.blob`, \ 337 ]): 338 Object to which future data is to be applied -- only relevant 339 in the context of a batch. 340 341 Returns: 342 dict 343 The JSON resource fetched 344 345 Raises: 346 google.cloud.exceptions.NotFound 347 If the bucket is not found. 348 """ 349 return self._connection.api_request( 350 method="GET", 351 path=path, 352 query_params=query_params, 353 headers=headers, 354 timeout=timeout, 355 retry=retry, 356 _target_object=_target_object, 357 ) 358 359 def _list_resource( 360 self, 361 path, 362 item_to_value, 363 page_token=None, 364 max_results=None, 365 extra_params=None, 366 page_start=page_iterator._do_nothing_page_start, 367 page_size=None, 368 timeout=_DEFAULT_TIMEOUT, 369 retry=DEFAULT_RETRY, 370 ): 371 api_request = functools.partial( 372 self._connection.api_request, timeout=timeout, retry=retry 373 ) 374 return page_iterator.HTTPIterator( 375 client=self, 376 api_request=api_request, 377 path=path, 378 item_to_value=item_to_value, 379 page_token=page_token, 380 max_results=max_results, 381 extra_params=extra_params, 382 page_start=page_start, 383 page_size=page_size, 384 ) 385 386 def _patch_resource( 387 self, 388 path, 389 data, 390 query_params=None, 391 headers=None, 392 timeout=_DEFAULT_TIMEOUT, 393 retry=None, 394 _target_object=None, 395 ): 396 """Helper for bucket / blob methods making API 'PATCH' calls. 397 398 Args: 399 path str: 400 The path of the resource to fetch. 401 402 data dict: 403 The data to be patched. 404 405 query_params Optional[dict]: 406 HTTP query parameters to be passed 407 408 headers Optional[dict]: 409 HTTP headers to be passed 410 411 timeout (Optional[Union[float, Tuple[float, float]]]): 412 The amount of time, in seconds, to wait for the server response. 413 414 Can also be passed as a tuple (connect_timeout, read_timeout). 415 See :meth:`requests.Session.request` documentation for details. 416 417 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 418 How to retry the RPC. A None value will disable retries. 419 A google.api_core.retry.Retry value will enable retries, and the object will 420 define retriable response codes and errors and configure backoff and timeout options. 421 422 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 423 activates it only if certain conditions are met. This class exists to provide safe defaults 424 for RPC calls that are not technically safe to retry normally (due to potential data 425 duplication or other side-effects) but become safe to retry if a condition such as 426 if_metageneration_match is set. 427 428 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 429 information on retry types and how to configure them. 430 431 _target_object (Union[ \ 432 :class:`~google.cloud.storage.bucket.Bucket`, \ 433 :class:`~google.cloud.storage.bucket.blob`, \ 434 ]): 435 Object to which future data is to be applied -- only relevant 436 in the context of a batch. 437 438 Returns: 439 dict 440 The JSON resource fetched 441 442 Raises: 443 google.cloud.exceptions.NotFound 444 If the bucket is not found. 445 """ 446 return self._connection.api_request( 447 method="PATCH", 448 path=path, 449 data=data, 450 query_params=query_params, 451 headers=headers, 452 timeout=timeout, 453 retry=retry, 454 _target_object=_target_object, 455 ) 456 457 def _put_resource( 458 self, 459 path, 460 data, 461 query_params=None, 462 headers=None, 463 timeout=_DEFAULT_TIMEOUT, 464 retry=None, 465 _target_object=None, 466 ): 467 """Helper for bucket / blob methods making API 'PUT' calls. 468 469 Args: 470 path str: 471 The path of the resource to fetch. 472 473 data dict: 474 The data to be patched. 475 476 query_params Optional[dict]: 477 HTTP query parameters to be passed 478 479 headers Optional[dict]: 480 HTTP headers to be passed 481 482 timeout (Optional[Union[float, Tuple[float, float]]]): 483 The amount of time, in seconds, to wait for the server response. 484 485 Can also be passed as a tuple (connect_timeout, read_timeout). 486 See :meth:`requests.Session.request` documentation for details. 487 488 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 489 How to retry the RPC. A None value will disable retries. 490 A google.api_core.retry.Retry value will enable retries, and the object will 491 define retriable response codes and errors and configure backoff and timeout options. 492 493 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 494 activates it only if certain conditions are met. This class exists to provide safe defaults 495 for RPC calls that are not technically safe to retry normally (due to potential data 496 duplication or other side-effects) but become safe to retry if a condition such as 497 if_metageneration_match is set. 498 499 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 500 information on retry types and how to configure them. 501 502 _target_object (Union[ \ 503 :class:`~google.cloud.storage.bucket.Bucket`, \ 504 :class:`~google.cloud.storage.bucket.blob`, \ 505 ]): 506 Object to which future data is to be applied -- only relevant 507 in the context of a batch. 508 509 Returns: 510 dict 511 The JSON resource fetched 512 513 Raises: 514 google.cloud.exceptions.NotFound 515 If the bucket is not found. 516 """ 517 return self._connection.api_request( 518 method="PUT", 519 path=path, 520 data=data, 521 query_params=query_params, 522 headers=headers, 523 timeout=timeout, 524 retry=retry, 525 _target_object=_target_object, 526 ) 527 528 def _post_resource( 529 self, 530 path, 531 data, 532 query_params=None, 533 headers=None, 534 timeout=_DEFAULT_TIMEOUT, 535 retry=None, 536 _target_object=None, 537 ): 538 """Helper for bucket / blob methods making API 'POST' calls. 539 540 Args: 541 path str: 542 The path of the resource to which to post. 543 544 data dict: 545 The data to be posted. 546 547 query_params Optional[dict]: 548 HTTP query parameters to be passed 549 550 headers Optional[dict]: 551 HTTP headers to be passed 552 553 timeout (Optional[Union[float, Tuple[float, float]]]): 554 The amount of time, in seconds, to wait for the server response. 555 556 Can also be passed as a tuple (connect_timeout, read_timeout). 557 See :meth:`requests.Session.request` documentation for details. 558 559 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 560 How to retry the RPC. A None value will disable retries. 561 A google.api_core.retry.Retry value will enable retries, and the object will 562 define retriable response codes and errors and configure backoff and timeout options. 563 564 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 565 activates it only if certain conditions are met. This class exists to provide safe defaults 566 for RPC calls that are not technically safe to retry normally (due to potential data 567 duplication or other side-effects) but become safe to retry if a condition such as 568 if_metageneration_match is set. 569 570 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 571 information on retry types and how to configure them. 572 573 _target_object (Union[ \ 574 :class:`~google.cloud.storage.bucket.Bucket`, \ 575 :class:`~google.cloud.storage.bucket.blob`, \ 576 ]): 577 Object to which future data is to be applied -- only relevant 578 in the context of a batch. 579 580 Returns: 581 dict 582 The JSON resource returned from the post. 583 584 Raises: 585 google.cloud.exceptions.NotFound 586 If the bucket is not found. 587 """ 588 return self._connection.api_request( 589 method="POST", 590 path=path, 591 data=data, 592 query_params=query_params, 593 headers=headers, 594 timeout=timeout, 595 retry=retry, 596 _target_object=_target_object, 597 ) 598 599 def _delete_resource( 600 self, 601 path, 602 query_params=None, 603 headers=None, 604 timeout=_DEFAULT_TIMEOUT, 605 retry=DEFAULT_RETRY, 606 _target_object=None, 607 ): 608 """Helper for bucket / blob methods making API 'DELETE' calls. 609 610 Args: 611 path str: 612 The path of the resource to delete. 613 614 query_params Optional[dict]: 615 HTTP query parameters to be passed 616 617 headers Optional[dict]: 618 HTTP headers to be passed 619 620 timeout (Optional[Union[float, Tuple[float, float]]]): 621 The amount of time, in seconds, to wait for the server response. 622 623 Can also be passed as a tuple (connect_timeout, read_timeout). 624 See :meth:`requests.Session.request` documentation for details. 625 626 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 627 How to retry the RPC. A None value will disable retries. 628 A google.api_core.retry.Retry value will enable retries, and the object will 629 define retriable response codes and errors and configure backoff and timeout options. 630 631 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 632 activates it only if certain conditions are met. This class exists to provide safe defaults 633 for RPC calls that are not technically safe to retry normally (due to potential data 634 duplication or other side-effects) but become safe to retry if a condition such as 635 if_metageneration_match is set. 636 637 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 638 information on retry types and how to configure them. 639 640 _target_object (Union[ \ 641 :class:`~google.cloud.storage.bucket.Bucket`, \ 642 :class:`~google.cloud.storage.bucket.blob`, \ 643 ]): 644 Object to which future data is to be applied -- only relevant 645 in the context of a batch. 646 647 Returns: 648 dict 649 The JSON resource fetched 650 651 Raises: 652 google.cloud.exceptions.NotFound 653 If the bucket is not found. 654 """ 655 return self._connection.api_request( 656 method="DELETE", 657 path=path, 658 query_params=query_params, 659 headers=headers, 660 timeout=timeout, 661 retry=retry, 662 _target_object=_target_object, 663 ) 664 665 def _bucket_arg_to_bucket(self, bucket_or_name): 666 """Helper to return given bucket or create new by name. 667 668 Args: 669 bucket_or_name (Union[ \ 670 :class:`~google.cloud.storage.bucket.Bucket`, \ 671 str, \ 672 ]): 673 The bucket resource to pass or name to create. 674 675 Returns: 676 google.cloud.storage.bucket.Bucket 677 The newly created bucket or the given one. 678 """ 679 if isinstance(bucket_or_name, Bucket): 680 bucket = bucket_or_name 681 if bucket.client is None: 682 bucket._client = self 683 else: 684 bucket = Bucket(self, name=bucket_or_name) 685 return bucket 686 687 def get_bucket( 688 self, 689 bucket_or_name, 690 timeout=_DEFAULT_TIMEOUT, 691 if_metageneration_match=None, 692 if_metageneration_not_match=None, 693 retry=DEFAULT_RETRY, 694 ): 695 """API call: retrieve a bucket via a GET request. 696 697 See 698 https://cloud.google.com/storage/docs/json_api/v1/buckets/get 699 700 Args: 701 bucket_or_name (Union[ \ 702 :class:`~google.cloud.storage.bucket.Bucket`, \ 703 str, \ 704 ]): 705 The bucket resource to pass or name to create. 706 707 timeout (Optional[Union[float, Tuple[float, float]]]): 708 The amount of time, in seconds, to wait for the server response. 709 710 Can also be passed as a tuple (connect_timeout, read_timeout). 711 See :meth:`requests.Session.request` documentation for details. 712 713 if_metageneration_match (Optional[long]): 714 Make the operation conditional on whether the 715 blob's current metageneration matches the given value. 716 717 if_metageneration_not_match (Optional[long]): 718 Make the operation conditional on whether the blob's 719 current metageneration does not match the given value. 720 721 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 722 How to retry the RPC. A None value will disable retries. 723 A google.api_core.retry.Retry value will enable retries, and the object will 724 define retriable response codes and errors and configure backoff and timeout options. 725 726 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 727 activates it only if certain conditions are met. This class exists to provide safe defaults 728 for RPC calls that are not technically safe to retry normally (due to potential data 729 duplication or other side-effects) but become safe to retry if a condition such as 730 if_metageneration_match is set. 731 732 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 733 information on retry types and how to configure them. 734 735 Returns: 736 google.cloud.storage.bucket.Bucket 737 The bucket matching the name provided. 738 739 Raises: 740 google.cloud.exceptions.NotFound 741 If the bucket is not found. 742 743 Examples: 744 Retrieve a bucket using a string. 745 746 .. literalinclude:: snippets.py 747 :start-after: [START get_bucket] 748 :end-before: [END get_bucket] 749 :dedent: 4 750 751 Get a bucket using a resource. 752 753 >>> from google.cloud import storage 754 >>> client = storage.Client() 755 756 >>> # Set properties on a plain resource object. 757 >>> bucket = client.get_bucket("my-bucket-name") 758 759 >>> # Time passes. Another program may have modified the bucket 760 ... # in the meantime, so you want to get the latest state. 761 >>> bucket = client.get_bucket(bucket) # API request. 762 763 """ 764 bucket = self._bucket_arg_to_bucket(bucket_or_name) 765 bucket.reload( 766 client=self, 767 timeout=timeout, 768 if_metageneration_match=if_metageneration_match, 769 if_metageneration_not_match=if_metageneration_not_match, 770 retry=retry, 771 ) 772 return bucket 773 774 def lookup_bucket( 775 self, 776 bucket_name, 777 timeout=_DEFAULT_TIMEOUT, 778 if_metageneration_match=None, 779 if_metageneration_not_match=None, 780 retry=DEFAULT_RETRY, 781 ): 782 """Get a bucket by name, returning None if not found. 783 784 You can use this if you would rather check for a None value 785 than catching an exception: 786 787 .. literalinclude:: snippets.py 788 :start-after: [START lookup_bucket] 789 :end-before: [END lookup_bucket] 790 :dedent: 4 791 792 :type bucket_name: str 793 :param bucket_name: The name of the bucket to get. 794 795 :type timeout: float or tuple 796 :param timeout: 797 (Optional) The amount of time, in seconds, to wait 798 for the server response. See: :ref:`configuring_timeouts` 799 800 :type if_metageneration_match: long 801 :param if_metageneration_match: (Optional) Make the operation conditional on whether the 802 blob's current metageneration matches the given value. 803 804 :type if_metageneration_not_match: long 805 :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the 806 blob's current metageneration does not match the given value. 807 808 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 809 :param retry: 810 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 811 812 :rtype: :class:`google.cloud.storage.bucket.Bucket` 813 :returns: The bucket matching the name provided or None if not found. 814 """ 815 try: 816 return self.get_bucket( 817 bucket_name, 818 timeout=timeout, 819 if_metageneration_match=if_metageneration_match, 820 if_metageneration_not_match=if_metageneration_not_match, 821 retry=retry, 822 ) 823 except NotFound: 824 return None 825 826 def create_bucket( 827 self, 828 bucket_or_name, 829 requester_pays=None, 830 project=None, 831 user_project=None, 832 location=None, 833 predefined_acl=None, 834 predefined_default_object_acl=None, 835 timeout=_DEFAULT_TIMEOUT, 836 retry=DEFAULT_RETRY, 837 ): 838 """API call: create a new bucket via a POST request. 839 840 See 841 https://cloud.google.com/storage/docs/json_api/v1/buckets/insert 842 843 Args: 844 bucket_or_name (Union[ \ 845 :class:`~google.cloud.storage.bucket.Bucket`, \ 846 str, \ 847 ]): 848 The bucket resource to pass or name to create. 849 requester_pays (bool): 850 DEPRECATED. Use Bucket().requester_pays instead. 851 (Optional) Whether requester pays for API requests for 852 this bucket and its blobs. 853 project (str): 854 (Optional) The project under which the bucket is to be created. 855 If not passed, uses the project set on the client. 856 user_project (str): 857 (Optional) The project ID to be billed for API requests 858 made via created bucket. 859 location (str): 860 (Optional) The location of the bucket. If not passed, 861 the default location, US, will be used. See 862 https://cloud.google.com/storage/docs/bucket-locations 863 predefined_acl (str): 864 (Optional) Name of predefined ACL to apply to bucket. See: 865 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl 866 predefined_default_object_acl (str): 867 (Optional) Name of predefined ACL to apply to bucket's objects. See: 868 https://cloud.google.com/storage/docs/access-control/lists#predefined-acl 869 timeout (Optional[Union[float, Tuple[float, float]]]): 870 The amount of time, in seconds, to wait for the server response. 871 872 Can also be passed as a tuple (connect_timeout, read_timeout). 873 See :meth:`requests.Session.request` documentation for details. 874 875 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 876 How to retry the RPC. A None value will disable retries. 877 A google.api_core.retry.Retry value will enable retries, and the object will 878 define retriable response codes and errors and configure backoff and timeout options. 879 880 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 881 activates it only if certain conditions are met. This class exists to provide safe defaults 882 for RPC calls that are not technically safe to retry normally (due to potential data 883 duplication or other side-effects) but become safe to retry if a condition such as 884 if_metageneration_match is set. 885 886 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 887 information on retry types and how to configure them. 888 889 Returns: 890 google.cloud.storage.bucket.Bucket 891 The newly created bucket. 892 893 Raises: 894 google.cloud.exceptions.Conflict 895 If the bucket already exists. 896 897 Examples: 898 Create a bucket using a string. 899 900 .. literalinclude:: snippets.py 901 :start-after: [START create_bucket] 902 :end-before: [END create_bucket] 903 :dedent: 4 904 905 Create a bucket using a resource. 906 907 >>> from google.cloud import storage 908 >>> client = storage.Client() 909 910 >>> # Set properties on a plain resource object. 911 >>> bucket = storage.Bucket("my-bucket-name") 912 >>> bucket.location = "europe-west6" 913 >>> bucket.storage_class = "COLDLINE" 914 915 >>> # Pass that resource object to the client. 916 >>> bucket = client.create_bucket(bucket) # API request. 917 918 """ 919 bucket = self._bucket_arg_to_bucket(bucket_or_name) 920 921 if project is None: 922 project = self.project 923 924 if project is None: 925 raise ValueError("Client project not set: pass an explicit project.") 926 927 if requester_pays is not None: 928 warnings.warn( 929 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", 930 PendingDeprecationWarning, 931 stacklevel=1, 932 ) 933 bucket.requester_pays = requester_pays 934 935 query_params = {"project": project} 936 937 if predefined_acl is not None: 938 predefined_acl = BucketACL.validate_predefined(predefined_acl) 939 query_params["predefinedAcl"] = predefined_acl 940 941 if predefined_default_object_acl is not None: 942 predefined_default_object_acl = DefaultObjectACL.validate_predefined( 943 predefined_default_object_acl 944 ) 945 query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl 946 947 if user_project is not None: 948 query_params["userProject"] = user_project 949 950 properties = {key: bucket._properties[key] for key in bucket._changes} 951 properties["name"] = bucket.name 952 953 if location is not None: 954 properties["location"] = location 955 956 api_response = self._post_resource( 957 "/b", 958 properties, 959 query_params=query_params, 960 timeout=timeout, 961 retry=retry, 962 _target_object=bucket, 963 ) 964 965 bucket._set_properties(api_response) 966 return bucket 967 968 def download_blob_to_file( 969 self, 970 blob_or_uri, 971 file_obj, 972 start=None, 973 end=None, 974 raw_download=False, 975 if_etag_match=None, 976 if_etag_not_match=None, 977 if_generation_match=None, 978 if_generation_not_match=None, 979 if_metageneration_match=None, 980 if_metageneration_not_match=None, 981 timeout=_DEFAULT_TIMEOUT, 982 checksum="md5", 983 retry=DEFAULT_RETRY, 984 ): 985 """Download the contents of a blob object or blob URI into a file-like object. 986 987 Args: 988 blob_or_uri (Union[ \ 989 :class:`~google.cloud.storage.blob.Blob`, \ 990 str, \ 991 ]): 992 The blob resource to pass or URI to download. 993 994 file_obj (file): 995 A file handle to which to write the blob's data. 996 997 start (int): 998 (Optional) The first byte in a range to be downloaded. 999 1000 end (int): 1001 (Optional) The last byte in a range to be downloaded. 1002 1003 raw_download (bool): 1004 (Optional) If true, download the object without any expansion. 1005 1006 if_etag_match (Union[str, Set[str]]): 1007 (Optional) See :ref:`using-if-etag-match` 1008 1009 if_etag_not_match (Union[str, Set[str]]): 1010 (Optional) See :ref:`using-if-etag-not-match` 1011 1012 if_generation_match (long): 1013 (Optional) See :ref:`using-if-generation-match` 1014 1015 if_generation_not_match (long): 1016 (Optional) See :ref:`using-if-generation-not-match` 1017 1018 if_metageneration_match (long): 1019 (Optional) See :ref:`using-if-metageneration-match` 1020 1021 if_metageneration_not_match (long): 1022 (Optional) See :ref:`using-if-metageneration-not-match` 1023 1024 timeout ([Union[float, Tuple[float, float]]]): 1025 (Optional) The amount of time, in seconds, to wait 1026 for the server response. See: :ref:`configuring_timeouts` 1027 1028 checksum (str): 1029 (Optional) The type of checksum to compute to verify the integrity 1030 of the object. The response headers must contain a checksum of the 1031 requested type. If the headers lack an appropriate checksum (for 1032 instance in the case of transcoded or ranged downloads where the 1033 remote service does not know the correct checksum, including 1034 downloads where chunk_size is set) an INFO-level log will be 1035 emitted. Supported values are "md5", "crc32c" and None. The default 1036 is "md5". 1037 retry (google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy) 1038 (Optional) How to retry the RPC. A None value will disable 1039 retries. A google.api_core.retry.Retry value will enable retries, 1040 and the object will define retriable response codes and errors and 1041 configure backoff and timeout options. 1042 1043 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a 1044 Retry object and activates it only if certain conditions are met. 1045 This class exists to provide safe defaults for RPC calls that are 1046 not technically safe to retry normally (due to potential data 1047 duplication or other side-effects) but become safe to retry if a 1048 condition such as if_metageneration_match is set. 1049 1050 See the retry.py source code and docstrings in this package 1051 (google.cloud.storage.retry) for information on retry types and how 1052 to configure them. 1053 1054 Media operations (downloads and uploads) do not support non-default 1055 predicates in a Retry object. The default will always be used. Other 1056 configuration changes for Retry objects such as delays and deadlines 1057 are respected. 1058 1059 Examples: 1060 Download a blob using a blob resource. 1061 1062 >>> from google.cloud import storage 1063 >>> client = storage.Client() 1064 1065 >>> bucket = client.get_bucket('my-bucket-name') 1066 >>> blob = storage.Blob('path/to/blob', bucket) 1067 1068 >>> with open('file-to-download-to') as file_obj: 1069 >>> client.download_blob_to_file(blob, file_obj) # API request. 1070 1071 1072 Download a blob using a URI. 1073 1074 >>> from google.cloud import storage 1075 >>> client = storage.Client() 1076 1077 >>> with open('file-to-download-to') as file_obj: 1078 >>> client.download_blob_to_file( 1079 >>> 'gs://bucket_name/path/to/blob', file_obj) 1080 1081 1082 """ 1083 1084 # Handle ConditionalRetryPolicy. 1085 if isinstance(retry, ConditionalRetryPolicy): 1086 # Conditional retries are designed for non-media calls, which change 1087 # arguments into query_params dictionaries. Media operations work 1088 # differently, so here we make a "fake" query_params to feed to the 1089 # ConditionalRetryPolicy. 1090 query_params = { 1091 "ifGenerationMatch": if_generation_match, 1092 "ifMetagenerationMatch": if_metageneration_match, 1093 } 1094 retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) 1095 1096 if not isinstance(blob_or_uri, Blob): 1097 blob_or_uri = Blob.from_string(blob_or_uri) 1098 download_url = blob_or_uri._get_download_url( 1099 self, 1100 if_generation_match=if_generation_match, 1101 if_generation_not_match=if_generation_not_match, 1102 if_metageneration_match=if_metageneration_match, 1103 if_metageneration_not_match=if_metageneration_not_match, 1104 ) 1105 headers = _get_encryption_headers(blob_or_uri._encryption_key) 1106 headers["accept-encoding"] = "gzip" 1107 _add_etag_match_headers( 1108 headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match, 1109 ) 1110 1111 transport = self._http 1112 try: 1113 blob_or_uri._do_download( 1114 transport, 1115 file_obj, 1116 download_url, 1117 headers, 1118 start, 1119 end, 1120 raw_download, 1121 timeout=timeout, 1122 checksum=checksum, 1123 retry=retry, 1124 ) 1125 except resumable_media.InvalidResponse as exc: 1126 _raise_from_invalid_response(exc) 1127 1128 def list_blobs( 1129 self, 1130 bucket_or_name, 1131 max_results=None, 1132 page_token=None, 1133 prefix=None, 1134 delimiter=None, 1135 start_offset=None, 1136 end_offset=None, 1137 include_trailing_delimiter=None, 1138 versions=None, 1139 projection="noAcl", 1140 fields=None, 1141 page_size=None, 1142 timeout=_DEFAULT_TIMEOUT, 1143 retry=DEFAULT_RETRY, 1144 ): 1145 """Return an iterator used to find blobs in the bucket. 1146 1147 If :attr:`user_project` is set, bills the API request to that project. 1148 1149 Args: 1150 bucket_or_name (Union[ \ 1151 :class:`~google.cloud.storage.bucket.Bucket`, \ 1152 str, \ 1153 ]): 1154 The bucket resource to pass or name to create. 1155 1156 max_results (int): 1157 (Optional) The maximum number of blobs to return. 1158 1159 page_token (str): 1160 (Optional) If present, return the next batch of blobs, using the 1161 value, which must correspond to the ``nextPageToken`` value 1162 returned in the previous response. Deprecated: use the ``pages`` 1163 property of the returned iterator instead of manually passing the 1164 token. 1165 1166 prefix (str): 1167 (Optional) Prefix used to filter blobs. 1168 1169 delimiter (str): 1170 (Optional) Delimiter, used with ``prefix`` to 1171 emulate hierarchy. 1172 1173 start_offset (str): 1174 (Optional) Filter results to objects whose names are 1175 lexicographically equal to or after ``startOffset``. If 1176 ``endOffset`` is also set, the objects listed will have names 1177 between ``startOffset`` (inclusive) and ``endOffset`` 1178 (exclusive). 1179 1180 end_offset (str): 1181 (Optional) Filter results to objects whose names are 1182 lexicographically before ``endOffset``. If ``startOffset`` is 1183 also set, the objects listed will have names between 1184 ``startOffset`` (inclusive) and ``endOffset`` (exclusive). 1185 1186 include_trailing_delimiter (boolean): 1187 (Optional) If true, objects that end in exactly one instance of 1188 ``delimiter`` will have their metadata included in ``items`` in 1189 addition to ``prefixes``. 1190 1191 versions (bool): 1192 (Optional) Whether object versions should be returned 1193 as separate blobs. 1194 1195 projection (str): 1196 (Optional) If used, must be 'full' or 'noAcl'. 1197 Defaults to ``'noAcl'``. Specifies the set of 1198 properties to return. 1199 1200 fields (str): 1201 (Optional) Selector specifying which fields to include 1202 in a partial response. Must be a list of fields. For 1203 example to get a partial response with just the next 1204 page token and the name and language of each blob returned: 1205 ``'items(name,contentLanguage),nextPageToken'``. 1206 See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields 1207 1208 page_size (int): 1209 (Optional) Maximum number of blobs to return in each page. 1210 Defaults to a value set by the API. 1211 1212 timeout (Optional[Union[float, Tuple[float, float]]]): 1213 The amount of time, in seconds, to wait for the server response. 1214 1215 Can also be passed as a tuple (connect_timeout, read_timeout). 1216 See :meth:`requests.Session.request` documentation for details. 1217 1218 retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): 1219 How to retry the RPC. A None value will disable retries. 1220 A google.api_core.retry.Retry value will enable retries, and the object will 1221 define retriable response codes and errors and configure backoff and timeout options. 1222 1223 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 1224 activates it only if certain conditions are met. This class exists to provide safe defaults 1225 for RPC calls that are not technically safe to retry normally (due to potential data 1226 duplication or other side-effects) but become safe to retry if a condition such as 1227 if_metageneration_match is set. 1228 1229 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 1230 information on retry types and how to configure them. 1231 1232 Returns: 1233 Iterator of all :class:`~google.cloud.storage.blob.Blob` 1234 in this bucket matching the arguments. 1235 1236 Example: 1237 List blobs in the bucket with user_project. 1238 1239 >>> from google.cloud import storage 1240 >>> client = storage.Client() 1241 1242 >>> bucket = storage.Bucket(client, "my-bucket-name", user_project="my-project") 1243 >>> all_blobs = list(client.list_blobs(bucket)) 1244 """ 1245 bucket = self._bucket_arg_to_bucket(bucket_or_name) 1246 1247 extra_params = {"projection": projection} 1248 1249 if prefix is not None: 1250 extra_params["prefix"] = prefix 1251 1252 if delimiter is not None: 1253 extra_params["delimiter"] = delimiter 1254 1255 if start_offset is not None: 1256 extra_params["startOffset"] = start_offset 1257 1258 if end_offset is not None: 1259 extra_params["endOffset"] = end_offset 1260 1261 if include_trailing_delimiter is not None: 1262 extra_params["includeTrailingDelimiter"] = include_trailing_delimiter 1263 1264 if versions is not None: 1265 extra_params["versions"] = versions 1266 1267 if fields is not None: 1268 extra_params["fields"] = fields 1269 1270 if bucket.user_project is not None: 1271 extra_params["userProject"] = bucket.user_project 1272 1273 path = bucket.path + "/o" 1274 iterator = self._list_resource( 1275 path, 1276 _item_to_blob, 1277 page_token=page_token, 1278 max_results=max_results, 1279 extra_params=extra_params, 1280 page_start=_blobs_page_start, 1281 page_size=page_size, 1282 timeout=timeout, 1283 retry=retry, 1284 ) 1285 iterator.bucket = bucket 1286 iterator.prefixes = set() 1287 return iterator 1288 1289 def list_buckets( 1290 self, 1291 max_results=None, 1292 page_token=None, 1293 prefix=None, 1294 projection="noAcl", 1295 fields=None, 1296 project=None, 1297 page_size=None, 1298 timeout=_DEFAULT_TIMEOUT, 1299 retry=DEFAULT_RETRY, 1300 ): 1301 """Get all buckets in the project associated to the client. 1302 1303 This will not populate the list of blobs available in each 1304 bucket. 1305 1306 .. literalinclude:: snippets.py 1307 :start-after: [START list_buckets] 1308 :end-before: [END list_buckets] 1309 :dedent: 4 1310 1311 This implements "storage.buckets.list". 1312 1313 :type max_results: int 1314 :param max_results: (Optional) The maximum number of buckets to return. 1315 1316 :type page_token: str 1317 :param page_token: 1318 (Optional) If present, return the next batch of buckets, using the 1319 value, which must correspond to the ``nextPageToken`` value 1320 returned in the previous response. Deprecated: use the ``pages`` 1321 property of the returned iterator instead of manually passing the 1322 token. 1323 1324 :type prefix: str 1325 :param prefix: (Optional) Filter results to buckets whose names begin 1326 with this prefix. 1327 1328 :type projection: str 1329 :param projection: 1330 (Optional) Specifies the set of properties to return. If used, must 1331 be 'full' or 'noAcl'. Defaults to 'noAcl'. 1332 1333 :type fields: str 1334 :param fields: 1335 (Optional) Selector specifying which fields to include in a partial 1336 response. Must be a list of fields. For example to get a partial 1337 response with just the next page token and the language of each 1338 bucket returned: 'items/id,nextPageToken' 1339 1340 :type project: str 1341 :param project: (Optional) The project whose buckets are to be listed. 1342 If not passed, uses the project set on the client. 1343 1344 :type page_size: int 1345 :param page_size: (Optional) Maximum number of buckets to return in each page. 1346 Defaults to a value set by the API. 1347 1348 :type timeout: float or tuple 1349 :param timeout: 1350 (Optional) The amount of time, in seconds, to wait 1351 for the server response. See: :ref:`configuring_timeouts` 1352 1353 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 1354 :param retry: 1355 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 1356 1357 :rtype: :class:`~google.api_core.page_iterator.Iterator` 1358 :raises ValueError: if both ``project`` is ``None`` and the client's 1359 project is also ``None``. 1360 :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket` 1361 belonging to this project. 1362 """ 1363 if project is None: 1364 project = self.project 1365 1366 if project is None: 1367 raise ValueError("Client project not set: pass an explicit project.") 1368 1369 extra_params = {"project": project} 1370 1371 if prefix is not None: 1372 extra_params["prefix"] = prefix 1373 1374 extra_params["projection"] = projection 1375 1376 if fields is not None: 1377 extra_params["fields"] = fields 1378 1379 return self._list_resource( 1380 "/b", 1381 _item_to_bucket, 1382 page_token=page_token, 1383 max_results=max_results, 1384 extra_params=extra_params, 1385 page_size=page_size, 1386 timeout=timeout, 1387 retry=retry, 1388 ) 1389 1390 def create_hmac_key( 1391 self, 1392 service_account_email, 1393 project_id=None, 1394 user_project=None, 1395 timeout=_DEFAULT_TIMEOUT, 1396 retry=None, 1397 ): 1398 """Create an HMAC key for a service account. 1399 1400 :type service_account_email: str 1401 :param service_account_email: e-mail address of the service account 1402 1403 :type project_id: str 1404 :param project_id: (Optional) Explicit project ID for the key. 1405 Defaults to the client's project. 1406 1407 :type user_project: str 1408 :param user_project: (Optional) This parameter is currently ignored. 1409 1410 :type timeout: float or tuple 1411 :param timeout: 1412 (Optional) The amount of time, in seconds, to wait 1413 for the server response. See: :ref:`configuring_timeouts` 1414 1415 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 1416 :param retry: (Optional) How to retry the RPC. A None value will disable retries. 1417 A google.api_core.retry.Retry value will enable retries, and the object will 1418 define retriable response codes and errors and configure backoff and timeout options. 1419 1420 A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and 1421 activates it only if certain conditions are met. This class exists to provide safe defaults 1422 for RPC calls that are not technically safe to retry normally (due to potential data 1423 duplication or other side-effects) but become safe to retry if a condition such as 1424 if_metageneration_match is set. 1425 1426 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for 1427 information on retry types and how to configure them. 1428 1429 :rtype: 1430 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] 1431 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. 1432 """ 1433 if project_id is None: 1434 project_id = self.project 1435 1436 path = "/projects/{}/hmacKeys".format(project_id) 1437 qs_params = {"serviceAccountEmail": service_account_email} 1438 1439 if user_project is not None: 1440 qs_params["userProject"] = user_project 1441 1442 api_response = self._post_resource( 1443 path, None, query_params=qs_params, timeout=timeout, retry=retry, 1444 ) 1445 metadata = HMACKeyMetadata(self) 1446 metadata._properties = api_response["metadata"] 1447 secret = api_response["secret"] 1448 return metadata, secret 1449 1450 def list_hmac_keys( 1451 self, 1452 max_results=None, 1453 service_account_email=None, 1454 show_deleted_keys=None, 1455 project_id=None, 1456 user_project=None, 1457 timeout=_DEFAULT_TIMEOUT, 1458 retry=DEFAULT_RETRY, 1459 ): 1460 """List HMAC keys for a project. 1461 1462 :type max_results: int 1463 :param max_results: 1464 (Optional) Max number of keys to return in a given page. 1465 1466 :type service_account_email: str 1467 :param service_account_email: 1468 (Optional) Limit keys to those created by the given service account. 1469 1470 :type show_deleted_keys: bool 1471 :param show_deleted_keys: 1472 (Optional) Included deleted keys in the list. Default is to 1473 exclude them. 1474 1475 :type project_id: str 1476 :param project_id: (Optional) Explicit project ID for the key. 1477 Defaults to the client's project. 1478 1479 :type user_project: str 1480 :param user_project: (Optional) This parameter is currently ignored. 1481 1482 :type timeout: float or tuple 1483 :param timeout: 1484 (Optional) The amount of time, in seconds, to wait 1485 for the server response. See: :ref:`configuring_timeouts` 1486 1487 :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy 1488 :param retry: 1489 (Optional) How to retry the RPC. See: :ref:`configuring_retries` 1490 1491 :rtype: 1492 Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] 1493 :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. 1494 """ 1495 if project_id is None: 1496 project_id = self.project 1497 1498 path = "/projects/{}/hmacKeys".format(project_id) 1499 extra_params = {} 1500 1501 if service_account_email is not None: 1502 extra_params["serviceAccountEmail"] = service_account_email 1503 1504 if show_deleted_keys is not None: 1505 extra_params["showDeletedKeys"] = show_deleted_keys 1506 1507 if user_project is not None: 1508 extra_params["userProject"] = user_project 1509 1510 return self._list_resource( 1511 path, 1512 _item_to_hmac_key_metadata, 1513 max_results=max_results, 1514 extra_params=extra_params, 1515 timeout=timeout, 1516 retry=retry, 1517 ) 1518 1519 def get_hmac_key_metadata( 1520 self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT 1521 ): 1522 """Return a metadata instance for the given HMAC key. 1523 1524 :type access_id: str 1525 :param access_id: Unique ID of an existing key. 1526 1527 :type project_id: str 1528 :param project_id: (Optional) Project ID of an existing key. 1529 Defaults to client's project. 1530 1531 :type timeout: float or tuple 1532 :param timeout: 1533 (Optional) The amount of time, in seconds, to wait 1534 for the server response. See: :ref:`configuring_timeouts` 1535 1536 :type user_project: str 1537 :param user_project: (Optional) This parameter is currently ignored. 1538 """ 1539 metadata = HMACKeyMetadata(self, access_id, project_id, user_project) 1540 metadata.reload(timeout=timeout) # raises NotFound for missing key 1541 return metadata 1542 1543 def generate_signed_post_policy_v4( 1544 self, 1545 bucket_name, 1546 blob_name, 1547 expiration, 1548 conditions=None, 1549 fields=None, 1550 credentials=None, 1551 virtual_hosted_style=False, 1552 bucket_bound_hostname=None, 1553 scheme="http", 1554 service_account_email=None, 1555 access_token=None, 1556 ): 1557 """Generate a V4 signed policy object. 1558 1559 .. note:: 1560 1561 Assumes ``credentials`` implements the 1562 :class:`google.auth.credentials.Signing` interface. Also assumes 1563 ``credentials`` has a ``service_account_email`` property which 1564 identifies the credentials. 1565 1566 Generated policy object allows user to upload objects with a POST request. 1567 1568 :type bucket_name: str 1569 :param bucket_name: Bucket name. 1570 1571 :type blob_name: str 1572 :param blob_name: Object name. 1573 1574 :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] 1575 :param expiration: Policy expiration time. If a ``datetime`` instance is 1576 passed without an explicit ``tzinfo`` set, it will be 1577 assumed to be ``UTC``. 1578 1579 :type conditions: list 1580 :param conditions: (Optional) List of POST policy conditions, which are 1581 used to restrict what is allowed in the request. 1582 1583 :type fields: dict 1584 :param fields: (Optional) Additional elements to include into request. 1585 1586 :type credentials: :class:`google.auth.credentials.Signing` 1587 :param credentials: (Optional) Credentials object with an associated private 1588 key to sign text. 1589 1590 :type virtual_hosted_style: bool 1591 :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket 1592 virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'. 1593 1594 :type bucket_bound_hostname: str 1595 :param bucket_bound_hostname: 1596 (Optional) If passed, construct the URL relative to the bucket-bound hostname. 1597 Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'. 1598 See: https://cloud.google.com/storage/docs/request-endpoints#cname 1599 1600 :type scheme: str 1601 :param scheme: 1602 (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use 1603 this value as a scheme. ``https`` will work only when using a CDN. 1604 Defaults to ``"http"``. 1605 1606 :type service_account_email: str 1607 :param service_account_email: (Optional) E-mail address of the service account. 1608 1609 :type access_token: str 1610 :param access_token: (Optional) Access token for a service account. 1611 1612 :rtype: dict 1613 :returns: Signed POST policy. 1614 1615 Example: 1616 Generate signed POST policy and upload a file. 1617 1618 >>> import datetime 1619 >>> from google.cloud import storage 1620 >>> client = storage.Client() 1621 >>> tz = datetime.timezone(datetime.timedelta(hours=1), 'CET') 1622 >>> policy = client.generate_signed_post_policy_v4( 1623 "bucket-name", 1624 "blob-name", 1625 expiration=datetime.datetime(2020, 3, 17, tzinfo=tz), 1626 conditions=[ 1627 ["content-length-range", 0, 255] 1628 ], 1629 fields=[ 1630 "x-goog-meta-hello" => "world" 1631 ], 1632 ) 1633 >>> with open("bucket-name", "rb") as f: 1634 files = {"file": ("bucket-name", f)} 1635 requests.post(policy["url"], data=policy["fields"], files=files) 1636 """ 1637 credentials = self._credentials if credentials is None else credentials 1638 ensure_signed_credentials(credentials) 1639 1640 # prepare policy conditions and fields 1641 timestamp, datestamp = get_v4_now_dtstamps() 1642 1643 x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format( 1644 email=credentials.signer_email, datestamp=datestamp 1645 ) 1646 required_conditions = [ 1647 {"bucket": bucket_name}, 1648 {"key": blob_name}, 1649 {"x-goog-date": timestamp}, 1650 {"x-goog-credential": x_goog_credential}, 1651 {"x-goog-algorithm": "GOOG4-RSA-SHA256"}, 1652 ] 1653 1654 conditions = conditions or [] 1655 policy_fields = {} 1656 for key, value in sorted((fields or {}).items()): 1657 if not key.startswith("x-ignore-"): 1658 policy_fields[key] = value 1659 conditions.append({key: value}) 1660 1661 conditions += required_conditions 1662 1663 # calculate policy expiration time 1664 now = _NOW() 1665 if expiration is None: 1666 expiration = now + datetime.timedelta(hours=1) 1667 1668 policy_expires = now + datetime.timedelta( 1669 seconds=get_expiration_seconds_v4(expiration) 1670 ) 1671 1672 # encode policy for signing 1673 policy = json.dumps( 1674 collections.OrderedDict( 1675 sorted( 1676 { 1677 "conditions": conditions, 1678 "expiration": policy_expires.isoformat() + "Z", 1679 }.items() 1680 ) 1681 ), 1682 separators=(",", ":"), 1683 ) 1684 str_to_sign = base64.b64encode(policy.encode("utf-8")) 1685 1686 # sign the policy and get its cryptographic signature 1687 if access_token and service_account_email: 1688 signature = _sign_message(str_to_sign, access_token, service_account_email) 1689 signature_bytes = base64.b64decode(signature) 1690 else: 1691 signature_bytes = credentials.sign_bytes(str_to_sign) 1692 1693 # get hexadecimal representation of the signature 1694 signature = binascii.hexlify(signature_bytes).decode("utf-8") 1695 1696 policy_fields.update( 1697 { 1698 "key": blob_name, 1699 "x-goog-algorithm": "GOOG4-RSA-SHA256", 1700 "x-goog-credential": x_goog_credential, 1701 "x-goog-date": timestamp, 1702 "x-goog-signature": signature, 1703 "policy": str_to_sign.decode("utf-8"), 1704 } 1705 ) 1706 # designate URL 1707 if virtual_hosted_style: 1708 url = "https://{}.storage.googleapis.com/".format(bucket_name) 1709 elif bucket_bound_hostname: 1710 url = _bucket_bound_hostname_url(bucket_bound_hostname, scheme) 1711 else: 1712 url = "https://storage.googleapis.com/{}/".format(bucket_name) 1713 1714 return {"url": url, "fields": policy_fields} 1715 1716 1717def _item_to_bucket(iterator, item): 1718 """Convert a JSON bucket to the native object. 1719 1720 :type iterator: :class:`~google.api_core.page_iterator.Iterator` 1721 :param iterator: The iterator that has retrieved the item. 1722 1723 :type item: dict 1724 :param item: An item to be converted to a bucket. 1725 1726 :rtype: :class:`.Bucket` 1727 :returns: The next bucket in the page. 1728 """ 1729 name = item.get("name") 1730 bucket = Bucket(iterator.client, name) 1731 bucket._set_properties(item) 1732 return bucket 1733 1734 1735def _item_to_hmac_key_metadata(iterator, item): 1736 """Convert a JSON key metadata resource to the native object. 1737 1738 :type iterator: :class:`~google.api_core.page_iterator.Iterator` 1739 :param iterator: The iterator that has retrieved the item. 1740 1741 :type item: dict 1742 :param item: An item to be converted to a key metadata instance. 1743 1744 :rtype: :class:`~google.cloud.storage.hmac_key.HMACKeyMetadata` 1745 :returns: The next key metadata instance in the page. 1746 """ 1747 metadata = HMACKeyMetadata(iterator.client) 1748 metadata._properties = item 1749 return metadata 1750