1# Copyright 2015 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import base64 16import io 17import json 18import mock 19import pytest 20import re 21import requests 22import unittest 23from six import string_types 24from six.moves import http_client 25from six.moves.urllib import parse as urlparse 26 27from google.api_core import exceptions 28 29from google.oauth2.service_account import Credentials 30from . import _read_local_json 31 32from google.cloud.storage.retry import DEFAULT_RETRY 33from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED 34 35 36_SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") 37_CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json")[ 38 "postPolicyV4Tests" 39] 40_POST_POLICY_TESTS = [test for test in _CONFORMANCE_TESTS if "policyInput" in test] 41_FAKE_CREDENTIALS = Credentials.from_service_account_info(_SERVICE_ACCOUNT_JSON) 42 43 44def _make_credentials(project=None): 45 import google.auth.credentials 46 47 if project is not None: 48 return mock.Mock(spec=google.auth.credentials.Credentials, project_id=project) 49 50 return mock.Mock(spec=google.auth.credentials.Credentials) 51 52 53def _create_signing_credentials(): 54 import google.auth.credentials 55 56 class _SigningCredentials( 57 google.auth.credentials.Credentials, google.auth.credentials.Signing 58 ): 59 pass 60 61 credentials = mock.Mock(spec=_SigningCredentials) 62 credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") 63 credentials.signer_email = "test@mail.com" 64 return credentials 65 66 67def _make_connection(*responses): 68 import google.cloud.storage._http 69 from google.cloud.exceptions import NotFound 70 71 mock_conn = mock.create_autospec(google.cloud.storage._http.Connection) 72 mock_conn.user_agent = "testing 1.2.3" 73 mock_conn.api_request.side_effect = list(responses) + [NotFound("miss")] 74 return mock_conn 75 76 77def _make_response(status=http_client.OK, content=b"", headers={}): 78 response = requests.Response() 79 response.status_code = status 80 response._content = content 81 response.headers = headers 82 response.request = requests.Request() 83 return response 84 85 86def _make_json_response(data, status=http_client.OK, headers=None): 87 headers = headers or {} 88 headers["Content-Type"] = "application/json" 89 return _make_response( 90 status=status, content=json.dumps(data).encode("utf-8"), headers=headers 91 ) 92 93 94def _make_requests_session(responses): 95 session = mock.create_autospec(requests.Session, instance=True) 96 session.request.side_effect = responses 97 session.is_mtls = False 98 return session 99 100 101class TestClient(unittest.TestCase): 102 @staticmethod 103 def _get_target_class(): 104 from google.cloud.storage.client import Client 105 106 return Client 107 108 @staticmethod 109 def _get_default_timeout(): 110 from google.cloud.storage.constants import _DEFAULT_TIMEOUT 111 112 return _DEFAULT_TIMEOUT 113 114 def _make_one(self, *args, **kw): 115 return self._get_target_class()(*args, **kw) 116 117 def test_ctor_connection_type(self): 118 from google.cloud._http import ClientInfo 119 from google.cloud.storage._http import Connection 120 121 PROJECT = "PROJECT" 122 credentials = _make_credentials() 123 124 client = self._make_one(project=PROJECT, credentials=credentials) 125 126 self.assertEqual(client.project, PROJECT) 127 self.assertIsInstance(client._connection, Connection) 128 self.assertIs(client._connection.credentials, credentials) 129 self.assertIsNone(client.current_batch) 130 self.assertEqual(list(client._batch_stack), []) 131 self.assertIsInstance(client._connection._client_info, ClientInfo) 132 self.assertEqual( 133 client._connection.API_BASE_URL, Connection.DEFAULT_API_ENDPOINT 134 ) 135 136 def test_ctor_w_empty_client_options(self): 137 from google.api_core.client_options import ClientOptions 138 139 PROJECT = "PROJECT" 140 credentials = _make_credentials() 141 client_options = ClientOptions() 142 143 client = self._make_one( 144 project=PROJECT, credentials=credentials, client_options=client_options 145 ) 146 147 self.assertEqual( 148 client._connection.API_BASE_URL, client._connection.DEFAULT_API_ENDPOINT 149 ) 150 151 def test_ctor_w_client_options_dict(self): 152 PROJECT = "PROJECT" 153 credentials = _make_credentials() 154 api_endpoint = "https://www.foo-googleapis.com" 155 client_options = {"api_endpoint": api_endpoint} 156 157 client = self._make_one( 158 project=PROJECT, credentials=credentials, client_options=client_options 159 ) 160 161 self.assertEqual(client._connection.API_BASE_URL, api_endpoint) 162 163 def test_ctor_w_client_options_object(self): 164 from google.api_core.client_options import ClientOptions 165 166 PROJECT = "PROJECT" 167 credentials = _make_credentials() 168 client_options = ClientOptions(api_endpoint="https://www.foo-googleapis.com") 169 170 client = self._make_one( 171 project=PROJECT, credentials=credentials, client_options=client_options 172 ) 173 174 self.assertEqual( 175 client._connection.API_BASE_URL, "https://www.foo-googleapis.com" 176 ) 177 178 def test_ctor_wo_project(self): 179 from google.cloud.storage._http import Connection 180 181 PROJECT = "PROJECT" 182 credentials = _make_credentials(project=PROJECT) 183 184 client = self._make_one(credentials=credentials) 185 186 self.assertEqual(client.project, PROJECT) 187 self.assertIsInstance(client._connection, Connection) 188 self.assertIs(client._connection.credentials, credentials) 189 self.assertIsNone(client.current_batch) 190 self.assertEqual(list(client._batch_stack), []) 191 192 def test_ctor_w_project_explicit_none(self): 193 from google.cloud.storage._http import Connection 194 195 credentials = _make_credentials() 196 197 client = self._make_one(project=None, credentials=credentials) 198 199 self.assertIsNone(client.project) 200 self.assertIsInstance(client._connection, Connection) 201 self.assertIs(client._connection.credentials, credentials) 202 self.assertIsNone(client.current_batch) 203 self.assertEqual(list(client._batch_stack), []) 204 205 def test_ctor_w_client_info(self): 206 from google.cloud._http import ClientInfo 207 from google.cloud.storage._http import Connection 208 209 credentials = _make_credentials() 210 client_info = ClientInfo() 211 212 client = self._make_one( 213 project=None, credentials=credentials, client_info=client_info 214 ) 215 216 self.assertIsNone(client.project) 217 self.assertIsInstance(client._connection, Connection) 218 self.assertIs(client._connection.credentials, credentials) 219 self.assertIsNone(client.current_batch) 220 self.assertEqual(list(client._batch_stack), []) 221 self.assertIs(client._connection._client_info, client_info) 222 223 def test_ctor_mtls(self): 224 PROJECT = "PROJECT" 225 credentials = _make_credentials(project=PROJECT) 226 227 client = self._make_one(credentials=credentials) 228 self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, True) 229 self.assertEqual( 230 client._connection.API_BASE_URL, "https://storage.googleapis.com" 231 ) 232 233 client = self._make_one( 234 credentials=credentials, client_options={"api_endpoint": "http://foo"} 235 ) 236 self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, False) 237 self.assertEqual(client._connection.API_BASE_URL, "http://foo") 238 239 def test_create_anonymous_client(self): 240 from google.auth.credentials import AnonymousCredentials 241 from google.cloud.storage._http import Connection 242 243 klass = self._get_target_class() 244 client = klass.create_anonymous_client() 245 246 self.assertIsNone(client.project) 247 self.assertIsInstance(client._connection, Connection) 248 self.assertIsInstance(client._connection.credentials, AnonymousCredentials) 249 250 def test__push_batch_and__pop_batch(self): 251 from google.cloud.storage.batch import Batch 252 253 PROJECT = "PROJECT" 254 CREDENTIALS = _make_credentials() 255 256 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 257 batch1 = Batch(client) 258 batch2 = Batch(client) 259 client._push_batch(batch1) 260 self.assertEqual(list(client._batch_stack), [batch1]) 261 self.assertIs(client.current_batch, batch1) 262 client._push_batch(batch2) 263 self.assertIs(client.current_batch, batch2) 264 # list(_LocalStack) returns in reverse order. 265 self.assertEqual(list(client._batch_stack), [batch2, batch1]) 266 self.assertIs(client._pop_batch(), batch2) 267 self.assertEqual(list(client._batch_stack), [batch1]) 268 self.assertIs(client._pop_batch(), batch1) 269 self.assertEqual(list(client._batch_stack), []) 270 271 def test__connection_setter(self): 272 PROJECT = "PROJECT" 273 CREDENTIALS = _make_credentials() 274 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 275 client._base_connection = None # Unset the value from the constructor 276 client._connection = connection = object() 277 self.assertIs(client._base_connection, connection) 278 279 def test__connection_setter_when_set(self): 280 PROJECT = "PROJECT" 281 CREDENTIALS = _make_credentials() 282 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 283 self.assertRaises(ValueError, setattr, client, "_connection", None) 284 285 def test__connection_getter_no_batch(self): 286 PROJECT = "PROJECT" 287 CREDENTIALS = _make_credentials() 288 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 289 self.assertIs(client._connection, client._base_connection) 290 self.assertIsNone(client.current_batch) 291 292 def test__connection_getter_with_batch(self): 293 from google.cloud.storage.batch import Batch 294 295 PROJECT = "PROJECT" 296 CREDENTIALS = _make_credentials() 297 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 298 batch = Batch(client) 299 client._push_batch(batch) 300 self.assertIsNot(client._connection, client._base_connection) 301 self.assertIs(client._connection, batch) 302 self.assertIs(client.current_batch, batch) 303 304 def test_get_service_account_email_wo_project(self): 305 PROJECT = "PROJECT" 306 CREDENTIALS = _make_credentials() 307 EMAIL = "storage-user-123@example.com" 308 RESOURCE = {"kind": "storage#serviceAccount", "email_address": EMAIL} 309 310 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 311 http = _make_requests_session([_make_json_response(RESOURCE)]) 312 client._http_internal = http 313 314 service_account_email = client.get_service_account_email(timeout=42) 315 316 self.assertEqual(service_account_email, EMAIL) 317 http.request.assert_called_once_with( 318 method="GET", url=mock.ANY, data=None, headers=mock.ANY, timeout=42 319 ) 320 _, kwargs = http.request.call_args 321 scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) 322 self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) 323 self.assertEqual( 324 path, 325 "/".join( 326 [ 327 "", 328 "storage", 329 client._connection.API_VERSION, 330 "projects", 331 PROJECT, 332 "serviceAccount", 333 ] 334 ), 335 ) 336 337 def test_get_service_account_email_w_project(self): 338 PROJECT = "PROJECT" 339 OTHER_PROJECT = "OTHER_PROJECT" 340 CREDENTIALS = _make_credentials() 341 EMAIL = "storage-user-123@example.com" 342 RESOURCE = {"kind": "storage#serviceAccount", "email_address": EMAIL} 343 344 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 345 http = _make_requests_session([_make_json_response(RESOURCE)]) 346 client._http_internal = http 347 348 service_account_email = client.get_service_account_email(project=OTHER_PROJECT) 349 350 self.assertEqual(service_account_email, EMAIL) 351 http.request.assert_called_once_with( 352 method="GET", 353 url=mock.ANY, 354 data=None, 355 headers=mock.ANY, 356 timeout=self._get_default_timeout(), 357 ) 358 _, kwargs = http.request.call_args 359 scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) 360 self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) 361 self.assertEqual( 362 path, 363 "/".join( 364 [ 365 "", 366 "storage", 367 client._connection.API_VERSION, 368 "projects", 369 OTHER_PROJECT, 370 "serviceAccount", 371 ] 372 ), 373 ) 374 375 def test_bucket(self): 376 from google.cloud.storage.bucket import Bucket 377 378 PROJECT = "PROJECT" 379 CREDENTIALS = _make_credentials() 380 BUCKET_NAME = "BUCKET_NAME" 381 382 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 383 bucket = client.bucket(BUCKET_NAME) 384 self.assertIsInstance(bucket, Bucket) 385 self.assertIs(bucket.client, client) 386 self.assertEqual(bucket.name, BUCKET_NAME) 387 self.assertIsNone(bucket.user_project) 388 389 def test_bucket_w_user_project(self): 390 from google.cloud.storage.bucket import Bucket 391 392 PROJECT = "PROJECT" 393 USER_PROJECT = "USER_PROJECT" 394 CREDENTIALS = _make_credentials() 395 BUCKET_NAME = "BUCKET_NAME" 396 397 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 398 bucket = client.bucket(BUCKET_NAME, user_project=USER_PROJECT) 399 self.assertIsInstance(bucket, Bucket) 400 self.assertIs(bucket.client, client) 401 self.assertEqual(bucket.name, BUCKET_NAME) 402 self.assertEqual(bucket.user_project, USER_PROJECT) 403 404 def test_batch(self): 405 from google.cloud.storage.batch import Batch 406 407 PROJECT = "PROJECT" 408 CREDENTIALS = _make_credentials() 409 410 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 411 batch = client.batch() 412 self.assertIsInstance(batch, Batch) 413 self.assertIs(batch._client, client) 414 415 def test__get_resource_miss_w_defaults(self): 416 from google.cloud.exceptions import NotFound 417 418 project = "PROJECT" 419 path = "/path/to/something" 420 credentials = _make_credentials() 421 422 client = self._make_one(project=project, credentials=credentials) 423 connection = client._base_connection = _make_connection() 424 425 with self.assertRaises(NotFound): 426 client._get_resource(path) 427 428 connection.api_request.assert_called_once_with( 429 method="GET", 430 path=path, 431 query_params=None, 432 headers=None, 433 timeout=self._get_default_timeout(), 434 retry=DEFAULT_RETRY, 435 _target_object=None, 436 ) 437 438 def test__get_resource_hit_w_explicit(self): 439 project = "PROJECT" 440 path = "/path/to/something" 441 query_params = {"foo": "Foo"} 442 headers = {"bar": "Bar"} 443 timeout = 100 444 retry = mock.Mock(spec=[]) 445 credentials = _make_credentials() 446 447 client = self._make_one(project=project, credentials=credentials) 448 expected = mock.Mock(spec={}) 449 connection = client._base_connection = _make_connection(expected) 450 target = mock.Mock(spec={}) 451 452 found = client._get_resource( 453 path, 454 query_params=query_params, 455 headers=headers, 456 timeout=timeout, 457 retry=retry, 458 _target_object=target, 459 ) 460 461 self.assertIs(found, expected) 462 463 connection.api_request.assert_called_once_with( 464 method="GET", 465 path=path, 466 query_params=query_params, 467 headers=headers, 468 timeout=timeout, 469 retry=retry, 470 _target_object=target, 471 ) 472 473 def test__list_resource_w_defaults(self): 474 import functools 475 from google.api_core.page_iterator import HTTPIterator 476 from google.api_core.page_iterator import _do_nothing_page_start 477 478 project = "PROJECT" 479 path = "/path/to/list/resource" 480 item_to_value = mock.Mock(spec=[]) 481 credentials = _make_credentials() 482 client = self._make_one(project=project, credentials=credentials) 483 connection = client._base_connection = _make_connection() 484 485 iterator = client._list_resource(path=path, item_to_value=item_to_value,) 486 487 self.assertIsInstance(iterator, HTTPIterator) 488 self.assertIs(iterator.client, client) 489 self.assertIsInstance(iterator.api_request, functools.partial) 490 self.assertIs(iterator.api_request.func, connection.api_request) 491 self.assertEqual(iterator.api_request.args, ()) 492 expected_keywords = { 493 "timeout": self._get_default_timeout(), 494 "retry": DEFAULT_RETRY, 495 } 496 self.assertEqual(iterator.api_request.keywords, expected_keywords) 497 self.assertEqual(iterator.path, path) 498 self.assertEqual(iterator.next_page_token, None) 499 self.assertEqual(iterator.max_results, None) 500 self.assertIs(iterator._page_start, _do_nothing_page_start) 501 502 def test__list_resource_w_explicit(self): 503 import functools 504 from google.api_core.page_iterator import HTTPIterator 505 506 project = "PROJECT" 507 path = "/path/to/list/resource" 508 item_to_value = mock.Mock(spec=[]) 509 page_token = "PAGE-TOKEN" 510 max_results = 47 511 extra_params = {"foo": "Foo"} 512 page_start = mock.Mock(spec=[]) 513 credentials = _make_credentials() 514 client = self._make_one(project=project, credentials=credentials) 515 connection = client._base_connection = _make_connection() 516 517 iterator = client._list_resource( 518 path=path, 519 item_to_value=item_to_value, 520 page_token=page_token, 521 max_results=max_results, 522 extra_params=extra_params, 523 page_start=page_start, 524 ) 525 526 self.assertIsInstance(iterator, HTTPIterator) 527 self.assertIs(iterator.client, client) 528 self.assertIsInstance(iterator.api_request, functools.partial) 529 self.assertIs(iterator.api_request.func, connection.api_request) 530 self.assertEqual(iterator.api_request.args, ()) 531 expected_keywords = { 532 "timeout": self._get_default_timeout(), 533 "retry": DEFAULT_RETRY, 534 } 535 self.assertEqual(iterator.api_request.keywords, expected_keywords) 536 self.assertEqual(iterator.path, path) 537 self.assertEqual(iterator.next_page_token, page_token) 538 self.assertEqual(iterator.max_results, max_results) 539 self.assertIs(iterator._page_start, page_start) 540 541 def test__patch_resource_miss_w_defaults(self): 542 from google.cloud.exceptions import NotFound 543 544 project = "PROJECT" 545 path = "/path/to/something" 546 credentials = _make_credentials() 547 data = {"baz": "Baz"} 548 549 client = self._make_one(project=project, credentials=credentials) 550 connection = client._base_connection = _make_connection() 551 552 with self.assertRaises(NotFound): 553 client._patch_resource(path, data) 554 555 connection.api_request.assert_called_once_with( 556 method="PATCH", 557 path=path, 558 data=data, 559 query_params=None, 560 headers=None, 561 timeout=self._get_default_timeout(), 562 retry=None, 563 _target_object=None, 564 ) 565 566 def test__patch_resource_hit_w_explicit(self): 567 project = "PROJECT" 568 path = "/path/to/something" 569 data = {"baz": "Baz"} 570 query_params = {"foo": "Foo"} 571 headers = {"bar": "Bar"} 572 timeout = 100 573 retry = mock.Mock(spec=[]) 574 credentials = _make_credentials() 575 576 client = self._make_one(project=project, credentials=credentials) 577 expected = mock.Mock(spec={}) 578 connection = client._base_connection = _make_connection(expected) 579 target = mock.Mock(spec={}) 580 581 found = client._patch_resource( 582 path, 583 data, 584 query_params=query_params, 585 headers=headers, 586 timeout=timeout, 587 retry=retry, 588 _target_object=target, 589 ) 590 591 self.assertIs(found, expected) 592 593 connection.api_request.assert_called_once_with( 594 method="PATCH", 595 path=path, 596 data=data, 597 query_params=query_params, 598 headers=headers, 599 timeout=timeout, 600 retry=retry, 601 _target_object=target, 602 ) 603 604 def test__put_resource_miss_w_defaults(self): 605 from google.cloud.exceptions import NotFound 606 607 project = "PROJECT" 608 path = "/path/to/something" 609 credentials = _make_credentials() 610 data = {"baz": "Baz"} 611 612 client = self._make_one(project=project, credentials=credentials) 613 connection = client._base_connection = _make_connection() 614 615 with self.assertRaises(NotFound): 616 client._put_resource(path, data) 617 618 connection.api_request.assert_called_once_with( 619 method="PUT", 620 path=path, 621 data=data, 622 query_params=None, 623 headers=None, 624 timeout=self._get_default_timeout(), 625 retry=None, 626 _target_object=None, 627 ) 628 629 def test__put_resource_hit_w_explicit(self): 630 project = "PROJECT" 631 path = "/path/to/something" 632 data = {"baz": "Baz"} 633 query_params = {"foo": "Foo"} 634 headers = {"bar": "Bar"} 635 timeout = 100 636 retry = mock.Mock(spec=[]) 637 credentials = _make_credentials() 638 639 client = self._make_one(project=project, credentials=credentials) 640 expected = mock.Mock(spec={}) 641 connection = client._base_connection = _make_connection(expected) 642 target = mock.Mock(spec={}) 643 644 found = client._put_resource( 645 path, 646 data, 647 query_params=query_params, 648 headers=headers, 649 timeout=timeout, 650 retry=retry, 651 _target_object=target, 652 ) 653 654 self.assertIs(found, expected) 655 656 connection.api_request.assert_called_once_with( 657 method="PUT", 658 path=path, 659 data=data, 660 query_params=query_params, 661 headers=headers, 662 timeout=timeout, 663 retry=retry, 664 _target_object=target, 665 ) 666 667 def test__post_resource_miss_w_defaults(self): 668 from google.cloud.exceptions import NotFound 669 670 project = "PROJECT" 671 path = "/path/to/something" 672 credentials = _make_credentials() 673 data = {"baz": "Baz"} 674 675 client = self._make_one(project=project, credentials=credentials) 676 connection = client._base_connection = _make_connection() 677 678 with self.assertRaises(NotFound): 679 client._post_resource(path, data) 680 681 connection.api_request.assert_called_once_with( 682 method="POST", 683 path=path, 684 data=data, 685 query_params=None, 686 headers=None, 687 timeout=self._get_default_timeout(), 688 retry=None, 689 _target_object=None, 690 ) 691 692 def test__post_resource_hit_w_explicit(self): 693 project = "PROJECT" 694 path = "/path/to/something" 695 data = {"baz": "Baz"} 696 query_params = {"foo": "Foo"} 697 headers = {"bar": "Bar"} 698 timeout = 100 699 retry = mock.Mock(spec=[]) 700 credentials = _make_credentials() 701 702 client = self._make_one(project=project, credentials=credentials) 703 expected = mock.Mock(spec={}) 704 connection = client._base_connection = _make_connection(expected) 705 target = mock.Mock(spec={}) 706 707 found = client._post_resource( 708 path, 709 data, 710 query_params=query_params, 711 headers=headers, 712 timeout=timeout, 713 retry=retry, 714 _target_object=target, 715 ) 716 717 self.assertIs(found, expected) 718 719 connection.api_request.assert_called_once_with( 720 method="POST", 721 path=path, 722 data=data, 723 query_params=query_params, 724 headers=headers, 725 timeout=timeout, 726 retry=retry, 727 _target_object=target, 728 ) 729 730 def test__delete_resource_miss_w_defaults(self): 731 from google.cloud.exceptions import NotFound 732 733 project = "PROJECT" 734 path = "/path/to/something" 735 credentials = _make_credentials() 736 737 client = self._make_one(project=project, credentials=credentials) 738 connection = client._base_connection = _make_connection() 739 740 with self.assertRaises(NotFound): 741 client._delete_resource(path) 742 743 connection.api_request.assert_called_once_with( 744 method="DELETE", 745 path=path, 746 query_params=None, 747 headers=None, 748 timeout=self._get_default_timeout(), 749 retry=DEFAULT_RETRY, 750 _target_object=None, 751 ) 752 753 def test__delete_resource_hit_w_explicit(self): 754 project = "PROJECT" 755 path = "/path/to/something" 756 query_params = {"foo": "Foo"} 757 headers = {"bar": "Bar"} 758 timeout = 100 759 retry = mock.Mock(spec=[]) 760 credentials = _make_credentials() 761 762 client = self._make_one(project=project, credentials=credentials) 763 expected = mock.Mock(spec={}) 764 connection = client._base_connection = _make_connection(expected) 765 target = mock.Mock(spec={}) 766 767 found = client._delete_resource( 768 path, 769 query_params=query_params, 770 headers=headers, 771 timeout=timeout, 772 retry=retry, 773 _target_object=target, 774 ) 775 776 self.assertIs(found, expected) 777 778 connection.api_request.assert_called_once_with( 779 method="DELETE", 780 path=path, 781 query_params=query_params, 782 headers=headers, 783 timeout=timeout, 784 retry=retry, 785 _target_object=target, 786 ) 787 788 def test__bucket_arg_to_bucket_w_bucket_w_client(self): 789 from google.cloud.storage.bucket import Bucket 790 791 project = "PROJECT" 792 credentials = _make_credentials() 793 client = self._make_one(project=project, credentials=credentials) 794 other_client = mock.Mock(spec=[]) 795 bucket_name = "w_client" 796 797 bucket = Bucket(other_client, name=bucket_name) 798 799 found = client._bucket_arg_to_bucket(bucket) 800 801 self.assertIs(found, bucket) 802 self.assertIs(found.client, other_client) 803 804 def test__bucket_arg_to_bucket_w_bucket_wo_client(self): 805 from google.cloud.storage.bucket import Bucket 806 807 project = "PROJECT" 808 credentials = _make_credentials() 809 client = self._make_one(project=project, credentials=credentials) 810 bucket_name = "wo_client" 811 812 bucket = Bucket(client=None, name=bucket_name) 813 814 found = client._bucket_arg_to_bucket(bucket) 815 816 self.assertIs(found, bucket) 817 self.assertIs(found.client, client) 818 819 def test__bucket_arg_to_bucket_w_bucket_name(self): 820 from google.cloud.storage.bucket import Bucket 821 822 project = "PROJECT" 823 credentials = _make_credentials() 824 client = self._make_one(project=project, credentials=credentials) 825 bucket_name = "string-name" 826 827 found = client._bucket_arg_to_bucket(bucket_name) 828 829 self.assertIsInstance(found, Bucket) 830 self.assertEqual(found.name, bucket_name) 831 self.assertIs(found.client, client) 832 833 def test_get_bucket_miss_w_string_w_defaults(self): 834 from google.cloud.exceptions import NotFound 835 from google.cloud.storage.bucket import Bucket 836 837 project = "PROJECT" 838 credentials = _make_credentials() 839 client = self._make_one(project=project, credentials=credentials) 840 client._get_resource = mock.Mock() 841 client._get_resource.side_effect = NotFound("testing") 842 bucket_name = "nonesuch" 843 844 with self.assertRaises(NotFound): 845 client.get_bucket(bucket_name) 846 847 expected_path = "/b/%s" % (bucket_name,) 848 expected_query_params = {"projection": "noAcl"} 849 expected_headers = {} 850 client._get_resource.assert_called_once_with( 851 expected_path, 852 query_params=expected_query_params, 853 headers=expected_headers, 854 timeout=self._get_default_timeout(), 855 retry=DEFAULT_RETRY, 856 _target_object=mock.ANY, 857 ) 858 859 target = client._get_resource.call_args[1]["_target_object"] 860 self.assertIsInstance(target, Bucket) 861 self.assertEqual(target.name, bucket_name) 862 863 def test_get_bucket_hit_w_string_w_timeout(self): 864 from google.cloud.storage.bucket import Bucket 865 866 project = "PROJECT" 867 bucket_name = "bucket-name" 868 timeout = 42 869 api_response = {"name": bucket_name} 870 credentials = _make_credentials() 871 client = self._make_one(project=project, credentials=credentials) 872 client._get_resource = mock.Mock(return_value=api_response) 873 874 bucket = client.get_bucket(bucket_name, timeout=timeout) 875 876 self.assertIsInstance(bucket, Bucket) 877 self.assertEqual(bucket.name, bucket_name) 878 879 expected_path = "/b/%s" % (bucket_name,) 880 expected_query_params = {"projection": "noAcl"} 881 expected_headers = {} 882 client._get_resource.assert_called_once_with( 883 expected_path, 884 query_params=expected_query_params, 885 headers=expected_headers, 886 timeout=timeout, 887 retry=DEFAULT_RETRY, 888 _target_object=bucket, 889 ) 890 891 def test_get_bucket_hit_w_string_w_metageneration_match(self): 892 from google.cloud.storage.bucket import Bucket 893 894 project = "PROJECT" 895 bucket_name = "bucket-name" 896 metageneration_number = 6 897 api_response = {"name": bucket_name} 898 credentials = _make_credentials() 899 client = self._make_one(project=project, credentials=credentials) 900 client._get_resource = mock.Mock(return_value=api_response) 901 902 bucket = client.get_bucket( 903 bucket_name, if_metageneration_match=metageneration_number 904 ) 905 906 self.assertIsInstance(bucket, Bucket) 907 self.assertEqual(bucket.name, bucket_name) 908 909 expected_path = "/b/%s" % (bucket_name,) 910 expected_query_params = { 911 "projection": "noAcl", 912 "ifMetagenerationMatch": metageneration_number, 913 } 914 expected_headers = {} 915 client._get_resource.assert_called_once_with( 916 expected_path, 917 query_params=expected_query_params, 918 headers=expected_headers, 919 timeout=self._get_default_timeout(), 920 retry=DEFAULT_RETRY, 921 _target_object=bucket, 922 ) 923 924 def test_get_bucket_miss_w_object_w_retry(self): 925 from google.cloud.exceptions import NotFound 926 from google.cloud.storage.bucket import Bucket 927 928 project = "PROJECT" 929 bucket_name = "nonesuch" 930 retry = mock.Mock(spec=[]) 931 credentials = _make_credentials() 932 client = self._make_one(project=project, credentials=credentials) 933 client._get_resource = mock.Mock(side_effect=NotFound("testing")) 934 bucket_obj = Bucket(client, bucket_name) 935 936 with self.assertRaises(NotFound): 937 client.get_bucket(bucket_obj, retry=retry) 938 939 expected_path = "/b/%s" % (bucket_name,) 940 expected_query_params = {"projection": "noAcl"} 941 expected_headers = {} 942 client._get_resource.assert_called_once_with( 943 expected_path, 944 query_params=expected_query_params, 945 headers=expected_headers, 946 timeout=self._get_default_timeout(), 947 retry=retry, 948 _target_object=mock.ANY, 949 ) 950 951 target = client._get_resource.call_args[1]["_target_object"] 952 self.assertIsInstance(target, Bucket) 953 self.assertEqual(target.name, bucket_name) 954 955 def test_get_bucket_hit_w_object_defaults(self): 956 from google.cloud.storage.bucket import Bucket 957 958 project = "PROJECT" 959 bucket_name = "bucket-name" 960 api_response = {"name": bucket_name} 961 credentials = _make_credentials() 962 client = self._make_one(project=project, credentials=credentials) 963 client._get_resource = mock.Mock(return_value=api_response) 964 bucket_obj = Bucket(client, bucket_name) 965 966 bucket = client.get_bucket(bucket_obj) 967 968 self.assertIsInstance(bucket, Bucket) 969 self.assertEqual(bucket.name, bucket_name) 970 971 expected_path = "/b/%s" % (bucket_name,) 972 expected_query_params = {"projection": "noAcl"} 973 expected_headers = {} 974 client._get_resource.assert_called_once_with( 975 expected_path, 976 query_params=expected_query_params, 977 headers=expected_headers, 978 timeout=self._get_default_timeout(), 979 retry=DEFAULT_RETRY, 980 _target_object=bucket, 981 ) 982 983 def test_get_bucket_hit_w_object_w_retry_none(self): 984 from google.cloud.storage.bucket import Bucket 985 986 project = "PROJECT" 987 bucket_name = "bucket-name" 988 api_response = {"name": bucket_name} 989 credentials = _make_credentials() 990 client = self._make_one(project=project, credentials=credentials) 991 client._get_resource = mock.Mock(return_value=api_response) 992 bucket_obj = Bucket(client, bucket_name) 993 994 bucket = client.get_bucket(bucket_obj, retry=None) 995 996 self.assertIsInstance(bucket, Bucket) 997 self.assertEqual(bucket.name, bucket_name) 998 999 expected_path = "/b/%s" % (bucket_name,) 1000 expected_query_params = {"projection": "noAcl"} 1001 expected_headers = {} 1002 client._get_resource.assert_called_once_with( 1003 expected_path, 1004 query_params=expected_query_params, 1005 headers=expected_headers, 1006 timeout=self._get_default_timeout(), 1007 retry=None, 1008 _target_object=bucket, 1009 ) 1010 1011 def test_lookup_bucket_miss_w_defaults(self): 1012 from google.cloud.exceptions import NotFound 1013 from google.cloud.storage.bucket import Bucket 1014 1015 project = "PROJECT" 1016 bucket_name = "nonesuch" 1017 credentials = _make_credentials() 1018 client = self._make_one(project=project, credentials=credentials) 1019 client._get_resource = mock.Mock(side_effect=NotFound("testing")) 1020 1021 bucket = client.lookup_bucket(bucket_name) 1022 1023 self.assertIsNone(bucket) 1024 1025 expected_path = "/b/%s" % (bucket_name,) 1026 expected_query_params = {"projection": "noAcl"} 1027 expected_headers = {} 1028 client._get_resource.assert_called_once_with( 1029 expected_path, 1030 query_params=expected_query_params, 1031 headers=expected_headers, 1032 timeout=self._get_default_timeout(), 1033 retry=DEFAULT_RETRY, 1034 _target_object=mock.ANY, 1035 ) 1036 1037 target = client._get_resource.call_args[1]["_target_object"] 1038 self.assertIsInstance(target, Bucket) 1039 self.assertEqual(target.name, bucket_name) 1040 1041 def test_lookup_bucket_hit_w_timeout(self): 1042 from google.cloud.storage.bucket import Bucket 1043 1044 project = "PROJECT" 1045 bucket_name = "bucket-name" 1046 timeout = 42 1047 api_response = {"name": bucket_name} 1048 credentials = _make_credentials() 1049 client = self._make_one(project=project, credentials=credentials) 1050 client._get_resource = mock.Mock(return_value=api_response) 1051 1052 bucket = client.lookup_bucket(bucket_name, timeout=timeout) 1053 1054 self.assertIsInstance(bucket, Bucket) 1055 self.assertEqual(bucket.name, bucket_name) 1056 1057 expected_path = "/b/%s" % (bucket_name,) 1058 expected_query_params = {"projection": "noAcl"} 1059 expected_headers = {} 1060 client._get_resource.assert_called_once_with( 1061 expected_path, 1062 query_params=expected_query_params, 1063 headers=expected_headers, 1064 timeout=timeout, 1065 retry=DEFAULT_RETRY, 1066 _target_object=bucket, 1067 ) 1068 1069 def test_lookup_bucket_hit_w_metageneration_match(self): 1070 from google.cloud.storage.bucket import Bucket 1071 1072 project = "PROJECT" 1073 bucket_name = "bucket-name" 1074 api_response = {"name": bucket_name} 1075 credentials = _make_credentials() 1076 metageneration_number = 6 1077 client = self._make_one(project=project, credentials=credentials) 1078 client._get_resource = mock.Mock(return_value=api_response) 1079 1080 bucket = client.lookup_bucket( 1081 bucket_name, if_metageneration_match=metageneration_number 1082 ) 1083 1084 self.assertIsInstance(bucket, Bucket) 1085 self.assertEqual(bucket.name, bucket_name) 1086 1087 expected_path = "/b/%s" % (bucket_name,) 1088 expected_query_params = { 1089 "projection": "noAcl", 1090 "ifMetagenerationMatch": metageneration_number, 1091 } 1092 expected_headers = {} 1093 client._get_resource.assert_called_once_with( 1094 expected_path, 1095 query_params=expected_query_params, 1096 headers=expected_headers, 1097 timeout=self._get_default_timeout(), 1098 retry=DEFAULT_RETRY, 1099 _target_object=bucket, 1100 ) 1101 1102 def test_lookup_bucket_hit_w_retry(self): 1103 from google.cloud.storage.bucket import Bucket 1104 1105 project = "PROJECT" 1106 bucket_name = "bucket-name" 1107 api_response = {"name": bucket_name} 1108 credentials = _make_credentials() 1109 client = self._make_one(project=project, credentials=credentials) 1110 client._get_resource = mock.Mock(return_value=api_response) 1111 bucket_obj = Bucket(client, bucket_name) 1112 1113 bucket = client.lookup_bucket(bucket_obj, retry=None) 1114 1115 self.assertIsInstance(bucket, Bucket) 1116 self.assertEqual(bucket.name, bucket_name) 1117 1118 expected_path = "/b/%s" % (bucket_name,) 1119 expected_query_params = {"projection": "noAcl"} 1120 expected_headers = {} 1121 client._get_resource.assert_called_once_with( 1122 expected_path, 1123 query_params=expected_query_params, 1124 headers=expected_headers, 1125 timeout=self._get_default_timeout(), 1126 retry=None, 1127 _target_object=bucket, 1128 ) 1129 1130 def test_create_bucket_w_missing_client_project(self): 1131 credentials = _make_credentials() 1132 client = self._make_one(project=None, credentials=credentials) 1133 1134 with self.assertRaises(ValueError): 1135 client.create_bucket("bucket") 1136 1137 def test_create_bucket_w_conflict_w_user_project(self): 1138 from google.cloud.exceptions import Conflict 1139 1140 project = "PROJECT" 1141 user_project = "USER_PROJECT" 1142 other_project = "OTHER_PROJECT" 1143 credentials = _make_credentials() 1144 client = self._make_one(project=project, credentials=credentials) 1145 client._post_resource = mock.Mock() 1146 client._post_resource.side_effect = Conflict("testing") 1147 1148 bucket_name = "bucket-name" 1149 1150 with self.assertRaises(Conflict): 1151 client.create_bucket( 1152 bucket_name, project=other_project, user_project=user_project 1153 ) 1154 1155 expected_path = "/b" 1156 expected_data = {"name": bucket_name} 1157 expected_query_params = { 1158 "project": other_project, 1159 "userProject": user_project, 1160 } 1161 client._post_resource.assert_called_once_with( 1162 expected_path, 1163 expected_data, 1164 query_params=expected_query_params, 1165 timeout=self._get_default_timeout(), 1166 retry=DEFAULT_RETRY, 1167 _target_object=mock.ANY, 1168 ) 1169 1170 @mock.patch("warnings.warn") 1171 def test_create_bucket_w_requester_pays_deprecated(self, mock_warn): 1172 from google.cloud.storage.bucket import Bucket 1173 1174 bucket_name = "bucket-name" 1175 project = "PROJECT" 1176 credentials = _make_credentials() 1177 api_respone = {"name": bucket_name, "billing": {"requesterPays": True}} 1178 client = self._make_one(project=project, credentials=credentials) 1179 client._post_resource = mock.Mock() 1180 client._post_resource.return_value = api_respone 1181 1182 bucket = client.create_bucket(bucket_name, requester_pays=True) 1183 1184 self.assertIsInstance(bucket, Bucket) 1185 self.assertEqual(bucket.name, bucket_name) 1186 self.assertTrue(bucket.requester_pays) 1187 1188 expected_path = "/b" 1189 expected_data = api_respone 1190 expected_query_params = {"project": project} 1191 client._post_resource.assert_called_once_with( 1192 expected_path, 1193 expected_data, 1194 query_params=expected_query_params, 1195 timeout=self._get_default_timeout(), 1196 retry=DEFAULT_RETRY, 1197 _target_object=mock.ANY, 1198 ) 1199 1200 mock_warn.assert_called_with( 1201 "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", 1202 PendingDeprecationWarning, 1203 stacklevel=1, 1204 ) 1205 1206 def test_create_bucket_w_predefined_acl_invalid(self): 1207 project = "PROJECT" 1208 bucket_name = "bucket-name" 1209 credentials = _make_credentials() 1210 client = self._make_one(project=project, credentials=credentials) 1211 client._post_resource = mock.Mock() 1212 1213 with self.assertRaises(ValueError): 1214 client.create_bucket(bucket_name, predefined_acl="bogus") 1215 1216 client._post_resource.assert_not_called() 1217 1218 def test_create_bucket_w_predefined_acl_valid_w_timeout(self): 1219 project = "PROJECT" 1220 bucket_name = "bucket-name" 1221 api_response = {"name": bucket_name} 1222 credentials = _make_credentials() 1223 client = self._make_one(project=project, credentials=credentials) 1224 client._post_resource = mock.Mock() 1225 client._post_resource.return_value = api_response 1226 timeout = 42 1227 1228 bucket = client.create_bucket( 1229 bucket_name, predefined_acl="publicRead", timeout=timeout, 1230 ) 1231 1232 expected_path = "/b" 1233 expected_data = api_response 1234 expected_query_params = { 1235 "project": project, 1236 "predefinedAcl": "publicRead", 1237 } 1238 client._post_resource.assert_called_once_with( 1239 expected_path, 1240 expected_data, 1241 query_params=expected_query_params, 1242 timeout=timeout, 1243 retry=DEFAULT_RETRY, 1244 _target_object=bucket, 1245 ) 1246 1247 def test_create_bucket_w_predefined_default_object_acl_invalid(self): 1248 project = "PROJECT" 1249 bucket_name = "bucket-name" 1250 1251 credentials = _make_credentials() 1252 client = self._make_one(project=project, credentials=credentials) 1253 client._post_resource = mock.Mock() 1254 1255 with self.assertRaises(ValueError): 1256 client.create_bucket(bucket_name, predefined_default_object_acl="bogus") 1257 1258 client._post_resource.assert_not_called() 1259 1260 def test_create_bucket_w_predefined_default_object_acl_valid_w_retry(self): 1261 project = "PROJECT" 1262 bucket_name = "bucket-name" 1263 api_response = {"name": bucket_name} 1264 credentials = _make_credentials() 1265 client = self._make_one(project=project, credentials=credentials) 1266 client._post_resource = mock.Mock() 1267 client._post_resource.return_value = api_response 1268 retry = mock.Mock(spec=[]) 1269 1270 bucket = client.create_bucket( 1271 bucket_name, predefined_default_object_acl="publicRead", retry=retry, 1272 ) 1273 1274 expected_path = "/b" 1275 expected_data = api_response 1276 expected_query_params = { 1277 "project": project, 1278 "predefinedDefaultObjectAcl": "publicRead", 1279 } 1280 client._post_resource.assert_called_once_with( 1281 expected_path, 1282 expected_data, 1283 query_params=expected_query_params, 1284 timeout=self._get_default_timeout(), 1285 retry=retry, 1286 _target_object=bucket, 1287 ) 1288 1289 def test_create_bucket_w_explicit_location(self): 1290 project = "PROJECT" 1291 bucket_name = "bucket-name" 1292 location = "us-central1" 1293 api_response = {"location": location, "name": bucket_name} 1294 credentials = _make_credentials() 1295 client = self._make_one(project=project, credentials=credentials) 1296 client._post_resource = mock.Mock() 1297 client._post_resource.return_value = api_response 1298 1299 bucket = client.create_bucket(bucket_name, location=location) 1300 1301 self.assertEqual(bucket.location, location) 1302 1303 expected_path = "/b" 1304 expected_data = {"location": location, "name": bucket_name} 1305 expected_query_params = {"project": project} 1306 client._post_resource.assert_called_once_with( 1307 expected_path, 1308 expected_data, 1309 query_params=expected_query_params, 1310 timeout=self._get_default_timeout(), 1311 retry=DEFAULT_RETRY, 1312 _target_object=bucket, 1313 ) 1314 1315 def test_create_bucket_w_explicit_project(self): 1316 project = "PROJECT" 1317 other_project = "other-project-123" 1318 bucket_name = "bucket-name" 1319 api_response = {"name": bucket_name} 1320 credentials = _make_credentials() 1321 client = self._make_one(project=project, credentials=credentials) 1322 client._post_resource = mock.Mock() 1323 client._post_resource.return_value = api_response 1324 1325 bucket = client.create_bucket(bucket_name, project=other_project) 1326 1327 expected_path = "/b" 1328 expected_data = api_response 1329 expected_query_params = {"project": other_project} 1330 client._post_resource.assert_called_once_with( 1331 expected_path, 1332 expected_data, 1333 query_params=expected_query_params, 1334 timeout=self._get_default_timeout(), 1335 retry=DEFAULT_RETRY, 1336 _target_object=bucket, 1337 ) 1338 1339 def test_create_bucket_w_extra_properties(self): 1340 from google.cloud.storage.bucket import Bucket 1341 1342 bucket_name = "bucket-name" 1343 project = "PROJECT" 1344 cors = [ 1345 { 1346 "maxAgeSeconds": 60, 1347 "methods": ["*"], 1348 "origin": ["https://example.com/frontend"], 1349 "responseHeader": ["X-Custom-Header"], 1350 } 1351 ] 1352 lifecycle_rules = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] 1353 location = "eu" 1354 labels = {"color": "red", "flavor": "cherry"} 1355 storage_class = "NEARLINE" 1356 api_response = { 1357 "name": bucket_name, 1358 "cors": cors, 1359 "lifecycle": {"rule": lifecycle_rules}, 1360 "location": location, 1361 "storageClass": storage_class, 1362 "versioning": {"enabled": True}, 1363 "billing": {"requesterPays": True}, 1364 "labels": labels, 1365 } 1366 credentials = _make_credentials() 1367 client = self._make_one(project=project, credentials=credentials) 1368 client._post_resource = mock.Mock() 1369 client._post_resource.return_value = api_response 1370 1371 bucket = Bucket(client=client, name=bucket_name) 1372 bucket.cors = cors 1373 bucket.lifecycle_rules = lifecycle_rules 1374 bucket.storage_class = storage_class 1375 bucket.versioning_enabled = True 1376 bucket.requester_pays = True 1377 bucket.labels = labels 1378 1379 client.create_bucket(bucket, location=location) 1380 1381 expected_path = "/b" 1382 expected_data = api_response 1383 expected_query_params = {"project": project} 1384 client._post_resource.assert_called_once_with( 1385 expected_path, 1386 expected_data, 1387 query_params=expected_query_params, 1388 timeout=self._get_default_timeout(), 1389 retry=DEFAULT_RETRY, 1390 _target_object=bucket, 1391 ) 1392 1393 def test_create_bucket_w_name_only(self): 1394 project = "PROJECT" 1395 bucket_name = "bucket-name" 1396 api_response = {"name": bucket_name} 1397 credentials = _make_credentials() 1398 client = self._make_one(project=project, credentials=credentials) 1399 client._post_resource = mock.Mock() 1400 client._post_resource.return_value = api_response 1401 1402 bucket = client.create_bucket(bucket_name) 1403 1404 expected_path = "/b" 1405 expected_data = api_response 1406 expected_query_params = {"project": project} 1407 client._post_resource.assert_called_once_with( 1408 expected_path, 1409 expected_data, 1410 query_params=expected_query_params, 1411 timeout=self._get_default_timeout(), 1412 retry=DEFAULT_RETRY, 1413 _target_object=bucket, 1414 ) 1415 1416 def test_download_blob_to_file_with_failure(self): 1417 from google.resumable_media import InvalidResponse 1418 from google.cloud.storage.blob import Blob 1419 from google.cloud.storage.constants import _DEFAULT_TIMEOUT 1420 1421 project = "PROJECT" 1422 raw_response = requests.Response() 1423 raw_response.status_code = http_client.NOT_FOUND 1424 raw_request = requests.Request("GET", "http://example.com") 1425 raw_response.request = raw_request.prepare() 1426 grmp_response = InvalidResponse(raw_response) 1427 credentials = _make_credentials(project=project) 1428 client = self._make_one(credentials=credentials) 1429 blob = mock.create_autospec(Blob) 1430 blob._encryption_key = None 1431 blob._get_download_url = mock.Mock() 1432 blob._do_download = mock.Mock() 1433 blob._do_download.side_effect = grmp_response 1434 1435 file_obj = io.BytesIO() 1436 with self.assertRaises(exceptions.NotFound): 1437 client.download_blob_to_file(blob, file_obj) 1438 1439 self.assertEqual(file_obj.tell(), 0) 1440 1441 headers = {"accept-encoding": "gzip"} 1442 blob._do_download.assert_called_once_with( 1443 client._http, 1444 file_obj, 1445 blob._get_download_url(), 1446 headers, 1447 None, 1448 None, 1449 False, 1450 checksum="md5", 1451 timeout=_DEFAULT_TIMEOUT, 1452 retry=DEFAULT_RETRY, 1453 ) 1454 1455 def test_download_blob_to_file_with_uri(self): 1456 from google.cloud.storage.constants import _DEFAULT_TIMEOUT 1457 1458 project = "PROJECT" 1459 credentials = _make_credentials(project=project) 1460 client = self._make_one(project=project, credentials=credentials) 1461 blob = mock.Mock() 1462 file_obj = io.BytesIO() 1463 blob._encryption_key = None 1464 blob._get_download_url = mock.Mock() 1465 blob._do_download = mock.Mock() 1466 1467 with mock.patch( 1468 "google.cloud.storage.client.Blob.from_string", return_value=blob 1469 ): 1470 client.download_blob_to_file("gs://bucket_name/path/to/object", file_obj) 1471 1472 headers = {"accept-encoding": "gzip"} 1473 blob._do_download.assert_called_once_with( 1474 client._http, 1475 file_obj, 1476 blob._get_download_url(), 1477 headers, 1478 None, 1479 None, 1480 False, 1481 checksum="md5", 1482 timeout=_DEFAULT_TIMEOUT, 1483 retry=DEFAULT_RETRY, 1484 ) 1485 1486 def test_download_blob_to_file_with_invalid_uri(self): 1487 project = "PROJECT" 1488 credentials = _make_credentials() 1489 client = self._make_one(project=project, credentials=credentials) 1490 file_obj = io.BytesIO() 1491 1492 with pytest.raises(ValueError, match="URI scheme must be gs"): 1493 client.download_blob_to_file("http://bucket_name/path/to/object", file_obj) 1494 1495 def test_download_blob_to_file_w_no_retry(self): 1496 self._download_blob_to_file_helper( 1497 use_chunks=True, raw_download=True, retry=None 1498 ) 1499 1500 def test_download_blob_to_file_w_conditional_etag_match_string(self): 1501 self._download_blob_to_file_helper( 1502 use_chunks=True, raw_download=True, retry=None, if_etag_match="kittens", 1503 ) 1504 1505 def test_download_blob_to_file_w_conditional_etag_match_list(self): 1506 self._download_blob_to_file_helper( 1507 use_chunks=True, 1508 raw_download=True, 1509 retry=None, 1510 if_etag_match=["kittens", "fluffy"], 1511 ) 1512 1513 def test_download_blob_to_file_w_conditional_etag_not_match_string(self): 1514 self._download_blob_to_file_helper( 1515 use_chunks=True, raw_download=True, retry=None, if_etag_not_match="kittens", 1516 ) 1517 1518 def test_download_blob_to_file_w_conditional_etag_not_match_list(self): 1519 self._download_blob_to_file_helper( 1520 use_chunks=True, 1521 raw_download=True, 1522 retry=None, 1523 if_etag_not_match=["kittens", "fluffy"], 1524 ) 1525 1526 def test_download_blob_to_file_w_conditional_retry_pass(self): 1527 self._download_blob_to_file_helper( 1528 use_chunks=True, 1529 raw_download=True, 1530 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 1531 if_generation_match=1, 1532 ) 1533 1534 def test_download_blob_to_file_w_conditional_retry_fail(self): 1535 self._download_blob_to_file_helper( 1536 use_chunks=True, 1537 raw_download=True, 1538 retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, 1539 expect_condition_fail=True, 1540 ) 1541 1542 def _download_blob_to_file_helper( 1543 self, use_chunks, raw_download, expect_condition_fail=False, **extra_kwargs 1544 ): 1545 from google.cloud.storage.blob import Blob 1546 from google.cloud.storage.constants import _DEFAULT_TIMEOUT 1547 1548 project = "PROJECT" 1549 credentials = _make_credentials(project=project) 1550 client = self._make_one(credentials=credentials) 1551 blob = mock.create_autospec(Blob) 1552 blob._encryption_key = None 1553 blob._get_download_url = mock.Mock() 1554 if use_chunks: 1555 blob._CHUNK_SIZE_MULTIPLE = 1 1556 blob.chunk_size = 3 1557 blob._do_download = mock.Mock() 1558 1559 file_obj = io.BytesIO() 1560 if raw_download: 1561 client.download_blob_to_file( 1562 blob, file_obj, raw_download=True, **extra_kwargs 1563 ) 1564 else: 1565 client.download_blob_to_file(blob, file_obj, **extra_kwargs) 1566 1567 expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY) 1568 if ( 1569 expected_retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED 1570 and not expect_condition_fail 1571 ): 1572 expected_retry = DEFAULT_RETRY 1573 elif expect_condition_fail: 1574 expected_retry = None 1575 1576 headers = {"accept-encoding": "gzip"} 1577 if_etag_match = extra_kwargs.get("if_etag_match") 1578 if if_etag_match is not None: 1579 if isinstance(if_etag_match, string_types): 1580 if_etag_match = [if_etag_match] 1581 headers["If-Match"] = ", ".join(if_etag_match) 1582 if_etag_not_match = extra_kwargs.get("if_etag_not_match") 1583 if if_etag_not_match is not None: 1584 if isinstance(if_etag_not_match, string_types): 1585 if_etag_not_match = [if_etag_not_match] 1586 headers["If-None-Match"] = ", ".join(if_etag_not_match) 1587 1588 blob._do_download.assert_called_once_with( 1589 client._http, 1590 file_obj, 1591 blob._get_download_url(), 1592 headers, 1593 None, 1594 None, 1595 raw_download, 1596 checksum="md5", 1597 timeout=_DEFAULT_TIMEOUT, 1598 retry=expected_retry, 1599 ) 1600 1601 def test_download_blob_to_file_wo_chunks_wo_raw(self): 1602 self._download_blob_to_file_helper(use_chunks=False, raw_download=False) 1603 1604 def test_download_blob_to_file_w_chunks_wo_raw(self): 1605 self._download_blob_to_file_helper(use_chunks=True, raw_download=False) 1606 1607 def test_download_blob_to_file_wo_chunks_w_raw(self): 1608 self._download_blob_to_file_helper(use_chunks=False, raw_download=True) 1609 1610 def test_download_blob_to_file_w_chunks_w_raw(self): 1611 self._download_blob_to_file_helper(use_chunks=True, raw_download=True) 1612 1613 def test_list_blobs_w_defaults_w_bucket_obj(self): 1614 from google.cloud.storage.bucket import Bucket 1615 from google.cloud.storage.bucket import _blobs_page_start 1616 from google.cloud.storage.bucket import _item_to_blob 1617 1618 project = "PROJECT" 1619 bucket_name = "bucket-name" 1620 credentials = _make_credentials() 1621 client = self._make_one(project=project, credentials=credentials) 1622 client._list_resource = mock.Mock(spec=[]) 1623 bucket = Bucket(client, bucket_name) 1624 1625 iterator = client.list_blobs(bucket) 1626 1627 self.assertIs(iterator, client._list_resource.return_value) 1628 self.assertIs(iterator.bucket, bucket) 1629 self.assertEqual(iterator.prefixes, set()) 1630 1631 expected_path = "/b/{}/o".format(bucket_name) 1632 expected_item_to_value = _item_to_blob 1633 expected_page_token = None 1634 expected_max_results = None 1635 expected_extra_params = {"projection": "noAcl"} 1636 expected_page_start = _blobs_page_start 1637 expected_page_size = None 1638 client._list_resource.assert_called_once_with( 1639 expected_path, 1640 expected_item_to_value, 1641 page_token=expected_page_token, 1642 max_results=expected_max_results, 1643 extra_params=expected_extra_params, 1644 page_start=expected_page_start, 1645 page_size=expected_page_size, 1646 timeout=self._get_default_timeout(), 1647 retry=DEFAULT_RETRY, 1648 ) 1649 1650 def test_list_blobs_w_explicit_w_user_project(self): 1651 from google.cloud.storage.bucket import _blobs_page_start 1652 from google.cloud.storage.bucket import _item_to_blob 1653 1654 project = "PROJECT" 1655 user_project = "user-project-123" 1656 bucket_name = "name" 1657 max_results = 10 1658 page_token = "ABCD" 1659 prefix = "subfolder" 1660 delimiter = "/" 1661 start_offset = "c" 1662 end_offset = "g" 1663 include_trailing_delimiter = True 1664 versions = True 1665 projection = "full" 1666 page_size = 2 1667 fields = "items/contentLanguage,nextPageToken" 1668 credentials = _make_credentials() 1669 client = self._make_one(project=project, credentials=credentials) 1670 client._list_resource = mock.Mock(spec=[]) 1671 client._bucket_arg_to_bucket = mock.Mock(spec=[]) 1672 bucket = client._bucket_arg_to_bucket.return_value = mock.Mock( 1673 spec=["path", "user_project"], 1674 ) 1675 bucket.path = "/b/{}".format(bucket_name) 1676 bucket.user_project = user_project 1677 timeout = 42 1678 retry = mock.Mock(spec=[]) 1679 1680 iterator = client.list_blobs( 1681 bucket_or_name=bucket_name, 1682 max_results=max_results, 1683 page_token=page_token, 1684 prefix=prefix, 1685 delimiter=delimiter, 1686 start_offset=start_offset, 1687 end_offset=end_offset, 1688 include_trailing_delimiter=include_trailing_delimiter, 1689 versions=versions, 1690 projection=projection, 1691 fields=fields, 1692 page_size=page_size, 1693 timeout=timeout, 1694 retry=retry, 1695 ) 1696 1697 self.assertIs(iterator, client._list_resource.return_value) 1698 self.assertIs(iterator.bucket, bucket) 1699 self.assertEqual(iterator.prefixes, set()) 1700 1701 expected_path = "/b/{}/o".format(bucket_name) 1702 expected_item_to_value = _item_to_blob 1703 expected_page_token = page_token 1704 expected_max_results = max_results 1705 expected_extra_params = { 1706 "projection": projection, 1707 "prefix": prefix, 1708 "delimiter": delimiter, 1709 "startOffset": start_offset, 1710 "endOffset": end_offset, 1711 "includeTrailingDelimiter": include_trailing_delimiter, 1712 "versions": versions, 1713 "fields": fields, 1714 "userProject": user_project, 1715 } 1716 expected_page_start = _blobs_page_start 1717 expected_page_size = 2 1718 client._list_resource.assert_called_once_with( 1719 expected_path, 1720 expected_item_to_value, 1721 page_token=expected_page_token, 1722 max_results=expected_max_results, 1723 extra_params=expected_extra_params, 1724 page_start=expected_page_start, 1725 page_size=expected_page_size, 1726 timeout=timeout, 1727 retry=retry, 1728 ) 1729 1730 def test_list_buckets_wo_project(self): 1731 credentials = _make_credentials() 1732 client = self._make_one(project=None, credentials=credentials) 1733 1734 with self.assertRaises(ValueError): 1735 client.list_buckets() 1736 1737 def test_list_buckets_w_defaults(self): 1738 from google.cloud.storage.client import _item_to_bucket 1739 1740 project = "PROJECT" 1741 credentials = _make_credentials() 1742 client = self._make_one(project=project, credentials=credentials) 1743 client._list_resource = mock.Mock(spec=[]) 1744 1745 iterator = client.list_buckets() 1746 1747 self.assertIs(iterator, client._list_resource.return_value) 1748 1749 expected_path = "/b" 1750 expected_item_to_value = _item_to_bucket 1751 expected_page_token = None 1752 expected_max_results = None 1753 expected_page_size = None 1754 expected_extra_params = { 1755 "project": project, 1756 "projection": "noAcl", 1757 } 1758 client._list_resource.assert_called_once_with( 1759 expected_path, 1760 expected_item_to_value, 1761 page_token=expected_page_token, 1762 max_results=expected_max_results, 1763 extra_params=expected_extra_params, 1764 page_size=expected_page_size, 1765 timeout=self._get_default_timeout(), 1766 retry=DEFAULT_RETRY, 1767 ) 1768 1769 def test_list_buckets_w_explicit(self): 1770 from google.cloud.storage.client import _item_to_bucket 1771 1772 project = "foo-bar" 1773 other_project = "OTHER_PROJECT" 1774 max_results = 10 1775 page_token = "ABCD" 1776 prefix = "subfolder" 1777 projection = "full" 1778 fields = "items/id,nextPageToken" 1779 credentials = _make_credentials() 1780 client = self._make_one(project=project, credentials=credentials) 1781 client._list_resource = mock.Mock(spec=[]) 1782 page_size = 2 1783 timeout = 42 1784 retry = mock.Mock(spec=[]) 1785 1786 iterator = client.list_buckets( 1787 project=other_project, 1788 max_results=max_results, 1789 page_token=page_token, 1790 prefix=prefix, 1791 projection=projection, 1792 fields=fields, 1793 page_size=page_size, 1794 timeout=timeout, 1795 retry=retry, 1796 ) 1797 1798 self.assertIs(iterator, client._list_resource.return_value) 1799 1800 expected_path = "/b" 1801 expected_item_to_value = _item_to_bucket 1802 expected_page_token = page_token 1803 expected_max_results = max_results 1804 expected_extra_params = { 1805 "project": other_project, 1806 "prefix": prefix, 1807 "projection": projection, 1808 "fields": fields, 1809 } 1810 expected_page_size = 2 1811 client._list_resource.assert_called_once_with( 1812 expected_path, 1813 expected_item_to_value, 1814 page_token=expected_page_token, 1815 max_results=expected_max_results, 1816 extra_params=expected_extra_params, 1817 page_size=expected_page_size, 1818 timeout=timeout, 1819 retry=retry, 1820 ) 1821 1822 def _create_hmac_key_helper( 1823 self, explicit_project=None, user_project=None, timeout=None, retry=None, 1824 ): 1825 import datetime 1826 from google.cloud._helpers import UTC 1827 from google.cloud.storage.hmac_key import HMACKeyMetadata 1828 1829 project = "PROJECT" 1830 access_id = "ACCESS-ID" 1831 credentials = _make_credentials() 1832 email = "storage-user-123@example.com" 1833 secret = "a" * 40 1834 now = datetime.datetime.utcnow().replace(tzinfo=UTC) 1835 now_stamp = "{}Z".format(now.isoformat()) 1836 1837 if explicit_project is not None: 1838 expected_project = explicit_project 1839 else: 1840 expected_project = project 1841 1842 api_response = { 1843 "kind": "storage#hmacKey", 1844 "metadata": { 1845 "accessId": access_id, 1846 "etag": "ETAG", 1847 "id": "projects/{}/hmacKeys/{}".format(project, access_id), 1848 "project": expected_project, 1849 "state": "ACTIVE", 1850 "serviceAccountEmail": email, 1851 "timeCreated": now_stamp, 1852 "updated": now_stamp, 1853 }, 1854 "secret": secret, 1855 } 1856 1857 client = self._make_one(project=project, credentials=credentials) 1858 client._post_resource = mock.Mock() 1859 client._post_resource.return_value = api_response 1860 1861 kwargs = {} 1862 if explicit_project is not None: 1863 kwargs["project_id"] = explicit_project 1864 1865 if user_project is not None: 1866 kwargs["user_project"] = user_project 1867 1868 if timeout is None: 1869 expected_timeout = self._get_default_timeout() 1870 else: 1871 expected_timeout = kwargs["timeout"] = timeout 1872 1873 if retry is None: 1874 expected_retry = None 1875 else: 1876 expected_retry = kwargs["retry"] = retry 1877 1878 metadata, secret = client.create_hmac_key(service_account_email=email, **kwargs) 1879 1880 self.assertIsInstance(metadata, HMACKeyMetadata) 1881 1882 self.assertIs(metadata._client, client) 1883 self.assertEqual(metadata._properties, api_response["metadata"]) 1884 self.assertEqual(secret, api_response["secret"]) 1885 1886 expected_path = "/projects/{}/hmacKeys".format(expected_project) 1887 expected_data = None 1888 expected_query_params = {"serviceAccountEmail": email} 1889 1890 if user_project is not None: 1891 expected_query_params["userProject"] = user_project 1892 1893 client._post_resource.assert_called_once_with( 1894 expected_path, 1895 expected_data, 1896 query_params=expected_query_params, 1897 timeout=expected_timeout, 1898 retry=expected_retry, 1899 ) 1900 1901 def test_create_hmac_key_defaults(self): 1902 self._create_hmac_key_helper() 1903 1904 def test_create_hmac_key_explicit_project(self): 1905 self._create_hmac_key_helper(explicit_project="other-project-456") 1906 1907 def test_create_hmac_key_w_user_project(self): 1908 self._create_hmac_key_helper(user_project="billed-project") 1909 1910 def test_create_hmac_key_w_timeout(self): 1911 self._create_hmac_key_helper(timeout=42) 1912 1913 def test_create_hmac_key_w_retry(self): 1914 self._create_hmac_key_helper(retry=mock.Mock(spec=[])) 1915 1916 def test_list_hmac_keys_w_defaults(self): 1917 from google.cloud.storage.client import _item_to_hmac_key_metadata 1918 1919 project = "PROJECT" 1920 credentials = _make_credentials() 1921 client = self._make_one(project=project, credentials=credentials) 1922 client._list_resource = mock.Mock(spec=[]) 1923 1924 iterator = client.list_hmac_keys() 1925 1926 self.assertIs(iterator, client._list_resource.return_value) 1927 1928 expected_path = "/projects/{}/hmacKeys".format(project) 1929 expected_item_to_value = _item_to_hmac_key_metadata 1930 expected_max_results = None 1931 expected_extra_params = {} 1932 client._list_resource.assert_called_once_with( 1933 expected_path, 1934 expected_item_to_value, 1935 max_results=expected_max_results, 1936 extra_params=expected_extra_params, 1937 timeout=self._get_default_timeout(), 1938 retry=DEFAULT_RETRY, 1939 ) 1940 1941 def test_list_hmac_keys_w_explicit(self): 1942 from google.cloud.storage.client import _item_to_hmac_key_metadata 1943 1944 project = "PROJECT" 1945 other_project = "other-project-456" 1946 max_results = 3 1947 show_deleted_keys = True 1948 service_account_email = "storage-user-123@example.com" 1949 user_project = "billed-project" 1950 credentials = _make_credentials() 1951 client = self._make_one(project=project, credentials=credentials) 1952 client._list_resource = mock.Mock(spec=[]) 1953 timeout = 42 1954 retry = mock.Mock(spec=[]) 1955 1956 iterator = client.list_hmac_keys( 1957 max_results=max_results, 1958 service_account_email=service_account_email, 1959 show_deleted_keys=show_deleted_keys, 1960 project_id=other_project, 1961 user_project=user_project, 1962 timeout=timeout, 1963 retry=retry, 1964 ) 1965 1966 self.assertIs(iterator, client._list_resource.return_value) 1967 1968 expected_path = "/projects/{}/hmacKeys".format(other_project) 1969 expected_item_to_value = _item_to_hmac_key_metadata 1970 expected_max_results = max_results 1971 expected_extra_params = { 1972 "serviceAccountEmail": service_account_email, 1973 "showDeletedKeys": show_deleted_keys, 1974 "userProject": user_project, 1975 } 1976 client._list_resource.assert_called_once_with( 1977 expected_path, 1978 expected_item_to_value, 1979 max_results=expected_max_results, 1980 extra_params=expected_extra_params, 1981 timeout=timeout, 1982 retry=retry, 1983 ) 1984 1985 def test_get_hmac_key_metadata_wo_project(self): 1986 from google.cloud.storage.hmac_key import HMACKeyMetadata 1987 1988 PROJECT = "PROJECT" 1989 EMAIL = "storage-user-123@example.com" 1990 ACCESS_ID = "ACCESS-ID" 1991 CREDENTIALS = _make_credentials() 1992 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 1993 1994 resource = { 1995 "kind": "storage#hmacKeyMetadata", 1996 "accessId": ACCESS_ID, 1997 "projectId": PROJECT, 1998 "serviceAccountEmail": EMAIL, 1999 } 2000 2001 http = _make_requests_session([_make_json_response(resource)]) 2002 client._http_internal = http 2003 2004 metadata = client.get_hmac_key_metadata(ACCESS_ID, timeout=42) 2005 2006 self.assertIsInstance(metadata, HMACKeyMetadata) 2007 self.assertIs(metadata._client, client) 2008 self.assertEqual(metadata.access_id, ACCESS_ID) 2009 self.assertEqual(metadata.project, PROJECT) 2010 2011 http.request.assert_called_once_with( 2012 method="GET", url=mock.ANY, data=None, headers=mock.ANY, timeout=42 2013 ) 2014 _, kwargs = http.request.call_args 2015 scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) 2016 self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) 2017 self.assertEqual( 2018 path, 2019 "/".join( 2020 [ 2021 "", 2022 "storage", 2023 client._connection.API_VERSION, 2024 "projects", 2025 PROJECT, 2026 "hmacKeys", 2027 ACCESS_ID, 2028 ] 2029 ), 2030 ) 2031 2032 def test_get_hmac_key_metadata_w_project(self): 2033 from google.cloud.storage.hmac_key import HMACKeyMetadata 2034 2035 PROJECT = "PROJECT" 2036 OTHER_PROJECT = "other-project-456" 2037 EMAIL = "storage-user-123@example.com" 2038 ACCESS_ID = "ACCESS-ID" 2039 USER_PROJECT = "billed-project" 2040 CREDENTIALS = _make_credentials() 2041 client = self._make_one(project=PROJECT, credentials=CREDENTIALS) 2042 2043 resource = { 2044 "kind": "storage#hmacKeyMetadata", 2045 "accessId": ACCESS_ID, 2046 "projectId": OTHER_PROJECT, 2047 "serviceAccountEmail": EMAIL, 2048 } 2049 2050 http = _make_requests_session([_make_json_response(resource)]) 2051 client._http_internal = http 2052 2053 metadata = client.get_hmac_key_metadata( 2054 ACCESS_ID, project_id=OTHER_PROJECT, user_project=USER_PROJECT 2055 ) 2056 2057 self.assertIsInstance(metadata, HMACKeyMetadata) 2058 self.assertIs(metadata._client, client) 2059 self.assertEqual(metadata.access_id, ACCESS_ID) 2060 self.assertEqual(metadata.project, OTHER_PROJECT) 2061 2062 http.request.assert_called_once_with( 2063 method="GET", 2064 url=mock.ANY, 2065 data=None, 2066 headers=mock.ANY, 2067 timeout=self._get_default_timeout(), 2068 ) 2069 _, kwargs = http.request.call_args 2070 scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) 2071 self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) 2072 self.assertEqual( 2073 path, 2074 "/".join( 2075 [ 2076 "", 2077 "storage", 2078 client._connection.API_VERSION, 2079 "projects", 2080 OTHER_PROJECT, 2081 "hmacKeys", 2082 ACCESS_ID, 2083 ] 2084 ), 2085 ) 2086 parms = dict(urlparse.parse_qsl(qs)) 2087 self.assertEqual(parms["userProject"], USER_PROJECT) 2088 2089 def test_get_signed_policy_v4(self): 2090 import datetime 2091 2092 BUCKET_NAME = "bucket-name" 2093 BLOB_NAME = "object-name" 2094 EXPECTED_SIGN = "5369676e61747572655f6279746573" 2095 EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9" 2096 2097 project = "PROJECT" 2098 credentials = _make_credentials(project=project) 2099 client = self._make_one(credentials=credentials) 2100 2101 dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() 2102 with dtstamps_patch, now_patch, expire_secs_patch: 2103 policy = client.generate_signed_post_policy_v4( 2104 BUCKET_NAME, 2105 BLOB_NAME, 2106 expiration=datetime.datetime(2020, 3, 12), 2107 conditions=[ 2108 {"bucket": BUCKET_NAME}, 2109 {"acl": "private"}, 2110 ["starts-with", "$Content-Type", "text/plain"], 2111 ], 2112 credentials=_create_signing_credentials(), 2113 ) 2114 self.assertEqual( 2115 policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" 2116 ) 2117 fields = policy["fields"] 2118 2119 self.assertEqual(fields["key"], BLOB_NAME) 2120 self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") 2121 self.assertEqual(fields["x-goog-date"], "20200312T114716Z") 2122 self.assertEqual( 2123 fields["x-goog-credential"], 2124 "test@mail.com/20200312/auto/storage/goog4_request", 2125 ) 2126 self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) 2127 self.assertEqual(fields["policy"], EXPECTED_POLICY) 2128 2129 def test_get_signed_policy_v4_without_credentials(self): 2130 import datetime 2131 2132 BUCKET_NAME = "bucket-name" 2133 BLOB_NAME = "object-name" 2134 EXPECTED_SIGN = "5369676e61747572655f6279746573" 2135 EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9" 2136 2137 client = self._make_one( 2138 project="PROJECT", credentials=_create_signing_credentials() 2139 ) 2140 2141 dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() 2142 with dtstamps_patch, now_patch, expire_secs_patch: 2143 policy = client.generate_signed_post_policy_v4( 2144 BUCKET_NAME, 2145 BLOB_NAME, 2146 expiration=datetime.datetime(2020, 3, 12), 2147 conditions=[ 2148 {"bucket": BUCKET_NAME}, 2149 {"acl": "private"}, 2150 ["starts-with", "$Content-Type", "text/plain"], 2151 ], 2152 ) 2153 self.assertEqual( 2154 policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" 2155 ) 2156 fields = policy["fields"] 2157 2158 self.assertEqual(fields["key"], BLOB_NAME) 2159 self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") 2160 self.assertEqual(fields["x-goog-date"], "20200312T114716Z") 2161 self.assertEqual( 2162 fields["x-goog-credential"], 2163 "test@mail.com/20200312/auto/storage/goog4_request", 2164 ) 2165 self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) 2166 self.assertEqual(fields["policy"], EXPECTED_POLICY) 2167 2168 def test_get_signed_policy_v4_with_fields(self): 2169 import datetime 2170 2171 BUCKET_NAME = "bucket-name" 2172 BLOB_NAME = "object-name" 2173 FIELD1_VALUE = "Value1" 2174 EXPECTED_SIGN = "5369676e61747572655f6279746573" 2175 EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiZmllbGQxIjoiVmFsdWUxIn0seyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" 2176 2177 project = "PROJECT" 2178 credentials = _make_credentials(project=project) 2179 client = self._make_one(credentials=credentials) 2180 2181 dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() 2182 with dtstamps_patch, now_patch, expire_secs_patch: 2183 policy = client.generate_signed_post_policy_v4( 2184 BUCKET_NAME, 2185 BLOB_NAME, 2186 expiration=datetime.datetime(2020, 3, 12), 2187 conditions=[ 2188 {"bucket": BUCKET_NAME}, 2189 {"acl": "private"}, 2190 ["starts-with", "$Content-Type", "text/plain"], 2191 ], 2192 fields={"field1": FIELD1_VALUE, "x-ignore-field": "Ignored_value"}, 2193 credentials=_create_signing_credentials(), 2194 ) 2195 self.assertEqual( 2196 policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" 2197 ) 2198 fields = policy["fields"] 2199 2200 self.assertEqual(fields["key"], BLOB_NAME) 2201 self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") 2202 self.assertEqual(fields["x-goog-date"], "20200312T114716Z") 2203 self.assertEqual(fields["field1"], FIELD1_VALUE) 2204 self.assertNotIn("x-ignore-field", fields.keys()) 2205 self.assertEqual( 2206 fields["x-goog-credential"], 2207 "test@mail.com/20200312/auto/storage/goog4_request", 2208 ) 2209 self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) 2210 self.assertEqual(fields["policy"], EXPECTED_POLICY) 2211 2212 def test_get_signed_policy_v4_virtual_hosted_style(self): 2213 import datetime 2214 2215 BUCKET_NAME = "bucket-name" 2216 2217 project = "PROJECT" 2218 credentials = _make_credentials(project=project) 2219 client = self._make_one(credentials=credentials) 2220 2221 dtstamps_patch, _, _ = _time_functions_patches() 2222 with dtstamps_patch: 2223 policy = client.generate_signed_post_policy_v4( 2224 BUCKET_NAME, 2225 "object-name", 2226 expiration=datetime.datetime(2020, 3, 12), 2227 virtual_hosted_style=True, 2228 credentials=_create_signing_credentials(), 2229 ) 2230 self.assertEqual( 2231 policy["url"], "https://{}.storage.googleapis.com/".format(BUCKET_NAME) 2232 ) 2233 2234 def test_get_signed_policy_v4_bucket_bound_hostname(self): 2235 import datetime 2236 2237 project = "PROJECT" 2238 credentials = _make_credentials(project=project) 2239 client = self._make_one(credentials=credentials) 2240 2241 dtstamps_patch, _, _ = _time_functions_patches() 2242 with dtstamps_patch: 2243 policy = client.generate_signed_post_policy_v4( 2244 "bucket-name", 2245 "object-name", 2246 expiration=datetime.datetime(2020, 3, 12), 2247 bucket_bound_hostname="https://bucket.bound_hostname", 2248 credentials=_create_signing_credentials(), 2249 ) 2250 self.assertEqual(policy["url"], "https://bucket.bound_hostname") 2251 2252 def test_get_signed_policy_v4_bucket_bound_hostname_with_scheme(self): 2253 import datetime 2254 2255 project = "PROJECT" 2256 credentials = _make_credentials(project=project) 2257 client = self._make_one(credentials=credentials) 2258 2259 dtstamps_patch, _, _ = _time_functions_patches() 2260 with dtstamps_patch: 2261 policy = client.generate_signed_post_policy_v4( 2262 "bucket-name", 2263 "object-name", 2264 expiration=datetime.datetime(2020, 3, 12), 2265 bucket_bound_hostname="bucket.bound_hostname", 2266 scheme="http", 2267 credentials=_create_signing_credentials(), 2268 ) 2269 self.assertEqual(policy["url"], "http://bucket.bound_hostname/") 2270 2271 def test_get_signed_policy_v4_no_expiration(self): 2272 BUCKET_NAME = "bucket-name" 2273 EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" 2274 2275 project = "PROJECT" 2276 credentials = _make_credentials(project=project) 2277 client = self._make_one(credentials=credentials) 2278 2279 dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() 2280 with dtstamps_patch, now_patch, expire_secs_patch: 2281 policy = client.generate_signed_post_policy_v4( 2282 BUCKET_NAME, 2283 "object-name", 2284 expiration=None, 2285 credentials=_create_signing_credentials(), 2286 ) 2287 2288 self.assertEqual( 2289 policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" 2290 ) 2291 self.assertEqual(policy["fields"]["policy"], EXPECTED_POLICY) 2292 2293 def test_get_signed_policy_v4_with_access_token(self): 2294 import datetime 2295 2296 BUCKET_NAME = "bucket-name" 2297 BLOB_NAME = "object-name" 2298 EXPECTED_SIGN = "0c4003044105" 2299 EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9" 2300 2301 project = "PROJECT" 2302 credentials = _make_credentials(project=project) 2303 client = self._make_one(credentials=credentials) 2304 2305 dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() 2306 with dtstamps_patch, now_patch, expire_secs_patch: 2307 with mock.patch( 2308 "google.cloud.storage.client._sign_message", return_value=b"DEADBEEF" 2309 ): 2310 policy = client.generate_signed_post_policy_v4( 2311 BUCKET_NAME, 2312 BLOB_NAME, 2313 expiration=datetime.datetime(2020, 3, 12), 2314 conditions=[ 2315 {"bucket": BUCKET_NAME}, 2316 {"acl": "private"}, 2317 ["starts-with", "$Content-Type", "text/plain"], 2318 ], 2319 credentials=_create_signing_credentials(), 2320 service_account_email="test@mail.com", 2321 access_token="token", 2322 ) 2323 self.assertEqual( 2324 policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" 2325 ) 2326 fields = policy["fields"] 2327 2328 self.assertEqual(fields["key"], BLOB_NAME) 2329 self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") 2330 self.assertEqual(fields["x-goog-date"], "20200312T114716Z") 2331 self.assertEqual( 2332 fields["x-goog-credential"], 2333 "test@mail.com/20200312/auto/storage/goog4_request", 2334 ) 2335 self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) 2336 self.assertEqual(fields["policy"], EXPECTED_POLICY) 2337 2338 2339class Test__item_to_bucket(unittest.TestCase): 2340 def _call_fut(self, iterator, item): 2341 from google.cloud.storage.client import _item_to_bucket 2342 2343 return _item_to_bucket(iterator, item) 2344 2345 def test_w_empty_item(self): 2346 from google.cloud.storage.bucket import Bucket 2347 2348 iterator = mock.Mock(spec=["client"]) 2349 item = {} 2350 2351 bucket = self._call_fut(iterator, item) 2352 2353 self.assertIsInstance(bucket, Bucket) 2354 self.assertIs(bucket.client, iterator.client) 2355 self.assertIsNone(bucket.name) 2356 2357 def test_w_name(self): 2358 from google.cloud.storage.bucket import Bucket 2359 2360 name = "name" 2361 iterator = mock.Mock(spec=["client"]) 2362 item = {"name": name} 2363 2364 bucket = self._call_fut(iterator, item) 2365 2366 self.assertIsInstance(bucket, Bucket) 2367 self.assertIs(bucket.client, iterator.client) 2368 self.assertEqual(bucket.name, name) 2369 2370 2371class Test__item_to_hmac_key_metadata(unittest.TestCase): 2372 def _call_fut(self, iterator, item): 2373 from google.cloud.storage.client import _item_to_hmac_key_metadata 2374 2375 return _item_to_hmac_key_metadata(iterator, item) 2376 2377 def test_it(self): 2378 from google.cloud.storage.hmac_key import HMACKeyMetadata 2379 2380 access_id = "ABCDE" 2381 iterator = mock.Mock(spec=["client"]) 2382 item = {"id": access_id} 2383 2384 metadata = self._call_fut(iterator, item) 2385 2386 self.assertIsInstance(metadata, HMACKeyMetadata) 2387 self.assertIs(metadata._client, iterator.client) 2388 self.assertEqual(metadata._properties, item) 2389 2390 2391@pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) 2392def test_conformance_post_policy(test_data): 2393 import datetime 2394 from google.cloud.storage.client import Client 2395 2396 in_data = test_data["policyInput"] 2397 timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") 2398 2399 client = Client(credentials=_FAKE_CREDENTIALS, project="PROJECT") 2400 2401 # mocking time functions 2402 with mock.patch("google.cloud.storage._signing.NOW", return_value=timestamp): 2403 with mock.patch( 2404 "google.cloud.storage.client.get_expiration_seconds_v4", 2405 return_value=in_data["expiration"], 2406 ): 2407 with mock.patch("google.cloud.storage.client._NOW", return_value=timestamp): 2408 2409 policy = client.generate_signed_post_policy_v4( 2410 bucket_name=in_data["bucket"], 2411 blob_name=in_data["object"], 2412 conditions=_prepare_conditions(in_data), 2413 fields=in_data.get("fields"), 2414 credentials=_FAKE_CREDENTIALS, 2415 expiration=in_data["expiration"], 2416 virtual_hosted_style=in_data.get("urlStyle") 2417 == "VIRTUAL_HOSTED_STYLE", 2418 bucket_bound_hostname=in_data.get("bucketBoundHostname"), 2419 scheme=in_data.get("scheme"), 2420 ) 2421 fields = policy["fields"] 2422 out_data = test_data["policyOutput"] 2423 2424 decoded_policy = base64.b64decode(fields["policy"]).decode("unicode_escape") 2425 assert decoded_policy == out_data["expectedDecodedPolicy"] 2426 2427 for field in ( 2428 "x-goog-algorithm", 2429 "x-goog-credential", 2430 "x-goog-date", 2431 "x-goog-signature", 2432 ): 2433 assert fields[field] == test_data["policyOutput"]["fields"][field] 2434 2435 assert policy["url"] == out_data["url"] 2436 2437 2438def _prepare_conditions(in_data): 2439 """Helper for V4 POST policy generation conformance tests. 2440 2441 Convert conformance test data conditions dict into list. 2442 2443 Args: 2444 in_data (dict): conditions arg from conformance test data. 2445 2446 Returns: 2447 list: conditions arg to pass into generate_signed_post_policy_v4(). 2448 """ 2449 if "conditions" in in_data: 2450 conditions = [] 2451 for key, value in in_data["conditions"].items(): 2452 # camel case to snake case with "-" separator 2453 field = re.sub(r"(?<!^)(?=[A-Z])", "-", key).lower() 2454 conditions.append([field] + value) 2455 2456 return conditions 2457 2458 2459def _time_functions_patches(): 2460 """Helper for POST policy generation - returns all necessary time functions patches.""" 2461 import datetime 2462 2463 dtstamps_patch = mock.patch( 2464 "google.cloud.storage.client.get_v4_now_dtstamps", 2465 return_value=("20200312T114716Z", "20200312"), 2466 ) 2467 now_patch = mock.patch( 2468 "google.cloud.storage.client._NOW", return_value=datetime.datetime(2020, 3, 26) 2469 ) 2470 expire_secs_patch = mock.patch( 2471 "google.cloud.storage.client.get_expiration_seconds_v4", return_value=10 2472 ) 2473 return dtstamps_patch, now_patch, expire_secs_patch 2474