1# -*- coding: utf-8 -*- 2# Copyright 2013 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Unit tests for parallel upload functions in copy_helper.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import datetime 23import logging 24import os 25from apitools.base.py import exceptions as apitools_exceptions 26 27from gslib.bucket_listing_ref import BucketListingObject 28from gslib.bucket_listing_ref import BucketListingPrefix 29from gslib.cloud_api import CloudApi 30from gslib.cloud_api import ResumableUploadAbortException 31from gslib.cloud_api import ResumableUploadException 32from gslib.cloud_api import ResumableUploadStartOverException 33from gslib.cloud_api import ServiceException 34from gslib.command import CreateOrGetGsutilLogger 35from gslib.discard_messages_queue import DiscardMessagesQueue 36from gslib.exception import CommandException 37from gslib.gcs_json_api import GcsJsonApi 38from gslib.parallel_tracker_file import ObjectFromTracker 39from gslib.storage_url import StorageUrlFromString 40from gslib.tests.mock_cloud_api import MockCloudApi 41from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase 42from gslib.tests.util import GSMockBucketStorageUri 43from gslib.tests.util import SetBotoConfigForTest 44from gslib.tests.util import unittest 45from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 46from gslib.utils import parallelism_framework_util 47from gslib.utils import posix_util 48from gslib.utils import system_util 49from gslib.utils import hashing_helper 50from gslib.utils.copy_helper import _DelegateUploadFileToObject 51from gslib.utils.copy_helper import _GetPartitionInfo 52from gslib.utils.copy_helper import _SelectUploadCompressionStrategy 53from gslib.utils.copy_helper import _SetContentTypeFromFile 54from gslib.utils.copy_helper import ExpandUrlToSingleBlr 55from gslib.utils.copy_helper import FilterExistingComponents 56from gslib.utils.copy_helper import GZIP_ALL_FILES 57from gslib.utils.copy_helper import PerformParallelUploadFileToObjectArgs 58from gslib.utils.copy_helper import WarnIfMvEarlyDeletionChargeApplies 59 60from six import add_move, MovedModule 61add_move(MovedModule('mock', 'mock', 'unittest.mock')) 62from six.moves import mock 63 64_CalculateB64EncodedMd5FromContents = ( 65 hashing_helper.CalculateB64EncodedMd5FromContents) 66 67 68class TestCpFuncs(GsUtilUnitTestCase): 69 """Unit tests for parallel upload functions in cp command.""" 70 71 def testGetPartitionInfo(self): 72 """Tests the _GetPartitionInfo function.""" 73 # Simplest case - threshold divides file_size. 74 (num_components, component_size) = _GetPartitionInfo(300, 200, 10) 75 self.assertEqual(30, num_components) 76 self.assertEqual(10, component_size) 77 78 # Threshold = 1 (mod file_size). 79 (num_components, component_size) = _GetPartitionInfo(301, 200, 10) 80 self.assertEqual(31, num_components) 81 self.assertEqual(10, component_size) 82 83 # Threshold = -1 (mod file_size). 84 (num_components, component_size) = _GetPartitionInfo(299, 200, 10) 85 self.assertEqual(30, num_components) 86 self.assertEqual(10, component_size) 87 88 # Too many components needed. 89 (num_components, component_size) = _GetPartitionInfo(301, 2, 10) 90 self.assertEqual(2, num_components) 91 self.assertEqual(151, component_size) 92 93 # Test num_components with huge numbers. 94 (num_components, component_size) = _GetPartitionInfo((10**150) + 1, 10**200, 95 10) 96 self.assertEqual((10**149) + 1, num_components) 97 self.assertEqual(10, component_size) 98 99 # Test component_size with huge numbers. 100 (num_components, component_size) = _GetPartitionInfo((10**150) + 1, 10, 10) 101 self.assertEqual(10, num_components) 102 self.assertEqual((10**149) + 1, component_size) 103 104 # Test component_size > file_size (make sure we get at least two components. 105 (num_components, component_size) = _GetPartitionInfo(100, 500, 51) 106 self.assertEquals(2, num_components) 107 self.assertEqual(50, component_size) 108 109 def testFilterExistingComponentsNonVersioned(self): 110 """Tests upload with a variety of component states.""" 111 mock_api = MockCloudApi() 112 bucket_name = self.MakeTempName('bucket') 113 tracker_file = self.CreateTempFile(file_name='foo', contents=b'asdf') 114 tracker_file_lock = parallelism_framework_util.CreateLock() 115 116 # dst_obj_metadata used for passing content-type. 117 empty_object = apitools_messages.Object() 118 119 # Already uploaded, contents still match, component still used. 120 fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', 121 contents=b'1') 122 fpath_uploaded_correctly_url = StorageUrlFromString( 123 str(fpath_uploaded_correctly)) 124 object_uploaded_correctly_url = StorageUrlFromString( 125 '%s://%s/%s' % 126 (self.default_provider, bucket_name, fpath_uploaded_correctly)) 127 with open(fpath_uploaded_correctly, 'rb') as f_in: 128 fpath_uploaded_correctly_md5 = _CalculateB64EncodedMd5FromContents(f_in) 129 mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( 130 bucket=bucket_name, 131 name=fpath_uploaded_correctly, 132 md5Hash=fpath_uploaded_correctly_md5), 133 contents=b'1') 134 135 args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( 136 fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, 137 object_uploaded_correctly_url, '', empty_object, tracker_file, 138 tracker_file_lock, None, False) 139 140 # Not yet uploaded, but needed. 141 fpath_not_uploaded = self.CreateTempFile(file_name='foo2', contents=b'2') 142 fpath_not_uploaded_url = StorageUrlFromString(str(fpath_not_uploaded)) 143 object_not_uploaded_url = StorageUrlFromString( 144 '%s://%s/%s' % (self.default_provider, bucket_name, fpath_not_uploaded)) 145 args_not_uploaded = PerformParallelUploadFileToObjectArgs( 146 fpath_not_uploaded, 0, 1, fpath_not_uploaded_url, 147 object_not_uploaded_url, '', empty_object, tracker_file, 148 tracker_file_lock, None, False) 149 150 # Already uploaded, but contents no longer match. Even though the contents 151 # differ, we don't delete this since the bucket is not versioned and it 152 # will be overwritten anyway. 153 fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents=b'4') 154 fpath_wrong_contents_url = StorageUrlFromString(str(fpath_wrong_contents)) 155 object_wrong_contents_url = StorageUrlFromString( 156 '%s://%s/%s' % 157 (self.default_provider, bucket_name, fpath_wrong_contents)) 158 with open(self.CreateTempFile(contents=b'_'), 'rb') as f_in: 159 fpath_wrong_contents_md5 = _CalculateB64EncodedMd5FromContents(f_in) 160 mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( 161 bucket=bucket_name, 162 name=fpath_wrong_contents, 163 md5Hash=fpath_wrong_contents_md5), 164 contents=b'1') 165 166 args_wrong_contents = PerformParallelUploadFileToObjectArgs( 167 fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, 168 object_wrong_contents_url, '', empty_object, tracker_file, 169 tracker_file_lock, None, False) 170 171 # Exists in tracker file, but component object no longer exists. 172 fpath_remote_deleted = self.CreateTempFile(file_name='foo5', contents=b'5') 173 fpath_remote_deleted_url = StorageUrlFromString(str(fpath_remote_deleted)) 174 args_remote_deleted = PerformParallelUploadFileToObjectArgs( 175 fpath_remote_deleted, 0, 1, fpath_remote_deleted_url, '', '', 176 empty_object, tracker_file, tracker_file_lock, None, False) 177 178 # Exists in tracker file and already uploaded, but no longer needed. 179 fpath_no_longer_used = self.CreateTempFile(file_name='foo6', contents=b'6') 180 with open(fpath_no_longer_used, 'rb') as f_in: 181 file_md5 = _CalculateB64EncodedMd5FromContents(f_in) 182 mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( 183 bucket=bucket_name, name='foo6', md5Hash=file_md5), 184 contents=b'6') 185 186 dst_args = { 187 fpath_uploaded_correctly: args_uploaded_correctly, 188 fpath_not_uploaded: args_not_uploaded, 189 fpath_wrong_contents: args_wrong_contents, 190 fpath_remote_deleted: args_remote_deleted 191 } 192 193 existing_components = [ 194 ObjectFromTracker(fpath_uploaded_correctly, ''), 195 ObjectFromTracker(fpath_wrong_contents, ''), 196 ObjectFromTracker(fpath_remote_deleted, ''), 197 ObjectFromTracker(fpath_no_longer_used, '') 198 ] 199 200 bucket_url = StorageUrlFromString('%s://%s' % 201 (self.default_provider, bucket_name)) 202 203 (components_to_upload, uploaded_components, 204 existing_objects_to_delete) = (FilterExistingComponents( 205 dst_args, existing_components, bucket_url, mock_api)) 206 uploaded_components = [i[0] for i in uploaded_components] 207 for arg in [args_not_uploaded, args_wrong_contents, args_remote_deleted]: 208 self.assertTrue(arg in components_to_upload) 209 self.assertEqual(1, len(uploaded_components)) 210 self.assertEqual(args_uploaded_correctly.dst_url.url_string, 211 uploaded_components[0].url_string) 212 self.assertEqual(1, len(existing_objects_to_delete)) 213 no_longer_used_url = StorageUrlFromString( 214 '%s://%s/%s' % 215 (self.default_provider, bucket_name, fpath_no_longer_used)) 216 self.assertEqual(no_longer_used_url.url_string, 217 existing_objects_to_delete[0].url_string) 218 219 def testFilterExistingComponentsVersioned(self): 220 """Tests upload with versionined parallel components.""" 221 222 mock_api = MockCloudApi() 223 bucket_name = self.MakeTempName('bucket') 224 mock_api.MockCreateVersionedBucket(bucket_name) 225 226 # dst_obj_metadata used for passing content-type. 227 empty_object = apitools_messages.Object() 228 229 tracker_file = self.CreateTempFile(file_name='foo', contents=b'asdf') 230 tracker_file_lock = parallelism_framework_util.CreateLock() 231 232 # Already uploaded, contents still match, component still used. 233 fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', 234 contents=b'1') 235 fpath_uploaded_correctly_url = StorageUrlFromString( 236 str(fpath_uploaded_correctly)) 237 with open(fpath_uploaded_correctly, 'rb') as f_in: 238 fpath_uploaded_correctly_md5 = _CalculateB64EncodedMd5FromContents(f_in) 239 object_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( 240 apitools_messages.Object(bucket=bucket_name, 241 name=fpath_uploaded_correctly, 242 md5Hash=fpath_uploaded_correctly_md5), 243 contents=b'1') 244 object_uploaded_correctly_url = StorageUrlFromString( 245 '%s://%s/%s#%s' % 246 (self.default_provider, bucket_name, fpath_uploaded_correctly, 247 object_uploaded_correctly.generation)) 248 args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( 249 fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, 250 object_uploaded_correctly_url, object_uploaded_correctly.generation, 251 empty_object, tracker_file, tracker_file_lock, None, False) 252 253 # Duplicate object name in tracker file, but uploaded correctly. 254 fpath_duplicate = fpath_uploaded_correctly 255 fpath_duplicate_url = StorageUrlFromString(str(fpath_duplicate)) 256 duplicate_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( 257 apitools_messages.Object(bucket=bucket_name, 258 name=fpath_duplicate, 259 md5Hash=fpath_uploaded_correctly_md5), 260 contents=b'1') 261 duplicate_uploaded_correctly_url = StorageUrlFromString( 262 '%s://%s/%s#%s' % 263 (self.default_provider, bucket_name, fpath_uploaded_correctly, 264 duplicate_uploaded_correctly.generation)) 265 args_duplicate = PerformParallelUploadFileToObjectArgs( 266 fpath_duplicate, 0, 1, fpath_duplicate_url, 267 duplicate_uploaded_correctly_url, 268 duplicate_uploaded_correctly.generation, empty_object, tracker_file, 269 tracker_file_lock, None, False) 270 271 # Already uploaded, but contents no longer match. 272 fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents=b'4') 273 fpath_wrong_contents_url = StorageUrlFromString(str(fpath_wrong_contents)) 274 with open(self.CreateTempFile(contents=b'_'), 'rb') as f_in: 275 fpath_wrong_contents_md5 = _CalculateB64EncodedMd5FromContents(f_in) 276 object_wrong_contents = mock_api.MockCreateObjectWithMetadata( 277 apitools_messages.Object(bucket=bucket_name, 278 name=fpath_wrong_contents, 279 md5Hash=fpath_wrong_contents_md5), 280 contents=b'_') 281 wrong_contents_url = StorageUrlFromString( 282 '%s://%s/%s#%s' % 283 (self.default_provider, bucket_name, fpath_wrong_contents, 284 object_wrong_contents.generation)) 285 args_wrong_contents = PerformParallelUploadFileToObjectArgs( 286 fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, 287 wrong_contents_url, '', empty_object, tracker_file, tracker_file_lock, 288 None, False) 289 290 dst_args = { 291 fpath_uploaded_correctly: args_uploaded_correctly, 292 fpath_wrong_contents: args_wrong_contents 293 } 294 295 existing_components = [ 296 ObjectFromTracker(fpath_uploaded_correctly, 297 object_uploaded_correctly_url.generation), 298 ObjectFromTracker(fpath_duplicate, 299 duplicate_uploaded_correctly_url.generation), 300 ObjectFromTracker(fpath_wrong_contents, wrong_contents_url.generation) 301 ] 302 303 bucket_url = StorageUrlFromString('%s://%s' % 304 (self.default_provider, bucket_name)) 305 306 (components_to_upload, uploaded_components, 307 existing_objects_to_delete) = (FilterExistingComponents( 308 dst_args, existing_components, bucket_url, mock_api)) 309 uploaded_components = [i[0] for i in uploaded_components] 310 self.assertEqual([args_wrong_contents], components_to_upload) 311 self.assertEqual(args_uploaded_correctly.dst_url.url_string, 312 uploaded_components[0].url_string) 313 expected_to_delete = [(args_wrong_contents.dst_url.object_name, 314 args_wrong_contents.dst_url.generation), 315 (args_duplicate.dst_url.object_name, 316 args_duplicate.dst_url.generation)] 317 for uri in existing_objects_to_delete: 318 self.assertTrue((uri.object_name, uri.generation) in expected_to_delete) 319 self.assertEqual(len(expected_to_delete), len(existing_objects_to_delete)) 320 321 # pylint: disable=protected-access 322 def testTranslateApitoolsResumableUploadException(self): 323 """Tests that _TranslateApitoolsResumableUploadException works correctly.""" 324 gsutil_api = GcsJsonApi(GSMockBucketStorageUri, 325 CreateOrGetGsutilLogger('copy_test'), 326 DiscardMessagesQueue()) 327 328 gsutil_api.http.disable_ssl_certificate_validation = True 329 exc = apitools_exceptions.HttpError({'status': 503}, None, None) 330 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 331 self.assertTrue(isinstance(translated_exc, ServiceException)) 332 333 gsutil_api.http.disable_ssl_certificate_validation = False 334 exc = apitools_exceptions.HttpError({'status': 503}, None, None) 335 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 336 self.assertTrue(isinstance(translated_exc, ResumableUploadException)) 337 338 gsutil_api.http.disable_ssl_certificate_validation = False 339 exc = apitools_exceptions.HttpError({'status': 429}, None, None) 340 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 341 self.assertTrue(isinstance(translated_exc, ResumableUploadException)) 342 343 exc = apitools_exceptions.HttpError({'status': 410}, None, None) 344 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 345 self.assertTrue( 346 isinstance(translated_exc, ResumableUploadStartOverException)) 347 348 exc = apitools_exceptions.HttpError({'status': 404}, None, None) 349 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 350 self.assertTrue( 351 isinstance(translated_exc, ResumableUploadStartOverException)) 352 353 exc = apitools_exceptions.HttpError({'status': 401}, None, None) 354 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 355 self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) 356 357 exc = apitools_exceptions.TransferError('Aborting transfer') 358 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 359 self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) 360 exc = apitools_exceptions.TransferError('additional bytes left in stream') 361 translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) 362 self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) 363 self.assertIn('this can happen if a file changes size', 364 translated_exc.reason) 365 366 def testSetContentTypeFromFile(self): 367 """Tests that content type is correctly determined for symlinks.""" 368 if system_util.IS_WINDOWS: 369 return unittest.skip('use_magicfile features not available on Windows') 370 371 surprise_html = b'<html><body>And you thought I was just text!</body></html>' 372 temp_dir_path = self.CreateTempDir() 373 txt_file_path = self.CreateTempFile(tmpdir=temp_dir_path, 374 contents=surprise_html, 375 file_name='html_in_disguise.txt') 376 link_name = 'link_to_realfile' # Notice no file extension was supplied. 377 os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name) 378 # Content-type of a symlink should be obtained from the link's target. 379 dst_obj_metadata_mock = mock.MagicMock(contentType=None) 380 src_url_stub = mock.MagicMock(object_name=temp_dir_path + os.path.sep + 381 link_name, 382 **{ 383 'IsFileUrl.return_value': True, 384 'IsStream.return_value': False, 385 'IsFifo.return_value': False 386 }) 387 388 # The file command should detect HTML in the real file. 389 with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]): 390 _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) 391 self.assertEqual('text/html; charset=us-ascii', 392 dst_obj_metadata_mock.contentType) 393 394 dst_obj_metadata_mock = mock.MagicMock(contentType=None) 395 # The mimetypes module should guess based on the real file's extension. 396 with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]): 397 _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) 398 self.assertEqual('text/plain', dst_obj_metadata_mock.contentType) 399 400 _PI_DAY = datetime.datetime(2016, 3, 14, 15, 9, 26) 401 402 @mock.patch('time.time', 403 new=mock.MagicMock( 404 return_value=posix_util.ConvertDatetimeToPOSIX(_PI_DAY))) 405 def testWarnIfMvEarlyDeletionChargeApplies(self): 406 """Tests that WarnIfEarlyDeletionChargeApplies warns when appropriate.""" 407 test_logger = logging.Logger('test') 408 src_url = StorageUrlFromString('gs://bucket/object') 409 410 # Recent nearline objects should generate a warning. 411 for object_time_created in (self._PI_DAY, self._PI_DAY - 412 datetime.timedelta(days=29, hours=23)): 413 recent_nearline_obj = apitools_messages.Object( 414 storageClass='NEARLINE', timeCreated=object_time_created) 415 416 with mock.patch.object(test_logger, 'warn') as mocked_warn: 417 WarnIfMvEarlyDeletionChargeApplies(src_url, recent_nearline_obj, 418 test_logger) 419 mocked_warn.assert_called_with( 420 'Warning: moving %s object %s may incur an early deletion ' 421 'charge, because the original object is less than %s days old ' 422 'according to the local system time.', 'nearline', 423 src_url.url_string, 30) 424 425 # Recent coldine objects should generate a warning. 426 for object_time_created in (self._PI_DAY, self._PI_DAY - 427 datetime.timedelta(days=89, hours=23)): 428 recent_nearline_obj = apitools_messages.Object( 429 storageClass='COLDLINE', timeCreated=object_time_created) 430 431 with mock.patch.object(test_logger, 'warn') as mocked_warn: 432 WarnIfMvEarlyDeletionChargeApplies(src_url, recent_nearline_obj, 433 test_logger) 434 mocked_warn.assert_called_with( 435 'Warning: moving %s object %s may incur an early deletion ' 436 'charge, because the original object is less than %s days old ' 437 'according to the local system time.', 'coldline', 438 src_url.url_string, 90) 439 440 # Recent archive objects should generate a warning. 441 for object_time_created in (self._PI_DAY, self._PI_DAY - 442 datetime.timedelta(days=364, hours=23)): 443 recent_archive_obj = apitools_messages.Object( 444 storageClass='ARCHIVE', timeCreated=object_time_created) 445 446 with mock.patch.object(test_logger, 'warn') as mocked_warn: 447 WarnIfMvEarlyDeletionChargeApplies(src_url, recent_archive_obj, 448 test_logger) 449 mocked_warn.assert_called_with( 450 'Warning: moving %s object %s may incur an early deletion ' 451 'charge, because the original object is less than %s days old ' 452 'according to the local system time.', 'archive', 453 src_url.url_string, 365) 454 455 # Sufficiently old objects should not generate a warning. 456 with mock.patch.object(test_logger, 'warn') as mocked_warn: 457 old_nearline_obj = apitools_messages.Object( 458 storageClass='NEARLINE', 459 timeCreated=self._PI_DAY - datetime.timedelta(days=30, seconds=1)) 460 WarnIfMvEarlyDeletionChargeApplies(src_url, old_nearline_obj, test_logger) 461 mocked_warn.assert_not_called() 462 with mock.patch.object(test_logger, 'warn') as mocked_warn: 463 old_coldline_obj = apitools_messages.Object( 464 storageClass='COLDLINE', 465 timeCreated=self._PI_DAY - datetime.timedelta(days=90, seconds=1)) 466 WarnIfMvEarlyDeletionChargeApplies(src_url, old_coldline_obj, test_logger) 467 mocked_warn.assert_not_called() 468 with mock.patch.object(test_logger, 'warn') as mocked_warn: 469 old_archive_obj = apitools_messages.Object( 470 storageClass='ARCHIVE', 471 timeCreated=self._PI_DAY - datetime.timedelta(days=365, seconds=1)) 472 WarnIfMvEarlyDeletionChargeApplies(src_url, old_archive_obj, test_logger) 473 mocked_warn.assert_not_called() 474 475 # Recent standard storage class object should not generate a warning. 476 with mock.patch.object(test_logger, 'warn') as mocked_warn: 477 not_old_enough_nearline_obj = apitools_messages.Object( 478 storageClass='STANDARD', timeCreated=self._PI_DAY) 479 WarnIfMvEarlyDeletionChargeApplies(src_url, not_old_enough_nearline_obj, 480 test_logger) 481 mocked_warn.assert_not_called() 482 483 def testSelectUploadCompressionStrategyAll(self): 484 paths = ('file://test', 'test.xml', 'test.py') 485 exts = GZIP_ALL_FILES 486 for path in paths: 487 zipped, gzip_encoded = _SelectUploadCompressionStrategy( 488 path, False, exts, False) 489 self.assertTrue(zipped) 490 self.assertFalse(gzip_encoded) 491 zipped, gzip_encoded = _SelectUploadCompressionStrategy( 492 path, False, exts, True) 493 self.assertFalse(zipped) 494 self.assertTrue(gzip_encoded) 495 496 def testSelectUploadCompressionStrategyFilter(self): 497 zipped, gzip_encoded = _SelectUploadCompressionStrategy( 498 'test.xml', False, ['xml'], False) 499 self.assertTrue(zipped) 500 self.assertFalse(gzip_encoded) 501 zipped, gzip_encoded = _SelectUploadCompressionStrategy( 502 'test.xml', False, ['yaml'], False) 503 self.assertFalse(zipped) 504 self.assertFalse(gzip_encoded) 505 506 def testSelectUploadCompressionStrategyComponent(self): 507 zipped, gzip_encoded = _SelectUploadCompressionStrategy( 508 'test.xml', True, ['not_matching'], True) 509 self.assertFalse(zipped) 510 self.assertTrue(gzip_encoded) 511 512 def testDelegateUploadFileToObjectNormal(self): 513 mock_stream = mock.Mock() 514 mock_stream.close = mock.Mock() 515 516 def DelegateUpload(): 517 return 'a', 'b' 518 519 elapsed_time, uploaded_object = _DelegateUploadFileToObject( 520 DelegateUpload, 'url', mock_stream, False, False, False, None) 521 # Ensure results are passed through. 522 self.assertEqual(elapsed_time, 'a') 523 self.assertEqual(uploaded_object, 'b') 524 # Ensure close was called. 525 self.assertTrue(mock_stream.close.called) 526 527 @mock.patch('os.unlink') 528 def testDelegateUploadFileToObjectZipped(self, mock_unlink): 529 mock_stream = mock.Mock() 530 mock_stream.close = mock.Mock() 531 mock_upload_url = mock.Mock() 532 mock_upload_url.object_name = 'Sample' 533 534 def DelegateUpload(): 535 return 'a', 'b' 536 537 elapsed_time, uploaded_object = _DelegateUploadFileToObject( 538 DelegateUpload, mock_upload_url, mock_stream, True, False, False, None) 539 # Ensure results are passed through. 540 self.assertEqual(elapsed_time, 'a') 541 self.assertEqual(uploaded_object, 'b') 542 # Ensure the file was unlinked. 543 self.assertTrue(mock_unlink.called) 544 # Ensure close was called. 545 self.assertTrue(mock_stream.close.called) 546 547 @mock.patch('gslib.command.concurrent_compressed_upload_lock') 548 def testDelegateUploadFileToObjectGzipEncoded(self, mock_lock): 549 mock_stream = mock.Mock() 550 mock_stream.close = mock.Mock() 551 552 def DelegateUpload(): 553 # Ensure the lock was aquired before the delegate was called. 554 self.assertTrue(mock_lock.__enter__.called) 555 return 'a', 'b' 556 557 elapsed_time, uploaded_object = _DelegateUploadFileToObject( 558 DelegateUpload, 'url', mock_stream, False, True, False, None) 559 # Ensure results are passed through. 560 self.assertEqual(elapsed_time, 'a') 561 self.assertEqual(uploaded_object, 'b') 562 # Ensure close was called. 563 self.assertTrue(mock_stream.close.called) 564 # Ensure the lock was released. 565 self.assertTrue(mock_lock.__exit__.called) 566 567 @mock.patch('gslib.command.concurrent_compressed_upload_lock') 568 def testDelegateUploadFileToObjectGzipEncodedComposite(self, mock_lock): 569 mock_stream = mock.Mock() 570 mock_stream.close = mock.Mock() 571 572 def DelegateUpload(): 573 # Ensure the lock was not aquired before the delegate was called. 574 self.assertFalse(mock_lock.__enter__.called) 575 return 'a', 'b' 576 577 elapsed_time, uploaded_object = _DelegateUploadFileToObject( 578 DelegateUpload, 'url', mock_stream, False, True, True, None) 579 # Ensure results are passed through. 580 self.assertEqual(elapsed_time, 'a') 581 self.assertEqual(uploaded_object, 'b') 582 # Ensure close was called. 583 self.assertTrue(mock_stream.close.called) 584 # Ensure the lock was released. 585 self.assertFalse(mock_lock.__exit__.called) 586 587 588class TestExpandUrlToSingleBlr(GsUtilUnitTestCase): 589 590 @mock.patch('gslib.cloud_api.CloudApi') 591 @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') 592 def testContainsWildcardMatchesNotObject(self, mock_CreateWildcardIterator, 593 mock_gsutil_api): 594 storage_url = StorageUrlFromString('gs://test/helloworld') 595 mock_CreateWildcardIterator.return_value = iter( 596 [BucketListingPrefix(storage_url)]) 597 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 598 'gs://test/hello*/', mock_gsutil_api, 'project_id', False, 599 CreateOrGetGsutilLogger('copy_test')) 600 601 self.assertTrue(have_existing_dst_container) 602 self.assertEqual(exp_url, storage_url) 603 604 @mock.patch('gslib.cloud_api.CloudApi') 605 @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') 606 def testContainsWildcardMatchesObject(self, mock_CreateWildcardIterator, 607 mock_gsutil_api): 608 storage_url = StorageUrlFromString('gs://test/helloworld') 609 mock_CreateWildcardIterator.return_value = iter( 610 [BucketListingObject(storage_url)]) 611 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 612 'gs://test/hello*/', mock_gsutil_api, 'project_id', False, 613 CreateOrGetGsutilLogger('copy_test')) 614 615 self.assertFalse(have_existing_dst_container) 616 self.assertEqual(exp_url, storage_url) 617 618 @mock.patch('gslib.cloud_api.CloudApi') 619 @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') 620 def testContainsWildcardMultipleMatches(self, mock_CreateWildcardIterator, 621 mock_gsutil_api): 622 mock_CreateWildcardIterator.return_value = iter([ 623 BucketListingObject(StorageUrlFromString('gs://test/helloworld')), 624 BucketListingObject(StorageUrlFromString('gs://test/helloworld2')) 625 ]) 626 with self.assertRaises(CommandException): 627 ExpandUrlToSingleBlr('gs://test/hello*/', mock_gsutil_api, 'project_id', 628 False, CreateOrGetGsutilLogger('copy_test')) 629 630 @mock.patch('gslib.cloud_api.CloudApi') 631 @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') 632 def testContainsWildcardNoMatches(self, mock_CreateWildcardIterator, 633 mock_gsutil_api): 634 mock_CreateWildcardIterator.return_value = iter([]) 635 with self.assertRaises(CommandException): 636 ExpandUrlToSingleBlr('gs://test/hello*/', mock_gsutil_api, 'project_id', 637 False, CreateOrGetGsutilLogger('copy_test')) 638 639 @mock.patch('gslib.cloud_api.CloudApi') 640 @mock.patch('gslib.utils.copy_helper.StorageUrlFromString') 641 def testLocalFileDirectory(self, mock_StorageUrlFromString, mock_gsutil_api): 642 mock_storage_url = mock.Mock() 643 mock_storage_url.isFileUrl.return_value = True 644 mock_storage_url.IsDirectory.return_value = True 645 mock_StorageUrlFromString.return_value = mock_storage_url 646 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 647 '/home/test', mock_gsutil_api, 'project_id', False, 648 CreateOrGetGsutilLogger('copy_test')) 649 650 self.assertTrue(have_existing_dst_container) 651 self.assertEqual(exp_url, mock_storage_url) 652 653 @mock.patch('gslib.cloud_api.CloudApi') 654 @mock.patch('gslib.utils.copy_helper.StorageUrlFromString') 655 def testLocalFileNotDirectory(self, mock_StorageUrlFromString, 656 mock_gsutil_api): 657 mock_storage_url = mock.Mock() 658 mock_storage_url.isFileUrl.return_value = True 659 mock_storage_url.IsDirectory.return_value = False 660 mock_StorageUrlFromString.return_value = mock_storage_url 661 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 662 '/home/test', mock_gsutil_api, 'project_id', False, 663 CreateOrGetGsutilLogger('copy_test')) 664 665 self.assertFalse(have_existing_dst_container) 666 self.assertEqual(exp_url, mock_storage_url) 667 668 @mock.patch('gslib.cloud_api.CloudApi') 669 def testNoSlashPrefixExactMatch(self, mock_gsutil_api): 670 mock_gsutil_api.ListObjects.return_value = iter([ 671 CloudApi.CsObjectOrPrefix('folder/', 672 CloudApi.CsObjectOrPrefixType.PREFIX) 673 ]) 674 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 675 'gs://test/folder', mock_gsutil_api, 'project_id', False, 676 CreateOrGetGsutilLogger('copy_test')) 677 678 self.assertTrue(have_existing_dst_container) 679 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) 680 681 @mock.patch('gslib.cloud_api.CloudApi') 682 def testNoSlashPrefixSubstringMatch(self, mock_gsutil_api): 683 mock_gsutil_api.ListObjects.return_value = iter([ 684 CloudApi.CsObjectOrPrefix('folderone/', 685 CloudApi.CsObjectOrPrefixType.PREFIX) 686 ]) 687 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 688 'gs://test/folder', mock_gsutil_api, 'project_id', False, 689 CreateOrGetGsutilLogger('copy_test')) 690 691 self.assertFalse(have_existing_dst_container) 692 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) 693 694 @mock.patch('gslib.cloud_api.CloudApi') 695 def testNoSlashFolderPlaceholder(self, mock_gsutil_api): 696 mock_gsutil_api.ListObjects.return_value = iter([ 697 CloudApi.CsObjectOrPrefix( 698 apitools_messages.Object(name='folder_$folder$'), 699 CloudApi.CsObjectOrPrefixType.OBJECT) 700 ]) 701 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 702 'gs://test/folder', mock_gsutil_api, 'project_id', False, 703 CreateOrGetGsutilLogger('copy_test')) 704 705 self.assertTrue(have_existing_dst_container) 706 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) 707 708 @mock.patch('gslib.cloud_api.CloudApi') 709 def testNoSlashNoMatch(self, mock_gsutil_api): 710 mock_gsutil_api.ListObjects.return_value = iter([]) 711 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 712 'gs://test/folder', mock_gsutil_api, 'project_id', False, 713 CreateOrGetGsutilLogger('copy_test')) 714 715 self.assertFalse(have_existing_dst_container) 716 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) 717 718 @mock.patch('gslib.cloud_api.CloudApi') 719 def testWithSlashPrefixExactMatch(self, mock_gsutil_api): 720 mock_gsutil_api.ListObjects.return_value = iter([ 721 CloudApi.CsObjectOrPrefix('folder/', 722 CloudApi.CsObjectOrPrefixType.PREFIX) 723 ]) 724 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 725 'gs://test/folder/', mock_gsutil_api, 'project_id', False, 726 CreateOrGetGsutilLogger('copy_test')) 727 728 self.assertTrue(have_existing_dst_container) 729 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder/')) 730 731 @mock.patch('gslib.cloud_api.CloudApi') 732 def testWithSlashNoMatch(self, mock_gsutil_api): 733 mock_gsutil_api.ListObjects.return_value = iter([]) 734 (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 735 'gs://test/folder/', mock_gsutil_api, 'project_id', False, 736 CreateOrGetGsutilLogger('copy_test')) 737 738 self.assertTrue(have_existing_dst_container) 739 self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder/')) 740