1# -*- coding: utf-8 -*- 2# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ 3# Copyright (c) 2010, Eucalyptus Systems, Inc. 4# Copyright (c) 2011, Nexenta Systems, Inc. 5# Copyright (c) 2012, Google, Inc. 6# All rights reserved. 7# 8# Permission is hereby granted, free of charge, to any person obtaining a 9# copy of this software and associated documentation files (the 10# "Software"), to deal in the Software without restriction, including 11# without limitation the rights to use, copy, modify, merge, publish, dis- 12# tribute, sublicense, and/or sell copies of the Software, and to permit 13# persons to whom the Software is furnished to do so, subject to the fol- 14# lowing conditions: 15# 16# The above copyright notice and this permission notice shall be included 17# in all copies or substantial portions of the Software. 18# 19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 21# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 22# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 25# IN THE SOFTWARE. 26 27""" 28Some integration tests for the GSConnection 29""" 30 31import os 32import re 33import StringIO 34import urllib 35import xml.sax 36 37from boto import handler 38from boto import storage_uri 39from boto.gs.acl import ACL 40from boto.gs.cors import Cors 41from boto.gs.lifecycle import LifecycleConfig 42from tests.integration.gs.testcase import GSTestCase 43 44 45CORS_EMPTY = '<CorsConfig></CorsConfig>' 46CORS_DOC = ('<CorsConfig><Cors><Origins><Origin>origin1.example.com' 47 '</Origin><Origin>origin2.example.com</Origin></Origins>' 48 '<Methods><Method>GET</Method><Method>PUT</Method>' 49 '<Method>POST</Method></Methods><ResponseHeaders>' 50 '<ResponseHeader>foo</ResponseHeader>' 51 '<ResponseHeader>bar</ResponseHeader></ResponseHeaders>' 52 '</Cors></CorsConfig>') 53 54ENCRYPTION_CONFIG_WITH_KEY = ( 55 '<?xml version="1.0" encoding="UTF-8"?>\n' 56 '<EncryptionConfiguration>' 57 '<DefaultKmsKeyName>%s</DefaultKmsKeyName>' 58 '</EncryptionConfiguration>') 59 60LIFECYCLE_EMPTY = ('<?xml version="1.0" encoding="UTF-8"?>' 61 '<LifecycleConfiguration></LifecycleConfiguration>') 62LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>' 63 '<LifecycleConfiguration><Rule>' 64 '<Action><Delete/></Action>' 65 '<Condition>''<IsLive>true</IsLive>' 66 '<MatchesStorageClass>STANDARD</MatchesStorageClass>' 67 '<Age>365</Age>' 68 '<CreatedBefore>2013-01-15</CreatedBefore>' 69 '<NumberOfNewerVersions>3</NumberOfNewerVersions>' 70 '</Condition></Rule><Rule>' 71 '<Action><SetStorageClass>NEARLINE</SetStorageClass></Action>' 72 '<Condition><Age>366</Age>' 73 '</Condition></Rule></LifecycleConfiguration>') 74LIFECYCLE_CONDITIONS_FOR_DELETE_RULE = { 75 'Age': '365', 76 'CreatedBefore': '2013-01-15', 77 'NumberOfNewerVersions': '3', 78 'IsLive': 'true', 79 'MatchesStorageClass': ['STANDARD']} 80LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE = {'Age': '366'} 81 82BILLING_EMPTY = {'BillingConfiguration': {}} 83BILLING_ENABLED = {'BillingConfiguration': {'RequesterPays': 'Enabled'}} 84BILLING_DISABLED = {'BillingConfiguration': {'RequesterPays': 'Disabled'}} 85 86# Regexp for matching project-private default object ACL. 87PROJECT_PRIVATE_RE = ('\s*<AccessControlList>\s*<Entries>\s*<Entry>' 88 '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' 89 '\s*(<Name>[^<]+</Name>)?\s*</Scope>' 90 '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' 91 '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' 92 '\s*(<Name>[^<]+</Name>)?\s*</Scope>' 93 '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' 94 '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' 95 '\s*(<Name>[^<]+</Name>)?\s*</Scope>' 96 '\s*<Permission>READ</Permission>\s*</Entry>\s*</Entries>' 97 '\s*</AccessControlList>\s*') 98 99 100class GSBasicTest(GSTestCase): 101 """Tests some basic GCS functionality.""" 102 103 def test_read_write(self): 104 """Tests basic read/write to keys.""" 105 bucket = self._MakeBucket() 106 bucket_name = bucket.name 107 # now try a get_bucket call and see if it's really there 108 bucket = self._GetConnection().get_bucket(bucket_name) 109 key_name = 'foobar' 110 k = bucket.new_key(key_name) 111 s1 = 'This is a test of file upload and download' 112 k.set_contents_from_string(s1) 113 tmpdir = self._MakeTempDir() 114 fpath = os.path.join(tmpdir, key_name) 115 fp = open(fpath, 'wb') 116 # now get the contents from gcs to a local file 117 k.get_contents_to_file(fp) 118 fp.close() 119 fp = open(fpath) 120 # check to make sure content read from gcs is identical to original 121 self.assertEqual(s1, fp.read()) 122 fp.close() 123 # Use generate_url to get the contents 124 url = self._conn.generate_url(900, 'GET', bucket=bucket.name, key=key_name) 125 f = urllib.urlopen(url) 126 self.assertEqual(s1, f.read()) 127 f.close() 128 # check to make sure set_contents_from_file is working 129 sfp = StringIO.StringIO('foo') 130 k.set_contents_from_file(sfp) 131 self.assertEqual(k.get_contents_as_string(), 'foo') 132 sfp2 = StringIO.StringIO('foo2') 133 k.set_contents_from_file(sfp2) 134 self.assertEqual(k.get_contents_as_string(), 'foo2') 135 136 def test_get_all_keys(self): 137 """Tests get_all_keys.""" 138 phony_mimetype = 'application/x-boto-test' 139 headers = {'Content-Type': phony_mimetype} 140 tmpdir = self._MakeTempDir() 141 fpath = os.path.join(tmpdir, 'foobar1') 142 fpath2 = os.path.join(tmpdir, 'foobar') 143 with open(fpath2, 'w') as f: 144 f.write('test-data') 145 bucket = self._MakeBucket() 146 147 # First load some data for the first one, overriding content type. 148 k = bucket.new_key('foobar') 149 s1 = 'test-contents' 150 s2 = 'test-contents2' 151 k.name = 'foo/bar' 152 k.set_contents_from_string(s1, headers) 153 k.name = 'foo/bas' 154 k.set_contents_from_filename(fpath2) 155 k.name = 'foo/bat' 156 k.set_contents_from_string(s1) 157 k.name = 'fie/bar' 158 k.set_contents_from_string(s1) 159 k.name = 'fie/bas' 160 k.set_contents_from_string(s1) 161 k.name = 'fie/bat' 162 k.set_contents_from_string(s1) 163 # try resetting the contents to another value 164 md5 = k.md5 165 k.set_contents_from_string(s2) 166 self.assertNotEqual(k.md5, md5) 167 168 fp2 = open(fpath2, 'rb') 169 k.md5 = None 170 k.base64md5 = None 171 k.set_contents_from_stream(fp2) 172 fp = open(fpath, 'wb') 173 k.get_contents_to_file(fp) 174 fp.close() 175 fp2.seek(0, 0) 176 fp = open(fpath, 'rb') 177 self.assertEqual(fp2.read(), fp.read()) 178 fp.close() 179 fp2.close() 180 all = bucket.get_all_keys() 181 self.assertEqual(len(all), 6) 182 rs = bucket.get_all_keys(prefix='foo') 183 self.assertEqual(len(rs), 3) 184 rs = bucket.get_all_keys(prefix='', delimiter='/') 185 self.assertEqual(len(rs), 2) 186 rs = bucket.get_all_keys(maxkeys=5) 187 self.assertEqual(len(rs), 5) 188 189 def test_bucket_lookup(self): 190 """Test the bucket lookup method.""" 191 bucket = self._MakeBucket() 192 k = bucket.new_key('foo/bar') 193 phony_mimetype = 'application/x-boto-test' 194 headers = {'Content-Type': phony_mimetype} 195 k.set_contents_from_string('testdata', headers) 196 197 k = bucket.lookup('foo/bar') 198 self.assertIsInstance(k, bucket.key_class) 199 self.assertEqual(k.content_type, phony_mimetype) 200 k = bucket.lookup('notthere') 201 self.assertIsNone(k) 202 203 def test_metadata(self): 204 """Test key metadata operations.""" 205 bucket = self._MakeBucket() 206 k = self._MakeKey(bucket=bucket) 207 key_name = k.name 208 s1 = 'This is a test of file upload and download' 209 210 mdkey1 = 'meta1' 211 mdval1 = 'This is the first metadata value' 212 k.set_metadata(mdkey1, mdval1) 213 mdkey2 = 'meta2' 214 mdval2 = 'This is the second metadata value' 215 k.set_metadata(mdkey2, mdval2) 216 217 # Test unicode character. 218 mdval3 = u'föö' 219 mdkey3 = 'meta3' 220 k.set_metadata(mdkey3, mdval3) 221 k.set_contents_from_string(s1) 222 223 k = bucket.lookup(key_name) 224 self.assertEqual(k.get_metadata(mdkey1), mdval1) 225 self.assertEqual(k.get_metadata(mdkey2), mdval2) 226 self.assertEqual(k.get_metadata(mdkey3), mdval3) 227 k = bucket.new_key(key_name) 228 k.get_contents_as_string() 229 self.assertEqual(k.get_metadata(mdkey1), mdval1) 230 self.assertEqual(k.get_metadata(mdkey2), mdval2) 231 self.assertEqual(k.get_metadata(mdkey3), mdval3) 232 233 def test_list_iterator(self): 234 """Test list and iterator.""" 235 bucket = self._MakeBucket() 236 num_iter = len([k for k in bucket.list()]) 237 rs = bucket.get_all_keys() 238 num_keys = len(rs) 239 self.assertEqual(num_iter, num_keys) 240 241 def test_acl(self): 242 """Test bucket and key ACLs.""" 243 bucket = self._MakeBucket() 244 245 # try some acl stuff 246 bucket.set_acl('public-read') 247 acl = bucket.get_acl() 248 self.assertEqual(len(acl.entries.entry_list), 2) 249 bucket.set_acl('private') 250 acl = bucket.get_acl() 251 self.assertEqual(len(acl.entries.entry_list), 1) 252 k = self._MakeKey(bucket=bucket) 253 k.set_acl('public-read') 254 acl = k.get_acl() 255 self.assertEqual(len(acl.entries.entry_list), 2) 256 k.set_acl('private') 257 acl = k.get_acl() 258 self.assertEqual(len(acl.entries.entry_list), 1) 259 260 # Test case-insensitivity of XML ACL parsing. 261 acl_xml = ( 262 '<ACCESSControlList><EntrIes><Entry>' + 263 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + 264 '</Entry></EntrIes></ACCESSControlList>') 265 acl = ACL() 266 h = handler.XmlHandler(acl, bucket) 267 xml.sax.parseString(acl_xml, h) 268 bucket.set_acl(acl) 269 self.assertEqual(len(acl.entries.entry_list), 1) 270 aclstr = k.get_xml_acl() 271 self.assertGreater(aclstr.count('/Entry', 1), 0) 272 273 def test_logging(self): 274 """Test set/get raw logging subresource.""" 275 bucket = self._MakeBucket() 276 empty_logging_str="<?xml version='1.0' encoding='UTF-8'?><Logging/>" 277 logging_str = ( 278 "<?xml version='1.0' encoding='UTF-8'?><Logging>" 279 "<LogBucket>log-bucket</LogBucket>" + 280 "<LogObjectPrefix>example</LogObjectPrefix>" + 281 "</Logging>") 282 bucket.set_subresource('logging', logging_str) 283 self.assertEqual(bucket.get_subresource('logging'), logging_str) 284 # try disable/enable logging 285 bucket.disable_logging() 286 self.assertEqual(bucket.get_subresource('logging'), empty_logging_str) 287 bucket.enable_logging('log-bucket', 'example') 288 self.assertEqual(bucket.get_subresource('logging'), logging_str) 289 290 def test_copy_key(self): 291 """Test copying a key from one bucket to another.""" 292 # create two new, empty buckets 293 bucket1 = self._MakeBucket() 294 bucket2 = self._MakeBucket() 295 bucket_name_1 = bucket1.name 296 bucket_name_2 = bucket2.name 297 # verify buckets got created 298 bucket1 = self._GetConnection().get_bucket(bucket_name_1) 299 bucket2 = self._GetConnection().get_bucket(bucket_name_2) 300 # create a key in bucket1 and give it some content 301 key_name = 'foobar' 302 k1 = bucket1.new_key(key_name) 303 self.assertIsInstance(k1, bucket1.key_class) 304 k1.name = key_name 305 s = 'This is a test.' 306 k1.set_contents_from_string(s) 307 # copy the new key from bucket1 to bucket2 308 k1.copy(bucket_name_2, key_name) 309 # now copy the contents from bucket2 to a local file 310 k2 = bucket2.lookup(key_name) 311 self.assertIsInstance(k2, bucket2.key_class) 312 tmpdir = self._MakeTempDir() 313 fpath = os.path.join(tmpdir, 'foobar') 314 fp = open(fpath, 'wb') 315 k2.get_contents_to_file(fp) 316 fp.close() 317 fp = open(fpath) 318 # check to make sure content read is identical to original 319 self.assertEqual(s, fp.read()) 320 fp.close() 321 # delete keys 322 bucket1.delete_key(k1) 323 bucket2.delete_key(k2) 324 325 def test_default_object_acls(self): 326 """Test default object acls.""" 327 # create a new bucket 328 bucket = self._MakeBucket() 329 # get default acl and make sure it's project-private 330 acl = bucket.get_def_acl() 331 self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml())) 332 # set default acl to a canned acl and verify it gets set 333 bucket.set_def_acl('public-read') 334 acl = bucket.get_def_acl() 335 # save public-read acl for later test 336 public_read_acl = acl 337 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 338 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 339 '</Entry></Entries></AccessControlList>')) 340 # back to private acl 341 bucket.set_def_acl('private') 342 acl = bucket.get_def_acl() 343 self.assertEqual(acl.to_xml(), 344 '<AccessControlList></AccessControlList>') 345 # set default acl to an xml acl and verify it gets set 346 bucket.set_def_acl(public_read_acl) 347 acl = bucket.get_def_acl() 348 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 349 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 350 '</Entry></Entries></AccessControlList>')) 351 # back to private acl 352 bucket.set_def_acl('private') 353 acl = bucket.get_def_acl() 354 self.assertEqual(acl.to_xml(), 355 '<AccessControlList></AccessControlList>') 356 357 def test_default_object_acls_storage_uri(self): 358 """Test default object acls using storage_uri.""" 359 # create a new bucket 360 bucket = self._MakeBucket() 361 bucket_name = bucket.name 362 uri = storage_uri('gs://' + bucket_name) 363 # get default acl and make sure it's project-private 364 acl = uri.get_def_acl() 365 self.assertIsNotNone( 366 re.search(PROJECT_PRIVATE_RE, acl.to_xml()), 367 'PROJECT_PRIVATE_RE not found in ACL XML:\n' + acl.to_xml()) 368 # set default acl to a canned acl and verify it gets set 369 uri.set_def_acl('public-read') 370 acl = uri.get_def_acl() 371 # save public-read acl for later test 372 public_read_acl = acl 373 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 374 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 375 '</Entry></Entries></AccessControlList>')) 376 # back to private acl 377 uri.set_def_acl('private') 378 acl = uri.get_def_acl() 379 self.assertEqual(acl.to_xml(), 380 '<AccessControlList></AccessControlList>') 381 # set default acl to an xml acl and verify it gets set 382 uri.set_def_acl(public_read_acl) 383 acl = uri.get_def_acl() 384 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 385 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 386 '</Entry></Entries></AccessControlList>')) 387 # back to private acl 388 uri.set_def_acl('private') 389 acl = uri.get_def_acl() 390 self.assertEqual(acl.to_xml(), 391 '<AccessControlList></AccessControlList>') 392 393 def test_cors_xml_bucket(self): 394 """Test setting and getting of CORS XML documents on Bucket.""" 395 # create a new bucket 396 bucket = self._MakeBucket() 397 bucket_name = bucket.name 398 # now call get_bucket to see if it's really there 399 bucket = self._GetConnection().get_bucket(bucket_name) 400 # get new bucket cors and make sure it's empty 401 cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) 402 self.assertEqual(cors, CORS_EMPTY) 403 # set cors document on new bucket 404 bucket.set_cors(CORS_DOC) 405 cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) 406 self.assertEqual(cors, CORS_DOC) 407 408 def test_cors_xml_storage_uri(self): 409 """Test setting and getting of CORS XML documents with storage_uri.""" 410 # create a new bucket 411 bucket = self._MakeBucket() 412 bucket_name = bucket.name 413 uri = storage_uri('gs://' + bucket_name) 414 # get new bucket cors and make sure it's empty 415 cors = re.sub(r'\s', '', uri.get_cors().to_xml()) 416 self.assertEqual(cors, CORS_EMPTY) 417 # set cors document on new bucket 418 cors_obj = Cors() 419 h = handler.XmlHandler(cors_obj, None) 420 xml.sax.parseString(CORS_DOC, h) 421 uri.set_cors(cors_obj) 422 cors = re.sub(r'\s', '', uri.get_cors().to_xml()) 423 self.assertEqual(cors, CORS_DOC) 424 425 def test_lifecycle_config_bucket(self): 426 """Test setting and getting of lifecycle config on Bucket.""" 427 # create a new bucket 428 bucket = self._MakeBucket() 429 bucket_name = bucket.name 430 # now call get_bucket to see if it's really there 431 bucket = self._GetConnection().get_bucket(bucket_name) 432 # get lifecycle config and make sure it's empty 433 xml = bucket.get_lifecycle_config().to_xml() 434 self.assertEqual(xml, LIFECYCLE_EMPTY) 435 # set lifecycle config 436 lifecycle_config = LifecycleConfig() 437 lifecycle_config.add_rule( 438 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE) 439 lifecycle_config.add_rule( 440 'SetStorageClass', 'NEARLINE', 441 LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE) 442 bucket.configure_lifecycle(lifecycle_config) 443 xml = bucket.get_lifecycle_config().to_xml() 444 self.assertEqual(xml, LIFECYCLE_DOC) 445 446 def test_lifecycle_config_storage_uri(self): 447 """Test setting and getting of lifecycle config with storage_uri.""" 448 # create a new bucket 449 bucket = self._MakeBucket() 450 bucket_name = bucket.name 451 uri = storage_uri('gs://' + bucket_name) 452 # get lifecycle config and make sure it's empty 453 xml = uri.get_lifecycle_config().to_xml() 454 self.assertEqual(xml, LIFECYCLE_EMPTY) 455 # set lifecycle config 456 lifecycle_config = LifecycleConfig() 457 lifecycle_config.add_rule( 458 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE) 459 lifecycle_config.add_rule( 460 'SetStorageClass', 'NEARLINE', 461 LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE) 462 uri.configure_lifecycle(lifecycle_config) 463 xml = uri.get_lifecycle_config().to_xml() 464 self.assertEqual(xml, LIFECYCLE_DOC) 465 466 def test_billing_config_bucket(self): 467 """Test setting and getting of billing config on Bucket.""" 468 # create a new bucket 469 bucket = self._MakeBucket() 470 bucket_name = bucket.name 471 # get billing config and make sure it's empty 472 billing = bucket.get_billing_config() 473 self.assertEqual(billing, BILLING_EMPTY) 474 # set requester pays to enabled 475 bucket.configure_billing(requester_pays=True) 476 billing = bucket.get_billing_config() 477 self.assertEqual(billing, BILLING_ENABLED) 478 # set requester pays to disabled 479 bucket.configure_billing(requester_pays=False) 480 billing = bucket.get_billing_config() 481 self.assertEqual(billing, BILLING_DISABLED) 482 483 def test_billing_config_storage_uri(self): 484 """Test setting and getting of billing config with storage_uri.""" 485 # create a new bucket 486 bucket = self._MakeBucket() 487 bucket_name = bucket.name 488 uri = storage_uri('gs://' + bucket_name) 489 # get billing config and make sure it's empty 490 billing = uri.get_billing_config() 491 self.assertEqual(billing, BILLING_EMPTY) 492 # set requester pays to enabled 493 uri.configure_billing(requester_pays=True) 494 billing = uri.get_billing_config() 495 self.assertEqual(billing, BILLING_ENABLED) 496 # set requester pays to disabled 497 uri.configure_billing(requester_pays=False) 498 billing = uri.get_billing_config() 499 self.assertEqual(billing, BILLING_DISABLED) 500 501 def test_encryption_config_bucket(self): 502 """Test setting and getting of EncryptionConfig on gs Bucket objects.""" 503 # Create a new bucket. 504 bucket = self._MakeBucket() 505 bucket_name = bucket.name 506 # Get EncryptionConfig and make sure it's empty. 507 encryption_config = bucket.get_encryption_config() 508 self.assertIsNone(encryption_config.default_kms_key_name) 509 # Testing set functionality would require having an existing Cloud KMS 510 # key. Since we can't hardcode a key name or dynamically create one, we 511 # only test here that we're creating the correct XML document to send to 512 # GCS. 513 xmldoc = bucket._construct_encryption_config_xml( 514 default_kms_key_name='dummykey') 515 self.assertEqual(xmldoc, ENCRYPTION_CONFIG_WITH_KEY % 'dummykey') 516 # Test that setting an empty encryption config works. 517 bucket.set_encryption_config() 518 519 def test_encryption_config_storage_uri(self): 520 """Test setting and getting of EncryptionConfig with storage_uri.""" 521 # Create a new bucket. 522 bucket = self._MakeBucket() 523 bucket_name = bucket.name 524 uri = storage_uri('gs://' + bucket_name) 525 # Get EncryptionConfig and make sure it's empty. 526 encryption_config = uri.get_encryption_config() 527 self.assertIsNone(encryption_config.default_kms_key_name) 528 529 # Test that setting an empty encryption config works. 530 uri.set_encryption_config() 531