1# -*- coding: utf-8 -*-
2# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/
3# Copyright (c) 2010, Eucalyptus Systems, Inc.
4# Copyright (c) 2011, Nexenta Systems, Inc.
5# Copyright (c) 2012, Google, Inc.
6# All rights reserved.
7#
8# Permission is hereby granted, free of charge, to any person obtaining a
9# copy of this software and associated documentation files (the
10# "Software"), to deal in the Software without restriction, including
11# without limitation the rights to use, copy, modify, merge, publish, dis-
12# tribute, sublicense, and/or sell copies of the Software, and to permit
13# persons to whom the Software is furnished to do so, subject to the fol-
14# lowing conditions:
15#
16# The above copyright notice and this permission notice shall be included
17# in all copies or substantial portions of the Software.
18#
19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
21# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
22# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
23# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25# IN THE SOFTWARE.
26
27"""
28Some integration tests for the GSConnection
29"""
30
31import os
32import re
33import StringIO
34import urllib
35import xml.sax
36
37from boto import handler
38from boto import storage_uri
39from boto.gs.acl import ACL
40from boto.gs.cors import Cors
41from boto.gs.lifecycle import LifecycleConfig
42from tests.integration.gs.testcase import GSTestCase
43
44
45CORS_EMPTY = '<CorsConfig></CorsConfig>'
46CORS_DOC = ('<CorsConfig><Cors><Origins><Origin>origin1.example.com'
47            '</Origin><Origin>origin2.example.com</Origin></Origins>'
48            '<Methods><Method>GET</Method><Method>PUT</Method>'
49            '<Method>POST</Method></Methods><ResponseHeaders>'
50            '<ResponseHeader>foo</ResponseHeader>'
51            '<ResponseHeader>bar</ResponseHeader></ResponseHeaders>'
52            '</Cors></CorsConfig>')
53
54ENCRYPTION_CONFIG_WITH_KEY = (
55    '<?xml version="1.0" encoding="UTF-8"?>\n'
56    '<EncryptionConfiguration>'
57    '<DefaultKmsKeyName>%s</DefaultKmsKeyName>'
58    '</EncryptionConfiguration>')
59
60LIFECYCLE_EMPTY = ('<?xml version="1.0" encoding="UTF-8"?>'
61                   '<LifecycleConfiguration></LifecycleConfiguration>')
62LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>'
63                 '<LifecycleConfiguration><Rule>'
64                 '<Action><Delete/></Action>'
65                 '<Condition>''<IsLive>true</IsLive>'
66                 '<MatchesStorageClass>STANDARD</MatchesStorageClass>'
67                 '<Age>365</Age>'
68                 '<CreatedBefore>2013-01-15</CreatedBefore>'
69                 '<NumberOfNewerVersions>3</NumberOfNewerVersions>'
70                 '</Condition></Rule><Rule>'
71                 '<Action><SetStorageClass>NEARLINE</SetStorageClass></Action>'
72                 '<Condition><Age>366</Age>'
73                 '</Condition></Rule></LifecycleConfiguration>')
74LIFECYCLE_CONDITIONS_FOR_DELETE_RULE = {
75    'Age': '365',
76    'CreatedBefore': '2013-01-15',
77    'NumberOfNewerVersions': '3',
78    'IsLive': 'true',
79    'MatchesStorageClass': ['STANDARD']}
80LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE = {'Age': '366'}
81
82BILLING_EMPTY = {'BillingConfiguration': {}}
83BILLING_ENABLED = {'BillingConfiguration': {'RequesterPays': 'Enabled'}}
84BILLING_DISABLED = {'BillingConfiguration': {'RequesterPays': 'Disabled'}}
85
86# Regexp for matching project-private default object ACL.
87PROJECT_PRIVATE_RE = ('\s*<AccessControlList>\s*<Entries>\s*<Entry>'
88  '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>'
89  '\s*(<Name>[^<]+</Name>)?\s*</Scope>'
90  '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>'
91  '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>'
92  '\s*(<Name>[^<]+</Name>)?\s*</Scope>'
93  '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>'
94  '\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>'
95  '\s*(<Name>[^<]+</Name>)?\s*</Scope>'
96  '\s*<Permission>READ</Permission>\s*</Entry>\s*</Entries>'
97  '\s*</AccessControlList>\s*')
98
99
100class GSBasicTest(GSTestCase):
101    """Tests some basic GCS functionality."""
102
103    def test_read_write(self):
104        """Tests basic read/write to keys."""
105        bucket = self._MakeBucket()
106        bucket_name = bucket.name
107        # now try a get_bucket call and see if it's really there
108        bucket = self._GetConnection().get_bucket(bucket_name)
109        key_name = 'foobar'
110        k = bucket.new_key(key_name)
111        s1 = 'This is a test of file upload and download'
112        k.set_contents_from_string(s1)
113        tmpdir = self._MakeTempDir()
114        fpath = os.path.join(tmpdir, key_name)
115        fp = open(fpath, 'wb')
116        # now get the contents from gcs to a local file
117        k.get_contents_to_file(fp)
118        fp.close()
119        fp = open(fpath)
120        # check to make sure content read from gcs is identical to original
121        self.assertEqual(s1, fp.read())
122        fp.close()
123        # Use generate_url to get the contents
124        url = self._conn.generate_url(900, 'GET', bucket=bucket.name, key=key_name)
125        f = urllib.urlopen(url)
126        self.assertEqual(s1, f.read())
127        f.close()
128        # check to make sure set_contents_from_file is working
129        sfp = StringIO.StringIO('foo')
130        k.set_contents_from_file(sfp)
131        self.assertEqual(k.get_contents_as_string(), 'foo')
132        sfp2 = StringIO.StringIO('foo2')
133        k.set_contents_from_file(sfp2)
134        self.assertEqual(k.get_contents_as_string(), 'foo2')
135
136    def test_get_all_keys(self):
137        """Tests get_all_keys."""
138        phony_mimetype = 'application/x-boto-test'
139        headers = {'Content-Type': phony_mimetype}
140        tmpdir = self._MakeTempDir()
141        fpath = os.path.join(tmpdir, 'foobar1')
142        fpath2 = os.path.join(tmpdir, 'foobar')
143        with open(fpath2, 'w') as f:
144            f.write('test-data')
145        bucket = self._MakeBucket()
146
147        # First load some data for the first one, overriding content type.
148        k = bucket.new_key('foobar')
149        s1 = 'test-contents'
150        s2 = 'test-contents2'
151        k.name = 'foo/bar'
152        k.set_contents_from_string(s1, headers)
153        k.name = 'foo/bas'
154        k.set_contents_from_filename(fpath2)
155        k.name = 'foo/bat'
156        k.set_contents_from_string(s1)
157        k.name = 'fie/bar'
158        k.set_contents_from_string(s1)
159        k.name = 'fie/bas'
160        k.set_contents_from_string(s1)
161        k.name = 'fie/bat'
162        k.set_contents_from_string(s1)
163        # try resetting the contents to another value
164        md5 = k.md5
165        k.set_contents_from_string(s2)
166        self.assertNotEqual(k.md5, md5)
167
168        fp2 = open(fpath2, 'rb')
169        k.md5 = None
170        k.base64md5 = None
171        k.set_contents_from_stream(fp2)
172        fp = open(fpath, 'wb')
173        k.get_contents_to_file(fp)
174        fp.close()
175        fp2.seek(0, 0)
176        fp = open(fpath, 'rb')
177        self.assertEqual(fp2.read(), fp.read())
178        fp.close()
179        fp2.close()
180        all = bucket.get_all_keys()
181        self.assertEqual(len(all), 6)
182        rs = bucket.get_all_keys(prefix='foo')
183        self.assertEqual(len(rs), 3)
184        rs = bucket.get_all_keys(prefix='', delimiter='/')
185        self.assertEqual(len(rs), 2)
186        rs = bucket.get_all_keys(maxkeys=5)
187        self.assertEqual(len(rs), 5)
188
189    def test_bucket_lookup(self):
190        """Test the bucket lookup method."""
191        bucket = self._MakeBucket()
192        k = bucket.new_key('foo/bar')
193        phony_mimetype = 'application/x-boto-test'
194        headers = {'Content-Type': phony_mimetype}
195        k.set_contents_from_string('testdata', headers)
196
197        k = bucket.lookup('foo/bar')
198        self.assertIsInstance(k, bucket.key_class)
199        self.assertEqual(k.content_type, phony_mimetype)
200        k = bucket.lookup('notthere')
201        self.assertIsNone(k)
202
203    def test_metadata(self):
204        """Test key metadata operations."""
205        bucket = self._MakeBucket()
206        k = self._MakeKey(bucket=bucket)
207        key_name = k.name
208        s1 = 'This is a test of file upload and download'
209
210        mdkey1 = 'meta1'
211        mdval1 = 'This is the first metadata value'
212        k.set_metadata(mdkey1, mdval1)
213        mdkey2 = 'meta2'
214        mdval2 = 'This is the second metadata value'
215        k.set_metadata(mdkey2, mdval2)
216
217        # Test unicode character.
218        mdval3 = u'föö'
219        mdkey3 = 'meta3'
220        k.set_metadata(mdkey3, mdval3)
221        k.set_contents_from_string(s1)
222
223        k = bucket.lookup(key_name)
224        self.assertEqual(k.get_metadata(mdkey1), mdval1)
225        self.assertEqual(k.get_metadata(mdkey2), mdval2)
226        self.assertEqual(k.get_metadata(mdkey3), mdval3)
227        k = bucket.new_key(key_name)
228        k.get_contents_as_string()
229        self.assertEqual(k.get_metadata(mdkey1), mdval1)
230        self.assertEqual(k.get_metadata(mdkey2), mdval2)
231        self.assertEqual(k.get_metadata(mdkey3), mdval3)
232
233    def test_list_iterator(self):
234        """Test list and iterator."""
235        bucket = self._MakeBucket()
236        num_iter = len([k for k in bucket.list()])
237        rs = bucket.get_all_keys()
238        num_keys = len(rs)
239        self.assertEqual(num_iter, num_keys)
240
241    def test_acl(self):
242        """Test bucket and key ACLs."""
243        bucket = self._MakeBucket()
244
245        # try some acl stuff
246        bucket.set_acl('public-read')
247        acl = bucket.get_acl()
248        self.assertEqual(len(acl.entries.entry_list), 2)
249        bucket.set_acl('private')
250        acl = bucket.get_acl()
251        self.assertEqual(len(acl.entries.entry_list), 1)
252        k = self._MakeKey(bucket=bucket)
253        k.set_acl('public-read')
254        acl = k.get_acl()
255        self.assertEqual(len(acl.entries.entry_list), 2)
256        k.set_acl('private')
257        acl = k.get_acl()
258        self.assertEqual(len(acl.entries.entry_list), 1)
259
260        # Test case-insensitivity of XML ACL parsing.
261        acl_xml = (
262            '<ACCESSControlList><EntrIes><Entry>'    +
263            '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' +
264            '</Entry></EntrIes></ACCESSControlList>')
265        acl = ACL()
266        h = handler.XmlHandler(acl, bucket)
267        xml.sax.parseString(acl_xml, h)
268        bucket.set_acl(acl)
269        self.assertEqual(len(acl.entries.entry_list), 1)
270        aclstr = k.get_xml_acl()
271        self.assertGreater(aclstr.count('/Entry', 1), 0)
272
273    def test_logging(self):
274        """Test set/get raw logging subresource."""
275        bucket = self._MakeBucket()
276        empty_logging_str="<?xml version='1.0' encoding='UTF-8'?><Logging/>"
277        logging_str = (
278            "<?xml version='1.0' encoding='UTF-8'?><Logging>"
279            "<LogBucket>log-bucket</LogBucket>" +
280            "<LogObjectPrefix>example</LogObjectPrefix>" +
281            "</Logging>")
282        bucket.set_subresource('logging', logging_str)
283        self.assertEqual(bucket.get_subresource('logging'), logging_str)
284        # try disable/enable logging
285        bucket.disable_logging()
286        self.assertEqual(bucket.get_subresource('logging'), empty_logging_str)
287        bucket.enable_logging('log-bucket', 'example')
288        self.assertEqual(bucket.get_subresource('logging'), logging_str)
289
290    def test_copy_key(self):
291        """Test copying a key from one bucket to another."""
292        # create two new, empty buckets
293        bucket1 = self._MakeBucket()
294        bucket2 = self._MakeBucket()
295        bucket_name_1 = bucket1.name
296        bucket_name_2 = bucket2.name
297        # verify buckets got created
298        bucket1 = self._GetConnection().get_bucket(bucket_name_1)
299        bucket2 = self._GetConnection().get_bucket(bucket_name_2)
300        # create a key in bucket1 and give it some content
301        key_name = 'foobar'
302        k1 = bucket1.new_key(key_name)
303        self.assertIsInstance(k1, bucket1.key_class)
304        k1.name = key_name
305        s = 'This is a test.'
306        k1.set_contents_from_string(s)
307        # copy the new key from bucket1 to bucket2
308        k1.copy(bucket_name_2, key_name)
309        # now copy the contents from bucket2 to a local file
310        k2 = bucket2.lookup(key_name)
311        self.assertIsInstance(k2, bucket2.key_class)
312        tmpdir = self._MakeTempDir()
313        fpath = os.path.join(tmpdir, 'foobar')
314        fp = open(fpath, 'wb')
315        k2.get_contents_to_file(fp)
316        fp.close()
317        fp = open(fpath)
318        # check to make sure content read is identical to original
319        self.assertEqual(s, fp.read())
320        fp.close()
321        # delete keys
322        bucket1.delete_key(k1)
323        bucket2.delete_key(k2)
324
325    def test_default_object_acls(self):
326        """Test default object acls."""
327        # create a new bucket
328        bucket = self._MakeBucket()
329        # get default acl and make sure it's project-private
330        acl = bucket.get_def_acl()
331        self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml()))
332        # set default acl to a canned acl and verify it gets set
333        bucket.set_def_acl('public-read')
334        acl = bucket.get_def_acl()
335        # save public-read acl for later test
336        public_read_acl = acl
337        self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
338          '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
339          '</Entry></Entries></AccessControlList>'))
340        # back to private acl
341        bucket.set_def_acl('private')
342        acl = bucket.get_def_acl()
343        self.assertEqual(acl.to_xml(),
344                         '<AccessControlList></AccessControlList>')
345        # set default acl to an xml acl and verify it gets set
346        bucket.set_def_acl(public_read_acl)
347        acl = bucket.get_def_acl()
348        self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
349          '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
350          '</Entry></Entries></AccessControlList>'))
351        # back to private acl
352        bucket.set_def_acl('private')
353        acl = bucket.get_def_acl()
354        self.assertEqual(acl.to_xml(),
355                         '<AccessControlList></AccessControlList>')
356
357    def test_default_object_acls_storage_uri(self):
358        """Test default object acls using storage_uri."""
359        # create a new bucket
360        bucket = self._MakeBucket()
361        bucket_name = bucket.name
362        uri = storage_uri('gs://' + bucket_name)
363        # get default acl and make sure it's project-private
364        acl = uri.get_def_acl()
365        self.assertIsNotNone(
366            re.search(PROJECT_PRIVATE_RE, acl.to_xml()),
367            'PROJECT_PRIVATE_RE not found in ACL XML:\n' + acl.to_xml())
368        # set default acl to a canned acl and verify it gets set
369        uri.set_def_acl('public-read')
370        acl = uri.get_def_acl()
371        # save public-read acl for later test
372        public_read_acl = acl
373        self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
374          '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
375          '</Entry></Entries></AccessControlList>'))
376        # back to private acl
377        uri.set_def_acl('private')
378        acl = uri.get_def_acl()
379        self.assertEqual(acl.to_xml(),
380                         '<AccessControlList></AccessControlList>')
381        # set default acl to an xml acl and verify it gets set
382        uri.set_def_acl(public_read_acl)
383        acl = uri.get_def_acl()
384        self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
385          '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
386          '</Entry></Entries></AccessControlList>'))
387        # back to private acl
388        uri.set_def_acl('private')
389        acl = uri.get_def_acl()
390        self.assertEqual(acl.to_xml(),
391                         '<AccessControlList></AccessControlList>')
392
393    def test_cors_xml_bucket(self):
394        """Test setting and getting of CORS XML documents on Bucket."""
395        # create a new bucket
396        bucket = self._MakeBucket()
397        bucket_name = bucket.name
398        # now call get_bucket to see if it's really there
399        bucket = self._GetConnection().get_bucket(bucket_name)
400        # get new bucket cors and make sure it's empty
401        cors = re.sub(r'\s', '', bucket.get_cors().to_xml())
402        self.assertEqual(cors, CORS_EMPTY)
403        # set cors document on new bucket
404        bucket.set_cors(CORS_DOC)
405        cors = re.sub(r'\s', '', bucket.get_cors().to_xml())
406        self.assertEqual(cors, CORS_DOC)
407
408    def test_cors_xml_storage_uri(self):
409        """Test setting and getting of CORS XML documents with storage_uri."""
410        # create a new bucket
411        bucket = self._MakeBucket()
412        bucket_name = bucket.name
413        uri = storage_uri('gs://' + bucket_name)
414        # get new bucket cors and make sure it's empty
415        cors = re.sub(r'\s', '', uri.get_cors().to_xml())
416        self.assertEqual(cors, CORS_EMPTY)
417        # set cors document on new bucket
418        cors_obj = Cors()
419        h = handler.XmlHandler(cors_obj, None)
420        xml.sax.parseString(CORS_DOC, h)
421        uri.set_cors(cors_obj)
422        cors = re.sub(r'\s', '', uri.get_cors().to_xml())
423        self.assertEqual(cors, CORS_DOC)
424
425    def test_lifecycle_config_bucket(self):
426        """Test setting and getting of lifecycle config on Bucket."""
427        # create a new bucket
428        bucket = self._MakeBucket()
429        bucket_name = bucket.name
430        # now call get_bucket to see if it's really there
431        bucket = self._GetConnection().get_bucket(bucket_name)
432        # get lifecycle config and make sure it's empty
433        xml = bucket.get_lifecycle_config().to_xml()
434        self.assertEqual(xml, LIFECYCLE_EMPTY)
435        # set lifecycle config
436        lifecycle_config = LifecycleConfig()
437        lifecycle_config.add_rule(
438            'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
439        lifecycle_config.add_rule(
440            'SetStorageClass', 'NEARLINE',
441            LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
442        bucket.configure_lifecycle(lifecycle_config)
443        xml = bucket.get_lifecycle_config().to_xml()
444        self.assertEqual(xml, LIFECYCLE_DOC)
445
446    def test_lifecycle_config_storage_uri(self):
447        """Test setting and getting of lifecycle config with storage_uri."""
448        # create a new bucket
449        bucket = self._MakeBucket()
450        bucket_name = bucket.name
451        uri = storage_uri('gs://' + bucket_name)
452        # get lifecycle config and make sure it's empty
453        xml = uri.get_lifecycle_config().to_xml()
454        self.assertEqual(xml, LIFECYCLE_EMPTY)
455        # set lifecycle config
456        lifecycle_config = LifecycleConfig()
457        lifecycle_config.add_rule(
458            'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
459        lifecycle_config.add_rule(
460            'SetStorageClass', 'NEARLINE',
461            LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
462        uri.configure_lifecycle(lifecycle_config)
463        xml = uri.get_lifecycle_config().to_xml()
464        self.assertEqual(xml, LIFECYCLE_DOC)
465
466    def test_billing_config_bucket(self):
467        """Test setting and getting of billing config on Bucket."""
468        # create a new bucket
469        bucket = self._MakeBucket()
470        bucket_name = bucket.name
471        # get billing config and make sure it's empty
472        billing = bucket.get_billing_config()
473        self.assertEqual(billing, BILLING_EMPTY)
474        # set requester pays to enabled
475        bucket.configure_billing(requester_pays=True)
476        billing = bucket.get_billing_config()
477        self.assertEqual(billing, BILLING_ENABLED)
478        # set requester pays to disabled
479        bucket.configure_billing(requester_pays=False)
480        billing = bucket.get_billing_config()
481        self.assertEqual(billing, BILLING_DISABLED)
482
483    def test_billing_config_storage_uri(self):
484        """Test setting and getting of billing config with storage_uri."""
485        # create a new bucket
486        bucket = self._MakeBucket()
487        bucket_name = bucket.name
488        uri = storage_uri('gs://' + bucket_name)
489        # get billing config and make sure it's empty
490        billing = uri.get_billing_config()
491        self.assertEqual(billing, BILLING_EMPTY)
492        # set requester pays to enabled
493        uri.configure_billing(requester_pays=True)
494        billing = uri.get_billing_config()
495        self.assertEqual(billing, BILLING_ENABLED)
496        # set requester pays to disabled
497        uri.configure_billing(requester_pays=False)
498        billing = uri.get_billing_config()
499        self.assertEqual(billing, BILLING_DISABLED)
500
501    def test_encryption_config_bucket(self):
502        """Test setting and getting of EncryptionConfig on gs Bucket objects."""
503        # Create a new bucket.
504        bucket = self._MakeBucket()
505        bucket_name = bucket.name
506        # Get EncryptionConfig and make sure it's empty.
507        encryption_config = bucket.get_encryption_config()
508        self.assertIsNone(encryption_config.default_kms_key_name)
509        # Testing set functionality would require having an existing Cloud KMS
510        # key. Since we can't hardcode a key name or dynamically create one, we
511        # only test here that we're creating the correct XML document to send to
512        # GCS.
513        xmldoc = bucket._construct_encryption_config_xml(
514            default_kms_key_name='dummykey')
515        self.assertEqual(xmldoc, ENCRYPTION_CONFIG_WITH_KEY % 'dummykey')
516        # Test that setting an empty encryption config works.
517        bucket.set_encryption_config()
518
519    def test_encryption_config_storage_uri(self):
520        """Test setting and getting of EncryptionConfig with storage_uri."""
521        # Create a new bucket.
522        bucket = self._MakeBucket()
523        bucket_name = bucket.name
524        uri = storage_uri('gs://' + bucket_name)
525        # Get EncryptionConfig and make sure it's empty.
526        encryption_config = uri.get_encryption_config()
527        self.assertIsNone(encryption_config.default_kms_key_name)
528
529        # Test that setting an empty encryption config works.
530        uri.set_encryption_config()
531