1# -*- coding: utf-8 -*- 2# Copyright 2017 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Integration tests for label command.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import json 23import xml 24from xml.dom.minidom import parseString 25from xml.sax import _exceptions as SaxExceptions 26 27import six 28import boto 29from boto import handler 30from boto.s3.tagging import Tags 31 32from gslib.exception import CommandException 33import gslib.tests.testcase as testcase 34from gslib.tests.testcase.integration_testcase import SkipForGS 35from gslib.tests.testcase.integration_testcase import SkipForS3 36from gslib.tests.util import ObjectToURI as suri 37from gslib.utils.retry_util import Retry 38from gslib.utils.constants import UTF8 39 40KEY1 = 'key_one' 41KEY2 = 'key_two' 42VALUE1 = 'value_one' 43VALUE2 = 'value_two' 44 45# Argument in this line should be formatted with bucket_uri. 46LABEL_SETTING_OUTPUT = 'Setting label configuration on %s/...' 47 48 49@SkipForGS('Tests use S3-style XML passthrough.') 50class TestLabelS3(testcase.GsUtilIntegrationTestCase): 51 """S3-specific tests. Most other test cases are covered in TestLabelGS.""" 52 53 _label_xml = parseString('<Tagging><TagSet>' + '<Tag><Key>' + KEY1 + 54 '</Key><Value>' + VALUE1 + '</Value></Tag>' + 55 '<Tag><Key>' + KEY2 + '</Key><Value>' + VALUE2 + 56 '</Value></Tag>' + 57 '</TagSet></Tagging>').toprettyxml(indent=' ') 58 59 def setUp(self): 60 super(TestLabelS3, self).setUp() 61 self.xml_fpath = self.CreateTempFile(contents=self._label_xml.encode(UTF8)) 62 63 def DoAssertItemsMatch(self, item1, item2): 64 if six.PY2: 65 self.assertItemsEqual(item1, item2) 66 else: 67 # The name was switched, and to a more misleading name, in PY3. Oh well. 68 self.assertCountEqual(item1, item2) 69 70 def _LabelDictFromXmlString(self, xml_str): 71 label_dict = {} 72 tags_list = Tags() 73 h = handler.XmlHandler(tags_list, None) 74 try: 75 xml.sax.parseString(xml_str, h) 76 except SaxExceptions.SAXParseException as e: 77 raise CommandException( 78 'Requested labels/tagging config is invalid: %s at line %s, column ' 79 '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) 80 for tagset_list in tags_list: 81 for tag in tagset_list: 82 label_dict[tag.key] = tag.value 83 return label_dict 84 85 def testSetAndGet(self): 86 bucket_uri = self.CreateBucket() 87 stderr = self.RunGsUtil(['label', 'set', self.xml_fpath, 88 suri(bucket_uri)], 89 return_stderr=True) 90 self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri)) 91 92 # Verify that the bucket is configured with the labels we just set. 93 # Work around eventual consistency for S3 tagging. 94 @Retry(AssertionError, tries=3, timeout_secs=1) 95 def _Check1(): 96 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 97 return_stdout=True) 98 self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), 99 self._LabelDictFromXmlString(self._label_xml)) 100 101 _Check1() 102 103 def testCh(self): 104 bucket_uri = self.CreateBucket() 105 self.RunGsUtil([ 106 'label', 'ch', '-l', 107 '%s:%s' % (KEY1, VALUE1), '-l', 108 '%s:%s' % (KEY2, VALUE2), 109 suri(bucket_uri) 110 ]) 111 112 # Verify that the bucket is configured with the labels we just set. 113 # Work around eventual consistency for S3 tagging. 114 @Retry(AssertionError, tries=3, timeout_secs=1) 115 def _Check1(): 116 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 117 return_stdout=True) 118 self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), 119 self._LabelDictFromXmlString(self._label_xml)) 120 121 _Check1() 122 123 # Remove KEY1, add a new key, and attempt to remove a nonexistent key 124 # with 'label ch'. 125 self.RunGsUtil([ 126 'label', 'ch', '-d', KEY1, '-l', 'new_key:new_value', '-d', 127 'nonexistent-key', 128 suri(bucket_uri) 129 ]) 130 expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'} 131 132 @Retry(AssertionError, tries=3, timeout_secs=1) 133 def _Check2(): 134 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 135 return_stdout=True) 136 self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), 137 expected_dict) 138 139 _Check2() 140 141 142@SkipForS3('Tests use GS-style ') 143class TestLabelGS(testcase.GsUtilIntegrationTestCase): 144 """Integration tests for label command.""" 145 146 _label_dict = {KEY1: VALUE1, KEY2: VALUE2} 147 148 def setUp(self): 149 super(TestLabelGS, self).setUp() 150 self.json_fpath = self.CreateTempFile( 151 contents=json.dumps(self._label_dict).encode(UTF8)) 152 153 def testSetAndGetOnOneBucket(self): 154 bucket_uri = self.CreateBucket() 155 156 # Try setting labels for one bucket. 157 stderr = self.RunGsUtil(['label', 'set', self.json_fpath, 158 suri(bucket_uri)], 159 return_stderr=True) 160 self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri)) 161 # Verify that the bucket is configured with the labels we just set. 162 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 163 return_stdout=True) 164 self.assertDictEqual(json.loads(stdout), self._label_dict) 165 166 def testSetOnMultipleBucketsInSameCommand(self): 167 bucket_uri = self.CreateBucket() 168 bucket2_uri = self.CreateBucket() 169 170 # Try setting labels for multiple buckets in one command. 171 stderr = self.RunGsUtil( 172 ['label', 'set', self.json_fpath, 173 suri(bucket_uri), 174 suri(bucket2_uri)], 175 return_stderr=True) 176 actual = set(stderr.splitlines()) 177 expected = set([ 178 LABEL_SETTING_OUTPUT % suri(bucket_uri), 179 LABEL_SETTING_OUTPUT % suri(bucket2_uri) 180 ]) 181 self.assertSetEqual(actual, expected) 182 183 def testSetOverwritesOldLabelConfig(self): 184 bucket_uri = self.CreateBucket() 185 # Try setting labels for one bucket. 186 self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)]) 187 new_key_1 = 'new_key_1' 188 new_key_2 = 'new_key_2' 189 new_value_1 = 'new_value_1' 190 new_value_2 = 'new_value_2' 191 new_json = { 192 new_key_1: new_value_1, 193 new_key_2: new_value_2, 194 KEY1: 'different_value_for_an_existing_key' 195 } 196 new_json_fpath = self.CreateTempFile( 197 contents=json.dumps(new_json).encode('ascii')) 198 self.RunGsUtil(['label', 'set', new_json_fpath, suri(bucket_uri)]) 199 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 200 return_stdout=True) 201 self.assertDictEqual(json.loads(stdout), new_json) 202 203 def testInitialAndSubsequentCh(self): 204 bucket_uri = self.CreateBucket() 205 ch_subargs = [ 206 '-l', '%s:%s' % (KEY1, VALUE1), '-l', 207 '%s:%s' % (KEY2, VALUE2) 208 ] 209 210 # Ensure 'ch' progress message shows in stderr. 211 stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + [suri(bucket_uri)], 212 return_stderr=True) 213 self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri)) 214 215 # Check the bucket to ensure it's configured with the labels we just 216 # specified. 217 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 218 return_stdout=True) 219 self.assertDictEqual(json.loads(stdout), self._label_dict) 220 221 # Ensure a subsequent 'ch' command works correctly. 222 new_key = 'new-key' 223 new_value = 'new-value' 224 self.RunGsUtil([ 225 'label', 'ch', '-l', 226 '%s:%s' % (new_key, new_value), '-d', KEY2, 227 suri(bucket_uri) 228 ]) 229 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 230 return_stdout=True) 231 actual = json.loads(stdout) 232 expected = {KEY1: VALUE1, new_key: new_value} 233 self.assertDictEqual(actual, expected) 234 235 def testChAppliesChangesToAllBucketArgs(self): 236 bucket_suris = [suri(self.CreateBucket()), suri(self.CreateBucket())] 237 ch_subargs = [ 238 '-l', '%s:%s' % (KEY1, VALUE1), '-l', 239 '%s:%s' % (KEY2, VALUE2) 240 ] 241 242 # Ensure 'ch' progress message appears for both buckets in stderr. 243 stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + bucket_suris, 244 return_stderr=True) 245 actual = set(stderr.splitlines()) 246 expected = set( 247 [LABEL_SETTING_OUTPUT % bucket_suri for bucket_suri in bucket_suris]) 248 self.assertSetEqual(actual, expected) 249 250 # Check the buckets to ensure both are configured with the labels we 251 # just specified. 252 for bucket_suri in bucket_suris: 253 stdout = self.RunGsUtil(['label', 'get', bucket_suri], return_stdout=True) 254 self.assertDictEqual(json.loads(stdout), self._label_dict) 255 256 def testChMinusDWorksWithoutExistingLabels(self): 257 bucket_uri = self.CreateBucket() 258 self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)]) 259 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 260 return_stdout=True) 261 self.assertIn('%s/ has no label configuration.' % suri(bucket_uri), stdout) 262 263 def testTooFewArgumentsFails(self): 264 """Ensures label commands fail with too few arguments.""" 265 invocations_missing_args = ( 266 # Neither arguments nor subcommand. 267 ['label'], 268 # Not enough arguments for 'set'. 269 ['label', 'set'], 270 ['label', 'set', 'filename'], 271 # Not enough arguments for 'get'. 272 ['label', 'get'], 273 # Not enough arguments for 'ch'. 274 ['label', 'ch'], 275 ['label', 'ch', '-l', 'key:val']) 276 for arg_list in invocations_missing_args: 277 stderr = self.RunGsUtil(arg_list, return_stderr=True, expected_status=1) 278 self.assertIn('command requires at least', stderr) 279 280 # Invoking 'ch' without any changes gives a slightly different message. 281 stderr = self.RunGsUtil( 282 ['label', 'ch', 'gs://some-nonexistent-foobar-bucket-name'], 283 return_stderr=True, 284 expected_status=1) 285 self.assertIn('Please specify at least one label change', stderr) 286