1# -*- coding: utf-8 -*- 2# Copyright 2017 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Integration tests for label command.""" 16 17from __future__ import absolute_import 18 19import json 20import xml 21from xml.dom.minidom import parseString 22from xml.sax import _exceptions as SaxExceptions 23 24import boto 25from boto import handler 26from boto.s3.tagging import Tags 27 28from gslib.exception import CommandException 29import gslib.tests.testcase as testcase 30from gslib.tests.testcase.integration_testcase import SkipForGS 31from gslib.tests.testcase.integration_testcase import SkipForS3 32from gslib.tests.util import ObjectToURI as suri 33from gslib.util import Retry 34 35KEY1 = 'key_one' 36KEY2 = 'key_two' 37VALUE1 = 'value_one' 38VALUE2 = 'value_two' 39 40# Argument in this line should be formatted with bucket_uri. 41LABEL_SETTING_OUTPUT = 'Setting label configuration on %s/...' 42 43 44@SkipForGS('Tests use S3-style XML passthrough.') 45class TestLabelS3(testcase.GsUtilIntegrationTestCase): 46 """S3-specific tests. Most other test cases are covered in TestLabelGS.""" 47 48 _label_xml = parseString( 49 '<Tagging><TagSet>' + 50 '<Tag><Key>' + KEY1 + '</Key><Value>' + VALUE1 + '</Value></Tag>' + 51 '<Tag><Key>' + KEY2 + '</Key><Value>' + VALUE2 + '</Value></Tag>' + 52 '</TagSet></Tagging>').toprettyxml(indent=' ') 53 54 def setUp(self): 55 super(TestLabelS3, self).setUp() 56 self.xml_fpath = self.CreateTempFile(contents=self._label_xml) 57 58 def _LabelDictFromXmlString(self, xml_str): 59 label_dict = {} 60 tags_list = Tags() 61 h = handler.XmlHandler(tags_list, None) 62 try: 63 xml.sax.parseString(xml_str, h) 64 except SaxExceptions.SAXParseException, e: 65 raise CommandException( 66 'Requested labels/tagging config is invalid: %s at line %s, column ' 67 '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) 68 for tagset_list in tags_list: 69 for tag in tagset_list: 70 label_dict[tag.key] = tag.value 71 return label_dict 72 73 def testSetAndGet(self): 74 bucket_uri = self.CreateBucket() 75 stderr = self.RunGsUtil( 76 ['label', 'set', self.xml_fpath, suri(bucket_uri)], 77 return_stderr=True) 78 self.assertEqual( 79 stderr.strip(), 80 LABEL_SETTING_OUTPUT % suri(bucket_uri)) 81 82 # Verify that the bucket is configured with the labels we just set. 83 # Work around eventual consistency for S3 tagging. 84 @Retry(AssertionError, tries=3, timeout_secs=1) 85 def _Check1(): 86 stdout = self.RunGsUtil( 87 ['label', 'get', suri(bucket_uri)], return_stdout=True) 88 self.assertItemsEqual( 89 self._LabelDictFromXmlString(stdout), 90 self._LabelDictFromXmlString(self._label_xml)) 91 _Check1() 92 93 def testCh(self): 94 bucket_uri = self.CreateBucket() 95 self.RunGsUtil(['label', 'ch', 96 '-l', '%s:%s' % (KEY1, VALUE1), 97 '-l', '%s:%s' % (KEY2, VALUE2), 98 suri(bucket_uri)]) 99 100 # Verify that the bucket is configured with the labels we just set. 101 # Work around eventual consistency for S3 tagging. 102 @Retry(AssertionError, tries=3, timeout_secs=1) 103 def _Check1(): 104 stdout = self.RunGsUtil( 105 ['label', 'get', suri(bucket_uri)], return_stdout=True) 106 self.assertItemsEqual( 107 self._LabelDictFromXmlString(stdout), 108 self._LabelDictFromXmlString(self._label_xml)) 109 _Check1() 110 111 # Remove KEY1, add a new key, and attempt to remove a nonexistent key 112 # with 'label ch'. 113 self.RunGsUtil(['label', 'ch', 114 '-d', KEY1, 115 '-l', 'new_key:new_value', 116 '-d', 'nonexistent-key', 117 suri(bucket_uri)]) 118 expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'} 119 120 @Retry(AssertionError, tries=3, timeout_secs=1) 121 def _Check2(): 122 stdout = self.RunGsUtil( 123 ['label', 'get', suri(bucket_uri)], return_stdout=True) 124 self.assertItemsEqual( 125 self._LabelDictFromXmlString(stdout), 126 expected_dict) 127 _Check2() 128 129 130@SkipForS3('Tests use GS-style ') 131class TestLabelGS(testcase.GsUtilIntegrationTestCase): 132 """Integration tests for label command.""" 133 134 _label_dict = {KEY1: VALUE1, KEY2: VALUE2} 135 136 def setUp(self): 137 super(TestLabelGS, self).setUp() 138 self.json_fpath = self.CreateTempFile(contents=json.dumps(self._label_dict)) 139 140 def testSetAndGetOnOneBucket(self): 141 bucket_uri = self.CreateBucket() 142 143 # Try setting labels for one bucket. 144 stderr = self.RunGsUtil( 145 ['label', 'set', self.json_fpath, suri(bucket_uri)], 146 return_stderr=True) 147 self.assertEqual( 148 stderr.strip(), 149 LABEL_SETTING_OUTPUT % suri(bucket_uri)) 150 # Verify that the bucket is configured with the labels we just set. 151 stdout = self.RunGsUtil( 152 ['label', 'get', suri(bucket_uri)], return_stdout=True) 153 self.assertItemsEqual(json.loads(stdout), self._label_dict) 154 155 def testSetOnMultipleBucketsInSameCommand(self): 156 bucket_uri = self.CreateBucket() 157 bucket2_uri = self.CreateBucket() 158 159 # Try setting labels for multiple buckets in one command. 160 stderr = self.RunGsUtil( 161 ['label', 'set', self.json_fpath, suri(bucket_uri), suri(bucket2_uri)], 162 return_stderr=True) 163 actual = set(stderr.splitlines()) 164 expected = set([ 165 LABEL_SETTING_OUTPUT % suri(bucket_uri), 166 LABEL_SETTING_OUTPUT % suri(bucket2_uri)]) 167 self.assertItemsEqual(actual, expected) 168 169 def testSetOverwritesOldLabelConfig(self): 170 bucket_uri = self.CreateBucket() 171 # Try setting labels for one bucket. 172 self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)]) 173 new_key_1 = 'new_key_1' 174 new_key_2 = 'new_key_2' 175 new_value_1 = 'new_value_1' 176 new_value_2 = 'new_value_2' 177 new_json = { 178 new_key_1: new_value_1, 179 new_key_2: new_value_2, 180 KEY1: 'different_value_for_an_existing_key' 181 } 182 new_json_fpath = self.CreateTempFile(contents=json.dumps(new_json)) 183 self.RunGsUtil(['label', 'set', new_json_fpath, suri(bucket_uri)]) 184 stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], 185 return_stdout=True) 186 self.assertItemsEqual(json.loads(stdout), new_json) 187 188 def testInitialAndSubsequentCh(self): 189 bucket_uri = self.CreateBucket() 190 ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1), 191 '-l', '%s:%s' % (KEY2, VALUE2)] 192 193 # Ensure 'ch' progress message shows in stderr. 194 stderr = self.RunGsUtil( 195 ['label', 'ch'] + ch_subargs + [suri(bucket_uri)], 196 return_stderr=True) 197 self.assertEqual( 198 stderr.strip(), 199 LABEL_SETTING_OUTPUT % suri(bucket_uri)) 200 201 # Check the bucket to ensure it's configured with the labels we just 202 # specified. 203 stdout = self.RunGsUtil( 204 ['label', 'get', suri(bucket_uri)], return_stdout=True) 205 self.assertItemsEqual(json.loads(stdout), self._label_dict) 206 207 # Ensure a subsequent 'ch' command works correctly. 208 new_key = 'new-key' 209 new_value = 'new-value' 210 self.RunGsUtil([ 211 'label', 'ch', '-l', '%s:%s' % (new_key, new_value), '-d', KEY2, 212 suri(bucket_uri)]) 213 stdout = self.RunGsUtil( 214 ['label', 'get', suri(bucket_uri)], return_stdout=True) 215 actual = json.loads(stdout) 216 expected = {KEY1: VALUE1, new_key: new_value} 217 self.assertItemsEqual(actual, expected) 218 219 def testChAppliesChangesToAllBucketArgs(self): 220 bucket_suris = [suri(self.CreateBucket()), suri(self.CreateBucket())] 221 ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1), 222 '-l', '%s:%s' % (KEY2, VALUE2)] 223 224 # Ensure 'ch' progress message appears for both buckets in stderr. 225 stderr = self.RunGsUtil( 226 ['label', 'ch'] + ch_subargs + bucket_suris, 227 return_stderr=True) 228 actual = set(stderr.splitlines()) 229 expected = set( 230 [LABEL_SETTING_OUTPUT % bucket_suri for bucket_suri in bucket_suris]) 231 self.assertItemsEqual(actual, expected) 232 233 # Check the buckets to ensure both are configured with the labels we 234 # just specified. 235 for bucket_suri in bucket_suris: 236 stdout = self.RunGsUtil( 237 ['label', 'get', bucket_suri], return_stdout=True) 238 self.assertItemsEqual(json.loads(stdout), self._label_dict) 239 240 def testChMinusDWorksWithoutExistingLabels(self): 241 bucket_uri = self.CreateBucket() 242 self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)]) 243 stdout = self.RunGsUtil( 244 ['label', 'get', suri(bucket_uri)], return_stdout=True) 245 self.assertIn('%s/ has no label configuration.' % suri(bucket_uri), stdout) 246 247 def testTooFewArgumentsFails(self): 248 """Ensures label commands fail with too few arguments.""" 249 invocations_missing_args = ( 250 # Neither arguments nor subcommand. 251 ['label'], 252 # Not enough arguments for 'set'. 253 ['label', 'set'], 254 ['label', 'set', 'filename'], 255 # Not enough arguments for 'get'. 256 ['label', 'get'], 257 # Not enough arguments for 'ch'. 258 ['label', 'ch'], 259 ['label', 'ch', '-l', 'key:val']) 260 for arg_list in invocations_missing_args: 261 stderr = self.RunGsUtil(arg_list, return_stderr=True, expected_status=1) 262 self.assertIn('command requires at least', stderr) 263 264 # Invoking 'ch' without any changes gives a slightly different message. 265 stderr = self.RunGsUtil( 266 ['label', 'ch', 'gs://some-nonexistent-foobar-bucket-name'], 267 return_stderr=True, expected_status=1) 268 self.assertIn('Please specify at least one label change', stderr) 269