1# -*- coding: utf-8 -*-
2# Copyright 2017 Google Inc. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Integration tests for label command."""
16
17from __future__ import absolute_import
18from __future__ import print_function
19from __future__ import division
20from __future__ import unicode_literals
21
22import json
23import xml
24from xml.dom.minidom import parseString
25from xml.sax import _exceptions as SaxExceptions
26
27import six
28import boto
29from boto import handler
30from boto.s3.tagging import Tags
31
32from gslib.exception import CommandException
33import gslib.tests.testcase as testcase
34from gslib.tests.testcase.integration_testcase import SkipForGS
35from gslib.tests.testcase.integration_testcase import SkipForS3
36from gslib.tests.util import ObjectToURI as suri
37from gslib.utils.retry_util import Retry
38from gslib.utils.constants import UTF8
39
40KEY1 = 'key_one'
41KEY2 = 'key_two'
42VALUE1 = 'value_one'
43VALUE2 = 'value_two'
44
45# Argument in this line should be formatted with bucket_uri.
46LABEL_SETTING_OUTPUT = 'Setting label configuration on %s/...'
47
48
49@SkipForGS('Tests use S3-style XML passthrough.')
50class TestLabelS3(testcase.GsUtilIntegrationTestCase):
51  """S3-specific tests. Most other test cases are covered in TestLabelGS."""
52
53  _label_xml = parseString('<Tagging><TagSet>' + '<Tag><Key>' + KEY1 +
54                           '</Key><Value>' + VALUE1 + '</Value></Tag>' +
55                           '<Tag><Key>' + KEY2 + '</Key><Value>' + VALUE2 +
56                           '</Value></Tag>' +
57                           '</TagSet></Tagging>').toprettyxml(indent='    ')
58
59  def setUp(self):
60    super(TestLabelS3, self).setUp()
61    self.xml_fpath = self.CreateTempFile(contents=self._label_xml.encode(UTF8))
62
63  def DoAssertItemsMatch(self, item1, item2):
64    if six.PY2:
65      self.assertItemsEqual(item1, item2)
66    else:
67      # The name was switched, and to a more misleading name, in PY3. Oh well.
68      self.assertCountEqual(item1, item2)
69
70  def _LabelDictFromXmlString(self, xml_str):
71    label_dict = {}
72    tags_list = Tags()
73    h = handler.XmlHandler(tags_list, None)
74    try:
75      xml.sax.parseString(xml_str, h)
76    except SaxExceptions.SAXParseException as e:
77      raise CommandException(
78          'Requested labels/tagging config is invalid: %s at line %s, column '
79          '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber()))
80    for tagset_list in tags_list:
81      for tag in tagset_list:
82        label_dict[tag.key] = tag.value
83    return label_dict
84
85  def testSetAndGet(self):
86    bucket_uri = self.CreateBucket()
87    stderr = self.RunGsUtil(['label', 'set', self.xml_fpath,
88                             suri(bucket_uri)],
89                            return_stderr=True)
90    self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri))
91
92    # Verify that the bucket is configured with the labels we just set.
93    # Work around eventual consistency for S3 tagging.
94    @Retry(AssertionError, tries=3, timeout_secs=1)
95    def _Check1():
96      stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
97                              return_stdout=True)
98      self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout),
99                              self._LabelDictFromXmlString(self._label_xml))
100
101    _Check1()
102
103  def testCh(self):
104    bucket_uri = self.CreateBucket()
105    self.RunGsUtil([
106        'label', 'ch', '-l',
107        '%s:%s' % (KEY1, VALUE1), '-l',
108        '%s:%s' % (KEY2, VALUE2),
109        suri(bucket_uri)
110    ])
111
112    # Verify that the bucket is configured with the labels we just set.
113    # Work around eventual consistency for S3 tagging.
114    @Retry(AssertionError, tries=3, timeout_secs=1)
115    def _Check1():
116      stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
117                              return_stdout=True)
118      self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout),
119                              self._LabelDictFromXmlString(self._label_xml))
120
121    _Check1()
122
123    # Remove KEY1, add a new key, and attempt to remove a nonexistent key
124    # with 'label ch'.
125    self.RunGsUtil([
126        'label', 'ch', '-d', KEY1, '-l', 'new_key:new_value', '-d',
127        'nonexistent-key',
128        suri(bucket_uri)
129    ])
130    expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'}
131
132    @Retry(AssertionError, tries=3, timeout_secs=1)
133    def _Check2():
134      stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
135                              return_stdout=True)
136      self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout),
137                              expected_dict)
138
139    _Check2()
140
141
142@SkipForS3('Tests use GS-style ')
143class TestLabelGS(testcase.GsUtilIntegrationTestCase):
144  """Integration tests for label command."""
145
146  _label_dict = {KEY1: VALUE1, KEY2: VALUE2}
147
148  def setUp(self):
149    super(TestLabelGS, self).setUp()
150    self.json_fpath = self.CreateTempFile(
151        contents=json.dumps(self._label_dict).encode(UTF8))
152
153  def testSetAndGetOnOneBucket(self):
154    bucket_uri = self.CreateBucket()
155
156    # Try setting labels for one bucket.
157    stderr = self.RunGsUtil(['label', 'set', self.json_fpath,
158                             suri(bucket_uri)],
159                            return_stderr=True)
160    self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri))
161    # Verify that the bucket is configured with the labels we just set.
162    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
163                            return_stdout=True)
164    self.assertDictEqual(json.loads(stdout), self._label_dict)
165
166  def testSetOnMultipleBucketsInSameCommand(self):
167    bucket_uri = self.CreateBucket()
168    bucket2_uri = self.CreateBucket()
169
170    # Try setting labels for multiple buckets in one command.
171    stderr = self.RunGsUtil(
172        ['label', 'set', self.json_fpath,
173         suri(bucket_uri),
174         suri(bucket2_uri)],
175        return_stderr=True)
176    actual = set(stderr.splitlines())
177    expected = set([
178        LABEL_SETTING_OUTPUT % suri(bucket_uri),
179        LABEL_SETTING_OUTPUT % suri(bucket2_uri)
180    ])
181    self.assertSetEqual(actual, expected)
182
183  def testSetOverwritesOldLabelConfig(self):
184    bucket_uri = self.CreateBucket()
185    # Try setting labels for one bucket.
186    self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)])
187    new_key_1 = 'new_key_1'
188    new_key_2 = 'new_key_2'
189    new_value_1 = 'new_value_1'
190    new_value_2 = 'new_value_2'
191    new_json = {
192        new_key_1: new_value_1,
193        new_key_2: new_value_2,
194        KEY1: 'different_value_for_an_existing_key'
195    }
196    new_json_fpath = self.CreateTempFile(
197        contents=json.dumps(new_json).encode('ascii'))
198    self.RunGsUtil(['label', 'set', new_json_fpath, suri(bucket_uri)])
199    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
200                            return_stdout=True)
201    self.assertDictEqual(json.loads(stdout), new_json)
202
203  def testInitialAndSubsequentCh(self):
204    bucket_uri = self.CreateBucket()
205    ch_subargs = [
206        '-l', '%s:%s' % (KEY1, VALUE1), '-l',
207        '%s:%s' % (KEY2, VALUE2)
208    ]
209
210    # Ensure 'ch' progress message shows in stderr.
211    stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + [suri(bucket_uri)],
212                            return_stderr=True)
213    self.assertEqual(stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri))
214
215    # Check the bucket to ensure it's configured with the labels we just
216    # specified.
217    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
218                            return_stdout=True)
219    self.assertDictEqual(json.loads(stdout), self._label_dict)
220
221    # Ensure a subsequent 'ch' command works correctly.
222    new_key = 'new-key'
223    new_value = 'new-value'
224    self.RunGsUtil([
225        'label', 'ch', '-l',
226        '%s:%s' % (new_key, new_value), '-d', KEY2,
227        suri(bucket_uri)
228    ])
229    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
230                            return_stdout=True)
231    actual = json.loads(stdout)
232    expected = {KEY1: VALUE1, new_key: new_value}
233    self.assertDictEqual(actual, expected)
234
235  def testChAppliesChangesToAllBucketArgs(self):
236    bucket_suris = [suri(self.CreateBucket()), suri(self.CreateBucket())]
237    ch_subargs = [
238        '-l', '%s:%s' % (KEY1, VALUE1), '-l',
239        '%s:%s' % (KEY2, VALUE2)
240    ]
241
242    # Ensure 'ch' progress message appears for both buckets in stderr.
243    stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + bucket_suris,
244                            return_stderr=True)
245    actual = set(stderr.splitlines())
246    expected = set(
247        [LABEL_SETTING_OUTPUT % bucket_suri for bucket_suri in bucket_suris])
248    self.assertSetEqual(actual, expected)
249
250    # Check the buckets to ensure both are configured with the labels we
251    # just specified.
252    for bucket_suri in bucket_suris:
253      stdout = self.RunGsUtil(['label', 'get', bucket_suri], return_stdout=True)
254      self.assertDictEqual(json.loads(stdout), self._label_dict)
255
256  def testChMinusDWorksWithoutExistingLabels(self):
257    bucket_uri = self.CreateBucket()
258    self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)])
259    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
260                            return_stdout=True)
261    self.assertIn('%s/ has no label configuration.' % suri(bucket_uri), stdout)
262
263  def testTooFewArgumentsFails(self):
264    """Ensures label commands fail with too few arguments."""
265    invocations_missing_args = (
266        # Neither arguments nor subcommand.
267        ['label'],
268        # Not enough arguments for 'set'.
269        ['label', 'set'],
270        ['label', 'set', 'filename'],
271        # Not enough arguments for 'get'.
272        ['label', 'get'],
273        # Not enough arguments for 'ch'.
274        ['label', 'ch'],
275        ['label', 'ch', '-l', 'key:val'])
276    for arg_list in invocations_missing_args:
277      stderr = self.RunGsUtil(arg_list, return_stderr=True, expected_status=1)
278      self.assertIn('command requires at least', stderr)
279
280    # Invoking 'ch' without any changes gives a slightly different message.
281    stderr = self.RunGsUtil(
282        ['label', 'ch', 'gs://some-nonexistent-foobar-bucket-name'],
283        return_stderr=True,
284        expected_status=1)
285    self.assertIn('Please specify at least one label change', stderr)
286