1# -*- coding: utf-8 -*-
2# Copyright 2017 Google Inc. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Integration tests for label command."""
16
17from __future__ import absolute_import
18
19import json
20import xml
21from xml.dom.minidom import parseString
22from xml.sax import _exceptions as SaxExceptions
23
24import boto
25from boto import handler
26from boto.s3.tagging import Tags
27
28from gslib.exception import CommandException
29import gslib.tests.testcase as testcase
30from gslib.tests.testcase.integration_testcase import SkipForGS
31from gslib.tests.testcase.integration_testcase import SkipForS3
32from gslib.tests.util import ObjectToURI as suri
33from gslib.util import Retry
34
35KEY1 = 'key_one'
36KEY2 = 'key_two'
37VALUE1 = 'value_one'
38VALUE2 = 'value_two'
39
40# Argument in this line should be formatted with bucket_uri.
41LABEL_SETTING_OUTPUT = 'Setting label configuration on %s/...'
42
43
44@SkipForGS('Tests use S3-style XML passthrough.')
45class TestLabelS3(testcase.GsUtilIntegrationTestCase):
46  """S3-specific tests. Most other test cases are covered in TestLabelGS."""
47
48  _label_xml = parseString(
49      '<Tagging><TagSet>' +
50      '<Tag><Key>' + KEY1 + '</Key><Value>' + VALUE1 + '</Value></Tag>' +
51      '<Tag><Key>' + KEY2 + '</Key><Value>' + VALUE2 + '</Value></Tag>' +
52      '</TagSet></Tagging>').toprettyxml(indent='    ')
53
54  def setUp(self):
55    super(TestLabelS3, self).setUp()
56    self.xml_fpath = self.CreateTempFile(contents=self._label_xml)
57
58  def _LabelDictFromXmlString(self, xml_str):
59    label_dict = {}
60    tags_list = Tags()
61    h = handler.XmlHandler(tags_list, None)
62    try:
63      xml.sax.parseString(xml_str, h)
64    except SaxExceptions.SAXParseException, e:
65      raise CommandException(
66          'Requested labels/tagging config is invalid: %s at line %s, column '
67          '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber()))
68    for tagset_list in tags_list:
69      for tag in tagset_list:
70        label_dict[tag.key] = tag.value
71    return label_dict
72
73  def testSetAndGet(self):
74    bucket_uri = self.CreateBucket()
75    stderr = self.RunGsUtil(
76        ['label', 'set', self.xml_fpath, suri(bucket_uri)],
77        return_stderr=True)
78    self.assertEqual(
79        stderr.strip(),
80        LABEL_SETTING_OUTPUT % suri(bucket_uri))
81
82    # Verify that the bucket is configured with the labels we just set.
83    # Work around eventual consistency for S3 tagging.
84    @Retry(AssertionError, tries=3, timeout_secs=1)
85    def _Check1():
86      stdout = self.RunGsUtil(
87          ['label', 'get', suri(bucket_uri)], return_stdout=True)
88      self.assertItemsEqual(
89          self._LabelDictFromXmlString(stdout),
90          self._LabelDictFromXmlString(self._label_xml))
91    _Check1()
92
93  def testCh(self):
94    bucket_uri = self.CreateBucket()
95    self.RunGsUtil(['label', 'ch',
96                    '-l', '%s:%s' % (KEY1, VALUE1),
97                    '-l', '%s:%s' % (KEY2, VALUE2),
98                    suri(bucket_uri)])
99
100    # Verify that the bucket is configured with the labels we just set.
101    # Work around eventual consistency for S3 tagging.
102    @Retry(AssertionError, tries=3, timeout_secs=1)
103    def _Check1():
104      stdout = self.RunGsUtil(
105          ['label', 'get', suri(bucket_uri)], return_stdout=True)
106      self.assertItemsEqual(
107          self._LabelDictFromXmlString(stdout),
108          self._LabelDictFromXmlString(self._label_xml))
109    _Check1()
110
111    # Remove KEY1, add a new key, and attempt to remove a nonexistent key
112    # with 'label ch'.
113    self.RunGsUtil(['label', 'ch',
114                    '-d', KEY1,
115                    '-l', 'new_key:new_value',
116                    '-d', 'nonexistent-key',
117                    suri(bucket_uri)])
118    expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'}
119
120    @Retry(AssertionError, tries=3, timeout_secs=1)
121    def _Check2():
122      stdout = self.RunGsUtil(
123          ['label', 'get', suri(bucket_uri)], return_stdout=True)
124      self.assertItemsEqual(
125          self._LabelDictFromXmlString(stdout),
126          expected_dict)
127    _Check2()
128
129
130@SkipForS3('Tests use GS-style ')
131class TestLabelGS(testcase.GsUtilIntegrationTestCase):
132  """Integration tests for label command."""
133
134  _label_dict = {KEY1: VALUE1, KEY2: VALUE2}
135
136  def setUp(self):
137    super(TestLabelGS, self).setUp()
138    self.json_fpath = self.CreateTempFile(contents=json.dumps(self._label_dict))
139
140  def testSetAndGetOnOneBucket(self):
141    bucket_uri = self.CreateBucket()
142
143    # Try setting labels for one bucket.
144    stderr = self.RunGsUtil(
145        ['label', 'set', self.json_fpath, suri(bucket_uri)],
146        return_stderr=True)
147    self.assertEqual(
148        stderr.strip(),
149        LABEL_SETTING_OUTPUT % suri(bucket_uri))
150    # Verify that the bucket is configured with the labels we just set.
151    stdout = self.RunGsUtil(
152        ['label', 'get', suri(bucket_uri)], return_stdout=True)
153    self.assertItemsEqual(json.loads(stdout), self._label_dict)
154
155  def testSetOnMultipleBucketsInSameCommand(self):
156    bucket_uri = self.CreateBucket()
157    bucket2_uri = self.CreateBucket()
158
159    # Try setting labels for multiple buckets in one command.
160    stderr = self.RunGsUtil(
161        ['label', 'set', self.json_fpath, suri(bucket_uri), suri(bucket2_uri)],
162        return_stderr=True)
163    actual = set(stderr.splitlines())
164    expected = set([
165        LABEL_SETTING_OUTPUT % suri(bucket_uri),
166        LABEL_SETTING_OUTPUT % suri(bucket2_uri)])
167    self.assertItemsEqual(actual, expected)
168
169  def testSetOverwritesOldLabelConfig(self):
170    bucket_uri = self.CreateBucket()
171    # Try setting labels for one bucket.
172    self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)])
173    new_key_1 = 'new_key_1'
174    new_key_2 = 'new_key_2'
175    new_value_1 = 'new_value_1'
176    new_value_2 = 'new_value_2'
177    new_json = {
178        new_key_1: new_value_1,
179        new_key_2: new_value_2,
180        KEY1: 'different_value_for_an_existing_key'
181    }
182    new_json_fpath = self.CreateTempFile(contents=json.dumps(new_json))
183    self.RunGsUtil(['label', 'set', new_json_fpath, suri(bucket_uri)])
184    stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)],
185                            return_stdout=True)
186    self.assertItemsEqual(json.loads(stdout), new_json)
187
188  def testInitialAndSubsequentCh(self):
189    bucket_uri = self.CreateBucket()
190    ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1),
191                  '-l', '%s:%s' % (KEY2, VALUE2)]
192
193    # Ensure 'ch' progress message shows in stderr.
194    stderr = self.RunGsUtil(
195        ['label', 'ch'] + ch_subargs + [suri(bucket_uri)],
196        return_stderr=True)
197    self.assertEqual(
198        stderr.strip(),
199        LABEL_SETTING_OUTPUT % suri(bucket_uri))
200
201    # Check the bucket to ensure it's configured with the labels we just
202    # specified.
203    stdout = self.RunGsUtil(
204        ['label', 'get', suri(bucket_uri)], return_stdout=True)
205    self.assertItemsEqual(json.loads(stdout), self._label_dict)
206
207    # Ensure a subsequent 'ch' command works correctly.
208    new_key = 'new-key'
209    new_value = 'new-value'
210    self.RunGsUtil([
211        'label', 'ch', '-l', '%s:%s' % (new_key, new_value), '-d', KEY2,
212        suri(bucket_uri)])
213    stdout = self.RunGsUtil(
214        ['label', 'get', suri(bucket_uri)], return_stdout=True)
215    actual = json.loads(stdout)
216    expected = {KEY1: VALUE1, new_key: new_value}
217    self.assertItemsEqual(actual, expected)
218
219  def testChAppliesChangesToAllBucketArgs(self):
220    bucket_suris = [suri(self.CreateBucket()), suri(self.CreateBucket())]
221    ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1),
222                  '-l', '%s:%s' % (KEY2, VALUE2)]
223
224    # Ensure 'ch' progress message appears for both buckets in stderr.
225    stderr = self.RunGsUtil(
226        ['label', 'ch'] + ch_subargs + bucket_suris,
227        return_stderr=True)
228    actual = set(stderr.splitlines())
229    expected = set(
230        [LABEL_SETTING_OUTPUT % bucket_suri for bucket_suri in bucket_suris])
231    self.assertItemsEqual(actual, expected)
232
233    # Check the buckets to ensure both are configured with the labels we
234    # just specified.
235    for bucket_suri in bucket_suris:
236      stdout = self.RunGsUtil(
237          ['label', 'get', bucket_suri], return_stdout=True)
238      self.assertItemsEqual(json.loads(stdout), self._label_dict)
239
240  def testChMinusDWorksWithoutExistingLabels(self):
241    bucket_uri = self.CreateBucket()
242    self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)])
243    stdout = self.RunGsUtil(
244        ['label', 'get', suri(bucket_uri)], return_stdout=True)
245    self.assertIn('%s/ has no label configuration.' % suri(bucket_uri), stdout)
246
247  def testTooFewArgumentsFails(self):
248    """Ensures label commands fail with too few arguments."""
249    invocations_missing_args = (
250        # Neither arguments nor subcommand.
251        ['label'],
252        # Not enough arguments for 'set'.
253        ['label', 'set'],
254        ['label', 'set', 'filename'],
255        # Not enough arguments for 'get'.
256        ['label', 'get'],
257        # Not enough arguments for 'ch'.
258        ['label', 'ch'],
259        ['label', 'ch', '-l', 'key:val'])
260    for arg_list in invocations_missing_args:
261      stderr = self.RunGsUtil(arg_list, return_stderr=True, expected_status=1)
262      self.assertIn('command requires at least', stderr)
263
264    # Invoking 'ch' without any changes gives a slightly different message.
265    stderr = self.RunGsUtil(
266        ['label', 'ch', 'gs://some-nonexistent-foobar-bucket-name'],
267        return_stderr=True, expected_status=1)
268    self.assertIn('Please specify at least one label change', stderr)
269