1# -*- coding: utf-8 -*- # 2# Copyright 2020 Google LLC. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""S3 API-specific resource subclasses.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import unicode_literals 20 21import collections 22 23from googlecloudsdk.api_lib.storage import errors 24from googlecloudsdk.command_lib.storage.resources import resource_reference 25from googlecloudsdk.command_lib.storage.resources import resource_util 26 27 28_INCOMPLETE_OBJECT_METADATA_WARNING = ( 29 'Use "-j", the JSON flag, to view additional S3 metadata.') 30 31 32def _json_dump_recursion_helper(metadata): 33 """See _get_json_dump docstring.""" 34 if isinstance(metadata, list): 35 return [_json_dump_recursion_helper(item) for item in metadata] 36 37 if not isinstance(metadata, dict): 38 return resource_util.convert_to_json_parsable_type(metadata) 39 40 # Sort by key to make sure dictionary always prints in correct order. 41 formatted_dict = collections.OrderedDict(sorted(metadata.items())) 42 for key, value in formatted_dict.items(): 43 if isinstance(value, dict): 44 # Recursively handle dictionaries. 45 formatted_dict[key] = _json_dump_recursion_helper(value) 46 elif isinstance(value, list): 47 # Recursively handled lists, which may contain more dicts, like ACLs. 48 formatted_list = [_json_dump_recursion_helper(item) for item in value] 49 if formatted_list: 50 # Ignore empty lists. 51 formatted_dict[key] = formatted_list 52 elif value or resource_util.should_preserve_falsy_metadata_value(value): 53 formatted_dict[key] = resource_util.convert_to_json_parsable_type(value) 54 55 return formatted_dict 56 57 58def _get_json_dump(resource): 59 """Formats S3 resource metadata as JSON. 60 61 Args: 62 resource (S3BucketResource|S3ObjectResource): Resource object. 63 64 Returns: 65 Formatted JSON string. 66 """ 67 return resource_util.configured_json_dumps( 68 collections.OrderedDict([ 69 ('url', resource.storage_url.url_string), 70 ('type', resource.TYPE_STRING), 71 ('metadata', _json_dump_recursion_helper(resource.metadata)), 72 ])) 73 74 75def _get_error_or_exists_string(value): 76 """Returns error if value is error or existence string.""" 77 if isinstance(value, errors.S3ApiError): 78 return value 79 else: 80 return resource_util.get_exists_string(value) 81 82 83def _get_formatted_acl_section(acl_metadata): 84 """Returns formatted ACLs, error, or formatted none value.""" 85 if isinstance(acl_metadata, errors.S3ApiError): 86 return resource_util.get_padded_metadata_key_value_line('ACL', acl_metadata) 87 elif acl_metadata: 88 return resource_util.get_metadata_json_section_string( 89 'ACL', acl_metadata, _json_dump_recursion_helper) 90 else: 91 return resource_util.get_padded_metadata_key_value_line('ACL', '[]') 92 93 94def _get_full_bucket_metadata_string(resource): 95 """Formats S3 resource metadata as string with rows. 96 97 Args: 98 resource (S3BucketResource): Resource with metadata. 99 100 Returns: 101 Formatted multi-line string. 102 """ 103 # Hardcoded strings found in Boto docs: 104 # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html 105 logging_enabled_value = _get_error_or_exists_string( 106 resource.metadata['LoggingEnabled']) 107 website_value = _get_error_or_exists_string(resource.metadata['Website']) 108 cors_value = _get_error_or_exists_string(resource.metadata['CORSRules']) 109 lifecycle_configuration_value = _get_error_or_exists_string( 110 resource.metadata['LifecycleConfiguration']) 111 112 if isinstance(resource.metadata['Versioning'], errors.S3ApiError): 113 versioning_enabled_value = resource.metadata['Versioning'] 114 else: 115 versioning_status = resource.metadata['Versioning'].get('Status') 116 if versioning_status == 'Enabled': 117 versioning_enabled_value = True 118 elif versioning_status == 'Suspended': 119 versioning_enabled_value = False 120 else: 121 versioning_enabled_value = None 122 123 if isinstance(resource.metadata['Payer'], errors.S3ApiError): 124 requester_pays_value = resource.metadata['Payer'] 125 elif resource.metadata['Payer'] == 'Requester': 126 requester_pays_value = True 127 elif resource.metadata['Payer'] == 'BucketOwner': 128 requester_pays_value = False 129 else: 130 requester_pays_value = None 131 132 return ( 133 '{bucket_url}:\n' 134 '{location_constraint_line}' 135 '{versioning_enabled_line}' 136 '{logging_config_line}' 137 '{website_config_line}' 138 '{cors_config_line}' 139 '{lifecycle_config_line}' 140 '{requester_pays_line}' 141 '{acl_section}' 142 ).format( 143 bucket_url=resource.storage_url.versionless_url_string, 144 location_constraint_line=resource_util.get_padded_metadata_key_value_line( 145 'Location constraint', resource.metadata['LocationConstraint']), 146 versioning_enabled_line=resource_util.get_padded_metadata_key_value_line( 147 'Versioning enabled', versioning_enabled_value), 148 logging_config_line=resource_util.get_padded_metadata_key_value_line( 149 'Logging configuration', logging_enabled_value), 150 website_config_line=resource_util.get_padded_metadata_key_value_line( 151 'Website configuration', website_value), 152 cors_config_line=resource_util.get_padded_metadata_key_value_line( 153 'CORS configuration', cors_value), 154 lifecycle_config_line=resource_util.get_padded_metadata_key_value_line( 155 'Lifecycle configuration', lifecycle_configuration_value), 156 requester_pays_line=resource_util.get_padded_metadata_key_value_line( 157 'Requester Pays enabled', requester_pays_value), 158 # Remove ending newline character because this is the last list item. 159 acl_section=_get_formatted_acl_section(resource.metadata['ACL'])[:-1]) 160 161 162def _get_full_object_metadata_string(resource): 163 """Formats S3 resource metadata as string with rows. 164 165 Args: 166 resource (S3ObjectResource): Resource with metadata. 167 168 Returns: 169 Formatted multi-line string. 170 """ 171 # Hardcoded strings found in Boto docs: 172 # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html 173 if 'LastModified' in resource.metadata: 174 optional_time_updated_line = resource_util.get_padded_metadata_time_line( 175 'Update time', resource.metadata['LastModified']) 176 else: 177 optional_time_updated_line = '' 178 179 if 'StorageClass' in resource.metadata: 180 optional_storage_class_line = resource_util.get_padded_metadata_key_value_line( 181 'Storage class', resource.metadata['StorageClass']) 182 else: 183 optional_storage_class_line = '' 184 185 if 'CacheControl' in resource.metadata: 186 optional_cache_control_line = resource_util.get_padded_metadata_key_value_line( 187 'Cache-Control', resource.metadata['CacheControl']) 188 else: 189 optional_cache_control_line = '' 190 191 if 'CacheDisposition' in resource.metadata: 192 optional_content_disposition_line = resource_util.get_padded_metadata_key_value_line( 193 'Cache-Disposition', resource.metadata['CacheDisposition']) 194 else: 195 optional_content_disposition_line = '' 196 197 if 'ContentEncoding' in resource.metadata: 198 optional_content_encoding_line = resource_util.get_padded_metadata_key_value_line( 199 'Cache-Encoding', resource.metadata['ContentEncoding']) 200 else: 201 optional_content_encoding_line = '' 202 203 if 'ContentLanguage' in resource.metadata: 204 optional_content_language_line = resource_util.get_padded_metadata_key_value_line( 205 'Cache-Language', resource.metadata['ContentLanguage']) 206 else: 207 optional_content_language_line = '' 208 209 if 'PartsCount' in resource.metadata: 210 optional_component_count_line = ( 211 resource_util.get_padded_metadata_key_value_line( 212 'Component-Count', resource.metadata['PartsCount'])) 213 else: 214 optional_component_count_line = '' 215 216 if 'SSECustomerAlgorithm' in resource.metadata: 217 optional_encryption_algorithm_line = ( 218 resource_util.get_padded_metadata_key_value_line( 219 'Encryption algorithm', resource.metadata['SSECustomerAlgorithm'])) 220 else: 221 optional_encryption_algorithm_line = '' 222 223 if resource.generation: 224 optional_generation_line = resource_util.get_padded_metadata_key_value_line( 225 'Generation', resource.generation) 226 else: 227 optional_generation_line = '' 228 229 return ( 230 '{object_url}:\n' 231 '{optional_time_updated_line}' 232 '{optional_storage_class_line}' 233 '{optional_cache_control_line}' 234 '{optional_content_disposition_line}' 235 '{optional_content_encoding_line}' 236 '{optional_content_language_line}' 237 '{content_length_line}' 238 '{content_type_line}' 239 '{optional_component_count_line}' 240 '{optional_encryption_algorithm_line}' 241 '{etag_line}' 242 '{optional_generation_line}' 243 '{acl_section}' 244 ' {incomplete_warning}').format( 245 object_url=resource.storage_url.versionless_url_string, 246 optional_time_updated_line=optional_time_updated_line, 247 optional_storage_class_line=optional_storage_class_line, 248 optional_cache_control_line=optional_cache_control_line, 249 optional_content_disposition_line=optional_content_disposition_line, 250 optional_content_encoding_line=optional_content_encoding_line, 251 optional_content_language_line=optional_content_language_line, 252 content_length_line=resource_util.get_padded_metadata_key_value_line( 253 'Content-Length', resource.size), 254 content_type_line=resource_util.get_padded_metadata_key_value_line( 255 'Content-Type', resource.metadata.get('ContentType')), 256 optional_component_count_line=optional_component_count_line, 257 optional_encryption_algorithm_line=optional_encryption_algorithm_line, 258 etag_line=resource_util.get_padded_metadata_key_value_line( 259 'ETag', resource.etag), 260 optional_generation_line=optional_generation_line, 261 acl_section=_get_formatted_acl_section(resource.metadata.get('ACL')), 262 incomplete_warning=_INCOMPLETE_OBJECT_METADATA_WARNING) 263 264 265class S3BucketResource(resource_reference.BucketResource): 266 """API-specific subclass for handling metadata.""" 267 268 def get_full_metadata_string(self): 269 return _get_full_bucket_metadata_string(self) 270 271 def get_json_dump(self): 272 return _get_json_dump(self) 273 274 275class S3ObjectResource(resource_reference.ObjectResource): 276 """API-specific subclass for handling metadata.""" 277 278 def __init__(self, 279 storage_url_object, 280 creation_time=None, 281 etag=None, 282 md5_hash=None, 283 metadata=None, 284 metageneration=None, 285 size=None): 286 """Initializes resource. Args are a subset of attributes.""" 287 # The S3 API returns etag wrapped in quotes in some cases. 288 if etag and etag.startswith('"') and etag.endswith('"'): 289 etag = etag[1:-1] 290 super(S3ObjectResource, 291 self).__init__(storage_url_object, creation_time, etag, md5_hash, 292 metadata, metageneration, size) 293 294 def get_full_metadata_string(self): 295 return _get_full_object_metadata_string(self) 296 297 def get_json_dump(self): 298 return _get_json_dump(self) 299