1# -*- coding: utf-8 -*- 2# Copyright 2018 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Shared utility methods for manipulating metadata of requests and resources.""" 16 17from __future__ import absolute_import 18from __future__ import print_function 19from __future__ import division 20from __future__ import unicode_literals 21 22import six 23from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 24 25 26def AddAcceptEncodingGzipIfNeeded(headers_dict, compressed_encoding=False): 27 if compressed_encoding: 28 # If we send accept-encoding: gzip with a range request, the service 29 # may respond with the whole object, which would be bad for resuming. 30 # So only accept gzip encoding if the object we are downloading has 31 # a gzip content encoding. 32 # TODO: If we want to support compressive transcoding fully in the client, 33 # condition on whether we are requesting the entire range of the object. 34 # In this case, we can accept the first bytes of the object compressively 35 # transcoded, but we must perform data integrity checking on bytes after 36 # they are decompressed on-the-fly, and any connection break must be 37 # resumed without compressive transcoding since we cannot specify an 38 # offset. We would also need to ensure that hashes for downloaded data 39 # from objects stored with content-encoding:gzip continue to be calculated 40 # prior to our own on-the-fly decompression so they match the stored hashes. 41 headers_dict['accept-encoding'] = 'gzip' 42 43 44def CreateCustomMetadata(entries=None, custom_metadata=None): 45 """Creates a custom MetadataValue object. 46 47 Inserts the key/value pairs in entries. 48 49 Args: 50 entries: (Dict[str, Any] or None) The dictionary containing key/value pairs 51 to insert into metadata. Both the key and value must be able to be 52 casted to a string type. 53 custom_metadata (apitools_messages.Object.MetadataValue or None): A 54 pre-existing custom metadata object to add to. If one is not provided, 55 a new one will be constructed. 56 57 Returns: 58 An apitools_messages.Object.MetadataValue. 59 """ 60 if custom_metadata is None: 61 custom_metadata = apitools_messages.Object.MetadataValue( 62 additionalProperties=[]) 63 if entries is None: 64 entries = {} 65 for key, value in six.iteritems(entries): 66 custom_metadata.additionalProperties.append( 67 apitools_messages.Object.MetadataValue.AdditionalProperty( 68 key=str(key), value=str(value))) 69 return custom_metadata 70 71 72def GetValueFromObjectCustomMetadata(obj_metadata, 73 search_key, 74 default_value=None): 75 """Filters a specific element out of an object's custom metadata. 76 77 Args: 78 obj_metadata: (apitools_messages.Object) The metadata for an object. 79 search_key: (str) The custom metadata key to search for. 80 default_value: (Any) The default value to use for the key if it cannot be 81 found. 82 83 Returns: 84 (Tuple(bool, Any)) A tuple indicating if the value could be found in 85 metadata and a value corresponding to search_key (the value at the specified 86 key in custom metadata, or the default value if the specified key does not 87 exist in the custom metadata). 88 """ 89 try: 90 value = next((attr.value 91 for attr in obj_metadata.metadata.additionalProperties 92 if attr.key == search_key), None) 93 if value is None: 94 return False, default_value 95 return True, value 96 except AttributeError: 97 return False, default_value 98 99 100def IsCustomMetadataHeader(header): 101 """Returns true if header (which must be lowercase) is a custom header.""" 102 return header.startswith('x-goog-meta-') or header.startswith('x-amz-meta-') 103 104 105def ObjectIsGzipEncoded(obj_metadata): 106 """Returns true if the apitools_messages.Object has gzip content-encoding.""" 107 return (obj_metadata.contentEncoding and 108 obj_metadata.contentEncoding.lower().endswith('gzip')) 109