1# -*- coding: utf-8 -*- # 2# Copyright 2018 Google LLC. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Wrapper module for ensuring consistent usage of yaml parsing. 17 18This module forces parsing to use version 1.1 of the YAML spec if not 19otherwise specified by the loading method arguments. 20However, dumping uses version 1.2. 21It also prevents use of unsafe loading and dumping. 22""" 23 24from __future__ import absolute_import 25from __future__ import division 26from __future__ import unicode_literals 27 28import collections 29 30from googlecloudsdk.core import exceptions 31from googlecloudsdk.core import yaml_location_value 32from googlecloudsdk.core.util import files 33 34from ruamel import yaml 35import six 36 37try: 38 # Python 3.3 and above. 39 collections_abc = collections.abc 40except AttributeError: 41 collections_abc = collections 42 43 44VERSION_1_1 = '1.1' 45VERSION_1_2 = '1.2' 46 47 48# YAML unfortunately uses a bunch of global class state for this kind of stuff. 49# We don't have to do it at import but the other option would be to do it every 50# time we try to dump something (which is worse for performance that just 51# doing it once). This allows OrderedDicts to be serialized as if they were 52# normal dicts. 53yaml.add_representer( 54 collections.OrderedDict, 55 yaml.dumper.SafeRepresenter.represent_dict, 56 Dumper=yaml.dumper.SafeDumper) 57yaml.add_representer( 58 collections.OrderedDict, 59 yaml.dumper.RoundTripRepresenter.represent_dict, 60 Dumper=yaml.dumper.RoundTripDumper) 61 62 63# Always output None as "null", instead of just empty. 64yaml.add_representer( 65 type(None), 66 lambda self, _: self.represent_scalar('tag:yaml.org,2002:null', 'null'), 67 Dumper=yaml.dumper.RoundTripDumper) 68 69 70class Error(exceptions.Error): 71 """Top level error for this module. 72 73 Attributes: 74 inner_error: Exception, The original exception that is being wrapped. This 75 will always be populated. 76 file: str, The path to the thing being loaded (if applicable). This is not 77 necessarily a literal file (it could be a URL or any hint the calling 78 code passes in). It should only be used for more descriptive error 79 messages. 80 """ 81 82 def __init__(self, e, verb, f=None): 83 file_text = ' from [{}]'.format(f) if f else '' 84 super(Error, self).__init__( 85 'Failed to {} YAML{}: {}'.format(verb, file_text, e)) 86 self.inner_error = e 87 self.file = f 88 89 90class YAMLParseError(Error): 91 """An error that wraps all YAML parsing errors.""" 92 93 def __init__(self, e, f=None): 94 super(YAMLParseError, self).__init__(e, verb='parse', f=f) 95 96 97class FileLoadError(Error): 98 """An error that wraps errors when loading/reading files.""" 99 100 def __init__(self, e, f): 101 super(FileLoadError, self).__init__(e, verb='load', f=f) 102 103 104def load(stream, 105 file_hint=None, 106 round_trip=False, 107 location_value=False, 108 version=VERSION_1_1): 109 """Loads YAML from the given steam. 110 111 Args: 112 stream: A file like object or string that can be read from. 113 file_hint: str, The name of a file or url that the stream data is coming 114 from. This is used for better error handling. If you have the actual file, 115 you should use load_file() instead. Sometimes the file cannot be read 116 directly so you can use a stream here and hint as to where the data is 117 coming from. 118 round_trip: bool, True to use the RoundTripLoader which preserves ordering 119 and line numbers. 120 location_value: bool, True to use a loader that preserves ordering and line 121 numbers for all values. Each YAML data item is an object with value and 122 lc attributes, where lc.line and lc.col are the line and column location 123 for the item in the YAML source file. 124 version: str, YAML version to use when parsing. 125 126 Raises: 127 YAMLParseError: If the data could not be parsed. 128 129 Returns: 130 The parsed YAML data. 131 """ 132 try: 133 if location_value: 134 return yaml_location_value.LocationValueLoad(stream) 135 loader = yaml.RoundTripLoader if round_trip else yaml.SafeLoader 136 return yaml.load(stream, loader, version=version) 137 except yaml.YAMLError as e: 138 raise YAMLParseError(e, f=file_hint) 139 140 141def load_all(stream, file_hint=None, version=VERSION_1_1, round_trip=False): 142 """Loads multiple YAML documents from the given steam. 143 144 Args: 145 stream: A file like object or string that can be read from. 146 file_hint: str, The name of a file or url that the stream data is coming 147 from. See load() for more information. 148 version: str, YAML version to use when parsing. 149 round_trip: bool, True to use the RoundTripLoader which preserves ordering 150 and line numbers. 151 152 Raises: 153 YAMLParseError: If the data could not be parsed. 154 155 Yields: 156 The parsed YAML data. 157 """ 158 loader = yaml.RoundTripLoader if round_trip else yaml.SafeLoader 159 try: 160 for x in yaml.load_all(stream, loader, version=version): 161 yield x 162 except yaml.YAMLError as e: 163 raise YAMLParseError(e, f=file_hint) 164 165 166def load_path(path, 167 round_trip=False, 168 location_value=False, 169 version=VERSION_1_1): 170 """Loads YAML from the given file path. 171 172 Args: 173 path: str, A file path to open and read from. 174 round_trip: bool, True to use the RoundTripLoader which preserves ordering 175 and line numbers. 176 location_value: bool, True to use a loader that preserves ordering and line 177 numbers for all values. Each YAML data item is an object with value and 178 lc attributes, where lc.line and lc.col are the line and column location 179 for the item in the YAML source file. 180 version: str, YAML version to use when parsing. 181 182 Raises: 183 YAMLParseError: If the data could not be parsed. 184 FileLoadError: If the file could not be opened or read. 185 186 Returns: 187 The parsed YAML data. 188 """ 189 try: 190 with files.FileReader(path) as fp: 191 return load( 192 fp, 193 file_hint=path, 194 round_trip=round_trip, 195 location_value=location_value, 196 version=version) 197 except files.Error as e: 198 raise FileLoadError(e, f=path) 199 200 201def load_all_path(path, version=VERSION_1_1, round_trip=False): 202 """Loads multiple YAML documents from the given file path. 203 204 Args: 205 path: str, A file path to open and read from. 206 version: str, YAML version to use when parsing. 207 round_trip: bool, True to use the RoundTripLoader which preserves ordering 208 and line numbers. 209 210 Raises: 211 YAMLParseError: If the data could not be parsed. 212 FileLoadError: If the file could not be opened or read. 213 214 Yields: 215 The parsed YAML data. 216 """ 217 try: 218 with files.FileReader(path) as fp: 219 for x in load_all(fp, 220 file_hint=path, 221 version=version, 222 round_trip=round_trip): 223 yield x 224 except files.Error as e: 225 # EnvironmentError is parent of IOError, OSError and WindowsError. 226 # Raised when file does not exist or can't be opened/read. 227 raise FileLoadError(e, f=path) 228 229 230def dump(data, stream=None, round_trip=False, **kwargs): 231 """Dumps the given YAML data to the stream. 232 233 Args: 234 data: The YAML serializable Python object to dump. 235 stream: The stream to write the data to or None to return it as a string. 236 round_trip: bool, True to use the RoundTripDumper which preserves ordering 237 and line numbers if the yaml was loaded in round trip mode. 238 **kwargs: Other arguments to the dump method. 239 240 Returns: 241 The string representation of the YAML data if stream is None. 242 """ 243 method = yaml.round_trip_dump if round_trip else yaml.safe_dump 244 return method(data, stream=stream, default_flow_style=False, indent=2, 245 **kwargs) 246 247 248def dump_all(documents, stream=None, **kwargs): 249 """Dumps multiple YAML documents to the stream. 250 251 Args: 252 documents: An iterable of YAML serializable Python objects to dump. 253 stream: The stream to write the data to or None to return it as a string. 254 **kwargs: Other arguments to the dump method. 255 256 Returns: 257 The string representation of the YAML data if stream is None. 258 """ 259 return yaml.safe_dump_all( 260 documents, stream=stream, default_flow_style=False, indent=2, **kwargs) 261 262 263def dump_all_round_trip(documents, stream=None, **kwargs): 264 """Dumps multiple YAML documents to the stream using the RoundTripDumper. 265 266 Args: 267 documents: An iterable of YAML serializable Python objects to dump. 268 stream: The stream to write the data to or None to return it as a string. 269 **kwargs: Other arguments to the dump method. 270 271 Returns: 272 The string representation of the YAML data if stream is None. 273 """ 274 return yaml.dump_all( 275 documents, stream=stream, default_flow_style=False, indent=2, 276 Dumper=yaml.RoundTripDumper, **kwargs) 277 278 279def convert_to_block_text(data): 280 r"""This processes the given dict or list so it will render as block text. 281 282 By default, the yaml dumper will write multiline strings out as a double 283 quoted string that just includes '\n'. Calling this on the data strucuture 284 will make it use the '|-' notation. 285 286 Args: 287 data: {} or [], The data structure to process. 288 """ 289 yaml.scalarstring.walk_tree(data) 290 291 292def list_like(item): 293 """Return True if the item is like a list: a MutableSequence.""" 294 return isinstance(item, collections_abc.MutableSequence) 295 296 297def dict_like(item): 298 """Return True if the item is like a dict: a MutableMapping.""" 299 return isinstance(item, collections_abc.MutableMapping) 300 301 302def strip_locations(obj): 303 if list_like(obj): 304 return [strip_locations(item) for item in obj] 305 if dict_like(obj): 306 return {key: strip_locations(value) for key, value in six.iteritems(obj)} 307 return obj.value 308